diff --git a/.BUILDING.txt.swp b/.BUILDING.txt.swp new file mode 100644 index 0000000000000..1fb0c25d0a516 Binary files /dev/null and b/.BUILDING.txt.swp differ diff --git a/BUILDING.txt b/BUILDING.txt index edf47c5f1337a..5f40a0d7dc3b3 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -57,7 +57,7 @@ Refer to dev-support/docker/Dockerfile): * Open JDK 1.8 $ sudo apt-get update - $ sudo apt-get -y install java-8-openjdk + $ sudo apt-get -y install openjdk-8-jdk * Maven $ sudo apt-get -y install maven * Native libraries diff --git a/LICENSE-binary b/LICENSE-binary index a9faf26db5a78..fe60ac3609c2c 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -214,16 +214,16 @@ com.aliyun:aliyun-java-sdk-core:3.4.0 com.aliyun:aliyun-java-sdk-ecs:4.2.0 com.aliyun:aliyun-java-sdk-ram:3.0.0 com.aliyun:aliyun-java-sdk-sts:3.0.0 -com.aliyun.oss:aliyun-sdk-oss:3.13.0 +com.aliyun.oss:aliyun-sdk-oss:3.13.2 com.amazonaws:aws-java-sdk-bundle:1.11.901 com.cedarsoftware:java-util:1.9.0 com.cedarsoftware:json-io:2.5.1 -com.fasterxml.jackson.core:jackson-annotations:2.9.9 -com.fasterxml.jackson.core:jackson-core:2.9.9 -com.fasterxml.jackson.core:jackson-databind:2.9.9.2 -com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.9.9 -com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.9.9 -com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.9.9 +com.fasterxml.jackson.core:jackson-annotations:2.13.2 +com.fasterxml.jackson.core:jackson-core:2.13.2 +com.fasterxml.jackson.core:jackson-databind:2.13.2.2 +com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.13.2 +com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.13.2 +com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.13.2 com.fasterxml.uuid:java-uuid-generator:3.1.4 com.fasterxml.woodstox:woodstox-core:5.3.0 com.github.davidmoten:rxjava-extras:0.8.0.17 @@ -283,7 +283,7 @@ log4j:log4j:1.2.17 net.java.dev.jna:jna:5.2.0 net.minidev:accessors-smart:1.2 net.minidev:json-smart:2.4.7 -org.apache.avro:avro:1.7.7 +org.apache.avro:avro:1.9.2 org.apache.commons:commons-collections4:4.2 org.apache.commons:commons-compress:1.21 org.apache.commons:commons-configuration2:2.1.1 @@ -297,10 +297,10 @@ org.apache.curator:curator-client:5.2.0 org.apache.curator:curator-framework:5.2.0 org.apache.curator:curator-recipes:5.2.0 org.apache.geronimo.specs:geronimo-jcache_1.0_spec:1.0-alpha-1 -org.apache.hbase:hbase-annotations:1.4.8 -org.apache.hbase:hbase-client:1.4.8 -org.apache.hbase:hbase-common:1.4.8 -org.apache.hbase:hbase-protocol:1.4.8 +org.apache.hbase:hbase-annotations:1.7.1 +org.apache.hbase:hbase-client:1.7.1 +org.apache.hbase:hbase-common:1.7.1 +org.apache.hbase:hbase-protocol:1.7.1 org.apache.htrace:htrace-core:3.1.0-incubating org.apache.htrace:htrace-core4:4.1.0-incubating org.apache.httpcomponents:httpclient:4.5.6 @@ -321,27 +321,24 @@ org.apache.kerby:kerby-pkix:1.0.1 org.apache.kerby:kerby-util:1.0.1 org.apache.kerby:kerby-xdr:1.0.1 org.apache.kerby:token-provider:1.0.1 +org.apache.solr:solr-solrj:8.8.2 org.apache.yetus:audience-annotations:0.5.0 org.apache.zookeeper:zookeeper:3.6.3 -org.codehaus.jackson:jackson-core-asl:1.9.13 -org.codehaus.jackson:jackson-jaxrs:1.9.13 -org.codehaus.jackson:jackson-mapper-asl:1.9.13 -org.codehaus.jackson:jackson-xc:1.9.13 org.codehaus.jettison:jettison:1.1 -org.eclipse.jetty:jetty-annotations:9.3.27.v20190418 -org.eclipse.jetty:jetty-http:9.3.27.v20190418 -org.eclipse.jetty:jetty-io:9.3.27.v20190418 -org.eclipse.jetty:jetty-jndi:9.3.27.v20190418 -org.eclipse.jetty:jetty-plus:9.3.27.v20190418 -org.eclipse.jetty:jetty-security:9.3.27.v20190418 -org.eclipse.jetty:jetty-server:9.3.27.v20190418 -org.eclipse.jetty:jetty-servlet:9.3.27.v20190418 -org.eclipse.jetty:jetty-util:9.3.27.v20190418 -org.eclipse.jetty:jetty-util-ajax:9.3.27.v20190418 -org.eclipse.jetty:jetty-webapp:9.3.27.v20190418 -org.eclipse.jetty:jetty-xml:9.3.27.v20190418 -org.eclipse.jetty.websocket:javax-websocket-client-impl:9.3.27.v20190418 -org.eclipse.jetty.websocket:javax-websocket-server-impl:9.3.27.v20190418 +org.eclipse.jetty:jetty-annotations:9.4.44.v20210927 +org.eclipse.jetty:jetty-http:9.4.44.v20210927 +org.eclipse.jetty:jetty-io:9.4.44.v20210927 +org.eclipse.jetty:jetty-jndi:9.4.44.v20210927 +org.eclipse.jetty:jetty-plus:9.4.44.v20210927 +org.eclipse.jetty:jetty-security:9.4.44.v20210927 +org.eclipse.jetty:jetty-server:9.4.44.v20210927 +org.eclipse.jetty:jetty-servlet:9.4.44.v20210927 +org.eclipse.jetty:jetty-util:9.4.44.v20210927 +org.eclipse.jetty:jetty-util-ajax:9.4.44.v20210927 +org.eclipse.jetty:jetty-webapp:9.4.44.v20210927 +org.eclipse.jetty:jetty-xml:9.4.44.v20210927 +org.eclipse.jetty.websocket:javax-websocket-client-impl:9.4.44.v20210927 +org.eclipse.jetty.websocket:javax-websocket-server-impl:9.4.44.v20210927 org.ehcache:ehcache:3.3.1 org.lz4:lz4-java:1.7.1 org.objenesis:objenesis:2.6 @@ -404,7 +401,7 @@ hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.css hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.js hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-full-2.0.0.min.js hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-helpers-1.1.1.min.js -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-3.5.1.min.js +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-3.6.0.min.js hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery.dataTables.min.js hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/moment.min.js hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/bootstrap.min.js @@ -467,8 +464,8 @@ com.microsoft.azure:azure-cosmosdb-gateway:2.4.5 com.microsoft.azure:azure-data-lake-store-sdk:2.3.3 com.microsoft.azure:azure-keyvault-core:1.0.0 com.microsoft.sqlserver:mssql-jdbc:6.2.1.jre7 -org.bouncycastle:bcpkix-jdk15on:1.60 -org.bouncycastle:bcprov-jdk15on:1.60 +org.bouncycastle:bcpkix-jdk15on:1.68 +org.bouncycastle:bcprov-jdk15on:1.68 org.checkerframework:checker-qual:2.5.2 org.codehaus.mojo:animal-sniffer-annotations:1.17 org.jruby.jcodings:jcodings:1.0.13 @@ -483,17 +480,18 @@ org.slf4j:slf4j-log4j12:1.7.25 CDDL 1.1 + GPLv2 with classpath exception ----------------------------------------- -com.sun.jersey:jersey-client:1.19 -com.sun.jersey:jersey-core:1.19 -com.sun.jersey:jersey-guice:1.19 -com.sun.jersey:jersey-json:1.19 -com.sun.jersey:jersey-server:1.19 -com.sun.jersey:jersey-servlet:1.19 +com.github.pjfanning:jersey-json:1.20 +com.sun.jersey:jersey-client:1.19.4 +com.sun.jersey:jersey-core:1.19.4 +com.sun.jersey:jersey-guice:1.19.4 +com.sun.jersey:jersey-server:1.19.4 +com.sun.jersey:jersey-servlet:1.19.4 com.sun.xml.bind:jaxb-impl:2.2.3-1 javax.annotation:javax.annotation-api:1.3.2 javax.servlet:javax.servlet-api:3.1.0 javax.servlet.jsp:jsp-api:2.1 javax.websocket:javax.websocket-api:1.0 +javax.ws.rs:javax.ws.rs-api:2.1.1 javax.ws.rs:jsr311-api:1.1.1 javax.xml.bind:jaxb-api:2.2.11 @@ -513,7 +511,7 @@ org.hsqldb:hsqldb:2.3.4 JDOM License ------------ -org.jdom:jdom:1.1 +org.jdom:jdom2:2.0.6.1 Public Domain diff --git a/LICENSE.txt b/LICENSE.txt index 3c079898b9071..763cf2ce53f4d 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -245,7 +245,7 @@ hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.css hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.js hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-full-2.0.0.min.js hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-helpers-1.1.1.min.js -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-3.5.1.min.js +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-3.6.0.min.js hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery.dataTables.min.js hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/moment.min.js hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/bootstrap.min.js diff --git a/NOTICE-binary b/NOTICE-binary index 2f8a9241a8d00..b96e052658876 100644 --- a/NOTICE-binary +++ b/NOTICE-binary @@ -66,7 +66,7 @@ available from http://www.digip.org/jansson/. AWS SDK for Java -Copyright 2010-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. +Copyright 2010-2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. This product includes software developed by Amazon Technologies, Inc (http://www.amazon.com/). diff --git a/dev-support/bin/create-release b/dev-support/bin/create-release index 31ae6ee1b0659..693b41c4f3910 100755 --- a/dev-support/bin/create-release +++ b/dev-support/bin/create-release @@ -293,6 +293,7 @@ function usage echo "--security Emergency security release" echo "--sign Use .gnupg dir to sign the artifacts and jars" echo "--version=[version] Use an alternative version string" + echo "--mvnargs=[args] Extra Maven args to be provided when running mvn commands" } function option_parse @@ -347,6 +348,9 @@ function option_parse --version=*) HADOOP_VERSION=${i#*=} ;; + --mvnargs=*) + MVNEXTRAARGS=${i#*=} + ;; esac done @@ -413,6 +417,9 @@ function option_parse MVN_ARGS=("-Dmaven.repo.local=${MVNCACHE}") fi fi + if [ -n "$MVNEXTRAARGS" ]; then + MVN_ARGS+=("$MVNEXTRAARGS") + fi if [[ "${SECURITYRELEASE}" = true ]]; then if [[ ! -d "${BASEDIR}/hadoop-common-project/hadoop-common/src/site/markdown/release/${HADOOP_VERSION}" ]]; then @@ -535,6 +542,10 @@ function makearelease big_console_header "Cleaning the Source Tree" + # Since CVE-2022-24765 in April 2022, git refuses to work in directories + # whose owner != the current user, unless explicitly told to trust it. + git config --global --add safe.directory /build/source + # git clean to clear any remnants from previous build run "${GIT}" clean -xdf -e /patchprocess diff --git a/dev-support/docker/Dockerfile_centos_8 b/dev-support/docker/Dockerfile_centos_8 index 7b82c4997dee6..8f3b008f7ba03 100644 --- a/dev-support/docker/Dockerfile_centos_8 +++ b/dev-support/docker/Dockerfile_centos_8 @@ -30,6 +30,13 @@ COPY pkg-resolver pkg-resolver RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \ && chmod a+r pkg-resolver/*.json +###### +# Centos 8 has reached its EOL and the packages +# are no longer available on mirror.centos.org site. +# Please see https://www.centos.org/centos-linux-eol/ +###### +RUN pkg-resolver/set-vault-as-baseurl-centos.sh centos:8 + ###### # Install packages from yum ###### diff --git a/dev-support/docker/pkg-resolver/set-vault-as-baseurl-centos.sh b/dev-support/docker/pkg-resolver/set-vault-as-baseurl-centos.sh new file mode 100644 index 0000000000000..4be4cd956b15b --- /dev/null +++ b/dev-support/docker/pkg-resolver/set-vault-as-baseurl-centos.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; then + echo "ERROR: Need at least 1 argument, $# were provided" + exit 1 +fi + +if [ "$1" == "centos:7" ] || [ "$1" == "centos:8" ]; then + cd /etc/yum.repos.d/ || exit && + sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* && + sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* && + yum update -y && + cd /root || exit +else + echo "ERROR: Setting the archived baseurl is only supported for centos 7 and 8 environments" + exit 1 +fi diff --git a/dev-support/git-jira-validation/README.md b/dev-support/git-jira-validation/README.md new file mode 100644 index 0000000000000..308c54228d17c --- /dev/null +++ b/dev-support/git-jira-validation/README.md @@ -0,0 +1,134 @@ + + +Apache Hadoop Git/Jira FixVersion validation +============================================================ + +Git commits in Apache Hadoop contains Jira number of the format +HADOOP-XXXX or HDFS-XXXX or YARN-XXXX or MAPREDUCE-XXXX. +While creating a release candidate, we also include changelist +and this changelist can be identified based on Fixed/Closed Jiras +with the correct fix versions. However, sometimes we face few +inconsistencies between fixed Jira and Git commit message. + +git_jira_fix_version_check.py script takes care of +identifying all git commits with commit +messages with any of these issues: + +1. commit is reverted as per commit message +2. commit does not contain Jira number format in message +3. Jira does not have expected fixVersion +4. Jira has expected fixVersion, but it is not yet resolved + +Moreover, this script also finds any resolved Jira with expected +fixVersion but without any corresponding commit present. + +This should be useful as part of RC preparation. + +git_jira_fix_version_check supports python3 and it required +installation of jira: + +``` +$ python3 --version +Python 3.9.7 + +$ python3 -m venv ./venv + +$ ./venv/bin/pip install -r dev-support/git-jira-validation/requirements.txt + +$ ./venv/bin/python dev-support/git-jira-validation/git_jira_fix_version_check.py + +``` + +The script also requires below inputs: +``` +1. First commit hash to start excluding commits from history: + Usually we can provide latest commit hash from last tagged release + so that the script will only loop through all commits in git commit + history before this commit hash. e.g for 3.3.2 release, we can provide + git hash: fa4915fdbbbec434ab41786cb17b82938a613f16 + because this commit bumps up hadoop pom versions to 3.3.2: + https://github.com/apache/hadoop/commit/fa4915fdbbbec434ab41786cb17b82938a613f16 + +2. Fix Version: + Exact fixVersion that we would like to compare all Jira's fixVersions + with. e.g for 3.3.2 release, it should be 3.3.2. + +3. JIRA Project Name: + The exact name of Project as case-sensitive e.g HADOOP / OZONE + +4. Path of project's working dir with release branch checked-in: + Path of project from where we want to compare git hashes from. Local fork + of the project should be up-to date with upstream and expected release + branch should be checked-in. + +5. Jira server url (default url: https://issues.apache.org/jira): + Default value of server points to ASF Jiras but this script can be + used outside of ASF Jira too. +``` + + +Example of script execution: +``` +JIRA Project Name (e.g HADOOP / OZONE etc): HADOOP +First commit hash to start excluding commits from history: fa4915fdbbbec434ab41786cb17b82938a613f16 +Fix Version: 3.3.2 +Jira server url (default: https://issues.apache.org/jira): +Path of project's working dir with release branch checked-in: /Users/vjasani/Documents/src/hadoop-3.3/hadoop + +Check git status output and verify expected branch + +On branch branch-3.3.2 +Your branch is up to date with 'origin/branch-3.3.2'. + +nothing to commit, working tree clean + + +Jira/Git commit message diff starting: ############################################## +Jira not present with version: 3.3.2. Commit: 8cd8e435fb43a251467ca74fadcb14f21a3e8163 HADOOP-17198. Support S3 Access Points (#3260) (branch-3.3.2) (#3955) +WARN: Jira not found. Commit: 8af28b7cca5c6020de94e739e5373afc69f399e5 Updated the index as per 3.3.2 release +WARN: Jira not found. Commit: e42e483d0085aa46543ebcb1196dd155ddb447d0 Make upstream aware of 3.3.1 release +Commit seems reverted. Commit: 6db1165380cd308fb74c9d17a35c1e57174d1e09 Revert "HDFS-14099. Unknown frame descriptor when decompressing multiple frames (#3836)" +Commit seems reverted. Commit: 1e3f94fa3c3d4a951d4f7438bc13e6f008f228f4 Revert "HDFS-16333. fix balancer bug when transfer an EC block (#3679)" +Jira not present with version: 3.3.2. Commit: ce0bc7b473a62a580c1227a4de6b10b64b045d3a HDFS-16344. Improve DirectoryScanner.Stats#toString (#3695) +Jira not present with version: 3.3.2. Commit: 30f0629d6e6f735c9f4808022f1a1827c5531f75 HDFS-16339. Show the threshold when mover threads quota is exceeded (#3689) +Jira not present with version: 3.3.2. Commit: e449daccf486219e3050254d667b74f92e8fc476 YARN-11007. Correct words in YARN documents (#3680) +Commit seems reverted. Commit: 5c189797828e60a3329fd920ecfb99bcbccfd82d Revert "HDFS-16336. Addendum: De-flake TestRollingUpgrade#testRollback (#3686)" +Jira not present with version: 3.3.2. Commit: 544dffd179ed756bc163e4899e899a05b93d9234 HDFS-16171. De-flake testDecommissionStatus (#3280) +Jira not present with version: 3.3.2. Commit: c6914b1cb6e4cab8263cd3ae5cc00bc7a8de25de HDFS-16350. Datanode start time should be set after RPC server starts successfully (#3711) +Jira not present with version: 3.3.2. Commit: 328d3b84dfda9399021ccd1e3b7afd707e98912d HDFS-16336. Addendum: De-flake TestRollingUpgrade#testRollback (#3686) +Jira not present with version: 3.3.2. Commit: 3ae8d4ccb911c9ababd871824a2fafbb0272c016 HDFS-16336. De-flake TestRollingUpgrade#testRollback (#3686) +Jira not present with version: 3.3.2. Commit: 15d3448e25c797b7d0d401afdec54683055d4bb5 HADOOP-17975. Fallback to simple auth does not work for a secondary DistributedFileSystem instance. (#3579) +Jira not present with version: 3.3.2. Commit: dd50261219de71eaa0a1ad28529953e12dfb92e0 YARN-10991. Fix to ignore the grouping "[]" for resourcesStr in parseResourcesString method (#3592) +Jira not present with version: 3.3.2. Commit: ef462b21bf03b10361d2f9ea7b47d0f7360e517f HDFS-16332. Handle invalid token exception in sasl handshake (#3677) +WARN: Jira not found. Commit: b55edde7071419410ea5bea4ce6462b980e48f5b Also update hadoop.version to 3.3.2 +... +... +... +Found first commit hash after which git history is redundant. commit: fa4915fdbbbec434ab41786cb17b82938a613f16 +Exiting successfully +Jira/Git commit message diff completed: ############################################## + +Any resolved Jira with fixVersion 3.3.2 but corresponding commit not present +Starting diff: ############################################## +HADOOP-18066 is marked resolved with fixVersion 3.3.2 but no corresponding commit found +HADOOP-17936 is marked resolved with fixVersion 3.3.2 but no corresponding commit found +Completed diff: ############################################## + + +``` + diff --git a/dev-support/git-jira-validation/git_jira_fix_version_check.py b/dev-support/git-jira-validation/git_jira_fix_version_check.py new file mode 100644 index 0000000000000..c2e12a13aae22 --- /dev/null +++ b/dev-support/git-jira-validation/git_jira_fix_version_check.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +############################################################################ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +############################################################################ +"""An application to assist Release Managers with ensuring that histories in +Git and fixVersions in JIRA are in agreement. See README.md for a detailed +explanation. +""" + + +import os +import re +import subprocess + +from jira import JIRA + +jira_project_name = input("JIRA Project Name (e.g HADOOP / OZONE etc): ") \ + or "HADOOP" +# Define project_jira_keys with - appended. e.g for HADOOP Jiras, +# project_jira_keys should include HADOOP-, HDFS-, YARN-, MAPREDUCE- +project_jira_keys = [jira_project_name + '-'] +if jira_project_name == 'HADOOP': + project_jira_keys.append('HDFS-') + project_jira_keys.append('YARN-') + project_jira_keys.append('MAPREDUCE-') + +first_exclude_commit_hash = input("First commit hash to start excluding commits from history: ") +fix_version = input("Fix Version: ") + +jira_server_url = input( + "Jira server url (default: https://issues.apache.org/jira): ") \ + or "https://issues.apache.org/jira" + +jira = JIRA(server=jira_server_url) + +local_project_dir = input("Path of project's working dir with release branch checked-in: ") +os.chdir(local_project_dir) + +GIT_STATUS_MSG = subprocess.check_output(['git', 'status']).decode("utf-8") +print('\nCheck git status output and verify expected branch\n') +print(GIT_STATUS_MSG) + +print('\nJira/Git commit message diff starting: ##############################################') + +issue_set_from_commit_msg = set() + +for commit in subprocess.check_output(['git', 'log', '--pretty=oneline']).decode( + "utf-8").splitlines(): + if commit.startswith(first_exclude_commit_hash): + print("Found first commit hash after which git history is redundant. commit: " + + first_exclude_commit_hash) + print("Exiting successfully") + break + if re.search('revert', commit, re.IGNORECASE): + print("Commit seems reverted. \t\t\t Commit: " + commit) + continue + ACTUAL_PROJECT_JIRA = None + for project_jira in project_jira_keys: + if project_jira in commit: + ACTUAL_PROJECT_JIRA = project_jira + break + if not ACTUAL_PROJECT_JIRA: + print("WARN: Jira not found. \t\t\t Commit: " + commit) + continue + JIRA_NUM = '' + for c in commit.split(ACTUAL_PROJECT_JIRA)[1]: + if c.isdigit(): + JIRA_NUM = JIRA_NUM + c + else: + break + issue = jira.issue(ACTUAL_PROJECT_JIRA + JIRA_NUM) + EXPECTED_FIX_VERSION = False + for version in issue.fields.fixVersions: + if version.name == fix_version: + EXPECTED_FIX_VERSION = True + break + if not EXPECTED_FIX_VERSION: + print("Jira not present with version: " + fix_version + ". \t Commit: " + commit) + continue + if issue.fields.status is None or issue.fields.status.name not in ('Resolved', 'Closed'): + print("Jira is not resolved yet? \t\t Commit: " + commit) + else: + # This means Jira corresponding to current commit message is resolved with expected + # fixVersion. + # This is no-op by default, if needed, convert to print statement. + issue_set_from_commit_msg.add(ACTUAL_PROJECT_JIRA + JIRA_NUM) + +print('Jira/Git commit message diff completed: ##############################################') + +print('\nAny resolved Jira with fixVersion ' + fix_version + + ' but corresponding commit not present') +print('Starting diff: ##############################################') +all_issues_with_fix_version = jira.search_issues( + 'project=' + jira_project_name + ' and status in (Resolved,Closed) and fixVersion=' + + fix_version) + +for issue in all_issues_with_fix_version: + if issue.key not in issue_set_from_commit_msg: + print(issue.key + ' is marked resolved with fixVersion ' + fix_version + + ' but no corresponding commit found') + +print('Completed diff: ##############################################') diff --git a/dev-support/git-jira-validation/requirements.txt b/dev-support/git-jira-validation/requirements.txt new file mode 100644 index 0000000000000..ae7535a119fa9 --- /dev/null +++ b/dev-support/git-jira-validation/requirements.txt @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +jira==3.1.1 diff --git a/dev-support/hadoop-vote.sh b/dev-support/hadoop-vote.sh new file mode 100755 index 0000000000000..3d381fb0b4be2 --- /dev/null +++ b/dev-support/hadoop-vote.sh @@ -0,0 +1,201 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script is useful to perform basic sanity tests for the given +# Hadoop RC. It checks for the Checksum, Signature, Rat check, +# Build from source and building tarball from the source. + +set -e -o pipefail + +usage() { + SCRIPT=$(basename "${BASH_SOURCE[@]}") + + cat << __EOF +hadoop-vote. A script for standard vote which verifies the following items +1. Checksum of sources and binaries +2. Signature of sources and binaries +3. Rat check +4. Built from source +5. Built tar from source + +Usage: ${SCRIPT} -s | --source [-k | --key ] [-f | --keys-file-url ] [-o | --output-dir ] [-D property[=value]] [-P profiles] + ${SCRIPT} -h | --help + + -h | --help Show this screen. + -s | --source '' A URL pointing to the release candidate sources and binaries + e.g. https://dist.apache.org/repos/dist/dev/hadoop/hadoop-RC0/ + -k | --key '' A signature of the public key, e.g. 9AD2AE49 + -f | --keys-file-url '' the URL of the key file, default is + https://downloads.apache.org/hadoop/common/KEYS + -o | --output-dir '' directory which has the stdout and stderr of each verification target + -D | list of maven properties to set for the mvn invocations, e.g. <-D hbase.profile=2.0 -D skipTests> Defaults to unset + -P | list of maven profiles to set for the build from source, e.g. <-P native -P yarn-ui> +__EOF +} + +MVN_PROPERTIES=() +MVN_PROFILES=() + +while ((${#})); do + case "${1}" in + -h | --help ) + usage; exit 0 ;; + -s | --source ) + SOURCE_URL="${2}"; shift 2 ;; + -k | --key ) + SIGNING_KEY="${2}"; shift 2 ;; + -f | --keys-file-url ) + KEY_FILE_URL="${2}"; shift 2 ;; + -o | --output-dir ) + OUTPUT_DIR="${2}"; shift 2 ;; + -D ) + MVN_PROPERTIES+=("-D ${2}"); shift 2 ;; + -P ) + MVN_PROFILES+=("-P ${2}"); shift 2 ;; + * ) + usage >&2; exit 1 ;; + esac +done + +# Source url must be provided +if [ -z "${SOURCE_URL}" ]; then + usage; + exit 1 +fi + +cat << __EOF +Although This tool helps verifying Hadoop RC build and unit tests, +operator may still consider verifying the following manually: +1. Verify the API compatibility report +2. Integration/performance/benchmark tests +3. Object store specific Integration tests against an endpoint +4. Verify overall unit test stability from Jenkins builds or locally +5. Other concerns if any +__EOF + +[[ "${SOURCE_URL}" != */ ]] && SOURCE_URL="${SOURCE_URL}/" +HADOOP_RC_VERSION=$(tr "/" "\n" <<< "${SOURCE_URL}" | tail -n2) +HADOOP_VERSION=$(echo "${HADOOP_RC_VERSION}" | sed -e 's/-RC[0-9]//g' | sed -e 's/hadoop-//g') +JAVA_VERSION=$(java -version 2>&1 | cut -f3 -d' ' | head -n1 | sed -e 's/"//g') +OUTPUT_DIR="${OUTPUT_DIR:-$(pwd)}" + +if [ ! -d "${OUTPUT_DIR}" ]; then + echo "Output directory ${OUTPUT_DIR} does not exist, please create it before running this script." + exit 1 +fi + +OUTPUT_PATH_PREFIX="${OUTPUT_DIR}"/"${HADOOP_RC_VERSION}" + +# default value for verification targets, 0 = failed +SIGNATURE_PASSED=0 +CHECKSUM_PASSED=0 +RAT_CHECK_PASSED=0 +BUILD_FROM_SOURCE_PASSED=0 +BUILD_TAR_FROM_SOURCE_PASSED=0 + +function download_and_import_keys() { + KEY_FILE_URL="${KEY_FILE_URL:-https://downloads.apache.org/hadoop/common/KEYS}" + echo "Obtain and import the publisher key(s) from ${KEY_FILE_URL}" + # download the keys file into file KEYS + wget -O KEYS "${KEY_FILE_URL}" + gpg --import KEYS + if [ -n "${SIGNING_KEY}" ]; then + gpg --list-keys "${SIGNING_KEY}" + fi +} + +function download_release_candidate () { + # get all files from release candidate repo + wget -r -np -N -nH --cut-dirs 4 "${SOURCE_URL}" +} + +function verify_signatures() { + rm -f "${OUTPUT_PATH_PREFIX}"_verify_signatures + for file in *.tar.gz; do + gpg --verify "${file}".asc "${file}" 2>&1 | tee -a "${OUTPUT_PATH_PREFIX}"_verify_signatures && SIGNATURE_PASSED=1 || SIGNATURE_PASSED=0 + done +} + +function verify_checksums() { + rm -f "${OUTPUT_PATH_PREFIX}"_verify_checksums + SHA_EXT=$(find . -name "*.sha*" | awk -F '.' '{ print $NF }' | head -n 1) + for file in *.tar.gz; do + sha512sum --tag "${file}" > "${file}"."${SHA_EXT}".tmp + diff "${file}"."${SHA_EXT}".tmp "${file}"."${SHA_EXT}" 2>&1 | tee -a "${OUTPUT_PATH_PREFIX}"_verify_checksums && CHECKSUM_PASSED=1 || CHECKSUM_PASSED=0 + rm -f "${file}"."${SHA_EXT}".tmp + done +} + +function unzip_from_source() { + tar -zxvf hadoop-"${HADOOP_VERSION}"-src.tar.gz + cd hadoop-"${HADOOP_VERSION}"-src +} + +function rat_test() { + rm -f "${OUTPUT_PATH_PREFIX}"_rat_test + mvn clean apache-rat:check "${MVN_PROPERTIES[@]}" 2>&1 | tee "${OUTPUT_PATH_PREFIX}"_rat_test && RAT_CHECK_PASSED=1 +} + +function build_from_source() { + rm -f "${OUTPUT_PATH_PREFIX}"_build_from_source + # No unit test run. + mvn clean install "${MVN_PROPERTIES[@]}" -DskipTests "${MVN_PROFILES[@]}" 2>&1 | tee "${OUTPUT_PATH_PREFIX}"_build_from_source && BUILD_FROM_SOURCE_PASSED=1 +} + +function build_tar_from_source() { + rm -f "${OUTPUT_PATH_PREFIX}"_build_tar_from_source + # No unit test run. + mvn clean package "${MVN_PROPERTIES[@]}" -Pdist -DskipTests -Dtar -Dmaven.javadoc.skip=true 2>&1 | tee "${OUTPUT_PATH_PREFIX}"_build_tar_from_source && BUILD_TAR_FROM_SOURCE_PASSED=1 +} + +function execute() { + ${1} || print_when_exit +} + +function print_when_exit() { + cat << __EOF + * Signature: $( ((SIGNATURE_PASSED)) && echo "ok" || echo "failed" ) + * Checksum : $( ((CHECKSUM_PASSED)) && echo "ok" || echo "failed" ) + * Rat check (${JAVA_VERSION}): $( ((RAT_CHECK_PASSED)) && echo "ok" || echo "failed" ) + - mvn clean apache-rat:check ${MVN_PROPERTIES[@]} + * Built from source (${JAVA_VERSION}): $( ((BUILD_FROM_SOURCE_PASSED)) && echo "ok" || echo "failed" ) + - mvn clean install ${MVN_PROPERTIES[@]} -DskipTests ${MVN_PROFILES[@]} + * Built tar from source (${JAVA_VERSION}): $( ((BUILD_TAR_FROM_SOURCE_PASSED)) && echo "ok" || echo "failed" ) + - mvn clean package ${MVN_PROPERTIES[@]} -Pdist -DskipTests -Dtar -Dmaven.javadoc.skip=true +__EOF + if ((CHECKSUM_PASSED)) && ((SIGNATURE_PASSED)) && ((RAT_CHECK_PASSED)) && ((BUILD_FROM_SOURCE_PASSED)) && ((BUILD_TAR_FROM_SOURCE_PASSED)) ; then + exit 0 + fi + exit 1 +} + +pushd "${OUTPUT_DIR}" + +download_and_import_keys +download_release_candidate + +execute verify_signatures +execute verify_checksums +execute unzip_from_source +execute rat_test +execute build_from_source +execute build_tar_from_source + +popd + +print_when_exit diff --git a/hadoop-client-modules/hadoop-client-api/pom.xml b/hadoop-client-modules/hadoop-client-api/pom.xml index 9b70f8c8b01e0..82f2f8a778087 100644 --- a/hadoop-client-modules/hadoop-client-api/pom.xml +++ b/hadoop-client-modules/hadoop-client-api/pom.xml @@ -161,6 +161,9 @@ org/xerial/snappy/* org/xerial/snappy/**/* + + org/wildfly/openssl/* + org/wildfly/openssl/**/* diff --git a/hadoop-client-modules/hadoop-client-check-invariants/pom.xml b/hadoop-client-modules/hadoop-client-check-invariants/pom.xml index 9d4bce1fddccd..b1c00678406d7 100644 --- a/hadoop-client-modules/hadoop-client-check-invariants/pom.xml +++ b/hadoop-client-modules/hadoop-client-check-invariants/pom.xml @@ -56,7 +56,7 @@ org.codehaus.mojo extra-enforcer-rules - 1.0-beta-3 + 1.5.1 diff --git a/hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml b/hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml index 635250ec1ae1f..0e576ac6f0666 100644 --- a/hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml +++ b/hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml @@ -60,7 +60,7 @@ org.codehaus.mojo extra-enforcer-rules - 1.0-beta-3 + 1.5.1 diff --git a/hadoop-client-modules/hadoop-client-integration-tests/pom.xml b/hadoop-client-modules/hadoop-client-integration-tests/pom.xml index 967782163f7e8..ba593ebd1b42d 100644 --- a/hadoop-client-modules/hadoop-client-integration-tests/pom.xml +++ b/hadoop-client-modules/hadoop-client-integration-tests/pom.xml @@ -184,6 +184,12 @@ hadoop-hdfs test test-jar + + + org.ow2.asm + asm-commons + + org.apache.hadoop diff --git a/hadoop-client-modules/hadoop-client-minicluster/pom.xml b/hadoop-client-modules/hadoop-client-minicluster/pom.xml index 066b518652441..06e36837a2098 100644 --- a/hadoop-client-modules/hadoop-client-minicluster/pom.xml +++ b/hadoop-client-modules/hadoop-client-minicluster/pom.xml @@ -332,6 +332,10 @@ org.apache.hadoop.thirdparty hadoop-shaded-guava + + org.ow2.asm + asm-commons + - com.sun.jersey - jersey-core + com.sun.jersey + jersey-core true @@ -419,29 +423,25 @@ true - com.sun.jersey + com.github.pjfanning jersey-json true - javax.xml.bind - jaxb-api - - - org.codehaus.jackson - jackson-core-asl + com.fasterxml.jackson.core + jackson-core - org.codehaus.jackson - jackson-mapper-asl + com.fasterxml.jackson.core + jackson-databind - org.codehaus.jackson - jackson-jaxrs + com.fasterxml.jackson.jaxrs + jackson-jaxrs-json-provider - org.codehaus.jackson - jackson-xc + javax.xml.bind + jaxb-api @@ -451,9 +451,23 @@ true - com.sun.jersey - jersey-servlet + com.sun.jersey + jersey-servlet true + + + javax.servlet + servlet-api + + + javax.enterprise + cdi-api + + + ch.qos.cal10n + cal10n-api + + @@ -690,6 +704,7 @@ org.bouncycastle:* org.xerial.snappy:* + javax.ws.rs:javax.ws.rs-api @@ -736,6 +751,12 @@ testdata/* + + com.fasterxml.jackson.*:* + + META-INF/versions/11/module-info.class + + diff --git a/hadoop-client-modules/hadoop-client-runtime/pom.xml b/hadoop-client-modules/hadoop-client-runtime/pom.xml index 6dae4f61969de..35fbd7665fb26 100644 --- a/hadoop-client-modules/hadoop-client-runtime/pom.xml +++ b/hadoop-client-modules/hadoop-client-runtime/pom.xml @@ -163,6 +163,7 @@ org.bouncycastle:* org.xerial.snappy:* + javax.ws.rs:javax.ws.rs-api @@ -242,6 +243,12 @@ google/protobuf/**/*.proto + + com.fasterxml.jackson.*:* + + META-INF/versions/11/module-info.class + + diff --git a/hadoop-client-modules/hadoop-client/pom.xml b/hadoop-client-modules/hadoop-client/pom.xml index dced359b286d9..b48a221bdf179 100644 --- a/hadoop-client-modules/hadoop-client/pom.xml +++ b/hadoop-client-modules/hadoop-client/pom.xml @@ -66,7 +66,7 @@ jersey-core - com.sun.jersey + com.github.pjfanning jersey-json @@ -167,7 +167,7 @@ jersey-core - com.sun.jersey + com.github.pjfanning jersey-json @@ -218,7 +218,7 @@ jersey-server - com.sun.jersey + com.github.pjfanning jersey-json @@ -275,7 +275,7 @@ guice-servlet - com.sun.jersey + com.github.pjfanning jersey-json diff --git a/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml b/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml index a8f45a7f3a222..33d3f9578172f 100644 --- a/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml +++ b/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml @@ -101,6 +101,10 @@ org.apache.zookeeper zookeeper + + org.projectlombok + lombok + diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/COSCredentialsProviderList.java b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/COSCredentialsProviderList.java index d2d2f8c9a7cab..66ef4b1c6fd87 100644 --- a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/COSCredentialsProviderList.java +++ b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/COSCredentialsProviderList.java @@ -24,7 +24,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import com.qcloud.cos.auth.AnonymousCOSCredentials; import com.qcloud.cos.auth.COSCredentials; import com.qcloud.cos.auth.COSCredentialsProvider; diff --git a/hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml b/hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml index 43360c11cd9d2..b96883b9ac80d 100755 --- a/hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml +++ b/hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml @@ -100,10 +100,14 @@ hadoop-common provided - - jdk.tools - jdk.tools - + + jdk.tools + jdk.tools + + + org.javassist + javassist + @@ -161,6 +165,14 @@ okio com.squareup.okio + + log4j-core + org.apache.logging.log4j + + + log4j-api + org.apache.logging.log4j + @@ -176,4 +188,4 @@ test - \ No newline at end of file + diff --git a/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSBlockOutputStream.java b/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSBlockOutputStream.java index cefa897927790..22c6cb5c350c9 100644 --- a/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSBlockOutputStream.java +++ b/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSBlockOutputStream.java @@ -19,7 +19,7 @@ package org.apache.hadoop.fs.obs; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Futures; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableFuture; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListeningExecutorService; diff --git a/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSCommonUtils.java b/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSCommonUtils.java index d477cec186b0e..3a06961d3acd9 100644 --- a/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSCommonUtils.java +++ b/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSCommonUtils.java @@ -18,7 +18,7 @@ package org.apache.hadoop.fs.obs; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import com.obs.services.ObsClient; import com.obs.services.exception.ObsException; import com.obs.services.model.AbortMultipartUploadRequest; diff --git a/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSDataBlocks.java b/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSDataBlocks.java index b58eaa00aa697..e347970ee8446 100644 --- a/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSDataBlocks.java +++ b/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSDataBlocks.java @@ -19,7 +19,7 @@ package org.apache.hadoop.fs.obs; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSExceptionMessages; diff --git a/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSIOException.java b/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSIOException.java index 29a92c71919a8..3f99fd610efa5 100644 --- a/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSIOException.java +++ b/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSIOException.java @@ -18,7 +18,7 @@ package org.apache.hadoop.fs.obs; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import com.obs.services.exception.ObsException; import java.io.IOException; diff --git a/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSInputStream.java b/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSInputStream.java index e94565a4d760a..3f7e9888889b5 100644 --- a/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSInputStream.java +++ b/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSInputStream.java @@ -18,7 +18,7 @@ package org.apache.hadoop.fs.obs; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import com.obs.services.ObsClient; import com.obs.services.exception.ObsException; import com.obs.services.model.GetObjectRequest; diff --git a/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSWriteOperationHelper.java b/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSWriteOperationHelper.java index 5cc3008f1dcfb..2b02f962a0598 100644 --- a/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSWriteOperationHelper.java +++ b/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSWriteOperationHelper.java @@ -18,7 +18,7 @@ package org.apache.hadoop.fs.obs; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import com.obs.services.ObsClient; import com.obs.services.exception.ObsException; import com.obs.services.model.AbortMultipartUploadRequest; diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index bcba2288300f3..6e762f567c180 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -141,12 +141,39 @@ com.sun.jersey jersey-servlet compile + + + javax.enterprise + cdi-api + + + javax.servlet + servlet-api + + + ch.qos.cal10n + cal10n-api + + - - com.sun.jersey + com.github.pjfanning jersey-json compile + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.jaxrs + jackson-jaxrs-json-provider + + com.sun.jersey @@ -187,6 +214,12 @@ org.apache.commons commons-configuration2 compile + + + javax.servlet + servlet-api + + org.apache.commons @@ -915,7 +948,6 @@ org.apache.maven.plugins maven-surefire-plugin - ${ignoreTestFailure} ${testsThreadCount} false ${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true diff --git a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties index 5a2ca4d922852..54d5c729848c7 100644 --- a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties +++ b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties @@ -251,30 +251,45 @@ log4j.appender.NMAUDIT.MaxBackupIndex=${nm.audit.log.maxbackupindex} #log4j.appender.HSAUDIT.DatePattern=.yyyy-MM-dd # Http Server Request Logs -#log4j.logger.http.requests.namenode=INFO,namenoderequestlog -#log4j.appender.namenoderequestlog=org.apache.hadoop.http.HttpRequestLogAppender -#log4j.appender.namenoderequestlog.Filename=${hadoop.log.dir}/jetty-namenode-yyyy_mm_dd.log -#log4j.appender.namenoderequestlog.RetainDays=3 - -#log4j.logger.http.requests.datanode=INFO,datanoderequestlog -#log4j.appender.datanoderequestlog=org.apache.hadoop.http.HttpRequestLogAppender -#log4j.appender.datanoderequestlog.Filename=${hadoop.log.dir}/jetty-datanode-yyyy_mm_dd.log -#log4j.appender.datanoderequestlog.RetainDays=3 - -#log4j.logger.http.requests.resourcemanager=INFO,resourcemanagerrequestlog -#log4j.appender.resourcemanagerrequestlog=org.apache.hadoop.http.HttpRequestLogAppender -#log4j.appender.resourcemanagerrequestlog.Filename=${hadoop.log.dir}/jetty-resourcemanager-yyyy_mm_dd.log -#log4j.appender.resourcemanagerrequestlog.RetainDays=3 - -#log4j.logger.http.requests.jobhistory=INFO,jobhistoryrequestlog -#log4j.appender.jobhistoryrequestlog=org.apache.hadoop.http.HttpRequestLogAppender -#log4j.appender.jobhistoryrequestlog.Filename=${hadoop.log.dir}/jetty-jobhistory-yyyy_mm_dd.log -#log4j.appender.jobhistoryrequestlog.RetainDays=3 - -#log4j.logger.http.requests.nodemanager=INFO,nodemanagerrequestlog -#log4j.appender.nodemanagerrequestlog=org.apache.hadoop.http.HttpRequestLogAppender -#log4j.appender.nodemanagerrequestlog.Filename=${hadoop.log.dir}/jetty-nodemanager-yyyy_mm_dd.log -#log4j.appender.nodemanagerrequestlog.RetainDays=3 +#log4j.appender.AccessNNDRFA=org.apache.log4j.DailyRollingFileAppender +#log4j.appender.AccessNNDRFA.File=${hadoop.log.dir}/jetty-namenode.log +#log4j.appender.AccessNNDRFA.DatePattern=.yyyy-MM-dd +#log4j.appender.AccessNNDRFA.layout=org.apache.log4j.PatternLayout +#log4j.appender.AccessNNDRFA.layout.ConversionPattern=%m%n + +#log4j.logger.http.requests.namenode=INFO,AccessNNDRFA + +#log4j.appender.AccessDNDRFA=org.apache.log4j.DailyRollingFileAppender +#log4j.appender.AccessDNDRFA.File=${hadoop.log.dir}/jetty-datanode.log +#log4j.appender.AccessDNDRFA.DatePattern=.yyyy-MM-dd +#log4j.appender.AccessDNDRFA.layout=org.apache.log4j.PatternLayout +#log4j.appender.AccessDNDRFA.layout.ConversionPattern=%m%n + +#log4j.logger.http.requests.datanode=INFO,AccessDNDRFA + +#log4j.appender.AccessRMDRFA=org.apache.log4j.DailyRollingFileAppender +#log4j.appender.AccessRMDRFA.File=${hadoop.log.dir}/jetty-resourcemanager.log +#log4j.appender.AccessRMDRFA.DatePattern=.yyyy-MM-dd +#log4j.appender.AccessRMDRFA.layout=org.apache.log4j.PatternLayout +#log4j.appender.AccessRMDRFA.layout.ConversionPattern=%m%n + +#log4j.logger.http.requests.resourcemanager=INFO,AccessRMDRFA + +#log4j.appender.AccessJHDRFA=org.apache.log4j.DailyRollingFileAppender +#log4j.appender.AccessJHDRFA.File=${hadoop.log.dir}/jetty-jobhistory.log +#log4j.appender.AccessJHDRFA.DatePattern=.yyyy-MM-dd +#log4j.appender.AccessJHDRFA.layout=org.apache.log4j.PatternLayout +#log4j.appender.AccessJHDRFA.layout.ConversionPattern=%m%n + +#log4j.logger.http.requests.jobhistory=INFO,AccessJHDRFA + +#log4j.appender.AccessNMDRFA=org.apache.log4j.DailyRollingFileAppender +#log4j.appender.AccessNMDRFA.File=${hadoop.log.dir}/jetty-jobhistory.log +#log4j.appender.AccessNMDRFA.DatePattern=.yyyy-MM-dd +#log4j.appender.AccessNMDRFA.layout=org.apache.log4j.PatternLayout +#log4j.appender.AccessNMDRFA.layout.ConversionPattern=%m%n + +#log4j.logger.http.requests.nodemanager=INFO,AccessNMDRFA # WebHdfs request log on datanodes # Specify -Ddatanode.webhdfs.logger=INFO,HTTPDRFA on datanode startup to diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index 7ceaad37b1dcd..1f809b7b54706 100755 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -1099,6 +1099,20 @@ private static int[] findSubVariable(String eval) { return result; } + /** + * Provides a public wrapper over substituteVars in order to avoid compatibility issues. + * See HADOOP-18021 for further details. + * + * @param expr the literal value of a config key + * @return null if expr is null, otherwise the value resulting from expanding + * expr using the algorithm above. + * @throws IllegalArgumentException when more than + * {@link Configuration#MAX_SUBST} replacements are required + */ + public String substituteCommonVariables(String expr) { + return substituteVars(expr); + } + /** * Attempts to repeatedly expand the value {@code expr} by replacing the * left-most substring of the form "${var}" in the following precedence order @@ -1120,13 +1134,17 @@ private static int[] findSubVariable(String eval) { * If a cycle is detected then the original expr is returned. Loops * involving multiple substitutions are not detected. * + * In order not to introduce breaking changes (as Oozie for example contains a method with the + * same name and same signature) do not make this method public, use substituteCommonVariables + * in this case. + * * @param expr the literal value of a config key * @return null if expr is null, otherwise the value resulting from expanding * expr using the algorithm above. * @throws IllegalArgumentException when more than * {@link Configuration#MAX_SUBST} replacements are required */ - public String substituteVars(String expr) { + private String substituteVars(String expr) { if (expr == null) { return null; } @@ -2960,11 +2978,13 @@ public Iterator> iterator() { // methods that allow non-strings to be put into configurations are removed, // we could replace properties with a Map and get rid of this // code. - Map result = new HashMap(); - for(Map.Entry item: getProps().entrySet()) { - if (item.getKey() instanceof String && - item.getValue() instanceof String) { + Properties props = getProps(); + Map result = new HashMap<>(); + synchronized (props) { + for (Map.Entry item : props.entrySet()) { + if (item.getKey() instanceof String && item.getValue() instanceof String) { result.put((String) item.getKey(), (String) item.getValue()); + } } } return result.entrySet().iterator(); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java index d22e91442cca4..0c65b74b2913b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java @@ -84,14 +84,14 @@ static int get(String padding) throws NoSuchPaddingException { String loadingFailure = null; try { if (!NativeCodeLoader.buildSupportsOpenssl()) { - PerformanceAdvisory.LOG.debug("Build does not support openssl"); + PerformanceAdvisory.LOG.warn("Build does not support openssl"); loadingFailure = "build does not support openssl."; } else { initIDs(); } } catch (Throwable t) { loadingFailure = t.getMessage(); - LOG.debug("Failed to load OpenSSL Cipher.", t); + LOG.warn("Failed to load OpenSSL Cipher.", t); } finally { loadingFailureReason = loadingFailure; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AvroFSInput.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AvroFSInput.java index b4a4a85674dfa..213fbc24c4db0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AvroFSInput.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AvroFSInput.java @@ -25,6 +25,10 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL; +import static org.apache.hadoop.util.functional.FutureIO.awaitFuture; + /** Adapts an {@link FSDataInputStream} to Avro's SeekableInput interface. */ @InterfaceAudience.Public @InterfaceStability.Stable @@ -42,7 +46,12 @@ public AvroFSInput(final FSDataInputStream in, final long len) { public AvroFSInput(final FileContext fc, final Path p) throws IOException { FileStatus status = fc.getFileStatus(p); this.len = status.getLen(); - this.stream = fc.open(p); + this.stream = awaitFuture(fc.openFile(p) + .opt(FS_OPTION_OPENFILE_READ_POLICY, + FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL) + .withFileStatus(status) + .build()); + fc.open(p); } @Override diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CachingGetSpaceUsed.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CachingGetSpaceUsed.java index 58dc82d2efb2d..362d125b09df5 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CachingGetSpaceUsed.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CachingGetSpaceUsed.java @@ -19,6 +19,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.classification.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -89,19 +90,19 @@ void init() { if (!shouldFirstRefresh) { // Skip initial refresh operation, so we need to do first refresh // operation immediately in refresh thread. - initRefeshThread(true); + initRefreshThread(true); return; } refresh(); } - initRefeshThread(false); + initRefreshThread(false); } /** * RunImmediately should set true, if we skip the first refresh. * @param runImmediately The param default should be false. */ - private void initRefeshThread (boolean runImmediately) { + private void initRefreshThread(boolean runImmediately) { if (refreshInterval > 0) { refreshUsed = new Thread(new RefreshThread(this, runImmediately), "refreshUsed-" + dirPath); @@ -154,10 +155,20 @@ boolean running() { /** * How long in between runs of the background refresh. */ - long getRefreshInterval() { + @VisibleForTesting + public long getRefreshInterval() { return refreshInterval; } + /** + * Randomize the refresh interval timing by this amount, the actual interval will be chosen + * uniformly between {@code interval-jitter} and {@code interval+jitter}. + */ + @VisibleForTesting + public long getJitter() { + return jitter; + } + /** * Reset the current used data amount. This should be called * when the cached value is re-computed. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java index c7f8e36c3f675..59ffe00bcb24d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java @@ -24,7 +24,6 @@ import java.io.InputStream; import java.nio.channels.ClosedChannelException; import java.util.Arrays; -import java.util.Collections; import java.util.EnumSet; import java.util.List; import java.util.concurrent.CompletableFuture; @@ -45,6 +44,7 @@ import org.apache.hadoop.util.LambdaUtils; import org.apache.hadoop.util.Progressable; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_STANDARD_OPTIONS; import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs; import static org.apache.hadoop.fs.impl.StoreImplementationUtils.isProbeForSyncable; @@ -889,7 +889,7 @@ protected CompletableFuture openFileWithOptions( final OpenFileParameters parameters) throws IOException { AbstractFSBuilderImpl.rejectUnknownMandatoryKeys( parameters.getMandatoryKeys(), - Collections.emptySet(), + FS_OPTION_OPENFILE_STANDARD_OPTIONS, "for " + path); return LambdaUtils.eval( new CompletableFuture<>(), diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java index 6949c67f278d1..db82498b3c15e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java @@ -399,6 +399,8 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic { public static final String ZK_ACL_DEFAULT = "world:anyone:rwcda"; /** Authentication for the ZooKeeper ensemble. */ public static final String ZK_AUTH = ZK_PREFIX + "auth"; + /** Principal name for zookeeper servers. */ + public static final String ZK_SERVER_PRINCIPAL = ZK_PREFIX + "server.principal"; /** Address of the ZooKeeper ensemble. */ public static final String ZK_ADDRESS = ZK_PREFIX + "address"; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonPathCapabilities.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonPathCapabilities.java index df932df43aebd..aa231554eb0cb 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonPathCapabilities.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonPathCapabilities.java @@ -146,4 +146,22 @@ private CommonPathCapabilities() { */ public static final String ABORTABLE_STREAM = "fs.capability.outputstream.abortable"; + + /** + * Does this FS support etags? + * That is: will FileStatus entries from listing/getFileStatus + * probes support EtagSource and return real values. + */ + public static final String ETAGS_AVAILABLE = + "fs.capability.etags.available"; + + /** + * Are etags guaranteed to be preserved across rename() operations.. + * FileSystems MUST NOT declare support for this feature + * unless this holds. + */ + public static final String ETAGS_PRESERVED_IN_RENAME = + "fs.capability.etags.preserved.in.rename"; + + } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestLocalMetadataStoreScale.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/EtagSource.java similarity index 58% rename from hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestLocalMetadataStoreScale.java rename to hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/EtagSource.java index 7477adeeb07b5..d7efdc705d8e5 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestLocalMetadataStoreScale.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/EtagSource.java @@ -16,23 +16,23 @@ * limitations under the License. */ -package org.apache.hadoop.fs.s3a.scale; - -import org.apache.hadoop.fs.s3a.s3guard.LocalMetadataStore; -import org.apache.hadoop.fs.s3a.s3guard.MetadataStore; -import org.apache.hadoop.fs.s3a.s3guard.S3Guard; - -import java.io.IOException; +package org.apache.hadoop.fs; /** - * Scale test for LocalMetadataStore. + * An optional interface for {@link FileStatus} subclasses to implement + * to provide access to etags. + * If available FS SHOULD also implement the matching PathCapabilities + * -- etag supported: {@link CommonPathCapabilities#ETAGS_AVAILABLE}. + * -- etag consistent over rename: + * {@link CommonPathCapabilities#ETAGS_PRESERVED_IN_RENAME}. */ -public class ITestLocalMetadataStoreScale - extends AbstractITestS3AMetadataStoreScale { - @Override - public MetadataStore createMetadataStore() throws IOException { - MetadataStore ms = new LocalMetadataStore(); - ms.initialize(getFileSystem(), new S3Guard.TtlTimeProvider(getConf())); - return ms; - } +public interface EtagSource { + + /** + * Return an etag of this file status. + * A return value of null or "" means "no etag" + * @return a possibly null or empty etag. + */ + String getEtag(); + } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSBuilder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSBuilder.java index b7757a62e28ad..a4c7254cfeb3c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSBuilder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSBuilder.java @@ -61,6 +61,13 @@ public interface FSBuilder> { */ B opt(@Nonnull String key, float value); + /** + * Set optional long parameter for the Builder. + * + * @see #opt(String, String) + */ + B opt(@Nonnull String key, long value); + /** * Set optional double parameter for the Builder. * @@ -104,6 +111,13 @@ public interface FSBuilder> { */ B must(@Nonnull String key, float value); + /** + * Set mandatory long option. + * + * @see #must(String, String) + */ + B must(@Nonnull String key, long value); + /** * Set mandatory double option. * diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java index 9922dfa0ac8b8..f3004ce7e03a3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java @@ -70,7 +70,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_BUFFER_SIZE; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_LENGTH; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE; import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs; +import static org.apache.hadoop.util.functional.FutureIO.awaitFuture; /** * The FileContext class provides an interface for users of the Hadoop @@ -2198,7 +2203,12 @@ public boolean copy(final Path src, final Path dst, boolean deleteSource, EnumSet createFlag = overwrite ? EnumSet.of( CreateFlag.CREATE, CreateFlag.OVERWRITE) : EnumSet.of(CreateFlag.CREATE); - InputStream in = open(qSrc); + InputStream in = awaitFuture(openFile(qSrc) + .opt(FS_OPTION_OPENFILE_READ_POLICY, + FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE) + .opt(FS_OPTION_OPENFILE_LENGTH, + fs.getLen()) // file length hint for object stores + .build()); try (OutputStream out = create(qDst, createFlag)) { IOUtils.copyBytes(in, out, conf, true); } finally { @@ -2930,9 +2940,11 @@ public CompletableFuture build() throws IOException { final Path absF = fixRelativePart(getPath()); OpenFileParameters parameters = new OpenFileParameters() .withMandatoryKeys(getMandatoryKeys()) + .withOptionalKeys(getOptionalKeys()) .withOptions(getOptions()) - .withBufferSize(getBufferSize()) - .withStatus(getStatus()); + .withStatus(getStatus()) + .withBufferSize( + getOptions().getInt(FS_OPTION_OPENFILE_BUFFER_SIZE, getBufferSize())); return new FSLinkResolver>() { @Override public CompletableFuture next( diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index fdb1a47552025..aa194e84a35d6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -88,6 +88,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_BUFFER_SIZE; import static org.apache.hadoop.util.Preconditions.checkArgument; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.*; import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs; @@ -4616,7 +4617,7 @@ protected CompletableFuture openFileWithOptions( final OpenFileParameters parameters) throws IOException { AbstractFSBuilderImpl.rejectUnknownMandatoryKeys( parameters.getMandatoryKeys(), - Collections.emptySet(), + Options.OpenFileOptions.FS_OPTION_OPENFILE_STANDARD_OPTIONS, "for " + path); return LambdaUtils.eval( new CompletableFuture<>(), () -> @@ -4644,7 +4645,7 @@ protected CompletableFuture openFileWithOptions( final OpenFileParameters parameters) throws IOException { AbstractFSBuilderImpl.rejectUnknownMandatoryKeys( parameters.getMandatoryKeys(), - Collections.emptySet(), ""); + Options.OpenFileOptions.FS_OPTION_OPENFILE_STANDARD_OPTIONS, ""); CompletableFuture result = new CompletableFuture<>(); try { result.complete(open(pathHandle, parameters.getBufferSize())); @@ -4751,9 +4752,11 @@ public CompletableFuture build() throws IOException { Optional optionalPath = getOptionalPath(); OpenFileParameters parameters = new OpenFileParameters() .withMandatoryKeys(getMandatoryKeys()) + .withOptionalKeys(getOptionalKeys()) .withOptions(getOptions()) - .withBufferSize(getBufferSize()) - .withStatus(super.getStatus()); // explicit to avoid IDE warnings + .withStatus(super.getStatus()) + .withBufferSize( + getOptions().getInt(FS_OPTION_OPENFILE_BUFFER_SIZE, getBufferSize())); if(optionalPath.isPresent()) { return getFS().openFileWithOptions(optionalPath.get(), parameters); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java index 3c88e5d21bf6b..7400ca36daa5c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java @@ -36,13 +36,18 @@ import java.nio.charset.CharsetEncoder; import java.nio.charset.StandardCharsets; import java.nio.file.AccessDeniedException; +import java.nio.file.attribute.PosixFilePermission; import java.nio.file.FileSystems; import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Enumeration; +import java.util.EnumSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -51,13 +56,13 @@ import java.util.jar.JarOutputStream; import java.util.jar.Manifest; import java.util.zip.GZIPInputStream; -import java.util.zip.ZipEntry; -import java.util.zip.ZipFile; -import java.util.zip.ZipInputStream; import org.apache.commons.collections.map.CaseInsensitiveMap; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; +import org.apache.commons.compress.archivers.zip.ZipFile; import org.apache.commons.io.FileUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -72,6 +77,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_LENGTH; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE; +import static org.apache.hadoop.util.functional.FutureIO.awaitFuture; + /** * A collection of file-processing util methods */ @@ -391,7 +401,32 @@ public static boolean copy(FileSystem srcFS, Path src, return copy(srcFS, fileStatus, dstFS, dst, deleteSource, overwrite, conf); } - /** Copy files between FileSystems. */ + /** + * Copy a file/directory tree within/between filesystems. + *

+ * returns true if the operation succeeded. When deleteSource is true, + * this means "after the copy, delete(source) returned true" + * If the destination is a directory, and mkdirs (dest) fails, + * the operation will return false rather than raise any exception. + *

+ * The overwrite flag is about overwriting files; it has no effect about + * handing an attempt to copy a file atop a directory (expect an IOException), + * or a directory over a path which contains a file (mkdir will fail, so + * "false"). + *

+ * The operation is recursive, and the deleteSource operation takes place + * as each subdirectory is copied. Therefore, if an operation fails partway + * through, the source tree may be partially deleted. + * @param srcFS source filesystem + * @param srcStatus status of source + * @param dstFS destination filesystem + * @param dst path of source + * @param deleteSource delete the source? + * @param overwrite overwrite files at destination? + * @param conf configuration to use when opening files + * @return true if the operation succeeded. + * @throws IOException failure + */ public static boolean copy(FileSystem srcFS, FileStatus srcStatus, FileSystem dstFS, Path dst, boolean deleteSource, @@ -404,22 +439,27 @@ public static boolean copy(FileSystem srcFS, FileStatus srcStatus, if (!dstFS.mkdirs(dst)) { return false; } - FileStatus contents[] = srcFS.listStatus(src); - for (int i = 0; i < contents.length; i++) { - copy(srcFS, contents[i], dstFS, - new Path(dst, contents[i].getPath().getName()), - deleteSource, overwrite, conf); + RemoteIterator contents = srcFS.listStatusIterator(src); + while (contents.hasNext()) { + FileStatus next = contents.next(); + copy(srcFS, next, dstFS, + new Path(dst, next.getPath().getName()), + deleteSource, overwrite, conf); } } else { - InputStream in=null; + InputStream in = null; OutputStream out = null; try { - in = srcFS.open(src); + in = awaitFuture(srcFS.openFile(src) + .opt(FS_OPTION_OPENFILE_READ_POLICY, + FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE) + .opt(FS_OPTION_OPENFILE_LENGTH, + srcStatus.getLen()) // file length hint for object stores + .build()); out = dstFS.create(dst, overwrite); IOUtils.copyBytes(in, out, conf, true); } catch (IOException e) { - IOUtils.closeStream(out); - IOUtils.closeStream(in); + IOUtils.cleanupWithLogger(LOG, in, out); throw e; } } @@ -498,7 +538,11 @@ private static boolean copy(FileSystem srcFS, FileStatus srcStatus, deleteSource, conf); } } else { - InputStream in = srcFS.open(src); + InputStream in = awaitFuture(srcFS.openFile(src) + .withFileStatus(srcStatus) + .opt(FS_OPTION_OPENFILE_READ_POLICY, + FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE) + .build()); IOUtils.copyBytes(in, Files.newOutputStream(dst.toPath()), conf); } if (deleteSource) { @@ -533,6 +577,26 @@ private static Path checkDest(String srcName, FileSystem dstFS, Path dst, return dst; } + public static boolean isRegularFile(File file) { + return isRegularFile(file, true); + } + + /** + * Check if the file is regular. + * @param file The file being checked. + * @param allowLinks Whether to allow matching links. + * @return Returns the result of checking whether the file is a regular file. + */ + public static boolean isRegularFile(File file, boolean allowLinks) { + if (file != null) { + if (allowLinks) { + return Files.isRegularFile(file.toPath()); + } + return Files.isRegularFile(file.toPath(), LinkOption.NOFOLLOW_LINKS); + } + return true; + } + /** * Convert a os-native filename to a path that works for the shell. * @param filename The filename to convert @@ -622,12 +686,12 @@ public static long getDU(File dir) { */ public static void unZip(InputStream inputStream, File toDir) throws IOException { - try (ZipInputStream zip = new ZipInputStream(inputStream)) { + try (ZipArchiveInputStream zip = new ZipArchiveInputStream(inputStream)) { int numOfFailedLastModifiedSet = 0; String targetDirPath = toDir.getCanonicalPath() + File.separator; - for(ZipEntry entry = zip.getNextEntry(); + for(ZipArchiveEntry entry = zip.getNextZipEntry(); entry != null; - entry = zip.getNextEntry()) { + entry = zip.getNextZipEntry()) { if (!entry.isDirectory()) { File file = new File(toDir, entry.getName()); if (!file.getCanonicalPath().startsWith(targetDirPath)) { @@ -646,6 +710,9 @@ public static void unZip(InputStream inputStream, File toDir) if (!file.setLastModified(entry.getTime())) { numOfFailedLastModifiedSet++; } + if (entry.getPlatform() == ZipArchiveEntry.PLATFORM_UNIX) { + Files.setPosixFilePermissions(file.toPath(), permissionsFromMode(entry.getUnixMode())); + } } } if (numOfFailedLastModifiedSet > 0) { @@ -655,6 +722,49 @@ public static void unZip(InputStream inputStream, File toDir) } } + /** + * The permission operation of this method only involves users, user groups, and others. + * If SUID is set, only executable permissions are reserved. + * @param mode Permissions are represented by numerical values + * @return The original permissions for files are stored in collections + */ + private static Set permissionsFromMode(int mode) { + EnumSet permissions = + EnumSet.noneOf(PosixFilePermission.class); + addPermissions(permissions, mode, PosixFilePermission.OTHERS_READ, + PosixFilePermission.OTHERS_WRITE, PosixFilePermission.OTHERS_EXECUTE); + addPermissions(permissions, mode >> 3, PosixFilePermission.GROUP_READ, + PosixFilePermission.GROUP_WRITE, PosixFilePermission.GROUP_EXECUTE); + addPermissions(permissions, mode >> 6, PosixFilePermission.OWNER_READ, + PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE); + return permissions; + } + + /** + * Assign the original permissions to the file + * @param permissions The original permissions for files are stored in collections + * @param mode Use a value of type int to indicate permissions + * @param r Read permission + * @param w Write permission + * @param x Execute permission + */ + private static void addPermissions( + Set permissions, + int mode, + PosixFilePermission r, + PosixFilePermission w, + PosixFilePermission x) { + if ((mode & 1L) == 1L) { + permissions.add(x); + } + if ((mode & 2L) == 2L) { + permissions.add(w); + } + if ((mode & 4L) == 4L) { + permissions.add(r); + } + } + /** * Given a File input it will unzip it in the unzip directory. * passed as the second parameter @@ -663,14 +773,14 @@ public static void unZip(InputStream inputStream, File toDir) * @throws IOException An I/O exception has occurred */ public static void unZip(File inFile, File unzipDir) throws IOException { - Enumeration entries; + Enumeration entries; ZipFile zipFile = new ZipFile(inFile); try { - entries = zipFile.entries(); + entries = zipFile.getEntries(); String targetDirPath = unzipDir.getCanonicalPath() + File.separator; while (entries.hasMoreElements()) { - ZipEntry entry = entries.nextElement(); + ZipArchiveEntry entry = entries.nextElement(); if (!entry.isDirectory()) { InputStream in = zipFile.getInputStream(entry); try { @@ -695,6 +805,9 @@ public static void unZip(File inFile, File unzipDir) throws IOException { } finally { out.close(); } + if (entry.getPlatform() == ZipArchiveEntry.PLATFORM_UNIX) { + Files.setPosixFilePermissions(file.toPath(), permissionsFromMode(entry.getUnixMode())); + } } finally { in.close(); } @@ -888,10 +1001,13 @@ private static void unTarUsingTar(InputStream inputStream, File untarDir, private static void unTarUsingTar(File inFile, File untarDir, boolean gzipped) throws IOException { StringBuffer untarCommand = new StringBuffer(); + // not using canonical path here; this postpones relative path + // resolution until bash is executed. + final String source = "'" + FileUtil.makeSecureShellPath(inFile) + "'"; if (gzipped) { - untarCommand.append(" gzip -dc '") - .append(FileUtil.makeSecureShellPath(inFile)) - .append("' | ("); + untarCommand.append(" gzip -dc ") + .append(source) + .append(" | ("); } untarCommand.append("cd '") .append(FileUtil.makeSecureShellPath(untarDir)) @@ -901,15 +1017,17 @@ private static void unTarUsingTar(File inFile, File untarDir, if (gzipped) { untarCommand.append(" -)"); } else { - untarCommand.append(FileUtil.makeSecureShellPath(inFile)); + untarCommand.append(source); } + LOG.debug("executing [{}]", untarCommand); String[] shellCmd = { "bash", "-c", untarCommand.toString() }; ShellCommandExecutor shexec = new ShellCommandExecutor(shellCmd); shexec.execute(); int exitcode = shexec.getExitCode(); if (exitcode != 0) { throw new IOException("Error untarring file " + inFile + - ". Tar process exited with exit code " + exitcode); + ". Tar process exited with exit code " + exitcode + + " from command " + untarCommand); } } @@ -966,6 +1084,14 @@ private static void unpackEntries(TarArchiveInputStream tis, + " would create entry outside of " + outputDir); } + if (entry.isSymbolicLink() || entry.isLink()) { + String canonicalTargetPath = getCanonicalPath(entry.getLinkName(), outputDir); + if (!canonicalTargetPath.startsWith(targetDirPath)) { + throw new IOException( + "expanding " + entry.getName() + " would create entry outside of " + outputDir); + } + } + if (entry.isDirectory()) { File subDir = new File(outputDir, entry.getName()); if (!subDir.mkdirs() && !subDir.isDirectory()) { @@ -981,10 +1107,12 @@ private static void unpackEntries(TarArchiveInputStream tis, } if (entry.isSymbolicLink()) { - // Create symbolic link relative to tar parent dir - Files.createSymbolicLink(FileSystems.getDefault() - .getPath(outputDir.getPath(), entry.getName()), - FileSystems.getDefault().getPath(entry.getLinkName())); + // Create symlink with canonical target path to ensure that we don't extract + // outside targetDirPath + String canonicalTargetPath = getCanonicalPath(entry.getLinkName(), outputDir); + Files.createSymbolicLink( + FileSystems.getDefault().getPath(outputDir.getPath(), entry.getName()), + FileSystems.getDefault().getPath(canonicalTargetPath)); return; } @@ -996,7 +1124,8 @@ private static void unpackEntries(TarArchiveInputStream tis, } if (entry.isLink()) { - File src = new File(outputDir, entry.getLinkName()); + String canonicalTargetPath = getCanonicalPath(entry.getLinkName(), outputDir); + File src = new File(canonicalTargetPath); HardLink.createHardLink(src, outputFile); return; } @@ -1004,6 +1133,20 @@ private static void unpackEntries(TarArchiveInputStream tis, org.apache.commons.io.FileUtils.copyToFile(tis, outputFile); } + /** + * Gets the canonical path for the given path. + * + * @param path The path for which the canonical path needs to be computed. + * @param parentDir The parent directory to use if the path is a relative path. + * @return The canonical path of the given path. + */ + private static String getCanonicalPath(String path, File parentDir) throws IOException { + java.nio.file.Path targetPath = Paths.get(path); + return (targetPath.isAbsolute() ? + new File(path) : + new File(parentDir, path)).getCanonicalPath(); + } + /** * Class for creating hardlinks. * Supports Unix, WindXP. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FutureDataInputStreamBuilder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FutureDataInputStreamBuilder.java index 27a522e593001..e7f441a75d3c8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FutureDataInputStreamBuilder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FutureDataInputStreamBuilder.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.fs; +import javax.annotation.Nullable; import java.io.IOException; import java.util.concurrent.CompletableFuture; @@ -34,7 +35,7 @@ * options accordingly, for example: * * If the option is not related to the file system, the option will be ignored. - * If the option is must, but not supported by the file system, a + * If the option is must, but not supported/known by the file system, an * {@link IllegalArgumentException} will be thrown. * */ @@ -51,10 +52,11 @@ CompletableFuture build() /** * A FileStatus may be provided to the open request. * It is up to the implementation whether to use this or not. - * @param status status. + * @param status status: may be null * @return the builder. */ - default FutureDataInputStreamBuilder withFileStatus(FileStatus status) { + default FutureDataInputStreamBuilder withFileStatus( + @Nullable FileStatus status) { return this; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java index 75bc12df8fdcf..9b457272fcb50 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java @@ -17,9 +17,13 @@ */ package org.apache.hadoop.fs; +import java.util.Collections; import java.util.Optional; +import java.util.Set; import java.util.function.Function; import java.util.function.BiFunction; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -518,4 +522,119 @@ public enum ChecksumCombineMode { MD5MD5CRC, // MD5 of block checksums, which are MD5 over chunk CRCs COMPOSITE_CRC // Block/chunk-independent composite CRC } + + /** + * The standard {@code openFile()} options. + */ + @InterfaceAudience.Public + @InterfaceStability.Evolving + public static final class OpenFileOptions { + + private OpenFileOptions() { + } + + /** + * Prefix for all standard filesystem options: {@value}. + */ + private static final String FILESYSTEM_OPTION = "fs.option."; + + /** + * Prefix for all openFile options: {@value}. + */ + public static final String FS_OPTION_OPENFILE = + FILESYSTEM_OPTION + "openfile."; + + /** + * OpenFile option for file length: {@value}. + */ + public static final String FS_OPTION_OPENFILE_LENGTH = + FS_OPTION_OPENFILE + "length"; + + /** + * OpenFile option for split start: {@value}. + */ + public static final String FS_OPTION_OPENFILE_SPLIT_START = + FS_OPTION_OPENFILE + "split.start"; + + /** + * OpenFile option for split end: {@value}. + */ + public static final String FS_OPTION_OPENFILE_SPLIT_END = + FS_OPTION_OPENFILE + "split.end"; + + /** + * OpenFile option for buffer size: {@value}. + */ + public static final String FS_OPTION_OPENFILE_BUFFER_SIZE = + FS_OPTION_OPENFILE + "buffer.size"; + + /** + * OpenFile option for read policies: {@value}. + */ + public static final String FS_OPTION_OPENFILE_READ_POLICY = + FS_OPTION_OPENFILE + "read.policy"; + + /** + * Set of standard options which openFile implementations + * MUST recognize, even if they ignore the actual values. + */ + public static final Set FS_OPTION_OPENFILE_STANDARD_OPTIONS = + Collections.unmodifiableSet(Stream.of( + FS_OPTION_OPENFILE_BUFFER_SIZE, + FS_OPTION_OPENFILE_READ_POLICY, + FS_OPTION_OPENFILE_LENGTH, + FS_OPTION_OPENFILE_SPLIT_START, + FS_OPTION_OPENFILE_SPLIT_END) + .collect(Collectors.toSet())); + + /** + * Read policy for adaptive IO: {@value}. + */ + public static final String FS_OPTION_OPENFILE_READ_POLICY_ADAPTIVE = + "adaptive"; + + /** + * Read policy {@value} -whateve the implementation does by default. + */ + public static final String FS_OPTION_OPENFILE_READ_POLICY_DEFAULT = + "default"; + + /** + * Read policy for random IO: {@value}. + */ + public static final String FS_OPTION_OPENFILE_READ_POLICY_RANDOM = + "random"; + + /** + * Read policy for sequential IO: {@value}. + */ + public static final String FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL = + "sequential"; + + /** + * Vectored IO API to be used: {@value}. + */ + public static final String FS_OPTION_OPENFILE_READ_POLICY_VECTOR = + "vector"; + + /** + * Whole file to be read, end-to-end: {@value}. + */ + public static final String FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE = + "whole-file"; + + /** + * All the current read policies as a set. + */ + public static final Set FS_OPTION_OPENFILE_READ_POLICIES = + Collections.unmodifiableSet(Stream.of( + FS_OPTION_OPENFILE_READ_POLICY_ADAPTIVE, + FS_OPTION_OPENFILE_READ_POLICY_DEFAULT, + FS_OPTION_OPENFILE_READ_POLICY_RANDOM, + FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL, + FS_OPTION_OPENFILE_READ_POLICY_VECTOR, + FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE) + .collect(Collectors.toSet())); + + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java index 99467f5633625..f4228dea69f49 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java @@ -191,8 +191,8 @@ public boolean moveToTrash(Path path) throws IOException { cause = e; } } - throw (IOException) - new IOException("Failed to move to trash: " + path).initCause(cause); + throw new IOException("Failed to move " + path + " to trash " + trashPath, + cause); } @SuppressWarnings("deprecation") diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/CommonAuditContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/CommonAuditContext.java index 11681546e3d0a..e188e168e5313 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/CommonAuditContext.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/CommonAuditContext.java @@ -24,6 +24,9 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.function.Supplier; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -69,11 +72,16 @@ * {@link #currentAuditContext()} to get the thread-local * context for the caller, which can then be manipulated. * + * For further information, especially related to memory consumption, + * read the document `auditing_architecture` in the `hadoop-aws` module. */ @InterfaceAudience.Public @InterfaceStability.Unstable public final class CommonAuditContext { + private static final Logger LOG = LoggerFactory.getLogger( + CommonAuditContext.class); + /** * Process ID; currently built from UUID and timestamp. */ @@ -92,7 +100,7 @@ public final class CommonAuditContext { * Supplier operations must themselves be thread safe. */ private final Map> evaluatedEntries = - new ConcurrentHashMap<>(); + new ConcurrentHashMap<>(1); static { // process ID is fixed. @@ -108,7 +116,7 @@ public final class CommonAuditContext { * the span is finalized. */ private static final ThreadLocal ACTIVE_CONTEXT = - ThreadLocal.withInitial(() -> createInstance()); + ThreadLocal.withInitial(CommonAuditContext::createInstance); private CommonAuditContext() { } @@ -125,11 +133,21 @@ public Supplier put(String key, String value) { /** * Put a context entry dynamically evaluated on demand. + * Important: as these supplier methods are long-lived, + * the supplier function MUST NOT be part of/refer to + * any object instance of significant memory size. + * Applications SHOULD remove references when they are + * no longer needed. + * When logged at TRACE, prints the key and stack trace of the caller, + * to allow for debugging of any problems. * @param key key * @param value new value * @return old value or null */ public Supplier put(String key, Supplier value) { + if (LOG.isTraceEnabled()) { + LOG.trace("Adding context entry {}", key, new Exception(key)); + } return evaluatedEntries.put(key, value); } @@ -138,6 +156,9 @@ public Supplier put(String key, Supplier value) { * @param key key */ public void remove(String key) { + if (LOG.isTraceEnabled()) { + LOG.trace("Remove context entry {}", key); + } evaluatedEntries.remove(key); } @@ -168,7 +189,7 @@ public void reset() { private void init() { // thread 1 is dynamic - put(PARAM_THREAD1, () -> currentThreadID()); + put(PARAM_THREAD1, CommonAuditContext::currentThreadID); } /** diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/AbstractFSBuilderImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/AbstractFSBuilderImpl.java index c69e7afe4e36e..9d3a46d633253 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/AbstractFSBuilderImpl.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/AbstractFSBuilderImpl.java @@ -46,7 +46,7 @@ * * .opt("foofs:option.a", true) * .opt("foofs:option.b", "value") - * .opt("barfs:cache", true) + * .opt("fs.s3a.open.option.etag", "9fe4c37c25b") * .must("foofs:cache", true) * .must("barfs:cache-size", 256 * 1024 * 1024) * .build(); @@ -88,6 +88,9 @@ /** Keep track of the keys for mandatory options. */ private final Set mandatoryKeys = new HashSet<>(); + /** Keep track of the optional keys. */ + private final Set optionalKeys = new HashSet<>(); + /** * Constructor with both optional path and path handle. * Either or both argument may be empty, but it is an error for @@ -163,6 +166,7 @@ public PathHandle getPathHandle() { @Override public B opt(@Nonnull final String key, @Nonnull final String value) { mandatoryKeys.remove(key); + optionalKeys.add(key); options.set(key, value); return getThisBuilder(); } @@ -175,6 +179,7 @@ public B opt(@Nonnull final String key, @Nonnull final String value) { @Override public B opt(@Nonnull final String key, boolean value) { mandatoryKeys.remove(key); + optionalKeys.add(key); options.setBoolean(key, value); return getThisBuilder(); } @@ -187,10 +192,19 @@ public B opt(@Nonnull final String key, boolean value) { @Override public B opt(@Nonnull final String key, int value) { mandatoryKeys.remove(key); + optionalKeys.add(key); options.setInt(key, value); return getThisBuilder(); } + @Override + public B opt(@Nonnull final String key, final long value) { + mandatoryKeys.remove(key); + optionalKeys.add(key); + options.setLong(key, value); + return getThisBuilder(); + } + /** * Set optional float parameter for the Builder. * @@ -199,6 +213,7 @@ public B opt(@Nonnull final String key, int value) { @Override public B opt(@Nonnull final String key, float value) { mandatoryKeys.remove(key); + optionalKeys.add(key); options.setFloat(key, value); return getThisBuilder(); } @@ -211,6 +226,7 @@ public B opt(@Nonnull final String key, float value) { @Override public B opt(@Nonnull final String key, double value) { mandatoryKeys.remove(key); + optionalKeys.add(key); options.setDouble(key, value); return getThisBuilder(); } @@ -223,6 +239,7 @@ public B opt(@Nonnull final String key, double value) { @Override public B opt(@Nonnull final String key, @Nonnull final String... values) { mandatoryKeys.remove(key); + optionalKeys.add(key); options.setStrings(key, values); return getThisBuilder(); } @@ -248,6 +265,7 @@ public B must(@Nonnull final String key, @Nonnull final String value) { @Override public B must(@Nonnull final String key, boolean value) { mandatoryKeys.add(key); + optionalKeys.remove(key); options.setBoolean(key, value); return getThisBuilder(); } @@ -260,10 +278,19 @@ public B must(@Nonnull final String key, boolean value) { @Override public B must(@Nonnull final String key, int value) { mandatoryKeys.add(key); + optionalKeys.remove(key); options.setInt(key, value); return getThisBuilder(); } + @Override + public B must(@Nonnull final String key, final long value) { + mandatoryKeys.add(key); + optionalKeys.remove(key); + options.setLong(key, value); + return getThisBuilder(); + } + /** * Set mandatory float option. * @@ -272,6 +299,7 @@ public B must(@Nonnull final String key, int value) { @Override public B must(@Nonnull final String key, float value) { mandatoryKeys.add(key); + optionalKeys.remove(key); options.setFloat(key, value); return getThisBuilder(); } @@ -284,6 +312,7 @@ public B must(@Nonnull final String key, float value) { @Override public B must(@Nonnull final String key, double value) { mandatoryKeys.add(key); + optionalKeys.remove(key); options.setDouble(key, value); return getThisBuilder(); } @@ -296,6 +325,7 @@ public B must(@Nonnull final String key, double value) { @Override public B must(@Nonnull final String key, @Nonnull final String... values) { mandatoryKeys.add(key); + optionalKeys.remove(key); options.setStrings(key, values); return getThisBuilder(); } @@ -314,6 +344,12 @@ public Configuration getOptions() { public Set getMandatoryKeys() { return Collections.unmodifiableSet(mandatoryKeys); } + /** + * Get all the keys that are set as optional keys. + */ + public Set getOptionalKeys() { + return Collections.unmodifiableSet(optionalKeys); + } /** * Reject a configuration if one or more mandatory keys are diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FileSystemMultipartUploader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FileSystemMultipartUploader.java index 2339e42128973..481d927672dc3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FileSystemMultipartUploader.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FileSystemMultipartUploader.java @@ -51,6 +51,7 @@ import org.apache.hadoop.fs.PathHandle; import org.apache.hadoop.fs.UploadHandle; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.util.functional.FutureIO; import static org.apache.hadoop.fs.Path.mergePaths; import static org.apache.hadoop.io.IOUtils.cleanupWithLogger; @@ -98,7 +99,7 @@ public FileSystemMultipartUploader( public CompletableFuture startUpload(Path filePath) throws IOException { checkPath(filePath); - return FutureIOSupport.eval(() -> { + return FutureIO.eval(() -> { Path collectorPath = createCollectorPath(filePath); fs.mkdirs(collectorPath, FsPermission.getDirDefault()); @@ -116,7 +117,7 @@ public CompletableFuture putPart(UploadHandle uploadId, throws IOException { checkPutArguments(filePath, inputStream, partNumber, uploadId, lengthInBytes); - return FutureIOSupport.eval(() -> innerPutPart(filePath, + return FutureIO.eval(() -> innerPutPart(filePath, inputStream, partNumber, uploadId, lengthInBytes)); } @@ -179,7 +180,7 @@ public CompletableFuture complete( Map handleMap) throws IOException { checkPath(filePath); - return FutureIOSupport.eval(() -> + return FutureIO.eval(() -> innerComplete(uploadId, filePath, handleMap)); } @@ -251,7 +252,7 @@ public CompletableFuture abort(UploadHandle uploadId, Path collectorPath = new Path(new String(uploadIdByteArray, 0, uploadIdByteArray.length, Charsets.UTF_8)); - return FutureIOSupport.eval(() -> { + return FutureIO.eval(() -> { // force a check for a file existing; raises FNFE if not found fs.getFileStatus(collectorPath); fs.delete(collectorPath, true); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FutureDataInputStreamBuilderImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FutureDataInputStreamBuilderImpl.java index 24a8d49747fe6..70e39de7388c3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FutureDataInputStreamBuilderImpl.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FutureDataInputStreamBuilderImpl.java @@ -19,6 +19,7 @@ package org.apache.hadoop.fs.impl; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.io.IOException; import java.util.concurrent.CompletableFuture; @@ -47,7 +48,7 @@ * options accordingly, for example: * * If the option is not related to the file system, the option will be ignored. - * If the option is must, but not supported by the file system, a + * If the option is must, but not supported/known by the file system, an * {@link IllegalArgumentException} will be thrown. * */ @@ -147,8 +148,9 @@ public FutureDataInputStreamBuilder getThisBuilder() { } @Override - public FutureDataInputStreamBuilder withFileStatus(FileStatus st) { - this.status = requireNonNull(st, "status"); + public FutureDataInputStreamBuilder withFileStatus( + @Nullable FileStatus st) { + this.status = st; return this; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FutureIOSupport.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FutureIOSupport.java index 18f5187cb6134..f47e5f4fbfbd6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FutureIOSupport.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FutureIOSupport.java @@ -20,7 +20,6 @@ import java.io.IOException; import java.io.InterruptedIOException; -import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutionException; @@ -37,14 +36,16 @@ /** * Support for future IO and the FS Builder subclasses. - * If methods in here are needed for applications, promote - * to {@link FutureIO} for public use -with the original - * method relaying to it. This is to ensure that external - * filesystem implementations can safely use these methods + * All methods in this class have been superceded by those in + * {@link FutureIO}. + * The methods here are retained but all marked as deprecated. + * This is to ensure that any external + * filesystem implementations can still use these methods * without linkage problems surfacing. */ @InterfaceAudience.Private @InterfaceStability.Unstable +@Deprecated public final class FutureIOSupport { private FutureIOSupport() { @@ -53,6 +54,7 @@ private FutureIOSupport() { /** * Given a future, evaluate it. Raised exceptions are * extracted and handled. + * See {@link FutureIO#awaitFuture(Future, long, TimeUnit)}. * @param future future to evaluate * @param type of the result. * @return the result, if all went well. @@ -60,7 +62,8 @@ private FutureIOSupport() { * @throws IOException if something went wrong * @throws RuntimeException any nested RTE thrown */ - public static T awaitFuture(final Future future) + @Deprecated + public static T awaitFuture(final Future future) throws InterruptedIOException, IOException, RuntimeException { return FutureIO.awaitFuture(future); } @@ -69,6 +72,7 @@ public static T awaitFuture(final Future future) /** * Given a future, evaluate it. Raised exceptions are * extracted and handled. + * See {@link FutureIO#awaitFuture(Future, long, TimeUnit)}. * @param future future to evaluate * @param type of the result. * @return the result, if all went well. @@ -77,6 +81,7 @@ public static T awaitFuture(final Future future) * @throws RuntimeException any nested RTE thrown * @throws TimeoutException the future timed out. */ + @Deprecated public static T awaitFuture(final Future future, final long timeout, final TimeUnit unit) @@ -88,10 +93,7 @@ public static T awaitFuture(final Future future, /** * From the inner cause of an execution exception, extract the inner cause * if it is an IOE or RTE. - * This will always raise an exception, either the inner IOException, - * an inner RuntimeException, or a new IOException wrapping the raised - * exception. - * + * See {@link FutureIO#raiseInnerCause(ExecutionException)}. * @param e exception. * @param type of return value. * @return nothing, ever. @@ -99,6 +101,7 @@ public static T awaitFuture(final Future future, * any non-Runtime-Exception * @throws RuntimeException if that is the inner cause. */ + @Deprecated public static T raiseInnerCause(final ExecutionException e) throws IOException { return FutureIO.raiseInnerCause(e); @@ -107,6 +110,7 @@ public static T raiseInnerCause(final ExecutionException e) /** * Extract the cause of a completion failure and rethrow it if an IOE * or RTE. + * See {@link FutureIO#raiseInnerCause(CompletionException)}. * @param e exception. * @param type of return value. * @return nothing, ever. @@ -114,20 +118,15 @@ public static T raiseInnerCause(final ExecutionException e) * any non-Runtime-Exception * @throws RuntimeException if that is the inner cause. */ + @Deprecated public static T raiseInnerCause(final CompletionException e) throws IOException { return FutureIO.raiseInnerCause(e); } /** - * Propagate options to any builder, converting everything with the - * prefix to an option where, if there were 2+ dot-separated elements, - * it is converted to a schema. - * 
{@code
-   *   fs.example.s3a.option => s3a:option
-   *   fs.example.fs.io.policy => s3a.io.policy
-   *   fs.example.something => something
-   * }
+ * Propagate options to any builder. + * {@link FutureIO#propagateOptions(FSBuilder, Configuration, String, String)} * @param builder builder to modify * @param conf configuration to read * @param optionalPrefix prefix for optional settings @@ -136,56 +135,39 @@ public static T raiseInnerCause(final CompletionException e) * @param type of builder * @return the builder passed in. */ + @Deprecated public static > FSBuilder propagateOptions( final FSBuilder builder, final Configuration conf, final String optionalPrefix, final String mandatoryPrefix) { - propagateOptions(builder, conf, - optionalPrefix, false); - propagateOptions(builder, conf, - mandatoryPrefix, true); - return builder; + return FutureIO.propagateOptions(builder, + conf, optionalPrefix, mandatoryPrefix); } /** - * Propagate options to any builder, converting everything with the - * prefix to an option where, if there were 2+ dot-separated elements, - * it is converted to a schema. - * 
{@code
-   *   fs.example.s3a.option => s3a:option
-   *   fs.example.fs.io.policy => s3a.io.policy
-   *   fs.example.something => something
-   * }
+ * Propagate options to any builder. + * {@link FutureIO#propagateOptions(FSBuilder, Configuration, String, boolean)} * @param builder builder to modify * @param conf configuration to read * @param prefix prefix to scan/strip * @param mandatory are the options to be mandatory or optional? */ + @Deprecated public static void propagateOptions( final FSBuilder builder, final Configuration conf, final String prefix, final boolean mandatory) { - - final String p = prefix.endsWith(".") ? prefix : (prefix + "."); - final Map propsWithPrefix = conf.getPropsWithPrefix(p); - for (Map.Entry entry : propsWithPrefix.entrySet()) { - // change the schema off each entry - String key = entry.getKey(); - String val = entry.getValue(); - if (mandatory) { - builder.must(key, val); - } else { - builder.opt(key, val); - } - } + FutureIO.propagateOptions(builder, conf, prefix, mandatory); } /** * Evaluate a CallableRaisingIOE in the current thread, * converting IOEs to RTEs and propagating. + * See {@link FutureIO#eval(CallableRaisingIOE)}. + * * @param callable callable to invoke * @param Return type. * @return the evaluated result. @@ -194,17 +176,6 @@ public static void propagateOptions( */ public static CompletableFuture eval( CallableRaisingIOE callable) { - CompletableFuture result = new CompletableFuture<>(); - try { - result.complete(callable.apply()); - } catch (UnsupportedOperationException | IllegalArgumentException tx) { - // fail fast here - throw tx; - } catch (Throwable tx) { - // fail lazily here to ensure callers expect all File IO operations to - // surface later - result.completeExceptionally(tx); - } - return result; + return FutureIO.eval(callable); } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/OpenFileParameters.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/OpenFileParameters.java index 77b4ff52696a3..a19c5faff4d90 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/OpenFileParameters.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/OpenFileParameters.java @@ -38,6 +38,9 @@ public class OpenFileParameters { */ private Set mandatoryKeys; + /** The optional keys. */ + private Set optionalKeys; + /** * Options set during the build sequence. */ @@ -61,6 +64,11 @@ public OpenFileParameters withMandatoryKeys(final Set keys) { return this; } + public OpenFileParameters withOptionalKeys(final Set keys) { + this.optionalKeys = requireNonNull(keys); + return this; + } + public OpenFileParameters withOptions(final Configuration opts) { this.options = requireNonNull(opts); return this; @@ -80,6 +88,10 @@ public Set getMandatoryKeys() { return mandatoryKeys; } + public Set getOptionalKeys() { + return optionalKeys; + } + public Configuration getOptions() { return options; } @@ -91,4 +103,5 @@ public int getBufferSize() { public FileStatus getStatus() { return status; } + } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/WeakReferenceThreadMap.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/WeakReferenceThreadMap.java new file mode 100644 index 0000000000000..b24bef2a816bf --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/WeakReferenceThreadMap.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.impl; + +import java.util.function.Consumer; +import java.util.function.Function; +import javax.annotation.Nullable; + +import org.apache.hadoop.util.WeakReferenceMap; + +/** + * A WeakReferenceMap for threads. + * @param value type of the map + */ +public class WeakReferenceThreadMap extends WeakReferenceMap { + + public WeakReferenceThreadMap(final Function factory, + @Nullable final Consumer referenceLost) { + super(factory, referenceLost); + } + + public V getForCurrentThread() { + return get(currentThreadId()); + } + + public V removeForCurrentThread() { + return remove(currentThreadId()); + } + + public long currentThreadId() { + return Thread.currentThread().getId(); + } + + public V setForCurrentThread(V newVal) { + return put(currentThreadId(), newVal); + } + +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/WrappedIOException.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/WrappedIOException.java index d2c999683c6c6..d5144b5e9c531 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/WrappedIOException.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/WrappedIOException.java @@ -20,20 +20,17 @@ import java.io.IOException; import java.io.UncheckedIOException; -import java.util.concurrent.ExecutionException; import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; /** - * A wrapper for an IOException which - * {@link FutureIOSupport#raiseInnerCause(ExecutionException)} knows to - * always extract the exception. + * A wrapper for an IOException. * * The constructor signature guarantees the cause will be an IOException, * and as it checks for a null-argument, non-null. - * @deprecated use the {@code UncheckedIOException}. + * @deprecated use the {@code UncheckedIOException} directly.] */ @Deprecated @InterfaceAudience.Private @@ -51,8 +48,4 @@ public WrappedIOException(final IOException cause) { super(Preconditions.checkNotNull(cause)); } - @Override - public synchronized IOException getCause() { - return (IOException) super.getCause(); - } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java index 90a709dffc0c1..678225f81e0e3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java @@ -55,6 +55,9 @@ import static org.apache.hadoop.fs.CreateFlag.CREATE; import static org.apache.hadoop.fs.CreateFlag.LAZY_PERSIST; import static org.apache.hadoop.fs.CreateFlag.OVERWRITE; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE; +import static org.apache.hadoop.util.functional.FutureIO.awaitFuture; /** * Provides: argument processing to ensure the destination is valid @@ -348,7 +351,11 @@ protected void copyFileToTarget(PathData src, PathData target) src.fs.setVerifyChecksum(verifyChecksum); InputStream in = null; try { - in = src.fs.open(src.path); + in = awaitFuture(src.fs.openFile(src.path) + .withFileStatus(src.stat) + .opt(FS_OPTION_OPENFILE_READ_POLICY, + FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE) + .build()); copyStreamToTarget(in, target); preserveAttributes(src, target, preserveRawXattrs); } finally { @@ -397,11 +404,11 @@ private boolean checkPathsForReservedRaw(Path src, Path target) /** * If direct write is disabled ,copies the stream contents to a temporary - * file "._COPYING_". If the copy is - * successful, the temporary file will be renamed to the real path, - * else the temporary file will be deleted. + * file "target._COPYING_". If the copy is successful, the temporary file + * will be renamed to the real path, else the temporary file will be deleted. * if direct write is enabled , then creation temporary file is skipped. - * @param in the input stream for the copy + * + * @param in the input stream for the copy * @param target where to store the contents of the stream * @throws IOException if copy fails */ diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommandWithMultiThread.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommandWithMultiThread.java new file mode 100644 index 0000000000000..aed4030540baf --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommandWithMultiThread.java @@ -0,0 +1,155 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.shell; + +import java.io.IOException; +import java.util.LinkedList; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import org.apache.hadoop.classification.VisibleForTesting; + +/** + * Abstract command to enable sub copy commands run with multi-thread. + */ +public abstract class CopyCommandWithMultiThread + extends CommandWithDestination { + + private int threadCount = 1; + private ThreadPoolExecutor executor = null; + private int threadPoolQueueSize = DEFAULT_QUEUE_SIZE; + + public static final int DEFAULT_QUEUE_SIZE = 1024; + + /** + * set thread count by option value, if the value less than 1, + * use 1 instead. + * + * @param optValue option value + */ + protected void setThreadCount(String optValue) { + if (optValue != null) { + threadCount = Math.max(Integer.parseInt(optValue), 1); + } + } + + /** + * set thread pool queue size by option value, if the value less than 1, + * use DEFAULT_QUEUE_SIZE instead. + * + * @param optValue option value + */ + protected void setThreadPoolQueueSize(String optValue) { + if (optValue != null) { + int size = Integer.parseInt(optValue); + threadPoolQueueSize = size < 1 ? DEFAULT_QUEUE_SIZE : size; + } + } + + @VisibleForTesting + protected int getThreadCount() { + return this.threadCount; + } + + @VisibleForTesting + protected int getThreadPoolQueueSize() { + return this.threadPoolQueueSize; + } + + @VisibleForTesting + protected ThreadPoolExecutor getExecutor() { + return this.executor; + } + + @Override + protected void processArguments(LinkedList args) + throws IOException { + + if (isMultiThreadNecessary(args)) { + initThreadPoolExecutor(); + } + + super.processArguments(args); + + if (executor != null) { + waitForCompletion(); + } + } + + // if thread count is 1 or the source is only one single file, + // don't init executor to avoid threading overhead. + @VisibleForTesting + protected boolean isMultiThreadNecessary(LinkedList args) + throws IOException { + return this.threadCount > 1 && hasMoreThanOneSourcePaths(args); + } + + // check if source is only one single file. + private boolean hasMoreThanOneSourcePaths(LinkedList args) + throws IOException { + if (args.size() > 1) { + return true; + } + if (args.size() == 1) { + PathData src = args.get(0); + if (src.stat == null) { + src.refreshStatus(); + } + return isPathRecursable(src); + } + return false; + } + + private void initThreadPoolExecutor() { + executor = + new ThreadPoolExecutor(threadCount, threadCount, 1, TimeUnit.SECONDS, + new ArrayBlockingQueue<>(threadPoolQueueSize), + new ThreadPoolExecutor.CallerRunsPolicy()); + } + + private void waitForCompletion() { + if (executor != null) { + executor.shutdown(); + try { + executor.awaitTermination(Long.MAX_VALUE, TimeUnit.MINUTES); + } catch (InterruptedException e) { + executor.shutdownNow(); + displayError(e); + Thread.currentThread().interrupt(); + } + } + } + + @Override + protected void copyFileToTarget(PathData src, PathData target) + throws IOException { + if (executor == null) { + super.copyFileToTarget(src, target); + } else { + executor.submit(() -> { + try { + super.copyFileToTarget(src, target); + } catch (IOException e) { + displayError(e); + } + }); + } + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java index 96d5cad2bf6bd..0643a2e983dcc 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java @@ -26,11 +26,7 @@ import java.util.Iterator; import java.util.LinkedList; import java.util.List; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.TimeUnit; -import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.FSDataInputStream; @@ -38,8 +34,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathIsDirectoryException; import org.apache.hadoop.io.IOUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** Various commands for copy files */ @InterfaceAudience.Private @@ -104,7 +98,8 @@ protected void processArguments(LinkedList items) try { for (PathData src : srcs) { if (src.stat.getLen() != 0) { - try (FSDataInputStream in = src.fs.open(src.path)) { + // Always do sequential reads. + try (FSDataInputStream in = src.openForSequentialIO()) { IOUtils.copyBytes(in, out, getConf(), false); writeDelimiter(out); } @@ -153,33 +148,40 @@ protected boolean isSorted() { } } - static class Cp extends CommandWithDestination { + static class Cp extends CopyCommandWithMultiThread { public static final String NAME = "cp"; public static final String USAGE = - "[-f] [-p | -p[topax]] [-d] ... "; + "[-f] [-p | -p[topax]] [-d] [-t ]" + + " [-q ] ... "; public static final String DESCRIPTION = - "Copy files that match the file pattern to a " + - "destination. When copying multiple files, the destination " + - "must be a directory. Passing -p preserves status " + - "[topax] (timestamps, ownership, permission, ACLs, XAttr). " + - "If -p is specified with no , then preserves " + - "timestamps, ownership, permission. If -pa is specified, " + - "then preserves permission also because ACL is a super-set of " + - "permission. Passing -f overwrites the destination if it " + - "already exists. raw namespace extended attributes are preserved " + - "if (1) they are supported (HDFS only) and, (2) all of the source and " + - "target pathnames are in the /.reserved/raw hierarchy. raw namespace " + - "xattr preservation is determined solely by the presence (or absence) " + - "of the /.reserved/raw prefix and not by the -p option. Passing -d "+ - "will skip creation of temporary file(._COPYING_).\n"; + "Copy files that match the file pattern to a destination." + + " When copying multiple files, the destination must be a " + + "directory.\nFlags :\n" + + " -p[topax] : Preserve file attributes [topx] (timestamps, " + + "ownership, permission, ACL, XAttr). If -p is specified with " + + "no arg, then preserves timestamps, ownership, permission. " + + "If -pa is specified, then preserves permission also because " + + "ACL is a super-set of permission. Determination of whether raw " + + "namespace extended attributes are preserved is independent of " + + "the -p flag.\n" + + " -f : Overwrite the destination if it already exists.\n" + + " -d : Skip creation of temporary file(._COPYING_).\n" + + " -t : Number of threads to be used, " + + "default is 1.\n" + + " -q : Thread pool queue size to be " + + "used, default is 1024.\n"; @Override protected void processOptions(LinkedList args) throws IOException { popPreserveOption(args); CommandFormat cf = new CommandFormat(2, Integer.MAX_VALUE, "f", "d"); + cf.addOptionWithValue("t"); + cf.addOptionWithValue("q"); cf.parse(args); setDirectWrite(cf.getOpt("d")); setOverwrite(cf.getOpt("f")); + setThreadCount(cf.getOptValue("t")); + setThreadPoolQueueSize(cf.getOptValue("q")); // should have a -r option setRecursive(true); getRemoteDestination(args); @@ -210,28 +212,37 @@ private void popPreserveOption(List args) { /** * Copy local files to a remote filesystem */ - public static class Get extends CommandWithDestination { + public static class Get extends CopyCommandWithMultiThread { public static final String NAME = "get"; public static final String USAGE = - "[-f] [-p] [-ignoreCrc] [-crc] ... "; + "[-f] [-p] [-crc] [-ignoreCrc] [-t ]" + + " [-q ] ... "; public static final String DESCRIPTION = - "Copy files that match the file pattern " + - "to the local name. is kept. When copying multiple " + - "files, the destination must be a directory. Passing " + - "-f overwrites the destination if it already exists and " + - "-p preserves access and modification times, " + - "ownership and the mode.\n"; + "Copy files that match the file pattern to the local name. " + + " is kept.\nWhen copying multiple files, the destination" + + " must be a directory.\nFlags:\n" + + " -p : Preserves timestamps, ownership and the mode.\n" + + " -f : Overwrites the destination if it already exists.\n" + + " -crc : write CRC checksums for the files downloaded.\n" + + " -ignoreCrc : Skip CRC checks on the file(s) downloaded.\n" + + " -t : Number of threads to be used," + + " default is 1.\n" + + " -q : Thread pool queue size to be" + + " used, default is 1024.\n"; @Override - protected void processOptions(LinkedList args) - throws IOException { - CommandFormat cf = new CommandFormat( - 1, Integer.MAX_VALUE, "crc", "ignoreCrc", "p", "f"); + protected void processOptions(LinkedList args) throws IOException { + CommandFormat cf = + new CommandFormat(1, Integer.MAX_VALUE, "crc", "ignoreCrc", "p", "f"); + cf.addOptionWithValue("t"); + cf.addOptionWithValue("q"); cf.parse(args); setWriteChecksum(cf.getOpt("crc")); setVerifyChecksum(!cf.getOpt("ignoreCrc")); setPreserve(cf.getOpt("p")); setOverwrite(cf.getOpt("f")); + setThreadCount(cf.getOptValue("t")); + setThreadPoolQueueSize(cf.getOptValue("q")); setRecursive(true); getLocalDestination(args); } @@ -240,21 +251,12 @@ protected void processOptions(LinkedList args) /** * Copy local files to a remote filesystem */ - public static class Put extends CommandWithDestination { - - public static final Logger LOG = LoggerFactory.getLogger(Put.class); - - private ThreadPoolExecutor executor = null; - private int threadPoolQueueSize = 1024; - private int numThreads = 1; - - private static final int MAX_THREADS = - Runtime.getRuntime().availableProcessors() * 2; + public static class Put extends CopyCommandWithMultiThread { public static final String NAME = "put"; public static final String USAGE = - "[-f] [-p] [-l] [-d] [-t ] [-q ] " + - " ... "; + "[-f] [-p] [-l] [-d] [-t ] [-q ]" + + " ... "; public static final String DESCRIPTION = "Copy files from the local file system " + "into fs. Copying fails if the file already " + @@ -263,11 +265,11 @@ public static class Put extends CommandWithDestination { " -p : Preserves timestamps, ownership and the mode.\n" + " -f : Overwrites the destination if it already exists.\n" + " -t : Number of threads to be used, default is 1.\n" + - " -q : ThreadPool queue size to be used, " + + " -q : Thread pool queue size to be used, " + "default is 1024.\n" + - " -l : Allow DataNode to lazily persist the file to disk. Forces" + - " replication factor of 1. This flag will result in reduced" + - " durability. Use with care.\n" + + " -l : Allow DataNode to lazily persist the file to disk. Forces " + + "replication factor of 1. This flag will result in reduced " + + "durability. Use with care.\n" + " -d : Skip creation of temporary file(._COPYING_).\n"; @Override @@ -277,7 +279,7 @@ protected void processOptions(LinkedList args) throws IOException { cf.addOptionWithValue("t"); cf.addOptionWithValue("q"); cf.parse(args); - setNumberThreads(cf.getOptValue("t")); + setThreadCount(cf.getOptValue("t")); setThreadPoolQueueSize(cf.getOptValue("q")); setOverwrite(cf.getOpt("f")); setPreserve(cf.getOpt("p")); @@ -308,92 +310,9 @@ protected void processArguments(LinkedList args) copyStreamToTarget(System.in, getTargetPath(args.get(0))); return; } - - executor = new ThreadPoolExecutor(numThreads, numThreads, 1, - TimeUnit.SECONDS, new ArrayBlockingQueue<>(threadPoolQueueSize), - new ThreadPoolExecutor.CallerRunsPolicy()); super.processArguments(args); - - // issue the command and then wait for it to finish - executor.shutdown(); - try { - executor.awaitTermination(Long.MAX_VALUE, TimeUnit.MINUTES); - } catch (InterruptedException e) { - executor.shutdownNow(); - displayError(e); - Thread.currentThread().interrupt(); - } - } - - private void setNumberThreads(String numberThreadsString) { - if (numberThreadsString == null) { - numThreads = 1; - } else { - int parsedValue = Integer.parseInt(numberThreadsString); - if (parsedValue <= 1) { - numThreads = 1; - } else if (parsedValue > MAX_THREADS) { - numThreads = MAX_THREADS; - } else { - numThreads = parsedValue; - } - } } - private void setThreadPoolQueueSize(String numThreadPoolQueueSize) { - if (numThreadPoolQueueSize != null) { - int parsedValue = Integer.parseInt(numThreadPoolQueueSize); - if (parsedValue < 1) { - LOG.warn("The value of the thread pool queue size cannot be " + - "less than 1, and the default value is used here. " + - "The default size is 1024."); - threadPoolQueueSize = 1024; - } else { - threadPoolQueueSize = parsedValue; - } - } - } - - @VisibleForTesting - protected int getThreadPoolQueueSize() { - return threadPoolQueueSize; - } - - private void copyFile(PathData src, PathData target) throws IOException { - if (isPathRecursable(src)) { - throw new PathIsDirectoryException(src.toString()); - } - super.copyFileToTarget(src, target); - } - - @Override - protected void copyFileToTarget(PathData src, PathData target) - throws IOException { - // if number of thread is 1, mimic put and avoid threading overhead - if (numThreads == 1) { - copyFile(src, target); - return; - } - - Runnable task = () -> { - try { - copyFile(src, target); - } catch (IOException e) { - displayError(e); - } - }; - executor.submit(task); - } - - @VisibleForTesting - public int getNumThreads() { - return numThreads; - } - - @VisibleForTesting - public ThreadPoolExecutor getExecutor() { - return executor; - } } public static class CopyFromLocal extends Put { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java index 670fa152f72ed..d3ca013a3f251 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java @@ -105,7 +105,8 @@ private void printToStdout(InputStream in) throws IOException { } protected InputStream getInputStream(PathData item) throws IOException { - return item.fs.open(item.path); + // Always do sequential reads; + return item.openForSequentialIO(); } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Head.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Head.java index 2280225b5ae32..7242f261801d6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Head.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Head.java @@ -28,6 +28,8 @@ import java.util.LinkedList; import java.util.List; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL; + /** * Show the first 1KB of the file. */ @@ -68,11 +70,9 @@ protected void processPath(PathData item) throws IOException { } private void dumpToOffset(PathData item) throws IOException { - FSDataInputStream in = item.fs.open(item.path); - try { + try (FSDataInputStream in = item.openFile( + FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL)) { IOUtils.copyBytes(in, System.out, endingOffset, false); - } finally { - in.close(); } } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java index 1ff8d8f0494a1..2071a16799a5c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java @@ -29,6 +29,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalFileSystem; @@ -39,6 +40,10 @@ import org.apache.hadoop.fs.PathNotFoundException; import org.apache.hadoop.fs.RemoteIterator; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_LENGTH; +import static org.apache.hadoop.util.functional.FutureIO.awaitFuture; import static org.apache.hadoop.util.functional.RemoteIterators.mappingRemoteIterator; /** @@ -601,4 +606,34 @@ public boolean equals(Object o) { public int hashCode() { return path.hashCode(); } + + + /** + * Open a file for sequential IO. + *

+ * This uses FileSystem.openFile() to request sequential IO; + * the file status is also passed in. + * Filesystems may use to optimize their IO. + * @return an input stream + * @throws IOException failure + */ + protected FSDataInputStream openForSequentialIO() + throws IOException { + return openFile(FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL); + } + + /** + * Open a file. + * @param policy fadvise policy. + * @return an input stream + * @throws IOException failure + */ + protected FSDataInputStream openFile(final String policy) throws IOException { + return awaitFuture(fs.openFile(path) + .opt(FS_OPTION_OPENFILE_READ_POLICY, + policy) + .opt(FS_OPTION_OPENFILE_LENGTH, + stat.getLen()) // file length hint for object stores + .build()); + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Tail.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Tail.java index 98db1c3b7b605..26ac3fee4713b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Tail.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Tail.java @@ -30,6 +30,8 @@ import org.apache.hadoop.classification.VisibleForTesting; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL; + /** * Get a listing of all files in that match the file patterns. */ @@ -107,16 +109,15 @@ private long dumpFromOffset(PathData item, long offset) throws IOException { if (offset < 0) { offset = Math.max(fileSize + offset, 0); } - - FSDataInputStream in = item.fs.open(item.path); - try { + // Always do sequential reads. + try (FSDataInputStream in = item.openFile( + FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL)) { in.seek(offset); // use conf so the system configured io block size is used IOUtils.copyBytes(in, System.out, getConf(), false); offset = in.getPos(); - } finally { - in.close(); } return offset; } + } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/TouchCommands.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/TouchCommands.java index 902eada98dbd1..457cd86ab703c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/TouchCommands.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/TouchCommands.java @@ -147,9 +147,6 @@ protected void processOptions(LinkedList args) { @Override protected void processPath(PathData item) throws IOException { - if (item.stat.isDirectory()) { - throw new PathIsDirectoryException(item.toString()); - } touch(item); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java index 9ec8dcdb3dc9b..c458269c3510d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java @@ -112,9 +112,15 @@ public final class StoreStatisticNames { /** {@value}. */ public static final String OP_MODIFY_ACL_ENTRIES = "op_modify_acl_entries"; + /** {@value}. */ + public static final String OP_MSYNC = "op_msync"; + /** {@value}. */ public static final String OP_OPEN = "op_open"; + /** Call to openFile() {@value}. */ + public static final String OP_OPENFILE = "op_openfile"; + /** {@value}. */ public static final String OP_REMOVE_ACL = "op_remove_acl"; @@ -172,6 +178,9 @@ public final class StoreStatisticNames { public static final String STORE_IO_THROTTLED = "store_io_throttled"; + /** Rate limiting was reported {@value}. */ + public static final String STORE_IO_RATE_LIMITED = "store_io_rate_limited"; + /** Requests made of a store: {@value}. */ public static final String STORE_IO_REQUEST = "store_io_request"; @@ -317,6 +326,12 @@ public final class StoreStatisticNames { public static final String ACTION_EXECUTOR_ACQUIRED = "action_executor_acquired"; + /** + * A file was opened: {@value}. + */ + public static final String ACTION_FILE_OPENED + = "action_file_opened"; + /** * An HTTP HEAD request was made: {@value}. */ diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StreamStatisticNames.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StreamStatisticNames.java index 7e9137294c1ef..ca755f0841914 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StreamStatisticNames.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StreamStatisticNames.java @@ -76,7 +76,7 @@ public final class StreamStatisticNames { public static final String STREAM_READ_CLOSED = "stream_read_closed"; /** - * Total count of times an attempt to close an input stream was made + * Total count of times an attempt to close an input stream was made. * Value: {@value}. */ public static final String STREAM_READ_CLOSE_OPERATIONS @@ -118,6 +118,23 @@ public final class StreamStatisticNames { public static final String STREAM_READ_OPERATIONS_INCOMPLETE = "stream_read_operations_incomplete"; + /** + * count/duration of aborting a remote stream during stream IO + * IO. + * Value: {@value}. + */ + public static final String STREAM_READ_REMOTE_STREAM_ABORTED + = "stream_read_remote_stream_aborted"; + + /** + * count/duration of closing a remote stream, + * possibly including draining the stream to recycle + * the HTTP connection. + * Value: {@value}. + */ + public static final String STREAM_READ_REMOTE_STREAM_DRAINED + = "stream_read_remote_stream_drain"; + /** * Count of version mismatches encountered while reading an input stream. * Value: {@value}. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsBinding.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsBinding.java index c2eab9d772a66..c45dfc21a1b1d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsBinding.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsBinding.java @@ -21,6 +21,7 @@ import javax.annotation.Nullable; import java.io.IOException; import java.io.Serializable; +import java.time.Duration; import java.util.Iterator; import java.util.Map; import java.util.concurrent.Callable; @@ -450,12 +451,37 @@ public static B trackDuration( * @param factory factory of duration trackers * @param statistic statistic key * @param input input callable. + * @throws IOException IO failure. */ public static void trackDurationOfInvocation( DurationTrackerFactory factory, String statistic, InvocationRaisingIOE input) throws IOException { + measureDurationOfInvocation(factory, statistic, input); + } + + /** + * Given an IOException raising callable/lambda expression, + * execute it and update the relevant statistic, + * returning the measured duration. + * + * {@link #trackDurationOfInvocation(DurationTrackerFactory, String, InvocationRaisingIOE)} + * with the duration returned for logging etc.; added as a new + * method to avoid linking problems with any code calling the existing + * method. + * + * @param factory factory of duration trackers + * @param statistic statistic key + * @param input input callable. + * @return the duration of the operation, as measured by the duration tracker. + * @throws IOException IO failure. + */ + public static Duration measureDurationOfInvocation( + DurationTrackerFactory factory, + String statistic, + InvocationRaisingIOE input) throws IOException { + // create the tracker outside try-with-resources so // that failures can be set in the catcher. DurationTracker tracker = createTracker(factory, statistic); @@ -473,6 +499,7 @@ public static void trackDurationOfInvocation( // set the failed flag. tracker.close(); } + return tracker.asDuration(); } /** @@ -494,23 +521,39 @@ public static CallableRaisingIOE trackDurationOfOperation( // create the tracker outside try-with-resources so // that failures can be set in the catcher. DurationTracker tracker = createTracker(factory, statistic); - try { - // exec the input function and return its value - return input.apply(); - } catch (IOException | RuntimeException e) { - // input function failed: note it - tracker.failed(); - // and rethrow - throw e; - } finally { - // update the tracker. - // this is called after the catch() call would have - // set the failed flag. - tracker.close(); - } + return invokeTrackingDuration(tracker, input); }; } + /** + * Given an IOException raising callable/lambda expression, + * execute it, updating the tracker on success/failure. + * @param tracker duration tracker. + * @param input input callable. + * @param return type. + * @return the result of the invocation + * @throws IOException on failure. + */ + public static B invokeTrackingDuration( + final DurationTracker tracker, + final CallableRaisingIOE input) + throws IOException { + try { + // exec the input function and return its value + return input.apply(); + } catch (IOException | RuntimeException e) { + // input function failed: note it + tracker.failed(); + // and rethrow + throw e; + } finally { + // update the tracker. + // this is called after the catch() call would have + // set the failed flag. + tracker.close(); + } + } + /** * Given an IOException raising Consumer, * return a new one which wraps the inner and tracks @@ -622,7 +665,7 @@ public static B trackDurationOfSupplier( * @param statistic statistic to track * @return a duration tracker. */ - private static DurationTracker createTracker( + public static DurationTracker createTracker( @Nullable final DurationTrackerFactory factory, final String statistic) { return factory != null diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsStore.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsStore.java index 1b4139e463a9e..c083ad8c3c2ed 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsStore.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsStore.java @@ -255,4 +255,15 @@ default long incrementCounter(String key) { */ void addTimedOperation(String prefix, Duration duration); + /** + * Add a statistics sample as a min, max and mean and count. + * @param key key to add. + * @param count count. + */ + default void addSample(String key, long count) { + incrementCounter(key, count); + addMeanStatisticSample(key, count); + addMaximumSample(key, count); + addMinimumSample(key, count); + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/PairedDurationTrackerFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/PairedDurationTrackerFactory.java index 33b13f78418a9..9bc01338a1497 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/PairedDurationTrackerFactory.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/PairedDurationTrackerFactory.java @@ -88,6 +88,11 @@ public void close() { public Duration asDuration() { return firstDuration.asDuration(); } + + @Override + public String toString() { + return firstDuration.toString(); + } } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/StatisticDurationTracker.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/StatisticDurationTracker.java index ef9e7cb107a0d..04d30135f6bd3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/StatisticDurationTracker.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/StatisticDurationTracker.java @@ -103,4 +103,11 @@ public void close() { } iostats.addTimedOperation(name, asDuration()); } + + @Override + public String toString() { + return " Duration of " + + (failed? (key + StoreStatisticNames.SUFFIX_FAILURES) : key) + + ": " + super.toString(); + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java index 8235e937bddbf..21f4d99f891c2 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java @@ -132,4 +132,11 @@ public interface Constants { Class DEFAULT_MOUNT_TABLE_CONFIG_LOADER_IMPL = HCFSMountTableConfigLoader.class; + + /** + * Force ViewFileSystem to return a trashRoot that is inside a mount point. + */ + String CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT = + "fs.viewfs.trash.force-inside-mount-point"; + boolean CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT_DEFAULT = false; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/FsGetter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/FsGetter.java index 071af11e63bf2..c72baac25fb75 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/FsGetter.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/FsGetter.java @@ -20,15 +20,17 @@ import java.io.IOException; import java.net.URI; -import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; /** * File system instance getter. */ -@Private -class FsGetter { +@InterfaceAudience.LimitedPrivate({"Common"}) +@InterfaceStability.Unstable +public class FsGetter { /** * Gets new file system instance of given uri. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java index e13c7366741c5..23ad053a67d5c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java @@ -61,7 +61,7 @@ @InterfaceAudience.Private @InterfaceStability.Unstable -abstract class InodeTree { +public abstract class InodeTree { private static final Logger LOGGER = LoggerFactory.getLogger(InodeTree.class.getName()); @@ -81,7 +81,7 @@ enum ResultKind { private List> regexMountPointList = new ArrayList>(); - static class MountPoint { + public static class MountPoint { String src; INodeLink target; @@ -89,6 +89,22 @@ static class MountPoint { src = srcPath; target = mountLink; } + + /** + * Returns the source of mount point. + * @return The source + */ + public String getSource() { + return this.src; + } + + /** + * Returns the target link. + * @return The target INode link + */ + public INodeLink getTarget() { + return this.target; + } } /** @@ -256,8 +272,8 @@ enum LinkType { * For a merge, each target is checked to be dir when created but if target * is changed later it is then ignored (a dir with null entries) */ - static class INodeLink extends INode { - final URI[] targetDirLinkList; + public static class INodeLink extends INode { + final String[] targetDirLinkList; private T targetFileSystem; // file system object created from the link. // Function to initialize file system. Only applicable for simple links private Function fileSystemInitMethod; @@ -267,7 +283,7 @@ static class INodeLink extends INode { * Construct a mergeLink or nfly. */ INodeLink(final String pathToNode, final UserGroupInformation aUgi, - final T targetMergeFs, final URI[] aTargetDirLinkList) { + final T targetMergeFs, final String[] aTargetDirLinkList) { super(pathToNode, aUgi); targetFileSystem = targetMergeFs; targetDirLinkList = aTargetDirLinkList; @@ -278,11 +294,11 @@ static class INodeLink extends INode { */ INodeLink(final String pathToNode, final UserGroupInformation aUgi, Function createFileSystemMethod, - final URI aTargetDirLink) { + final String aTargetDirLink) throws URISyntaxException { super(pathToNode, aUgi); targetFileSystem = null; - targetDirLinkList = new URI[1]; - targetDirLinkList[0] = aTargetDirLink; + targetDirLinkList = new String[1]; + targetDirLinkList[0] = new URI(aTargetDirLink).toString(); this.fileSystemInitMethod = createFileSystemMethod; } @@ -290,7 +306,7 @@ static class INodeLink extends INode { * Get the target of the link. If a merge link then it returned * as "," separated URI list. */ - Path getTargetLink() { + public Path getTargetLink() { StringBuilder result = new StringBuilder(targetDirLinkList[0].toString()); // If merge link, use "," as separator between the merged URIs for (int i = 1; i < targetDirLinkList.length; ++i) { @@ -320,7 +336,8 @@ public T getTargetFileSystem() throws IOException { if (targetFileSystem != null) { return targetFileSystem; } - targetFileSystem = fileSystemInitMethod.apply(targetDirLinkList[0]); + targetFileSystem = + fileSystemInitMethod.apply(URI.create(targetDirLinkList[0])); if (targetFileSystem == null) { throw new IOException( "Could not initialize target File System for URI : " + @@ -388,7 +405,7 @@ private void createLink(final String src, final String target, switch (linkType) { case SINGLE: newLink = new INodeLink(fullPath, aUgi, - initAndGetTargetFs(), new URI(target)); + initAndGetTargetFs(), target); break; case SINGLE_FALLBACK: case MERGE_SLASH: @@ -397,10 +414,10 @@ private void createLink(final String src, final String target, throw new IllegalArgumentException("Unexpected linkType: " + linkType); case MERGE: case NFLY: - final URI[] targetUris = StringUtils.stringToURI( - StringUtils.getStrings(target)); + final String[] targetUris = StringUtils.getStrings(target); newLink = new INodeLink(fullPath, aUgi, - getTargetFileSystem(settings, targetUris), targetUris); + getTargetFileSystem(settings, StringUtils.stringToURI(targetUris)), + targetUris); break; default: throw new IllegalArgumentException(linkType + ": Infeasible linkType"); @@ -441,11 +458,11 @@ private boolean hasFallbackLink() { * there will be root to root mapping. So, root does not represent as * internalDir. */ - protected boolean isRootInternalDir() { + public boolean isRootInternalDir() { return root.isInternalDir(); } - protected INodeLink getRootFallbackLink() { + public INodeLink getRootFallbackLink() { Preconditions.checkState(root.isInternalDir()); return rootFallbackLink; } @@ -617,8 +634,7 @@ protected InodeTree(final Configuration config, final String viewName, if (isMergeSlashConfigured) { Preconditions.checkNotNull(mergeSlashTarget); root = new INodeLink(mountTableName, ugi, - initAndGetTargetFs(), - new URI(mergeSlashTarget)); + initAndGetTargetFs(), mergeSlashTarget); mountPoints.add(new MountPoint("/", (INodeLink) root)); rootFallbackLink = null; } else { @@ -636,7 +652,7 @@ protected InodeTree(final Configuration config, final String viewName, + "not allowed."); } fallbackLink = new INodeLink(mountTableName, ugi, - initAndGetTargetFs(), new URI(le.getTarget())); + initAndGetTargetFs(), le.getTarget()); continue; case REGEX: addRegexMountEntry(le); @@ -661,7 +677,7 @@ protected InodeTree(final Configuration config, final String viewName, .info("Empty mount table detected for {} and considering itself " + "as a linkFallback.", theUri); rootFallbackLink = new INodeLink(mountTableName, ugi, - initAndGetTargetFs(), theUri); + initAndGetTargetFs(), theUri.toString()); getRootDir().addFallbackLink(rootFallbackLink); } } @@ -726,7 +742,7 @@ private LinkEntry buildLinkRegexEntry( * If the input pathname leads to an internal mount-table entry then * the target file system is one that represents the internal inode. */ - static class ResolveResult { + public static class ResolveResult { final ResultKind kind; final T targetFileSystem; final String resolvedPath; @@ -761,7 +777,7 @@ boolean isLastInternalDirLink() { * @return ResolveResult which allows further resolution of the remaining path * @throws IOException */ - ResolveResult resolve(final String p, final boolean resolveLastComponent) + public ResolveResult resolve(final String p, final boolean resolveLastComponent) throws IOException { ResolveResult resolveResult = null; String[] path = breakIntoPathComponents(p); @@ -932,7 +948,7 @@ protected ResolveResult buildResolveResultForRegexMountPoint( } } - List> getMountPoints() { + public List> getMountPoints() { return mountPoints; } @@ -941,7 +957,7 @@ List> getMountPoints() { * @return home dir value from mount table; null if no config value * was found. */ - String getHomeDirPrefixValue() { + public String getHomeDirPrefixValue() { return homedirPrefix; } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index 538f03a1300a3..b30e086cf06b0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -25,6 +25,8 @@ import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_MOUNT_LINKS_AS_SYMLINKS; import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_MOUNT_LINKS_AS_SYMLINKS_DEFAULT; import static org.apache.hadoop.fs.viewfs.Constants.PERMISSION_555; +import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT; +import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT_DEFAULT; import java.util.function.Function; import java.io.FileNotFoundException; @@ -214,11 +216,11 @@ public static class MountPoint { /** * Array of target FileSystem URIs. */ - private final URI[] targetFileSystemURIs; + private final String[] targetFileSystemPaths; - MountPoint(Path srcPath, URI[] targetFs) { + MountPoint(Path srcPath, String[] targetFs) { mountedOnPath = srcPath; - targetFileSystemURIs = targetFs; + targetFileSystemPaths = targetFs; } public Path getMountedOnPath() { @@ -226,7 +228,15 @@ public Path getMountedOnPath() { } public URI[] getTargetFileSystemURIs() { - return targetFileSystemURIs; + URI[] targetUris = new URI[targetFileSystemPaths.length]; + for (int i = 0; i < targetFileSystemPaths.length; i++) { + targetUris[i] = URI.create(targetFileSystemPaths[i]); + } + return targetUris; + } + + public String[] getTargetFileSystemPaths() { + return targetFileSystemPaths; } } @@ -1130,16 +1140,79 @@ public Collection getAllStoragePolicies() * Get the trash root directory for current user when the path * specified is deleted. * + * If FORCE_INSIDE_MOUNT_POINT flag is not set, return the default trash root + * from targetFS. + * + * When FORCE_INSIDE_MOUNT_POINT is set to true, + *
    + *
  1. + * If the trash root for path p is in the same mount point as path p, + * and one of: + *
      + *
    1. The mount point isn't at the top of the target fs.
      2. + *
    2. The resolved path of path is root (in fallback FS).
      3. + *
    3. The trash isn't in user's target fs home directory + * get the corresponding viewFS path for the trash root and return + * it. + *
      4. + *
    + *
    2. + *
  2. + * else, return the trash root under the root of the mount point + * (/{mntpoint}/.Trash/{user}). + *
    3. + *
+ * + * These conditions handle several different important cases: + *
    + *
  • File systems may need to have more local trash roots, such as + * encryption zones or snapshot roots.
    • + *
  • The fallback mount should use the user's home directory.
    • + *
  • Cloud storage systems should not use trash in an implicity defined + * home directory, per a container, unless it is the fallback fs.
    • + *
+ * * @param path the trash root of the path to be determined. * @return the trash root path. */ @Override public Path getTrashRoot(Path path) { + try { InodeTree.ResolveResult res = fsState.resolve(getUriPath(path), true); - return res.targetFileSystem.getTrashRoot(res.remainingPath); - } catch (Exception e) { + Path targetFSTrashRoot = + res.targetFileSystem.getTrashRoot(res.remainingPath); + + // Allow clients to use old behavior of delegating to target fs. + if (!config.getBoolean(CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT, + CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT_DEFAULT)) { + return targetFSTrashRoot; + } + + // The trash root path from the target fs + String targetFSTrashRootPath = targetFSTrashRoot.toUri().getPath(); + // The mount point path in the target fs + String mountTargetPath = res.targetFileSystem.getUri().getPath(); + if (!mountTargetPath.endsWith("/")) { + mountTargetPath = mountTargetPath + "/"; + } + + Path targetFsUserHome = res.targetFileSystem.getHomeDirectory(); + if (targetFSTrashRootPath.startsWith(mountTargetPath) && + !(mountTargetPath.equals(ROOT_PATH.toString()) && + !res.resolvedPath.equals(ROOT_PATH.toString()) && + (targetFsUserHome != null && targetFSTrashRootPath.startsWith( + targetFsUserHome.toUri().getPath())))) { + String relativeTrashRoot = + targetFSTrashRootPath.substring(mountTargetPath.length()); + return makeQualified(new Path(res.resolvedPath, relativeTrashRoot)); + } else { + // Return the trash root for the mount point. + return makeQualified(new Path(res.resolvedPath, + TRASH_PREFIX + "/" + ugi.getShortUserName())); + } + } catch (IOException | IllegalArgumentException e) { throw new NotInMountpointException(path, "getTrashRoot"); } } @@ -1147,16 +1220,78 @@ public Path getTrashRoot(Path path) { /** * Get all the trash roots for current user or all users. * + * When FORCE_INSIDE_MOUNT_POINT is set to true, we also return trash roots + * under the root of each mount point, with their viewFS paths. + * * @param allUsers return trash roots for all users if true. * @return all Trash root directories. */ @Override public Collection getTrashRoots(boolean allUsers) { - List trashRoots = new ArrayList<>(); + // A map from targetFSPath -> FileStatus. + // FileStatus can be from targetFS or viewFS. + HashMap trashRoots = new HashMap<>(); for (FileSystem fs : getChildFileSystems()) { - trashRoots.addAll(fs.getTrashRoots(allUsers)); + for (FileStatus trash : fs.getTrashRoots(allUsers)) { + trashRoots.put(trash.getPath(), trash); + } + } + + // Return trashRoots if FORCE_INSIDE_MOUNT_POINT is disabled. + if (!config.getBoolean(CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT, + CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT_DEFAULT)) { + return trashRoots.values(); } - return trashRoots; + + // Get trash roots in TRASH_PREFIX dir inside mount points and fallback FS. + List> mountPoints = + fsState.getMountPoints(); + // If we have a fallback FS, add a mount point for it as <"", fallback FS>. + // The source path of a mount point shall not end with '/', thus for + // fallback fs, we set its mount point src as "". + if (fsState.getRootFallbackLink() != null) { + mountPoints.add(new InodeTree.MountPoint<>("", + fsState.getRootFallbackLink())); + } + + try { + for (InodeTree.MountPoint mountPoint : mountPoints) { + + Path trashRoot = + makeQualified(new Path(mountPoint.src + "/" + TRASH_PREFIX)); + + // Continue if trashRoot does not exist for this mount point + if (!exists(trashRoot)) { + continue; + } + + FileSystem targetFS = mountPoint.target.getTargetFileSystem(); + if (!allUsers) { + Path userTrashRoot = new Path(trashRoot, ugi.getShortUserName()); + if (exists(userTrashRoot)) { + Path targetFSUserTrashRoot = targetFS.makeQualified( + new Path(targetFS.getUri().getPath(), + TRASH_PREFIX + "/" + ugi.getShortUserName())); + trashRoots.put(targetFSUserTrashRoot, getFileStatus(userTrashRoot)); + } + } else { + FileStatus[] mountPointTrashRoots = listStatus(trashRoot); + for (FileStatus trash : mountPointTrashRoots) { + // Remove the mountPoint and the leading '/' to get the + // relative targetFsTrash path + String targetFsTrash = trash.getPath().toUri().getPath() + .substring(mountPoint.src.length() + 1); + Path targetFsTrashPath = targetFS.makeQualified( + new Path(targetFS.getUri().getPath(), targetFsTrash)); + trashRoots.put(targetFsTrashPath, trash); + } + } + } + } catch (IOException e) { + LOG.warn("Exception in get all trash roots for mount points", e); + } + + return trashRoots.values(); } @Override @@ -1574,11 +1709,6 @@ public boolean mkdirs(Path dir, FsPermission permission) throw readOnlyMountTable("mkdirs", dir); } - @Override - public boolean mkdirs(Path dir) throws IOException { - return mkdirs(dir, null); - } - @Override public FSDataInputStream open(Path f, int bufferSize) throws AccessControlException, FileNotFoundException, IOException { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java index deaf4f4108a3e..6e35ddf19053b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java @@ -185,16 +185,18 @@ static AccessControlException readOnlyMountTable(final String operation, static public class MountPoint { - private Path src; // the src of the mount - private URI[] targets; // target of the mount; Multiple targets imply mergeMount - MountPoint(Path srcPath, URI[] targetURIs) { + // the src of the mount + private Path src; + // Target of the mount; Multiple targets imply mergeMount + private String[] targets; + MountPoint(Path srcPath, String[] targetURIs) { src = srcPath; targets = targetURIs; } Path getSrc() { return src; } - URI[] getTargets() { + String[] getTargets() { return targets; } } @@ -744,6 +746,17 @@ public List> getDelegationTokens(String renewer) throws IOException { result.addAll(tokens); } } + + // Add tokens from fallback FS + if (this.fsState.getRootFallbackLink() != null) { + AbstractFileSystem rootFallbackFs = + this.fsState.getRootFallbackLink().getTargetFileSystem(); + List> tokens = rootFallbackFs.getDelegationTokens(renewer); + if (tokens != null) { + result.addAll(tokens); + } + } + return result; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLog.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLog.java index b2f18538b6c7d..863afdf1f2e53 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLog.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLog.java @@ -17,16 +17,12 @@ */ package org.apache.hadoop.http; +import java.util.Collections; import java.util.HashMap; - -import org.apache.commons.logging.impl.Log4JLogger; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogConfigurationException; -import org.apache.commons.logging.LogFactory; -import org.apache.log4j.Appender; -import org.eclipse.jetty.server.AsyncRequestLogWriter; +import java.util.Map; import org.eclipse.jetty.server.CustomRequestLog; import org.eclipse.jetty.server.RequestLog; +import org.eclipse.jetty.server.Slf4jRequestLogWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,67 +33,27 @@ public class HttpRequestLog { public static final Logger LOG = LoggerFactory.getLogger(HttpRequestLog.class); - private static final HashMap serverToComponent; + private static final Map serverToComponent; static { - serverToComponent = new HashMap(); - serverToComponent.put("cluster", "resourcemanager"); - serverToComponent.put("hdfs", "namenode"); - serverToComponent.put("node", "nodemanager"); + Map map = new HashMap(); + map.put("cluster", "resourcemanager"); + map.put("hdfs", "namenode"); + map.put("node", "nodemanager"); + serverToComponent = Collections.unmodifiableMap(map); } public static RequestLog getRequestLog(String name) { - String lookup = serverToComponent.get(name); if (lookup != null) { name = lookup; } String loggerName = "http.requests." + name; - String appenderName = name + "requestlog"; - Log logger = LogFactory.getLog(loggerName); - - boolean isLog4JLogger;; - try { - isLog4JLogger = logger instanceof Log4JLogger; - } catch (NoClassDefFoundError err) { - // In some dependent projects, log4j may not even be on the classpath at - // runtime, in which case the above instanceof check will throw - // NoClassDefFoundError. - LOG.debug("Could not load Log4JLogger class", err); - isLog4JLogger = false; - } - if (isLog4JLogger) { - Log4JLogger httpLog4JLog = (Log4JLogger)logger; - org.apache.log4j.Logger httpLogger = httpLog4JLog.getLogger(); - Appender appender = null; - - try { - appender = httpLogger.getAppender(appenderName); - } catch (LogConfigurationException e) { - LOG.warn("Http request log for {} could not be created", loggerName); - throw e; - } - - if (appender == null) { - LOG.info("Http request log for {} is not defined", loggerName); - return null; - } + Slf4jRequestLogWriter writer = new Slf4jRequestLogWriter(); + writer.setLoggerName(loggerName); + return new CustomRequestLog(writer, CustomRequestLog.EXTENDED_NCSA_FORMAT); + } - if (appender instanceof HttpRequestLogAppender) { - HttpRequestLogAppender requestLogAppender - = (HttpRequestLogAppender)appender; - AsyncRequestLogWriter logWriter = new AsyncRequestLogWriter(); - logWriter.setFilename(requestLogAppender.getFilename()); - logWriter.setRetainDays(requestLogAppender.getRetainDays()); - return new CustomRequestLog(logWriter, - CustomRequestLog.EXTENDED_NCSA_FORMAT); - } else { - LOG.warn("Jetty request log for {} was of the wrong class", loggerName); - return null; - } - } else { - LOG.warn("Jetty request log can only be enabled using Log4j"); - return null; - } + private HttpRequestLog() { } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java index 76e77560a58bf..49807ac4b4597 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java @@ -27,6 +27,7 @@ import java.net.MalformedURLException; import java.net.URI; import java.net.URL; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.List; @@ -771,6 +772,28 @@ private void initializeWebServer(String name, String hostName, addDefaultServlets(); addPrometheusServlet(conf); + addAsyncProfilerServlet(contexts, conf); + } + + private void addAsyncProfilerServlet(ContextHandlerCollection contexts, Configuration conf) + throws IOException { + final String asyncProfilerHome = ProfileServlet.getAsyncProfilerHome(); + if (asyncProfilerHome != null && !asyncProfilerHome.trim().isEmpty()) { + addServlet("prof", "/prof", ProfileServlet.class); + Path tmpDir = Paths.get(ProfileServlet.OUTPUT_DIR); + if (Files.notExists(tmpDir)) { + Files.createDirectories(tmpDir); + } + ServletContextHandler genCtx = new ServletContextHandler(contexts, "/prof-output-hadoop"); + genCtx.addServlet(ProfileOutputServlet.class, "/*"); + genCtx.setResourceBase(tmpDir.toAbsolutePath().toString()); + genCtx.setDisplayName("prof-output-hadoop"); + setContextAttributes(genCtx, conf); + } else { + addServlet("prof", "/prof", ProfilerDisabledServlet.class); + LOG.info("ASYNC_PROFILER_HOME environment variable and async.profiler.home system property " + + "not specified. Disabling /prof endpoint."); + } } private void addPrometheusServlet(Configuration conf) { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfileOutputServlet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfileOutputServlet.java new file mode 100644 index 0000000000000..1ecc21f3753ce --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfileOutputServlet.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.http; + +import java.io.File; +import java.io.IOException; +import java.util.regex.Pattern; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.eclipse.jetty.servlet.DefaultServlet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.classification.InterfaceAudience; + +/** + * Servlet to serve files generated by {@link ProfileServlet}. + */ +@InterfaceAudience.Private +public class ProfileOutputServlet extends DefaultServlet { + + private static final long serialVersionUID = 1L; + + private static final Logger LOG = LoggerFactory.getLogger(ProfileOutputServlet.class); + // default refresh period 2 sec + private static final int REFRESH_PERIOD = 2; + // Alphanumeric characters, plus percent (url-encoding), equals, ampersand, dot and hyphen + private static final Pattern ALPHA_NUMERIC = Pattern.compile("[a-zA-Z0-9%=&.\\-]*"); + + @Override + protected void doGet(final HttpServletRequest req, final HttpServletResponse resp) + throws ServletException, IOException { + if (!HttpServer2.isInstrumentationAccessAllowed(getServletContext(), req, resp)) { + resp.setStatus(HttpServletResponse.SC_UNAUTHORIZED); + ProfileServlet.setResponseHeader(resp); + resp.getWriter().write("Unauthorized: Instrumentation access is not allowed!"); + return; + } + + String absoluteDiskPath = getServletContext().getRealPath(req.getPathInfo()); + File requestedFile = new File(absoluteDiskPath); + // async-profiler version 1.4 writes 'Started [cpu] profiling' to output file when profiler is + // running which gets replaced by final output. If final output is not ready yet, the file size + // will be <100 bytes (in all modes). + if (requestedFile.length() < 100) { + LOG.info("{} is incomplete. Sending auto-refresh header.", requestedFile); + String refreshUrl = req.getRequestURI(); + // Rebuild the query string (if we have one) + if (req.getQueryString() != null) { + refreshUrl += "?" + sanitize(req.getQueryString()); + } + ProfileServlet.setResponseHeader(resp); + resp.setHeader("Refresh", REFRESH_PERIOD + ";" + refreshUrl); + resp.getWriter().write("This page will be auto-refreshed every " + REFRESH_PERIOD + + " seconds until the output file is ready. Redirecting to " + refreshUrl); + } else { + super.doGet(req, resp); + } + } + + static String sanitize(String input) { + // Basic test to try to avoid any XSS attacks or HTML content showing up. + // Duplicates HtmlQuoting a little, but avoid destroying ampersand. + if (ALPHA_NUMERIC.matcher(input).matches()) { + return input; + } + throw new RuntimeException("Non-alphanumeric data found in input, aborting."); + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfileServlet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfileServlet.java new file mode 100644 index 0000000000000..ce532741512fc --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfileServlet.java @@ -0,0 +1,403 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.http; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.util.ProcessUtils; + +/** + * Servlet that runs async-profiler as web-endpoint. + *

+ * Following options from async-profiler can be specified as query paramater. + * // -e event profiling event: cpu|alloc|lock|cache-misses etc. + * // -d duration run profiling for 'duration' seconds (integer) + * // -i interval sampling interval in nanoseconds (long) + * // -j jstackdepth maximum Java stack depth (integer) + * // -b bufsize frame buffer size (long) + * // -t profile different threads separately + * // -s simple class names instead of FQN + * // -o fmt[,fmt...] output format: summary|traces|flat|collapsed|svg|tree|jfr|html + * // --width px SVG width pixels (integer) + * // --height px SVG frame height pixels (integer) + * // --minwidth px skip frames smaller than px (double) + * // --reverse generate stack-reversed FlameGraph / Call tree + *

+ * Example: + * If Namenode http address is localhost:9870, and ResourceManager http address is localhost:8088, + * ProfileServlet running with async-profiler setup can be accessed with + * http://localhost:9870/prof and http://localhost:8088/prof for Namenode and ResourceManager + * processes respectively. + * Deep dive into some params: + * - To collect 10 second CPU profile of current process i.e. Namenode (returns FlameGraph svg) + * curl "http://localhost:9870/prof" + * - To collect 10 second CPU profile of pid 12345 (returns FlameGraph svg) + * curl "http://localhost:9870/prof?pid=12345" (For instance, provide pid of Datanode) + * - To collect 30 second CPU profile of pid 12345 (returns FlameGraph svg) + * curl "http://localhost:9870/prof?pid=12345&duration=30" + * - To collect 1 minute CPU profile of current process and output in tree format (html) + * curl "http://localhost:9870/prof?output=tree&duration=60" + * - To collect 10 second heap allocation profile of current process (returns FlameGraph svg) + * curl "http://localhost:9870/prof?event=alloc" + * - To collect lock contention profile of current process (returns FlameGraph svg) + * curl "http://localhost:9870/prof?event=lock" + *

+ * Following event types are supported (default is 'cpu') (NOTE: not all OS'es support all events) + * // Perf events: + * // cpu + * // page-faults + * // context-switches + * // cycles + * // instructions + * // cache-references + * // cache-misses + * // branches + * // branch-misses + * // bus-cycles + * // L1-dcache-load-misses + * // LLC-load-misses + * // dTLB-load-misses + * // mem:breakpoint + * // trace:tracepoint + * // Java events: + * // alloc + * // lock + */ +@InterfaceAudience.Private +public class ProfileServlet extends HttpServlet { + + private static final long serialVersionUID = 1L; + private static final Logger LOG = LoggerFactory.getLogger(ProfileServlet.class); + + static final String ACCESS_CONTROL_ALLOW_METHODS = "Access-Control-Allow-Methods"; + static final String ACCESS_CONTROL_ALLOW_ORIGIN = "Access-Control-Allow-Origin"; + private static final String ALLOWED_METHODS = "GET"; + private static final String CONTENT_TYPE_TEXT = "text/plain; charset=utf-8"; + private static final String ASYNC_PROFILER_HOME_ENV = "ASYNC_PROFILER_HOME"; + private static final String ASYNC_PROFILER_HOME_SYSTEM_PROPERTY = "async.profiler.home"; + private static final String PROFILER_SCRIPT = "/profiler.sh"; + private static final int DEFAULT_DURATION_SECONDS = 10; + private static final AtomicInteger ID_GEN = new AtomicInteger(0); + + static final String OUTPUT_DIR = System.getProperty("java.io.tmpdir") + "/prof-output-hadoop"; + + // This flag is only allowed to be reset by tests. + private static boolean isTestRun = false; + + private enum Event { + + CPU("cpu"), + ALLOC("alloc"), + LOCK("lock"), + PAGE_FAULTS("page-faults"), + CONTEXT_SWITCHES("context-switches"), + CYCLES("cycles"), + INSTRUCTIONS("instructions"), + CACHE_REFERENCES("cache-references"), + CACHE_MISSES("cache-misses"), + BRANCHES("branches"), + BRANCH_MISSES("branch-misses"), + BUS_CYCLES("bus-cycles"), + L1_DCACHE_LOAD_MISSES("L1-dcache-load-misses"), + LLC_LOAD_MISSES("LLC-load-misses"), + DTLB_LOAD_MISSES("dTLB-load-misses"), + MEM_BREAKPOINT("mem:breakpoint"), + TRACE_TRACEPOINT("trace:tracepoint"); + + private final String internalName; + + Event(final String internalName) { + this.internalName = internalName; + } + + public String getInternalName() { + return internalName; + } + + public static Event fromInternalName(final String name) { + for (Event event : values()) { + if (event.getInternalName().equalsIgnoreCase(name)) { + return event; + } + } + + return null; + } + } + + private enum Output { + SUMMARY, + TRACES, + FLAT, + COLLAPSED, + // No SVG in 2.x asyncprofiler. + SVG, + TREE, + JFR, + // In 2.x asyncprofiler, this is how you get flamegraphs. + HTML + } + + private final Lock profilerLock = new ReentrantLock(); + private transient volatile Process process; + private final String asyncProfilerHome; + private Integer pid; + + public ProfileServlet() { + this.asyncProfilerHome = getAsyncProfilerHome(); + this.pid = ProcessUtils.getPid(); + LOG.info("Servlet process PID: {} asyncProfilerHome: {}", pid, asyncProfilerHome); + } + + static void setIsTestRun(boolean isTestRun) { + ProfileServlet.isTestRun = isTestRun; + } + + @Override + protected void doGet(final HttpServletRequest req, final HttpServletResponse resp) + throws IOException { + if (!HttpServer2.isInstrumentationAccessAllowed(getServletContext(), req, resp)) { + resp.setStatus(HttpServletResponse.SC_UNAUTHORIZED); + setResponseHeader(resp); + resp.getWriter().write("Unauthorized: Instrumentation access is not allowed!"); + return; + } + + // make sure async profiler home is set + if (asyncProfilerHome == null || asyncProfilerHome.trim().isEmpty()) { + resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + setResponseHeader(resp); + resp.getWriter().write("ASYNC_PROFILER_HOME env is not set.\n\n" + + "Please ensure the prerequisites for the Profiler Servlet have been installed and the\n" + + "environment is properly configured."); + return; + } + + // if pid is explicitly specified, use it else default to current process + pid = getInteger(req, "pid", pid); + + // if pid is not specified in query param and if current process pid cannot be determined + if (pid == null) { + resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + setResponseHeader(resp); + resp.getWriter().write( + "'pid' query parameter unspecified or unable to determine PID of current process."); + return; + } + + final int duration = getInteger(req, "duration", DEFAULT_DURATION_SECONDS); + final Output output = getOutput(req); + final Event event = getEvent(req); + final Long interval = getLong(req, "interval"); + final Integer jstackDepth = getInteger(req, "jstackdepth", null); + final Long bufsize = getLong(req, "bufsize"); + final boolean thread = req.getParameterMap().containsKey("thread"); + final boolean simple = req.getParameterMap().containsKey("simple"); + final Integer width = getInteger(req, "width", null); + final Integer height = getInteger(req, "height", null); + final Double minwidth = getMinWidth(req); + final boolean reverse = req.getParameterMap().containsKey("reverse"); + + if (process == null || !process.isAlive()) { + try { + int lockTimeoutSecs = 3; + if (profilerLock.tryLock(lockTimeoutSecs, TimeUnit.SECONDS)) { + try { + File outputFile = new File(OUTPUT_DIR, + "async-prof-pid-" + pid + "-" + event.name().toLowerCase() + "-" + ID_GEN + .incrementAndGet() + "." + output.name().toLowerCase()); + List cmd = new ArrayList<>(); + cmd.add(asyncProfilerHome + PROFILER_SCRIPT); + cmd.add("-e"); + cmd.add(event.getInternalName()); + cmd.add("-d"); + cmd.add("" + duration); + cmd.add("-o"); + cmd.add(output.name().toLowerCase()); + cmd.add("-f"); + cmd.add(outputFile.getAbsolutePath()); + if (interval != null) { + cmd.add("-i"); + cmd.add(interval.toString()); + } + if (jstackDepth != null) { + cmd.add("-j"); + cmd.add(jstackDepth.toString()); + } + if (bufsize != null) { + cmd.add("-b"); + cmd.add(bufsize.toString()); + } + if (thread) { + cmd.add("-t"); + } + if (simple) { + cmd.add("-s"); + } + if (width != null) { + cmd.add("--width"); + cmd.add(width.toString()); + } + if (height != null) { + cmd.add("--height"); + cmd.add(height.toString()); + } + if (minwidth != null) { + cmd.add("--minwidth"); + cmd.add(minwidth.toString()); + } + if (reverse) { + cmd.add("--reverse"); + } + cmd.add(pid.toString()); + if (!isTestRun) { + process = ProcessUtils.runCmdAsync(cmd); + } + + // set response and set refresh header to output location + setResponseHeader(resp); + resp.setStatus(HttpServletResponse.SC_ACCEPTED); + String relativeUrl = "/prof-output-hadoop/" + outputFile.getName(); + resp.getWriter().write("Started [" + event.getInternalName() + + "] profiling. This page will automatically redirect to " + relativeUrl + " after " + + duration + " seconds. " + + "If empty diagram and Linux 4.6+, see 'Basic Usage' section on the Async " + + "Profiler Home Page, https://github.com/jvm-profiling-tools/async-profiler." + + "\n\nCommand:\n" + Joiner.on(" ").join(cmd)); + + // to avoid auto-refresh by ProfileOutputServlet, refreshDelay can be specified + // via url param + int refreshDelay = getInteger(req, "refreshDelay", 0); + + // instead of sending redirect, set auto-refresh so that browsers will refresh + // with redirected url + resp.setHeader("Refresh", (duration + refreshDelay) + ";" + relativeUrl); + resp.getWriter().flush(); + } finally { + profilerLock.unlock(); + } + } else { + setResponseHeader(resp); + resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + resp.getWriter() + .write("Unable to acquire lock. Another instance of profiler might be running."); + LOG.warn("Unable to acquire lock in {} seconds. Another instance of profiler might be" + + " running.", lockTimeoutSecs); + } + } catch (InterruptedException e) { + LOG.warn("Interrupted while acquiring profile lock.", e); + resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + } + } else { + setResponseHeader(resp); + resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + resp.getWriter().write("Another instance of profiler is already running."); + } + } + + private Integer getInteger(final HttpServletRequest req, final String param, + final Integer defaultValue) { + final String value = req.getParameter(param); + if (value != null) { + try { + return Integer.valueOf(value); + } catch (NumberFormatException e) { + return defaultValue; + } + } + return defaultValue; + } + + private Long getLong(final HttpServletRequest req, final String param) { + final String value = req.getParameter(param); + if (value != null) { + try { + return Long.valueOf(value); + } catch (NumberFormatException e) { + return null; + } + } + return null; + } + + private Double getMinWidth(final HttpServletRequest req) { + final String value = req.getParameter("minwidth"); + if (value != null) { + try { + return Double.valueOf(value); + } catch (NumberFormatException e) { + return null; + } + } + return null; + } + + private Event getEvent(final HttpServletRequest req) { + final String eventArg = req.getParameter("event"); + if (eventArg != null) { + Event event = Event.fromInternalName(eventArg); + return event == null ? Event.CPU : event; + } + return Event.CPU; + } + + private Output getOutput(final HttpServletRequest req) { + final String outputArg = req.getParameter("output"); + if (req.getParameter("output") != null) { + try { + return Output.valueOf(outputArg.trim().toUpperCase()); + } catch (IllegalArgumentException e) { + return Output.HTML; + } + } + return Output.HTML; + } + + static void setResponseHeader(final HttpServletResponse response) { + response.setHeader(ACCESS_CONTROL_ALLOW_METHODS, ALLOWED_METHODS); + response.setHeader(ACCESS_CONTROL_ALLOW_ORIGIN, "*"); + response.setContentType(CONTENT_TYPE_TEXT); + } + + static String getAsyncProfilerHome() { + String asyncProfilerHome = System.getenv(ASYNC_PROFILER_HOME_ENV); + // if ENV is not set, see if -Dasync.profiler.home=/path/to/async/profiler/home is set + if (asyncProfilerHome == null || asyncProfilerHome.trim().isEmpty()) { + asyncProfilerHome = System.getProperty(ASYNC_PROFILER_HOME_SYSTEM_PROPERTY); + } + + return asyncProfilerHome; + } + +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfilerDisabledServlet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfilerDisabledServlet.java new file mode 100644 index 0000000000000..c488b574990cc --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfilerDisabledServlet.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.http; + +import java.io.IOException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.hadoop.classification.InterfaceAudience; + +/** + * Servlet for disabled async-profiler. + */ +@InterfaceAudience.Private +public class ProfilerDisabledServlet extends HttpServlet { + + @Override + protected void doGet(final HttpServletRequest req, final HttpServletResponse resp) + throws IOException { + resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + ProfileServlet.setResponseHeader(resp); + // TODO : Replace github.com link with + // https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/ + // AsyncProfilerServlet.html once Async profiler changes are released + // in 3.x (3.4.0 as of today). + resp.getWriter().write("The profiler servlet was disabled at startup.\n\n" + + "Please ensure the prerequisites for the Profiler Servlet have been installed and the\n" + + "environment is properly configured. \n\n" + + "For more details, please refer to: https://github.com/apache/hadoop/blob/trunk/" + + "hadoop-common-project/hadoop-common/src/site/markdown/AsyncProfilerServlet.md"); + } + +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java index 037bbd3fd0f21..890e7916ab076 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java @@ -57,6 +57,11 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_SEQFILE_COMPRESS_BLOCKSIZE_KEY; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_SKIP_CHECKSUM_ERRORS_DEFAULT; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_SKIP_CHECKSUM_ERRORS_KEY; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_BUFFER_SIZE; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_LENGTH; +import static org.apache.hadoop.util.functional.FutureIO.awaitFuture; /** * SequenceFiles are flat files consisting of binary key/value @@ -1948,7 +1953,14 @@ private void initialize(Path filename, FSDataInputStream in, */ protected FSDataInputStream openFile(FileSystem fs, Path file, int bufferSize, long length) throws IOException { - return fs.open(file, bufferSize); + FutureDataInputStreamBuilder builder = fs.openFile(file) + .opt(FS_OPTION_OPENFILE_READ_POLICY, + FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL) + .opt(FS_OPTION_OPENFILE_BUFFER_SIZE, bufferSize); + if (length >= 0) { + builder.opt(FS_OPTION_OPENFILE_LENGTH, length); + } + return awaitFuture(builder.build()); } /** diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java index 1fa7fd4b52be5..a195ed4e77fd4 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java @@ -40,7 +40,7 @@ public class CompressionCodecFactory { LoggerFactory.getLogger(CompressionCodecFactory.class.getName()); private static final ServiceLoader CODEC_PROVIDERS = - ServiceLoader.load(CompressionCodec.class); + ServiceLoader.load(CompressionCodec.class); /** * A map from the reversed filename suffixes to the codecs. @@ -49,15 +49,15 @@ public class CompressionCodecFactory { */ private SortedMap codecs = null; - /** - * A map from the reversed filename suffixes to the codecs. - * This is probably overkill, because the maps should be small, but it - * automatically supports finding the longest matching suffix. - */ - private Map codecsByName = null; + /** + * A map from the reversed filename suffixes to the codecs. + * This is probably overkill, because the maps should be small, but it + * automatically supports finding the longest matching suffix. + */ + private Map codecsByName = null; /** - * A map from class names to the codecs + * A map from class names to the codecs. */ private HashMap codecsByClassName = null; @@ -80,8 +80,8 @@ private void addCodec(CompressionCodec codec) { @Override public String toString() { StringBuilder buf = new StringBuilder(); - Iterator> itr = - codecs.entrySet().iterator(); + Iterator> itr = + codecs.entrySet().iterator(); buf.append("{ "); if (itr.hasNext()) { Map.Entry entry = itr.next(); @@ -110,8 +110,8 @@ public String toString() { */ public static List> getCodecClasses( Configuration conf) { - List> result - = new ArrayList>(); + List> result = + new ArrayList>(); // Add codec classes discovered via service loading synchronized (CODEC_PROVIDERS) { // CODEC_PROVIDERS is a lazy collection. Synchronize so it is @@ -200,11 +200,13 @@ public CompressionCodec getCodec(Path file) { String filename = file.getName(); String reversedFilename = new StringBuilder(filename).reverse().toString(); - SortedMap subMap = - codecs.headMap(reversedFilename); + String lowerReversedFilename = + StringUtils.toLowerCase(reversedFilename); + SortedMap subMap = + codecs.headMap(lowerReversedFilename); if (!subMap.isEmpty()) { String potentialSuffix = subMap.lastKey(); - if (reversedFilename.startsWith(potentialSuffix)) { + if (lowerReversedFilename.startsWith(potentialSuffix)) { result = codecs.get(potentialSuffix); } } @@ -224,57 +226,57 @@ public CompressionCodec getCodecByClassName(String classname) { return codecsByClassName.get(classname); } - /** - * Find the relevant compression codec for the codec's canonical class name - * or by codec alias. - *

- * Codec aliases are case insensitive. - *

- * The code alias is the short class name (without the package name). - * If the short class name ends with 'Codec', then there are two aliases for - * the codec, the complete short class name and the short class name without - * the 'Codec' ending. For example for the 'GzipCodec' codec class name the - * alias are 'gzip' and 'gzipcodec'. - * - * @param codecName the canonical class name of the codec - * @return the codec object - */ - public CompressionCodec getCodecByName(String codecName) { - if (codecsByClassName == null) { - return null; - } - CompressionCodec codec = getCodecByClassName(codecName); - if (codec == null) { - // trying to get the codec by name in case the name was specified - // instead a class - codec = codecsByName.get(StringUtils.toLowerCase(codecName)); - } - return codec; + /** + * Find the relevant compression codec for the codec's canonical class name + * or by codec alias. + *

+ * Codec aliases are case insensitive. + *

+ * The code alias is the short class name (without the package name). + * If the short class name ends with 'Codec', then there are two aliases for + * the codec, the complete short class name and the short class name without + * the 'Codec' ending. For example for the 'GzipCodec' codec class name the + * alias are 'gzip' and 'gzipcodec'. + * + * @param codecName the canonical class name of the codec + * @return the codec object + */ + public CompressionCodec getCodecByName(String codecName) { + if (codecsByClassName == null) { + return null; } + CompressionCodec codec = getCodecByClassName(codecName); + if (codec == null) { + // trying to get the codec by name in case the name was specified + // instead a class + codec = codecsByName.get(StringUtils.toLowerCase(codecName)); + } + return codec; + } - /** - * Find the relevant compression codec for the codec's canonical class name - * or by codec alias and returns its implemetation class. - *

- * Codec aliases are case insensitive. - *

- * The code alias is the short class name (without the package name). - * If the short class name ends with 'Codec', then there are two aliases for - * the codec, the complete short class name and the short class name without - * the 'Codec' ending. For example for the 'GzipCodec' codec class name the - * alias are 'gzip' and 'gzipcodec'. - * - * @param codecName the canonical class name of the codec - * @return the codec class - */ - public Class getCodecClassByName( - String codecName) { - CompressionCodec codec = getCodecByName(codecName); - if (codec == null) { - return null; - } - return codec.getClass(); + /** + * Find the relevant compression codec for the codec's canonical class name + * or by codec alias and returns its implemetation class. + *

+ * Codec aliases are case insensitive. + *

+ * The code alias is the short class name (without the package name). + * If the short class name ends with 'Codec', then there are two aliases for + * the codec, the complete short class name and the short class name without + * the 'Codec' ending. For example for the 'GzipCodec' codec class name the + * alias are 'gzip' and 'gzipcodec'. + * + * @param codecName the canonical class name of the codec + * @return the codec class + */ + public Class getCodecClassByName( + String codecName) { + CompressionCodec codec = getCodecByName(codecName); + if (codec == null) { + return null; } + return codec.getClass(); + } /** * Removes a suffix from a filename, if it has it. @@ -323,8 +325,12 @@ public static void main(String[] args) throws Exception { len = in.read(buffer); } } finally { - if(out != null) { out.close(); } - if(in != null) { in.close(); } + if(out != null) { + out.close(); + } + if(in != null) { + in.close(); + } } } else { CompressionInputStream in = null; @@ -338,7 +344,9 @@ public static void main(String[] args) throws Exception { len = in.read(buffer); } } finally { - if(in != null) { in.close(); } + if(in != null) { + in.close(); + } } } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.java index bc9d29cb4f294..adf2fe629f8f7 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.java @@ -113,6 +113,12 @@ private void setInputFromSavedData() { compressedDirectBuf.put( userBuf, userBufOff, bytesInCompressedBuffer); + // Set the finished to false when compressedDirectBuf still + // contains some bytes. + if (compressedDirectBuf.position() > 0 && finished) { + finished = false; + } + userBufOff += bytesInCompressedBuffer; userBufferBytesToConsume -= bytesInCompressedBuffer; } @@ -186,6 +192,13 @@ public int decompress(byte[] b, int off, int len) 0, directBufferSize ); + + // Set the finished to false when compressedDirectBuf still + // contains some bytes. + if (remaining > 0 && finished) { + finished = false; + } + uncompressedDirectBuf.limit(n); // Get at most 'len' bytes diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureDecoder.java index 249930ebe3f22..2ebe94b0385ab 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureDecoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureDecoder.java @@ -81,7 +81,7 @@ public RawErasureDecoder(ErasureCoderOptions coderOptions) { * @param outputs output buffers to put decoded data into according to * erasedIndexes, ready for read after the call */ - public void decode(ByteBuffer[] inputs, int[] erasedIndexes, + public synchronized void decode(ByteBuffer[] inputs, int[] erasedIndexes, ByteBuffer[] outputs) throws IOException { ByteBufferDecodingState decodingState = new ByteBufferDecodingState(this, inputs, erasedIndexes, outputs); @@ -130,7 +130,7 @@ protected abstract void doDecode(ByteBufferDecodingState decodingState) * erasedIndexes, ready for read after the call * @throws IOException if the decoder is closed. */ - public void decode(byte[][] inputs, int[] erasedIndexes, byte[][] outputs) + public synchronized void decode(byte[][] inputs, int[] erasedIndexes, byte[][] outputs) throws IOException { ByteArrayDecodingState decodingState = new ByteArrayDecodingState(this, inputs, erasedIndexes, outputs); @@ -163,7 +163,7 @@ protected abstract void doDecode(ByteArrayDecodingState decodingState) * erasedIndexes, ready for read after the call * @throws IOException if the decoder is closed */ - public void decode(ECChunk[] inputs, int[] erasedIndexes, + public synchronized void decode(ECChunk[] inputs, int[] erasedIndexes, ECChunk[] outputs) throws IOException { ByteBuffer[] newInputs = CoderUtil.toBuffers(inputs); ByteBuffer[] newOutputs = CoderUtil.toBuffers(outputs); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java index 79b489b3d1a98..ebe7f213ceeb1 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java @@ -141,7 +141,7 @@ public String getMessage() { } } - // Denotes the state of supporting PMDK. The value is set by JNI. + // Denotes the state of supporting PMDK. The actual value is set via JNI. private static SupportState pmdkSupportState = SupportState.UNSUPPORTED; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java index fcc5975987ebc..3960b189665f6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java @@ -387,12 +387,12 @@ private RetryInfo handleException(final Method method, final int callId, throw retryInfo.getFailException(); } - log(method, retryInfo.isFailover(), counters.failovers, retryInfo.delay, e); + log(method, retryInfo.isFailover(), counters.failovers, counters.retries, retryInfo.delay, e); return retryInfo; } - private void log(final Method method, final boolean isFailover, - final int failovers, final long delay, final Exception ex) { + private void log(final Method method, final boolean isFailover, final int failovers, + final int retries, final long delay, final Exception ex) { boolean info = true; // If this is the first failover to this proxy, skip logging at INFO level if (!failedAtLeastOnce.contains(proxyDescriptor.getProxyInfo().toString())) @@ -408,13 +408,15 @@ private void log(final Method method, final boolean isFailover, } final StringBuilder b = new StringBuilder() - .append(ex + ", while invoking ") + .append(ex) + .append(", while invoking ") .append(proxyDescriptor.getProxyInfo().getString(method.getName())); if (failovers > 0) { b.append(" after ").append(failovers).append(" failover attempts"); } b.append(isFailover? ". Trying to failover ": ". Retrying "); b.append(delay > 0? "after sleeping for " + delay + "ms.": "immediately."); + b.append(" Current retry count: ").append(retries).append("."); if (info) { LOG.info(b.toString()); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java index 9f9c9741f36bb..dbd9184a2b91e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java @@ -43,6 +43,11 @@ @InterfaceStability.Evolving public final class CallerContext { public static final Charset SIGNATURE_ENCODING = StandardCharsets.UTF_8; + + // field names + public static final String CLIENT_IP_STR = "clientIp"; + public static final String CLIENT_PORT_STR = "clientPort"; + /** The caller context. * * It will be truncated if it exceeds the maximum allowed length in @@ -116,7 +121,7 @@ public String toString() { /** The caller context builder. */ public static final class Builder { - private static final String KEY_VALUE_SEPARATOR = ":"; + public static final String KEY_VALUE_SEPARATOR = ":"; /** * The illegal separators include '\t', '\n', '='. * User should not set illegal separator. @@ -164,8 +169,6 @@ private void checkFieldSeparator(String separator) { /** * Whether the field is valid. - * The field should not contain '\t', '\n', '='. - * Because the context could be written to audit log. * @param field one of the fields in context. * @return true if the field is not null or empty. */ diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java index 96f925f0f2c0f..49432aff11789 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java @@ -807,17 +807,18 @@ public Object run() throws IOException, InterruptedException { */ private synchronized void setupIOstreams( AtomicBoolean fallbackToSimpleAuth) { - if (socket != null || shouldCloseConnection.get()) { - return; - } - UserGroupInformation ticket = remoteId.getTicket(); - if (ticket != null) { - final UserGroupInformation realUser = ticket.getRealUser(); - if (realUser != null) { - ticket = realUser; - } - } try { + if (socket != null || shouldCloseConnection.get()) { + setFallBackToSimpleAuth(fallbackToSimpleAuth); + return; + } + UserGroupInformation ticket = remoteId.getTicket(); + if (ticket != null) { + final UserGroupInformation realUser = ticket.getRealUser(); + if (realUser != null) { + ticket = realUser; + } + } connectingThread.set(Thread.currentThread()); if (LOG.isDebugEnabled()) { LOG.debug("Connecting to "+server); @@ -863,20 +864,8 @@ public AuthMethod run() remoteId.saslQop = (String)saslRpcClient.getNegotiatedProperty(Sasl.QOP); LOG.debug("Negotiated QOP is :" + remoteId.saslQop); - if (fallbackToSimpleAuth != null) { - fallbackToSimpleAuth.set(false); - } - } else if (UserGroupInformation.isSecurityEnabled()) { - if (!fallbackAllowed) { - throw new AccessControlException( - "Server asks us to fall back to SIMPLE " + - "auth, but this client is configured to only allow secure " + - "connections."); - } - if (fallbackToSimpleAuth != null) { - fallbackToSimpleAuth.set(true); - } } + setFallBackToSimpleAuth(fallbackToSimpleAuth); } if (doPing) { @@ -909,7 +898,41 @@ public AuthMethod run() connectingThread.set(null); } } - + + private void setFallBackToSimpleAuth(AtomicBoolean fallbackToSimpleAuth) + throws AccessControlException { + if (authMethod == null || authProtocol != AuthProtocol.SASL) { + if (authProtocol == AuthProtocol.SASL) { + LOG.trace("Auth method is not set, yield from setting auth fallback."); + } + return; + } + if (fallbackToSimpleAuth == null) { + // this should happen only during testing. + LOG.trace("Connection {} will skip to set fallbackToSimpleAuth as it is null.", remoteId); + } else { + if (fallbackToSimpleAuth.get()) { + // we already set the value to true, we do not need to examine again. + return; + } + } + if (authMethod != AuthMethod.SIMPLE) { + if (fallbackToSimpleAuth != null) { + LOG.trace("Disabling fallbackToSimpleAuth, target does not use SIMPLE authentication."); + fallbackToSimpleAuth.set(false); + } + } else if (UserGroupInformation.isSecurityEnabled()) { + if (!fallbackAllowed) { + throw new AccessControlException("Server asks us to fall back to SIMPLE auth, but this " + + "client is configured to only allow secure connections."); + } + if (fallbackToSimpleAuth != null) { + LOG.trace("Enabling fallbackToSimpleAuth for target, as we are allowed to fall back."); + fallbackToSimpleAuth.set(true); + } + } + } + private void closeConnection() { if (socket == null) { return; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java index 7a391c4994cdc..c5732c68b1517 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java @@ -498,6 +498,7 @@ protected ResponseBuffer initialValue() { private Map auxiliaryListenerMap; private Responder responder = null; private Handler[] handlers = null; + private final AtomicInteger numInProcessHandler = new AtomicInteger(); private boolean logSlowRPC = false; @@ -509,6 +510,10 @@ protected boolean isLogSlowRPC() { return logSlowRPC; } + public int getNumInProcessHandler() { + return numInProcessHandler.get(); + } + /** * Sets slow RPC flag. * @param logSlowRPCFlag @@ -3063,6 +3068,7 @@ public void run() { try { call = callQueue.take(); // pop the queue; maybe blocked here + numInProcessHandler.incrementAndGet(); startTimeNanos = Time.monotonicNowNanos(); if (alignmentContext != null && call.isCallCoordinated() && call.getClientStateId() > alignmentContext.getLastSeenStateId()) { @@ -3116,6 +3122,7 @@ public void run() { } } finally { CurCall.set(null); + numInProcessHandler.decrementAndGet(); IOUtils.cleanupWithLogger(LOG, traceScope); if (call != null) { updateMetrics(call, startTimeNanos, connDropped); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java index cbb4bb36d1da5..a67530b3c97b2 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java @@ -133,6 +133,11 @@ public static RpcMetrics create(Server server, Configuration conf) { return server.getNumOpenConnections(); } + @Metric("Number of in process handlers") + public int getNumInProcessHandler() { + return server.getNumInProcessHandler(); + } + @Metric("Number of open connections per user") public String numOpenConnectionsPerUser() { return server.getNumOpenConnectionsPerUser(); @@ -288,6 +293,7 @@ public void incrClientBackoff() { public void incrSlowRpc() { rpcSlowCalls.incr(); } + /** * Returns a MutableRate Counter. * @return Mutable Rate diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsJsonBuilder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsJsonBuilder.java index 1d62c0a29fca4..3a9be12803143 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsJsonBuilder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsJsonBuilder.java @@ -21,8 +21,9 @@ import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.map.ObjectWriter; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRollingAverages.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRollingAverages.java index 193ed0f71d7c7..aa4d4b9ca0c64 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRollingAverages.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRollingAverages.java @@ -167,7 +167,7 @@ synchronized void replaceScheduledTask(int windows, long interval, } @Override - public void snapshot(MetricsRecordBuilder builder, boolean all) { + public synchronized void snapshot(MetricsRecordBuilder builder, boolean all) { if (all || changed()) { for (final Entry> entry : averages.entrySet()) { @@ -179,8 +179,11 @@ public void snapshot(MetricsRecordBuilder builder, boolean all) { long totalCount = 0; for (final SumAndCount sumAndCount : entry.getValue()) { - totalCount += sumAndCount.getCount(); - totalSum += sumAndCount.getSum(); + if (Time.monotonicNow() - sumAndCount.getSnapshotTimeStamp() + < recordValidityMs) { + totalCount += sumAndCount.getCount(); + totalSum += sumAndCount.getSum(); + } } if (totalCount != 0) { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java index 4b924af03c196..fead87d7907d7 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java @@ -1053,6 +1053,32 @@ public static int getFreeSocketPort() { return port; } + /** + * Return free ports. There is no guarantee they will remain free, so + * ports should be used immediately. The number of free ports returned by + * this method should match argument {@code numOfPorts}. Num of ports + * provided in the argument should not exceed 25. + * + * @param numOfPorts Number of free ports to acquire. + * @return Free ports for binding a local socket. + */ + public static Set getFreeSocketPorts(int numOfPorts) { + Preconditions.checkArgument(numOfPorts > 0 && numOfPorts <= 25, + "Valid range for num of ports is between 0 and 26"); + final Set freePorts = new HashSet<>(numOfPorts); + for (int i = 0; i < numOfPorts * 5; i++) { + int port = getFreeSocketPort(); + if (port == 0) { + continue; + } + freePorts.add(port); + if (freePorts.size() == numOfPorts) { + return freePorts; + } + } + throw new IllegalStateException(numOfPorts + " free ports could not be acquired."); + } + /** * Return an @{@link InetAddress} to bind to. If bindWildCardAddress is true * than returns null. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java index 6e0d88f2a7b7b..137c940001c0c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java @@ -101,6 +101,13 @@ protected NetworkTopology init(InnerNode.Factory factory) { private int depthOfAllLeaves = -1; /** rack counter */ protected int numOfRacks = 0; + /** empty rack map, rackname->nodenumber. */ + private HashMap> rackMap = + new HashMap>(); + /** decommission nodes, contained stoped nodes. */ + private HashSet decommissionNodes = new HashSet<>(); + /** empty rack counter. */ + private int numOfEmptyRacks = 0; /** * Whether or not this cluster has ever consisted of more than 1 rack, @@ -150,6 +157,7 @@ public void add(Node node) { if (rack == null) { incrementRacks(); } + interAddNodeWithEmptyRack(node); if (depthOfAllLeaves == -1) { depthOfAllLeaves = node.getLevel(); } @@ -224,6 +232,7 @@ public void remove(Node node) { if (rack == null) { numOfRacks--; } + interRemoveNodeWithEmptyRack(node); } LOG.debug("NetworkTopology became:\n{}", this); } finally { @@ -1015,4 +1024,108 @@ protected static boolean isNodeInScope(Node node, String scope) { String nodeLocation = NodeBase.getPath(node) + NodeBase.PATH_SEPARATOR_STR; return nodeLocation.startsWith(scope); } -} \ No newline at end of file + + /** @return the number of nonempty racks */ + public int getNumOfNonEmptyRacks() { + return numOfRacks - numOfEmptyRacks; + } + + /** + * Update empty rack number when add a node like recommission. + * @param node node to be added; can be null + */ + public void recommissionNode(Node node) { + if (node == null) { + return; + } + if (node instanceof InnerNode) { + throw new IllegalArgumentException( + "Not allow to remove an inner node: " + NodeBase.getPath(node)); + } + netlock.writeLock().lock(); + try { + decommissionNodes.remove(node.getName()); + interAddNodeWithEmptyRack(node); + } finally { + netlock.writeLock().unlock(); + } + } + + /** + * Update empty rack number when remove a node like decommission. + * @param node node to be added; can be null + */ + public void decommissionNode(Node node) { + if (node == null) { + return; + } + if (node instanceof InnerNode) { + throw new IllegalArgumentException( + "Not allow to remove an inner node: " + NodeBase.getPath(node)); + } + netlock.writeLock().lock(); + try { + decommissionNodes.add(node.getName()); + interRemoveNodeWithEmptyRack(node); + } finally { + netlock.writeLock().unlock(); + } + } + + /** + * Internal function for update empty rack number + * for add or recommission a node. + * @param node node to be added; can be null + */ + private void interAddNodeWithEmptyRack(Node node) { + if (node == null) { + return; + } + String rackname = node.getNetworkLocation(); + Set nodes = rackMap.get(rackname); + if (nodes == null) { + nodes = new HashSet(); + } + if (!decommissionNodes.contains(node.getName())) { + nodes.add(node.getName()); + } + rackMap.put(rackname, nodes); + countEmptyRacks(); + } + + /** + * Internal function for update empty rack number + * for remove or decommission a node. + * @param node node to be removed; can be null + */ + private void interRemoveNodeWithEmptyRack(Node node) { + if (node == null) { + return; + } + String rackname = node.getNetworkLocation(); + Set nodes = rackMap.get(rackname); + if (nodes != null) { + InnerNode rack = (InnerNode) getNode(node.getNetworkLocation()); + if (rack == null) { + // this node and its rack are both removed. + rackMap.remove(rackname); + } else if (nodes.contains(node.getName())) { + // this node is decommissioned or removed. + nodes.remove(node.getName()); + rackMap.put(rackname, nodes); + } + countEmptyRacks(); + } + } + + private void countEmptyRacks() { + int count = 0; + for (Set nodes : rackMap.values()) { + if (nodes != null && nodes.isEmpty()) { + count++; + } + } + numOfEmptyRacks = count; + LOG.debug("Current numOfEmptyRacks is {}", numOfEmptyRacks); + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java index a3b1cbd14d822..b2efe502144cb 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java @@ -1508,7 +1508,19 @@ public UserGroupInformation getRealUser() { return null; } - + /** + * If this is a proxy user, get the real user. Otherwise, return + * this user. + * @param user the user to check + * @return the real user or self + */ + public static UserGroupInformation getRealUserOrSelf(UserGroupInformation user) { + if (user == null) { + return null; + } + UserGroupInformation real = user.getRealUser(); + return real != null ? real : user; + } /** * This class is used for storing the groups for testing. It stores a local diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java index b7283adcada56..6d2b1136da8c9 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java @@ -36,7 +36,17 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.statistics.DurationTracker; +import org.apache.hadoop.fs.statistics.DurationTrackerFactory; +import org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding; +import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore; import org.apache.hadoop.io.Text; +import org.apache.hadoop.metrics2.annotation.Metric; +import org.apache.hadoop.metrics2.annotation.Metrics; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.metrics2.lib.MetricsRegistry; +import org.apache.hadoop.metrics2.lib.MutableCounterLong; +import org.apache.hadoop.metrics2.lib.MutableRate; import org.apache.hadoop.metrics2.util.Metrics2Util.NameValuePair; import org.apache.hadoop.metrics2.util.Metrics2Util.TopN; import org.apache.hadoop.security.AccessControlException; @@ -47,6 +57,7 @@ import org.apache.hadoop.util.Time; import org.apache.hadoop.util.Preconditions; +import org.apache.hadoop.util.functional.InvocationRaisingIOE; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -59,6 +70,12 @@ class AbstractDelegationTokenSecretManager storeToken(identifier, tokenInfo)); } catch (IOException ioe) { LOG.error("Could not store token " + formatTokenId(identifier) + "!!", ioe); } return password; } - + /** @@ -555,7 +572,7 @@ public synchronized long renewToken(Token token, throw new InvalidToken("Renewal request for unknown token " + formatTokenId(id)); } - updateToken(id, info); + METRICS.trackUpdateToken(() -> updateToken(id, info)); return renewTime; } @@ -591,8 +608,10 @@ public synchronized TokenIdent cancelToken(Token token, if (info == null) { throw new InvalidToken("Token not found " + formatTokenId(id)); } - removeTokenForOwnerStats(id); - removeStoredToken(id); + METRICS.trackRemoveToken(() -> { + removeTokenForOwnerStats(id); + removeStoredToken(id); + }); return id; } @@ -825,4 +844,97 @@ protected void syncTokenOwnerStats() { addTokenForOwnerStats(id); } } + + protected DelegationTokenSecretManagerMetrics getMetrics() { + return METRICS; + } + + /** + * DelegationTokenSecretManagerMetrics tracks token management operations + * and publishes them through the metrics interfaces. + */ + @Metrics(about="Delegation token secret manager metrics", context="token") + static class DelegationTokenSecretManagerMetrics implements DurationTrackerFactory { + private static final Logger LOG = LoggerFactory.getLogger( + DelegationTokenSecretManagerMetrics.class); + + final static String STORE_TOKEN_STAT = "storeToken"; + final static String UPDATE_TOKEN_STAT = "updateToken"; + final static String REMOVE_TOKEN_STAT = "removeToken"; + final static String TOKEN_FAILURE_STAT = "tokenFailure"; + + private final MetricsRegistry registry; + private final IOStatisticsStore ioStatistics; + + @Metric("Rate of storage of delegation tokens and latency (milliseconds)") + private MutableRate storeToken; + @Metric("Rate of update of delegation tokens and latency (milliseconds)") + private MutableRate updateToken; + @Metric("Rate of removal of delegation tokens and latency (milliseconds)") + private MutableRate removeToken; + @Metric("Counter of delegation tokens operation failures") + private MutableCounterLong tokenFailure; + + static DelegationTokenSecretManagerMetrics create() { + return DefaultMetricsSystem.instance().register(new DelegationTokenSecretManagerMetrics()); + } + + DelegationTokenSecretManagerMetrics() { + ioStatistics = IOStatisticsBinding.iostatisticsStore() + .withDurationTracking(STORE_TOKEN_STAT, UPDATE_TOKEN_STAT, REMOVE_TOKEN_STAT) + .withCounters(TOKEN_FAILURE_STAT) + .build(); + registry = new MetricsRegistry("DelegationTokenSecretManagerMetrics"); + LOG.debug("Initialized {}", registry); + } + + public void trackStoreToken(InvocationRaisingIOE invocation) throws IOException { + trackInvocation(invocation, STORE_TOKEN_STAT, storeToken); + } + + public void trackUpdateToken(InvocationRaisingIOE invocation) throws IOException { + trackInvocation(invocation, UPDATE_TOKEN_STAT, updateToken); + } + + public void trackRemoveToken(InvocationRaisingIOE invocation) throws IOException { + trackInvocation(invocation, REMOVE_TOKEN_STAT, removeToken); + } + + public void trackInvocation(InvocationRaisingIOE invocation, String statistic, + MutableRate metric) throws IOException { + try { + long start = Time.monotonicNow(); + IOStatisticsBinding.trackDurationOfInvocation(this, statistic, invocation); + metric.add(Time.monotonicNow() - start); + } catch (Exception ex) { + tokenFailure.incr(); + throw ex; + } + } + + @Override + public DurationTracker trackDuration(String key, long count) { + return ioStatistics.trackDuration(key, count); + } + + protected MutableRate getStoreToken() { + return storeToken; + } + + protected MutableRate getUpdateToken() { + return updateToken; + } + + protected MutableRate getRemoveToken() { + return removeToken; + } + + protected MutableCounterLong getTokenFailure() { + return tokenFailure; + } + + protected IOStatisticsStore getIoStatistics() { + return ioStatistics; + } + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java index 5e5ea8cebc6f7..452565676a028 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java @@ -55,6 +55,7 @@ import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.delegation.web.DelegationTokenManager; import static org.apache.hadoop.util.Time.now; +import org.apache.hadoop.util.curator.ZKCuratorManager; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.KeeperException.NoNodeException; @@ -98,6 +99,8 @@ public abstract class ZKDelegationTokenSecretManager> consumer) { + consumer.accept(super.values().iterator()); + } + /** * Resize the internal table to given capacity. */ @SuppressWarnings("unchecked") - protected void resize(int cap) { + protected synchronized void resize(int cap) { int newCapacity = actualArrayLength(cap); if (newCapacity == this.capacity) { return; @@ -121,7 +143,7 @@ protected void resize(int cap) { /** * Checks if we need to expand, and expands if necessary. */ - protected void expandIfNecessary() { + protected synchronized void expandIfNecessary() { if (size > this.threshold && capacity < MAX_ARRAY_LENGTH) { resize(capacity * 2); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Lists.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Lists.java index b6d74ee679281..5d9cc0502afaa 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Lists.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Lists.java @@ -232,4 +232,28 @@ private static boolean addAll(Collection addTo, return addAll(addTo, elementsToAdd.iterator()); } + /** + * Returns consecutive sub-lists of a list, each of the same size + * (the final list may be smaller). + * @param originalList original big list. + * @param pageSize desired size of each sublist ( last one + * may be smaller) + * @return a list of sub lists. + */ + public static List> partition(List originalList, int pageSize) { + + Preconditions.checkArgument(originalList != null && originalList.size() > 0, + "Invalid original list"); + Preconditions.checkArgument(pageSize > 0, "Page size should " + + "be greater than 0 for performing partition"); + + List> result = new ArrayList<>(); + int i=0; + while (i < originalList.size()) { + result.add(originalList.subList(i, + Math.min(i + pageSize, originalList.size()))); + i = i + pageSize; + } + return result; + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ProcessUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ProcessUtils.java new file mode 100644 index 0000000000000..cf653b9c912c4 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ProcessUtils.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.util; + +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.classification.InterfaceAudience; + +/** + * Process related utilities. + */ +@InterfaceAudience.Private +public final class ProcessUtils { + + private static final Logger LOG = LoggerFactory.getLogger(ProcessUtils.class); + + private ProcessUtils() { + // no-op + } + + public static Integer getPid() { + // JVM_PID can be exported in service start script + String pidStr = System.getenv("JVM_PID"); + + // In case if it is not set correctly, fallback to mxbean which is implementation specific. + if (pidStr == null || pidStr.trim().isEmpty()) { + String name = ManagementFactory.getRuntimeMXBean().getName(); + if (name != null) { + int idx = name.indexOf("@"); + if (idx != -1) { + pidStr = name.substring(0, name.indexOf("@")); + } + } + } + try { + if (pidStr != null) { + return Integer.valueOf(pidStr); + } + } catch (NumberFormatException ignored) { + // ignore + } + return null; + } + + public static Process runCmdAsync(List cmd) { + try { + LOG.info("Running command async: {}", cmd); + return new ProcessBuilder(cmd).inheritIO().start(); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RateLimiting.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RateLimiting.java new file mode 100644 index 0000000000000..ae119c0e630f4 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RateLimiting.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.util; + +import java.time.Duration; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * Minimal subset of google rate limiter class. + * Can be used to throttle use of object stores where excess load + * will trigger cluster-wide throttling, backoff etc. and so collapse + * performance. + * The time waited is returned as a Duration type. + * The google rate limiter implements this by allowing a caller to ask for + * more capacity than is available. This will be granted + * but the subsequent request will be blocked if the bucket of + * capacity hasn't let refilled to the point where there is + * capacity again. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public interface RateLimiting { + + /** + * Acquire rate limiter capacity. + * If there is not enough space, the permits will be acquired, + * but the subsequent call will block until the capacity has been + * refilled. + * @param requestedCapacity capacity to acquire. + * @return time spent waiting for output. + */ + Duration acquire(int requestedCapacity); + +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RateLimitingFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RateLimitingFactory.java new file mode 100644 index 0000000000000..621415456e125 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RateLimitingFactory.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.util; + +import java.time.Duration; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.RateLimiter; + +/** + * Factory for Rate Limiting. + * This should be only place in the code where the guava RateLimiter is imported. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public final class RateLimitingFactory { + + private static final RateLimiting UNLIMITED = new NoRateLimiting(); + + /** + * No waiting took place. + */ + private static final Duration INSTANTLY = Duration.ofMillis(0); + + private RateLimitingFactory() { + } + + /** + * No Rate Limiting. + */ + private static class NoRateLimiting implements RateLimiting { + + + @Override + public Duration acquire(int requestedCapacity) { + return INSTANTLY; + } + } + + /** + * Rate limiting restricted to that of a google rate limiter. + */ + private static final class RestrictedRateLimiting implements RateLimiting { + private final RateLimiter limiter; + + /** + * Constructor. + * @param capacityPerSecond capacity in permits/second. + */ + private RestrictedRateLimiting(int capacityPerSecond) { + this.limiter = RateLimiter.create(capacityPerSecond); + } + + @Override + public Duration acquire(int requestedCapacity) { + final double delayMillis = limiter.acquire(requestedCapacity); + return delayMillis == 0 + ? INSTANTLY + : Duration.ofMillis((long) (delayMillis * 1000)); + } + + } + + /** + * Get the unlimited rate. + * @return a rate limiter which always has capacity. + */ + public static RateLimiting unlimitedRate() { + return UNLIMITED; + } + + /** + * Create an instance. + * If the rate is 0; return the unlimited rate. + * @param capacity capacity in permits/second. + * @return limiter restricted to the given capacity. + */ + public static RateLimiting create(int capacity) { + + return capacity == 0 + ? unlimitedRate() + : new RestrictedRateLimiting(capacity); + } + +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java index 49c5a38765eed..084e2b8f5e3b6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java @@ -38,7 +38,6 @@ import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.security.alias.AbstractJavaKeyStoreProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/WeakReferenceMap.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/WeakReferenceMap.java new file mode 100644 index 0000000000000..cdc85dcb7cafa --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/WeakReferenceMap.java @@ -0,0 +1,261 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.util; + +import java.lang.ref.WeakReference; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; +import java.util.function.Function; + +import javax.annotation.Nullable; + +import org.apache.hadoop.classification.InterfaceAudience; + +import static java.util.Objects.requireNonNull; + +/** + * A map of keys type K to objects of type V which uses weak references, + * so does lot leak memory through long-lived references + * at the expense of losing references when GC takes place.. + * + * This class is intended be used instead of ThreadLocal storage when + * references are to be cleaned up when the instance holding. + * In this use case, the key is the Long key. + * + * Concurrency. + * The class assumes that map entries are rarely contended for when writing, + * and that not blocking other threads is more important than atomicity. + * - a ConcurrentHashMap is used to map keys to weak references, with + * all its guarantees. + * - there is no automatic pruning. + * - see {@link #create(Object)} for the concurrency semantics on entry creation. + */ +@InterfaceAudience.Private +public class WeakReferenceMap { + + /** + * The reference map. + */ + private final Map> map = new ConcurrentHashMap<>(); + + /** + * Supplier of new instances. + */ + private final Function factory; + + /** + * Nullable callback when a get on a key got a weak reference back. + * The assumption is that this is for logging/stats, which is why + * no attempt is made to use the call as a supplier of a new value. + */ + private final Consumer referenceLost; + + /** + * Counter of references lost. + */ + private final AtomicLong referenceLostCount = new AtomicLong(); + + /** + * Counter of entries created. + */ + private final AtomicLong entriesCreatedCount = new AtomicLong(); + + /** + * instantiate. + * @param factory supplier of new instances + * @param referenceLost optional callback on lost references. + */ + public WeakReferenceMap( + Function factory, + @Nullable final Consumer referenceLost) { + + this.factory = requireNonNull(factory); + this.referenceLost = referenceLost; + } + + @Override + public String toString() { + return "WeakReferenceMap{" + + "size=" + size() + + ", referenceLostCount=" + referenceLostCount + + ", entriesCreatedCount=" + entriesCreatedCount + + '}'; + } + + /** + * Map size. + * @return the current map size. + */ + public int size() { + return map.size(); + } + + /** + * Clear all entries. + */ + public void clear() { + map.clear(); + } + + /** + * look up the value, returning the possibly empty weak reference + * to a value, or null if no value was found. + * @param key key to look up + * @return null if there is no entry, a weak reference if found + */ + public WeakReference lookup(K key) { + return map.get(key); + } + + /** + * Get the value, creating if needed. + * @param key key. + * @return an instance. + */ + public V get(K key) { + final WeakReference current = lookup(key); + V val = resolve(current); + if (val != null) { + // all good. + return val; + } + + // here, either no ref, or the value is null + if (current != null) { + noteLost(key); + } + return create(key); + } + + /** + * Create a new instance under a key. + * The instance is created, added to the map and then the + * map value retrieved. + * This ensures that the reference returned is that in the map, + * even if there is more than one entry being created at the same time. + * @param key key + * @return the value + */ + public V create(K key) { + entriesCreatedCount.incrementAndGet(); + WeakReference newRef = new WeakReference<>( + requireNonNull(factory.apply(key))); + map.put(key, newRef); + return map.get(key).get(); + } + + /** + * Put a value under the key. + * A null value can be put, though on a get() call + * a new entry is generated + * + * @param key key + * @param value value + * @return any old non-null reference. + */ + public V put(K key, V value) { + return resolve(map.put(key, new WeakReference<>(value))); + } + + /** + * Remove any value under the key. + * @param key key + * @return any old non-null reference. + */ + public V remove(K key) { + return resolve(map.remove(key)); + } + + /** + * Does the map have a valid reference for this object? + * no-side effects: there's no attempt to notify or cleanup + * if the reference is null. + * @param key key to look up + * @return true if there is a valid reference. + */ + public boolean containsKey(K key) { + final WeakReference current = lookup(key); + return resolve(current) != null; + } + + /** + * Given a possibly null weak reference, resolve + * its value. + * @param r reference to resolve + * @return the value or null + */ + private V resolve(WeakReference r) { + return r == null ? null : r.get(); + } + + /** + * Prune all null weak references, calling the referenceLost + * callback for each one. + * + * non-atomic and non-blocking. + * @return the number of entries pruned. + */ + public int prune() { + int count = 0; + final Iterator>> it = map.entrySet().iterator(); + while (it.hasNext()) { + final Map.Entry> next = it.next(); + if (next.getValue().get() == null) { + it.remove(); + count++; + noteLost(next.getKey()); + } + } + return count; + } + + /** + * Notify the reference lost callback. + * @param key key of lost reference + */ + private void noteLost(final K key) { + // incrment local counter + referenceLostCount.incrementAndGet(); + + // and call any notification function supplied in the constructor + if (referenceLost != null) { + referenceLost.accept(key); + } + } + + /** + * Get count of references lost as detected + * during prune() or get() calls. + * @return count of references lost + */ + public final long getReferenceLostCount() { + return referenceLostCount.get(); + } + + /** + * Get count of entries created on demand. + * @return count of entries created + */ + public final long getEntriesCreatedCount() { + return entriesCreatedCount.get(); + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/ExecutorHelper.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/ExecutorHelper.java index 0b349518e4c2b..5cc92eb71bb64 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/ExecutorHelper.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/ExecutorHelper.java @@ -47,12 +47,12 @@ static void logThrowableFromAfterExecute(Runnable r, Throwable t) { try { ((Future) r).get(); } catch (ExecutionException ee) { - LOG.warn( - "Execution exception when running task in " + Thread.currentThread() + LOG.debug( + "Execution exception when running task in {}", Thread.currentThread() .getName()); t = ee.getCause(); } catch (InterruptedException ie) { - LOG.warn("Thread (" + Thread.currentThread() + ") interrupted: ", ie); + LOG.debug("Thread ( {} ) interrupted: ", Thread.currentThread(), ie); Thread.currentThread().interrupt(); } catch (Throwable throwable) { t = throwable; @@ -60,8 +60,8 @@ static void logThrowableFromAfterExecute(Runnable r, Throwable t) { } if (t != null) { - LOG.warn("Caught exception in thread " + Thread - .currentThread().getName() + ": ", t); + LOG.warn("Caught exception in thread {} + : ", Thread + .currentThread().getName(), t); } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java index edb4eb6ac1dea..ef9cec6677302 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java @@ -28,12 +28,16 @@ import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.framework.api.transaction.CuratorOp; import org.apache.curator.retry.RetryNTimes; +import org.apache.curator.utils.ZookeeperFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.util.ZKUtil; import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.client.ZKClientConfig; import org.apache.zookeeper.data.ACL; import org.apache.zookeeper.data.Stat; import org.slf4j.Logger; @@ -148,6 +152,8 @@ public void start(List authInfos) throws IOException { CuratorFramework client = CuratorFrameworkFactory.builder() .connectString(zkHostPort) + .zookeeperFactory(new HadoopZookeeperFactory( + conf.get(CommonConfigurationKeys.ZK_SERVER_PRINCIPAL))) .sessionTimeoutMs(zkSessionTimeout) .retryPolicy(retryPolicy) .authorization(authInfos) @@ -428,4 +434,27 @@ public void setData(String path, byte[] data, int version) .forPath(path, data)); } } + + public static class HadoopZookeeperFactory implements ZookeeperFactory { + private final String zkPrincipal; + + public HadoopZookeeperFactory(String zkPrincipal) { + this.zkPrincipal = zkPrincipal; + } + + @Override + public ZooKeeper newZooKeeper(String connectString, int sessionTimeout, + Watcher watcher, boolean canBeReadOnly + ) throws Exception { + ZKClientConfig zkClientConfig = new ZKClientConfig(); + if (zkPrincipal != null) { + LOG.info("Configuring zookeeper to use {} as the server principal", + zkPrincipal); + zkClientConfig.setProperty(ZKClientConfig.ZK_SASL_CLIENT_USERNAME, + zkPrincipal); + } + return new ZooKeeper(connectString, sessionTimeout, watcher, + canBeReadOnly, zkClientConfig); + } + } } \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/CloseableTaskPoolSubmitter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/CloseableTaskPoolSubmitter.java new file mode 100644 index 0000000000000..26b687a3c5610 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/CloseableTaskPoolSubmitter.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.util.functional; + +import java.io.Closeable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +import static java.util.Objects.requireNonNull; + +/** + * A task submitter which is closeable, and whose close() call + * shuts down the pool. This can help manage + * thread pool lifecycles. + */ +@InterfaceAudience.Public +@InterfaceStability.Unstable +public final class CloseableTaskPoolSubmitter implements TaskPool.Submitter, + Closeable { + + /** Executors. */ + private ExecutorService pool; + + /** + * Constructor. + * @param pool non-null executor. + */ + public CloseableTaskPoolSubmitter(final ExecutorService pool) { + this.pool = requireNonNull(pool); + } + + /** + * Get the pool. + * @return the pool. + */ + public ExecutorService getPool() { + return pool; + } + + /** + * Shut down the pool. + */ + @Override + public void close() { + if (pool != null) { + pool.shutdown(); + pool = null; + } + } + + @Override + public Future submit(final Runnable task) { + return pool.submit(task); + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/CommonCallableSupplier.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/CommonCallableSupplier.java index e2cdc0fd41472..32e299b4d45b1 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/CommonCallableSupplier.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/CommonCallableSupplier.java @@ -34,7 +34,7 @@ import org.apache.hadoop.util.DurationInfo; -import static org.apache.hadoop.fs.impl.FutureIOSupport.raiseInnerCause; +import static org.apache.hadoop.util.functional.FutureIO.raiseInnerCause; /** * A bridge from Callable to Supplier; catching exceptions diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/FutureIO.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/FutureIO.java index 3f7218baa759f..c3fda19d8d73b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/FutureIO.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/FutureIO.java @@ -21,6 +21,8 @@ import java.io.IOException; import java.io.InterruptedIOException; import java.io.UncheckedIOException; +import java.util.Map; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; @@ -29,6 +31,8 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSBuilder; /** * Future IO Helper methods. @@ -86,6 +90,8 @@ public static T awaitFuture(final Future future) * extracted and rethrown. *

* @param future future to evaluate + * @param timeout timeout to wait + * @param unit time unit. * @param type of the result. * @return the result, if all went well. * @throws InterruptedIOException future was interrupted @@ -185,4 +191,88 @@ public static IOException unwrapInnerException(final Throwable e) { } } + /** + * Propagate options to any builder, converting everything with the + * prefix to an option where, if there were 2+ dot-separated elements, + * it is converted to a schema. + * See {@link #propagateOptions(FSBuilder, Configuration, String, boolean)}. + * @param builder builder to modify + * @param conf configuration to read + * @param optionalPrefix prefix for optional settings + * @param mandatoryPrefix prefix for mandatory settings + * @param type of result + * @param type of builder + * @return the builder passed in. + */ + public static > + FSBuilder propagateOptions( + final FSBuilder builder, + final Configuration conf, + final String optionalPrefix, + final String mandatoryPrefix) { + propagateOptions(builder, conf, + optionalPrefix, false); + propagateOptions(builder, conf, + mandatoryPrefix, true); + return builder; + } + + /** + * Propagate options to any builder, converting everything with the + * prefix to an option where, if there were 2+ dot-separated elements, + * it is converted to a schema. + * 
+   *   fs.example.s3a.option becomes "s3a.option"
+   *   fs.example.fs.io.policy becomes "fs.io.policy"
+   *   fs.example.something becomes "something"
+   *
+ * @param builder builder to modify + * @param conf configuration to read + * @param prefix prefix to scan/strip + * @param mandatory are the options to be mandatory or optional? + */ + public static void propagateOptions( + final FSBuilder builder, + final Configuration conf, + final String prefix, + final boolean mandatory) { + + final String p = prefix.endsWith(".") ? prefix : (prefix + "."); + final Map propsWithPrefix = conf.getPropsWithPrefix(p); + for (Map.Entry entry : propsWithPrefix.entrySet()) { + // change the schema off each entry + String key = entry.getKey(); + String val = entry.getValue(); + if (mandatory) { + builder.must(key, val); + } else { + builder.opt(key, val); + } + } + } + + /** + * Evaluate a CallableRaisingIOE in the current thread, + * converting IOEs to RTEs and propagating. + * @param callable callable to invoke + * @param Return type. + * @return the evaluated result. + * @throws UnsupportedOperationException fail fast if unsupported + * @throws IllegalArgumentException invalid argument + */ + public static CompletableFuture eval( + CallableRaisingIOE callable) { + CompletableFuture result = new CompletableFuture<>(); + try { + result.complete(callable.apply()); + } catch (UnsupportedOperationException | IllegalArgumentException tx) { + // fail fast here + throw tx; + } catch (Throwable tx) { + // fail lazily here to ensure callers expect all File IO operations to + // surface later + result.completeExceptionally(tx); + } + return result; + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/TaskPool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/TaskPool.java new file mode 100644 index 0000000000000..0abaab211de04 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/TaskPool.java @@ -0,0 +1,613 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.util.functional; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicBoolean; + +import javax.annotation.Nullable; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.RemoteIterator; + +import static java.util.Objects.requireNonNull; +import static org.apache.hadoop.util.functional.RemoteIterators.remoteIteratorFromIterable; + +/** + * Utility class for parallel execution, takes closures for the various + * actions. + * There is no retry logic: it is expected to be handled by the closures. + * From {@code org.apache.hadoop.fs.s3a.commit.Tasks} which came from + * the Netflix committer patch. + * Apache Iceberg has its own version of this, with a common ancestor + * at some point in its history. + * A key difference with this class is that the iterator is always, + * internally, an {@link RemoteIterator}. + * This is to allow tasks to be scheduled while incremental operations + * such as paged directory listings are still collecting in results. + * + * While awaiting completion, this thread spins and sleeps a time of + * {@link #SLEEP_INTERVAL_AWAITING_COMPLETION}, which, being a + * busy-wait, is inefficient. + * There's an implicit assumption that remote IO is being performed, and + * so this is not impacting throughput/performance. + * + * History: + * This class came with the Netflix contributions to the S3A committers + * in HADOOP-13786. + * It was moved into hadoop-common for use in the manifest committer and + * anywhere else it is needed, and renamed in the process as + * "Tasks" has too many meanings in the hadoop source. + * The iterator was then changed from a normal java iterable + * to a hadoop {@link org.apache.hadoop.fs.RemoteIterator}. + * This allows a task pool to be supplied with incremental listings + * from object stores, scheduling work as pages of listing + * results come in, rather than blocking until the entire + * directory/directory tree etc has been enumerated. + * + * There is a variant of this in Apache Iceberg in + * {@code org.apache.iceberg.util.Tasks} + * That is not derived from any version in the hadoop codebase, it + * just shares a common ancestor somewhere in the Netflix codebase. + * It is the more sophisticated version. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public final class TaskPool { + private static final Logger LOG = + LoggerFactory.getLogger(TaskPool.class); + + /** + * Interval in milliseconds to await completion. + */ + private static final int SLEEP_INTERVAL_AWAITING_COMPLETION = 10; + + private TaskPool() { + } + + /** + * Callback invoked to process an item. + * @param item type being processed + * @param exception class which may be raised + */ + @FunctionalInterface + public interface Task { + void run(I item) throws E; + } + + /** + * Callback invoked on a failure. + * @param item type being processed + * @param exception class which may be raised + */ + @FunctionalInterface + public interface FailureTask { + + /** + * process a failure. + * @param item item the task is processing + * @param exception the exception which was raised. + * @throws E Exception of type E + */ + void run(I item, Exception exception) throws E; + } + + /** + * Builder for task execution. + * @param item type + */ + public static class Builder { + private final RemoteIterator items; + private Submitter service = null; + private FailureTask onFailure = null; + private boolean stopOnFailure = false; + private boolean suppressExceptions = false; + private Task revertTask = null; + private boolean stopRevertsOnFailure = false; + private Task abortTask = null; + private boolean stopAbortsOnFailure = false; + private int sleepInterval = SLEEP_INTERVAL_AWAITING_COMPLETION; + + /** + * Create the builder. + * @param items items to process + */ + Builder(RemoteIterator items) { + this.items = requireNonNull(items, "items"); + } + + /** + * Create the builder. + * @param items items to process + */ + Builder(Iterable items) { + this(remoteIteratorFromIterable(items)); + } + + /** + * Declare executor service: if null, the tasks are executed in a single + * thread. + * @param submitter service to schedule tasks with. + * @return this builder. + */ + public Builder executeWith(@Nullable Submitter submitter) { + + this.service = submitter; + return this; + } + + /** + * Task to invoke on failure. + * @param task task + * @return the builder + */ + public Builder onFailure(FailureTask task) { + this.onFailure = task; + return this; + } + + public Builder stopOnFailure() { + this.stopOnFailure = true; + return this; + } + + /** + * Suppress exceptions from tasks. + * RemoteIterator exceptions are not suppressable. + * @return the builder. + */ + public Builder suppressExceptions() { + return suppressExceptions(true); + } + + /** + * Suppress exceptions from tasks. + * RemoteIterator exceptions are not suppressable. + * @param suppress new value + * @return the builder. + */ + public Builder suppressExceptions(boolean suppress) { + this.suppressExceptions = suppress; + return this; + } + + /** + * Task to revert with after another task failed. + * @param task task to execute + * @return the builder + */ + public Builder revertWith(Task task) { + this.revertTask = task; + return this; + } + + /** + * Stop trying to revert if one operation fails. + * @return the builder + */ + public Builder stopRevertsOnFailure() { + this.stopRevertsOnFailure = true; + return this; + } + + /** + * Task to abort with after another task failed. + * @param task task to execute + * @return the builder + */ + public Builder abortWith(Task task) { + this.abortTask = task; + return this; + } + + /** + * Stop trying to abort if one operation fails. + * @return the builder + */ + public Builder stopAbortsOnFailure() { + this.stopAbortsOnFailure = true; + return this; + } + + /** + * Set the sleep interval. + * @param value new value + * @return the builder + */ + public Builder sleepInterval(final int value) { + sleepInterval = value; + return this; + } + + /** + * Execute the task across the data. + * @param task task to execute + * @param exception which may be raised in execution. + * @return true if the operation executed successfully + * @throws E any exception raised. + * @throws IOException IOExceptions raised by remote iterator or in execution. + */ + public boolean run(Task task) throws E, IOException { + requireNonNull(items, "items"); + if (!items.hasNext()) { + // if there are no items, return without worrying about + // execution pools, errors etc. + return true; + } + if (service != null) { + // thread pool, so run in parallel + return runParallel(task); + } else { + // single threaded execution. + return runSingleThreaded(task); + } + } + + /** + * Single threaded execution. + * @param task task to execute + * @param exception which may be raised in execution. + * @return true if the operation executed successfully + * @throws E any exception raised. + * @throws IOException IOExceptions raised by remote iterator or in execution. + */ + private boolean runSingleThreaded(Task task) + throws E, IOException { + List succeeded = new ArrayList<>(); + List exceptions = new ArrayList<>(); + + RemoteIterator iterator = items; + boolean threw = true; + try { + while (iterator.hasNext()) { + I item = iterator.next(); + try { + task.run(item); + succeeded.add(item); + + } catch (Exception e) { + exceptions.add(e); + + if (onFailure != null) { + try { + onFailure.run(item, e); + } catch (Exception failException) { + LOG.error("Failed to clean up on failure", e); + // keep going + } + } + + if (stopOnFailure) { + break; + } + } + } + + threw = false; + } catch (IOException iteratorIOE) { + // an IOE is reaised here during iteration + LOG.debug("IOException when iterating through {}", iterator, iteratorIOE); + throw iteratorIOE; + } finally { + // threw handles exceptions that were *not* caught by the catch block, + // and exceptions that were caught and possibly handled by onFailure + // are kept in exceptions. + if (threw || !exceptions.isEmpty()) { + if (revertTask != null) { + boolean failed = false; + for (I item : succeeded) { + try { + revertTask.run(item); + } catch (Exception e) { + LOG.error("Failed to revert task", e); + failed = true; + // keep going + } + if (stopRevertsOnFailure && failed) { + break; + } + } + } + + if (abortTask != null) { + boolean failed = false; + while (iterator.hasNext()) { + try { + abortTask.run(iterator.next()); + } catch (Exception e) { + failed = true; + LOG.error("Failed to abort task", e); + // keep going + } + if (stopAbortsOnFailure && failed) { + break; + } + } + } + } + } + + if (!suppressExceptions && !exceptions.isEmpty()) { + TaskPool.throwOne(exceptions); + } + + return exceptions.isEmpty(); + } + + /** + * Parallel execution. + * @param task task to execute + * @param exception which may be raised in execution. + * @return true if the operation executed successfully + * @throws E any exception raised. + * @throws IOException IOExceptions raised by remote iterator or in execution. + */ + private boolean runParallel(final Task task) + throws E, IOException { + final Queue succeeded = new ConcurrentLinkedQueue<>(); + final Queue exceptions = new ConcurrentLinkedQueue<>(); + final AtomicBoolean taskFailed = new AtomicBoolean(false); + final AtomicBoolean abortFailed = new AtomicBoolean(false); + final AtomicBoolean revertFailed = new AtomicBoolean(false); + + List> futures = new ArrayList<>(); + + IOException iteratorIOE = null; + final RemoteIterator iterator = this.items; + try { + while(iterator.hasNext()) { + final I item = iterator.next(); + // submit a task for each item that will either run or abort the task + futures.add(service.submit(() -> { + if (!(stopOnFailure && taskFailed.get())) { + // run the task + boolean threw = true; + try { + LOG.debug("Executing task"); + task.run(item); + succeeded.add(item); + LOG.debug("Task succeeded"); + + threw = false; + + } catch (Exception e) { + taskFailed.set(true); + exceptions.add(e); + LOG.info("Task failed {}", e.toString()); + LOG.debug("Task failed", e); + + if (onFailure != null) { + try { + onFailure.run(item, e); + } catch (Exception failException) { + LOG.warn("Failed to clean up on failure", e); + // swallow the exception + } + } + } finally { + if (threw) { + taskFailed.set(true); + } + } + + } else if (abortTask != null) { + // abort the task instead of running it + if (stopAbortsOnFailure && abortFailed.get()) { + return; + } + + boolean failed = true; + try { + LOG.info("Aborting task"); + abortTask.run(item); + failed = false; + } catch (Exception e) { + LOG.error("Failed to abort task", e); + // swallow the exception + } finally { + if (failed) { + abortFailed.set(true); + } + } + } + })); + } + } catch (IOException e) { + // iterator failure. + LOG.debug("IOException when iterating through {}", iterator, e); + iteratorIOE = e; + // mark as a task failure so all submitted tasks will halt/abort + taskFailed.set(true); + } + + // let the above tasks complete (or abort) + waitFor(futures, sleepInterval); + int futureCount = futures.size(); + futures.clear(); + + if (taskFailed.get() && revertTask != null) { + // at least one task failed, revert any that succeeded + LOG.info("Reverting all {} succeeded tasks from {} futures", + succeeded.size(), futureCount); + for (final I item : succeeded) { + futures.add(service.submit(() -> { + if (stopRevertsOnFailure && revertFailed.get()) { + return; + } + + boolean failed = true; + try { + revertTask.run(item); + failed = false; + } catch (Exception e) { + LOG.error("Failed to revert task", e); + // swallow the exception + } finally { + if (failed) { + revertFailed.set(true); + } + } + })); + } + + // let the revert tasks complete + waitFor(futures, sleepInterval); + } + + // give priority to execution exceptions over + // iterator exceptions. + if (!suppressExceptions && !exceptions.isEmpty()) { + // there's an exception list to build up, cast and throw. + TaskPool.throwOne(exceptions); + } + + // raise any iterator exception. + // this can not be suppressed. + if (iteratorIOE != null) { + throw iteratorIOE; + } + + // return true if all tasks succeeded. + return !taskFailed.get(); + } + } + + /** + * Wait for all the futures to complete; there's a small sleep between + * each iteration; enough to yield the CPU. + * @param futures futures. + * @param sleepInterval Interval in milliseconds to await completion. + */ + private static void waitFor(Collection> futures, int sleepInterval) { + int size = futures.size(); + LOG.debug("Waiting for {} tasks to complete", size); + int oldNumFinished = 0; + while (true) { + int numFinished = (int) futures.stream().filter(Future::isDone).count(); + + if (oldNumFinished != numFinished) { + LOG.debug("Finished count -> {}/{}", numFinished, size); + oldNumFinished = numFinished; + } + + if (numFinished == size) { + // all of the futures are done, stop looping + break; + } else { + try { + Thread.sleep(sleepInterval); + } catch (InterruptedException e) { + futures.forEach(future -> future.cancel(true)); + Thread.currentThread().interrupt(); + break; + } + } + } + } + + /** + * Create a task builder for the iterable. + * @param items item source. + * @param type of result. + * @return builder. + */ + public static Builder foreach(Iterable items) { + return new Builder<>(requireNonNull(items, "items")); + } + + /** + * Create a task builder for the remote iterator. + * @param items item source. + * @param type of result. + * @return builder. + */ + public static Builder foreach(RemoteIterator items) { + return new Builder<>(items); + } + + public static Builder foreach(I[] items) { + return new Builder<>(Arrays.asList(requireNonNull(items, "items"))); + } + + /** + * Throw one exception, adding the others as suppressed + * exceptions attached to the one thrown. + * This method never completes normally. + * @param exceptions collection of exceptions + * @param class of exceptions + * @throws E an extracted exception. + */ + private static void throwOne( + Collection exceptions) + throws E { + Iterator iter = exceptions.iterator(); + Exception e = iter.next(); + Class exceptionClass = e.getClass(); + + while (iter.hasNext()) { + Exception other = iter.next(); + if (!exceptionClass.isInstance(other)) { + e.addSuppressed(other); + } + } + + TaskPool.castAndThrow(e); + } + + /** + * Raise an exception of the declared type. + * This method never completes normally. + * @param e exception + * @param class of exceptions + * @throws E a recast exception. + */ + @SuppressWarnings("unchecked") + private static void castAndThrow(Exception e) throws E { + if (e instanceof RuntimeException) { + throw (RuntimeException) e; + } + throw (E) e; + } + + /** + * Interface to whatever lets us submit tasks. + */ + public interface Submitter { + + /** + * Submit work. + * @param task task to execute + * @return the future of the submitted task. + */ + Future submit(Runnable task); + } + +} diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/pmdk_load.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/pmdk_load.c index 502508cbf3b86..f1a1df5c9dbe3 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/pmdk_load.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/pmdk_load.c @@ -35,13 +35,14 @@ #endif PmdkLibLoader * pmdkLoader; +// 1 represents loaded. Otherwise, not loaded. +int pmdkLoaded; /** * pmdk_load.c * Utility of loading the libpmem library and the required functions. * Building of this codes won't rely on any libpmem source codes, but running * into this will rely on successfully loading of the dynamic library. - * */ static const char* load_functions() { @@ -56,6 +57,10 @@ static const char* load_functions() { return NULL; } +/** + * It should be idempotent to call this function for checking + * whether PMDK lib is successfully loaded. + */ void load_pmdk_lib(char* err, size_t err_len) { const char* errMsg; const char* library = NULL; @@ -67,10 +72,13 @@ void load_pmdk_lib(char* err, size_t err_len) { err[0] = '\0'; - if (pmdkLoader != NULL) { + if (pmdkLoaded == 1) { return; } - pmdkLoader = calloc(1, sizeof(PmdkLibLoader)); + + if (pmdkLoader == NULL) { + pmdkLoader = calloc(1, sizeof(PmdkLibLoader)); + } // Load PMDK library #ifdef UNIX @@ -103,4 +111,5 @@ void load_pmdk_lib(char* err, size_t err_len) { } pmdkLoader->libname = strdup(library); + pmdkLoaded = 1; } diff --git a/hadoop-common-project/hadoop-common/src/main/proto/FSProtos.proto b/hadoop-common-project/hadoop-common/src/main/proto/FSProtos.proto index c895bce757b77..17bbcf8f48707 100644 --- a/hadoop-common-project/hadoop-common/src/main/proto/FSProtos.proto +++ b/hadoop-common-project/hadoop-common/src/main/proto/FSProtos.proto @@ -18,7 +18,7 @@ /** * These .proto interfaces are private and stable. - * Please see http://wiki.apache.org/hadoop/Compatibility + * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html * for what changes are allowed for a *stable* .proto interface. */ syntax = "proto2"; diff --git a/hadoop-common-project/hadoop-common/src/main/proto/GenericRefreshProtocol.proto b/hadoop-common-project/hadoop-common/src/main/proto/GenericRefreshProtocol.proto index 6296f88da69b8..91d2e2e6c4c4e 100644 --- a/hadoop-common-project/hadoop-common/src/main/proto/GenericRefreshProtocol.proto +++ b/hadoop-common-project/hadoop-common/src/main/proto/GenericRefreshProtocol.proto @@ -18,7 +18,7 @@ /** * These .proto interfaces are private and stable. - * Please see http://wiki.apache.org/hadoop/Compatibility + * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html * for what changes are allowed for a *stable* .proto interface. */ syntax = "proto2"; diff --git a/hadoop-common-project/hadoop-common/src/main/proto/GetUserMappingsProtocol.proto b/hadoop-common-project/hadoop-common/src/main/proto/GetUserMappingsProtocol.proto index cb91a13b04875..bccb57dd86832 100644 --- a/hadoop-common-project/hadoop-common/src/main/proto/GetUserMappingsProtocol.proto +++ b/hadoop-common-project/hadoop-common/src/main/proto/GetUserMappingsProtocol.proto @@ -18,7 +18,7 @@ /** * These .proto interfaces are private and stable. - * Please see http://wiki.apache.org/hadoop/Compatibility + * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html * for what changes are allowed for a *stable* .proto interface. */ syntax = "proto2"; diff --git a/hadoop-common-project/hadoop-common/src/main/proto/HAServiceProtocol.proto b/hadoop-common-project/hadoop-common/src/main/proto/HAServiceProtocol.proto index 5a88a7ff03f02..d9c7d70c0e904 100644 --- a/hadoop-common-project/hadoop-common/src/main/proto/HAServiceProtocol.proto +++ b/hadoop-common-project/hadoop-common/src/main/proto/HAServiceProtocol.proto @@ -18,7 +18,7 @@ /** * These .proto interfaces are private and stable. - * Please see http://wiki.apache.org/hadoop/Compatibility + * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html * for what changes are allowed for a *stable* .proto interface. */ syntax = "proto2"; diff --git a/hadoop-common-project/hadoop-common/src/main/proto/IpcConnectionContext.proto b/hadoop-common-project/hadoop-common/src/main/proto/IpcConnectionContext.proto index 16e2fb7c4db75..d853cf3afb3ca 100644 --- a/hadoop-common-project/hadoop-common/src/main/proto/IpcConnectionContext.proto +++ b/hadoop-common-project/hadoop-common/src/main/proto/IpcConnectionContext.proto @@ -18,7 +18,7 @@ /** * These .proto interfaces are private and stable. - * Please see http://wiki.apache.org/hadoop/Compatibility + * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html * for what changes are allowed for a *stable* .proto interface. */ syntax = "proto2"; diff --git a/hadoop-common-project/hadoop-common/src/main/proto/ProtobufRpcEngine.proto b/hadoop-common-project/hadoop-common/src/main/proto/ProtobufRpcEngine.proto index f72cf1a8da16e..8cace2d454ae9 100644 --- a/hadoop-common-project/hadoop-common/src/main/proto/ProtobufRpcEngine.proto +++ b/hadoop-common-project/hadoop-common/src/main/proto/ProtobufRpcEngine.proto @@ -18,7 +18,7 @@ /** * These .proto interfaces are private and stable. - * Please see http://wiki.apache.org/hadoop/Compatibility + * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html * for what changes are allowed for a *stable* .proto interface. */ syntax = "proto2"; diff --git a/hadoop-common-project/hadoop-common/src/main/proto/ProtobufRpcEngine2.proto b/hadoop-common-project/hadoop-common/src/main/proto/ProtobufRpcEngine2.proto index c3023ec26dfd4..0e38070ab33fa 100644 --- a/hadoop-common-project/hadoop-common/src/main/proto/ProtobufRpcEngine2.proto +++ b/hadoop-common-project/hadoop-common/src/main/proto/ProtobufRpcEngine2.proto @@ -18,7 +18,7 @@ /** * These .proto interfaces are private and stable. - * Please see http://wiki.apache.org/hadoop/Compatibility + * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html * for what changes are allowed for a *stable* .proto interface. */ syntax = "proto2"; diff --git a/hadoop-common-project/hadoop-common/src/main/proto/ProtocolInfo.proto b/hadoop-common-project/hadoop-common/src/main/proto/ProtocolInfo.proto index 0e9d0d4baa413..77d883227e5fe 100644 --- a/hadoop-common-project/hadoop-common/src/main/proto/ProtocolInfo.proto +++ b/hadoop-common-project/hadoop-common/src/main/proto/ProtocolInfo.proto @@ -18,7 +18,7 @@ /** * These .proto interfaces are private and stable. - * Please see http://wiki.apache.org/hadoop/Compatibility + * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html * for what changes are allowed for a *stable* .proto interface. */ syntax = "proto2"; diff --git a/hadoop-common-project/hadoop-common/src/main/proto/RefreshAuthorizationPolicyProtocol.proto b/hadoop-common-project/hadoop-common/src/main/proto/RefreshAuthorizationPolicyProtocol.proto index f57c6d6303916..7bf69a70ef07f 100644 --- a/hadoop-common-project/hadoop-common/src/main/proto/RefreshAuthorizationPolicyProtocol.proto +++ b/hadoop-common-project/hadoop-common/src/main/proto/RefreshAuthorizationPolicyProtocol.proto @@ -18,7 +18,7 @@ /** * These .proto interfaces are private and stable. - * Please see http://wiki.apache.org/hadoop/Compatibility + * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html * for what changes are allowed for a *stable* .proto interface. */ syntax = "proto2"; diff --git a/hadoop-common-project/hadoop-common/src/main/proto/RefreshCallQueueProtocol.proto b/hadoop-common-project/hadoop-common/src/main/proto/RefreshCallQueueProtocol.proto index 463b7c548fe22..138ede842d554 100644 --- a/hadoop-common-project/hadoop-common/src/main/proto/RefreshCallQueueProtocol.proto +++ b/hadoop-common-project/hadoop-common/src/main/proto/RefreshCallQueueProtocol.proto @@ -18,7 +18,7 @@ /** * These .proto interfaces are private and stable. - * Please see http://wiki.apache.org/hadoop/Compatibility + * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html * for what changes are allowed for a *stable* .proto interface. */ syntax = "proto2"; diff --git a/hadoop-common-project/hadoop-common/src/main/proto/RefreshUserMappingsProtocol.proto b/hadoop-common-project/hadoop-common/src/main/proto/RefreshUserMappingsProtocol.proto index a1130f5c2d96d..a0f9e41b7082d 100644 --- a/hadoop-common-project/hadoop-common/src/main/proto/RefreshUserMappingsProtocol.proto +++ b/hadoop-common-project/hadoop-common/src/main/proto/RefreshUserMappingsProtocol.proto @@ -18,7 +18,7 @@ /** * These .proto interfaces are private and stable. - * Please see http://wiki.apache.org/hadoop/Compatibility + * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html * for what changes are allowed for a *stable* .proto interface. */ syntax = "proto2"; diff --git a/hadoop-common-project/hadoop-common/src/main/proto/RpcHeader.proto b/hadoop-common-project/hadoop-common/src/main/proto/RpcHeader.proto index 760e8261b4ea7..042928c2aee18 100644 --- a/hadoop-common-project/hadoop-common/src/main/proto/RpcHeader.proto +++ b/hadoop-common-project/hadoop-common/src/main/proto/RpcHeader.proto @@ -18,7 +18,7 @@ /** * These .proto interfaces are private and stable. - * Please see http://wiki.apache.org/hadoop/Compatibility + * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html * for what changes are allowed for a *stable* .proto interface. */ syntax = "proto2"; diff --git a/hadoop-common-project/hadoop-common/src/main/proto/Security.proto b/hadoop-common-project/hadoop-common/src/main/proto/Security.proto index 5177a86ef113e..37dbf7f18a9c7 100644 --- a/hadoop-common-project/hadoop-common/src/main/proto/Security.proto +++ b/hadoop-common-project/hadoop-common/src/main/proto/Security.proto @@ -18,7 +18,7 @@ /** * These .proto interfaces are private and stable. - * Please see http://wiki.apache.org/hadoop/Compatibility + * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html * for what changes are allowed for a *stable* .proto interface. */ syntax = "proto2"; diff --git a/hadoop-common-project/hadoop-common/src/main/proto/TraceAdmin.proto b/hadoop-common-project/hadoop-common/src/main/proto/TraceAdmin.proto index 8cf131bfb460a..390040cc9d152 100644 --- a/hadoop-common-project/hadoop-common/src/main/proto/TraceAdmin.proto +++ b/hadoop-common-project/hadoop-common/src/main/proto/TraceAdmin.proto @@ -18,7 +18,7 @@ /** * These .proto interfaces are private and stable. - * Please see http://wiki.apache.org/hadoop/Compatibility + * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html * for what changes are allowed for a *stable* .proto interface. */ syntax = "proto2"; diff --git a/hadoop-common-project/hadoop-common/src/main/proto/ZKFCProtocol.proto b/hadoop-common-project/hadoop-common/src/main/proto/ZKFCProtocol.proto index 98bc05f4a360e..8f9d9f0d9e7f4 100644 --- a/hadoop-common-project/hadoop-common/src/main/proto/ZKFCProtocol.proto +++ b/hadoop-common-project/hadoop-common/src/main/proto/ZKFCProtocol.proto @@ -18,7 +18,7 @@ /** * These .proto interfaces are private and stable. - * Please see http://wiki.apache.org/hadoop/Compatibility + * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html * for what changes are allowed for a *stable* .proto interface. */ syntax = "proto2"; diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 27c86bbc9ac8b..85138611c3b4e 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -78,7 +78,7 @@ false Indicates if administrator ACLs are required to access - instrumentation servlets (JMX, METRICS, CONF, STACKS). + instrumentation servlets (JMX, METRICS, CONF, STACKS, PROF). @@ -793,7 +793,8 @@ The size of buffer for use in sequence files. The size of this buffer should probably be a multiple of hardware page size (4096 on Intel x86), and it determines how much data is - buffered during read and write operations. + buffered during read and write operations. Must be greater than zero. + @@ -1229,7 +1230,7 @@ com.amazonaws.auth.AWSCredentialsProvider. When S3A delegation tokens are not enabled, this list will be used - to directly authenticate with S3 and DynamoDB services. + to directly authenticate with S3 and other AWS services. When S3A Delegation tokens are enabled, depending upon the delegation token binding it may be used to communicate wih the STS endpoint to request session/role @@ -1358,12 +1359,6 @@ - - fs.s3a.delegation.tokens.enabled - false - - - fs.s3a.delegation.token.binding @@ -1623,9 +1618,12 @@ fs.s3a.buffer.dir - ${hadoop.tmp.dir}/s3a + ${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/s3a Comma separated list of directories that will be used to buffer file - uploads to. + uploads to. + Yarn container path will be used as default value on yarn applications, + otherwise fall back to hadoop.tmp.dir + @@ -1692,180 +1690,18 @@ - - fs.s3a.metadatastore.authoritative - false - - When true, allow MetadataStore implementations to act as source of - truth for getting file status and directory listings. Even if this - is set to true, MetadataStore implementations may choose not to - return authoritative results. If the configured MetadataStore does - not support being authoritative, this setting will have no effect. - - - - - fs.s3a.metadatastore.metadata.ttl - 15m - - This value sets how long an entry in a MetadataStore is valid. - - - - - fs.s3a.metadatastore.impl - org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore - - Fully-qualified name of the class that implements the MetadataStore - to be used by s3a. The default class, NullMetadataStore, has no - effect: s3a will continue to treat the backing S3 service as the one - and only source of truth for file and directory metadata. - - - - - fs.s3a.metadatastore.fail.on.write.error - true - - When true (default), FileSystem write operations generate - org.apache.hadoop.fs.s3a.MetadataPersistenceException if the metadata - cannot be saved to the metadata store. When false, failures to save to - metadata store are logged at ERROR level, but the overall FileSystem - write operation succeeds. - - - - - fs.s3a.s3guard.cli.prune.age - 86400000 - - Default age (in milliseconds) after which to prune metadata from the - metadatastore when the prune command is run. Can be overridden on the - command-line. - - - - fs.s3a.impl org.apache.hadoop.fs.s3a.S3AFileSystem The implementation class of the S3A Filesystem - - fs.s3a.s3guard.ddb.region - - - AWS DynamoDB region to connect to. An up-to-date list is - provided in the AWS Documentation: regions and endpoints. Without this - property, the S3Guard will operate table in the associated S3 bucket region. - - - - - fs.s3a.s3guard.ddb.table - - - The DynamoDB table name to operate. Without this property, the respective - S3 bucket name will be used. - - - - - fs.s3a.s3guard.ddb.table.create - false - - If true, the S3A client will create the table if it does not already exist. - - - - - fs.s3a.s3guard.ddb.table.capacity.read - 0 - - Provisioned throughput requirements for read operations in terms of capacity - units for the DynamoDB table. This config value will only be used when - creating a new DynamoDB table. - If set to 0 (the default), new tables are created with "per-request" capacity. - If a positive integer is provided for this and the write capacity, then - a table with "provisioned capacity" will be created. - You can change the capacity of an existing provisioned-capacity table - through the "s3guard set-capacity" command. - - - - - fs.s3a.s3guard.ddb.table.capacity.write - 0 - - Provisioned throughput requirements for write operations in terms of - capacity units for the DynamoDB table. - If set to 0 (the default), new tables are created with "per-request" capacity. - Refer to related configuration option fs.s3a.s3guard.ddb.table.capacity.read - - - - - fs.s3a.s3guard.ddb.table.sse.enabled - false - - Whether server-side encryption (SSE) is enabled or disabled on the table. - By default it's disabled, meaning SSE is set to AWS owned CMK. - - - - - fs.s3a.s3guard.ddb.table.sse.cmk - - - The KMS Customer Master Key (CMK) used for the KMS encryption on the table. - To specify a CMK, this config value can be its key ID, Amazon Resource Name - (ARN), alias name, or alias ARN. Users only need to provide this config if - the key is different from the default DynamoDB KMS Master Key, which is - alias/aws/dynamodb. - - - - - fs.s3a.s3guard.ddb.max.retries - 9 - - Max retries on throttled/incompleted DynamoDB operations - before giving up and throwing an IOException. - Each retry is delayed with an exponential - backoff timer which starts at 100 milliseconds and approximately - doubles each time. The minimum wait before throwing an exception is - sum(100, 200, 400, 800, .. 100*2^N-1 ) == 100 * ((2^N)-1) - - - - - fs.s3a.s3guard.ddb.throttle.retry.interval - 100ms - - Initial interval to retry after a request is throttled events; - the back-off policy is exponential until the number of retries of - fs.s3a.s3guard.ddb.max.retries is reached. - - - - - fs.s3a.s3guard.ddb.background.sleep - 25ms - - Length (in milliseconds) of pause between each batch of deletes when - pruning metadata. Prevents prune operations (which can typically be low - priority background operations) from overly interfering with other I/O - operations. - - - fs.s3a.retry.limit 7 Number of times to retry any repeatable S3 client request on failure, - excluding throttling requests and S3Guard inconsistency resolution. + excluding throttling requests. @@ -1874,7 +1710,7 @@ 500ms Initial retry interval when retrying operations for any reason other - than S3 throttle errors and S3Guard inconsistency resolution. + than S3 throttle errors. @@ -1897,27 +1733,6 @@ - - fs.s3a.s3guard.consistency.retry.limit - 7 - - Number of times to retry attempts to read/open/copy files when - S3Guard believes a specific version of the file to be available, - but the S3 request does not find any version of a file, or a different - version. - - - - - fs.s3a.s3guard.consistency.retry.interval - 2s - - Initial interval between attempts to retry operations while waiting for S3 - to become consistent with the S3Guard data. - An exponential back-off is used here: every failure doubles the delay. - - - fs.s3a.committer.name file @@ -2214,6 +2029,17 @@ + + + fs.s3a.audit.enabled + true + + Should auditing of S3A requests be enabled? + + + fs.AbstractFileSystem.wasb.impl diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/AdminCompatibilityGuide.md b/hadoop-common-project/hadoop-common/src/site/markdown/AdminCompatibilityGuide.md index 67f9c907ff97c..5d2c38d4c5646 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/AdminCompatibilityGuide.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/AdminCompatibilityGuide.md @@ -137,7 +137,8 @@ internal state stores: * The internal MapReduce state data will remain compatible across minor releases within the same major version to facilitate rolling upgrades while MapReduce workloads execute. * HDFS maintains metadata about the data stored in HDFS in a private, internal format that is versioned. In the event of an incompatible change, the store's version number will be incremented. When upgrading an existing cluster, the metadata store will automatically be upgraded if possible. After the metadata store has been upgraded, it is always possible to reverse the upgrade process. -* The AWS S3A guard keeps a private, internal metadata store that is versioned. Incompatible changes will cause the version number to be incremented. If an upgrade requires reformatting the store, it will be indicated in the release notes. +* The AWS S3A guard kept a private, internal metadata store. + Now that the feature has been removed, the store is obsolete and can be deleted. * The YARN resource manager keeps a private, internal state store of application and scheduler information that is versioned. Incompatible changes will cause the version number to be incremented. If an upgrade requires reformatting the store, it will be indicated in the release notes. * The YARN node manager keeps a private, internal state store of application information that is versioned. Incompatible changes will cause the version number to be incremented. If an upgrade requires reformatting the store, it will be indicated in the release notes. * The YARN federation service keeps a private, internal state store of application and cluster information that is versioned. Incompatible changes will cause the version number to be incremented. If an upgrade requires reformatting the store, it will be indicated in the release notes. diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/AsyncProfilerServlet.md b/hadoop-common-project/hadoop-common/src/site/markdown/AsyncProfilerServlet.md new file mode 100644 index 0000000000000..4b93cc219a5ee --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/site/markdown/AsyncProfilerServlet.md @@ -0,0 +1,145 @@ + + +Async Profiler Servlet for Hadoop +======================================== + + + +Purpose +------- + +This document describes how to configure and use async profiler +with Hadoop applications. +Async profiler is a low overhead sampling profiler for Java that +does not suffer from Safepoint bias problem. It features +HotSpot-specific APIs to collect stack traces and to track memory +allocations. The profiler works with OpenJDK, Oracle JDK and other +Java runtimes based on the HotSpot JVM. + +Hadoop profiler servlet supports Async Profiler major versions +1.x and 2.x. + +Prerequisites +------------- + +Make sure Hadoop is installed, configured and setup correctly. +For more information see: + +* [Single Node Setup](./SingleCluster.html) for first-time users. +* [Cluster Setup](./ClusterSetup.html) for large, distributed clusters. + +Go to https://github.com/jvm-profiling-tools/async-profiler, +download a release appropriate for your platform, and install +on every cluster host. + +Set `ASYNC_PROFILER_HOME` in the environment (put it in hadoop-env.sh) +to the root directory of the async-profiler install location, or pass +it on the Hadoop daemon's command line as a system property as +`-Dasync.profiler.home=/path/to/async-profiler`. + + +Usage +-------- + +Once the prerequisites have been satisfied, access to the async-profiler +is available by using Namenode or ResourceManager UI. + +Following options from async-profiler can be specified as query paramater. +* `-e event` profiling event: cpu|alloc|lock|cache-misses etc. +* `-d duration` run profiling for 'duration' seconds (integer) +* `-i interval` sampling interval in nanoseconds (long) +* `-j jstackdepth` maximum Java stack depth (integer) +* `-b bufsize` frame buffer size (long) +* `-t` profile different threads separately +* `-s` simple class names instead of FQN +* `-o fmt[,fmt...]` output format: summary|traces|flat|collapsed|svg|tree|jfr|html +* `--width px` SVG width pixels (integer) +* `--height px` SVG frame height pixels (integer) +* `--minwidth px` skip frames smaller than px (double) +* `--reverse` generate stack-reversed FlameGraph / Call tree + + +Example: +If Namenode http address is localhost:9870, and ResourceManager http +address is localhost:8088, ProfileServlet running with async-profiler +setup can be accessed with http://localhost:9870/prof and +http://localhost:8088/prof for Namenode and ResourceManager processes +respectively. + +Diving deep into some params: + +* To collect 10 second CPU profile of current process + (returns FlameGraph svg) + * `curl http://localhost:9870/prof` (FlameGraph svg for Namenode) + * `curl http://localhost:8088/prof` (FlameGraph svg for ResourceManager) +* To collect 10 second CPU profile of pid 12345 (returns FlameGraph svg) + * `curl http://localhost:9870/prof?pid=12345` (For instance, provide + pid of Datanode here) +* To collect 30 second CPU profile of pid 12345 (returns FlameGraph svg) + * `curl http://localhost:9870/prof?pid=12345&duration=30` +* To collect 1 minute CPU profile of current process and output in tree + format (html) + * `curl http://localhost:9870/prof?output=tree&duration=60` +* To collect 10 second heap allocation profile of current process + (returns FlameGraph svg) + * `curl http://localhost:9870/prof?event=alloc` +* To collect lock contention profile of current process + (returns FlameGraph svg) + * `curl http://localhost:9870/prof?event=lock` + + +The following event types are supported by async-profiler. +Use the 'event' parameter to specify. Default is 'cpu'. +Not all operating systems will support all types. + +Perf events: + +* cpu +* page-faults +* context-switches +* cycles +* instructions +* cache-references +* cache-misses +* branches +* branch-misses +* bus-cycles +* L1-dcache-load-misses +* LLC-load-misses +* dTLB-load-misses + +Java events: + +* alloc +* lock + +The following output formats are supported. +Use the 'output' parameter to specify. Default is 'flamegraph'. + +Output formats: + +* summary: A dump of basic profiling statistics. +* traces: Call traces. +* flat: Flat profile (top N hot methods). +* collapsed: Collapsed call traces in the format used by FlameGraph + script. This is a collection of call stacks, where each line is a + semicolon separated list of frames followed by a counter. +* svg: FlameGraph in SVG format. +* tree: Call tree in HTML format. +* jfr: Call traces in Java Flight Recorder format. + +The 'duration' parameter specifies how long to collect trace data +before generating output, specified in seconds. The default is 10 seconds. + diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Compatibility.md b/hadoop-common-project/hadoop-common/src/site/markdown/Compatibility.md index 03d162a18acc2..0ccb6a8b5c3ca 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/Compatibility.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/Compatibility.md @@ -477,19 +477,12 @@ rolled back to the older layout. ##### AWS S3A Guard Metadata -For each operation in the Hadoop S3 client (s3a) that reads or modifies -file metadata, a shadow copy of that file metadata is stored in a separate -metadata store, which offers HDFS-like consistency for the metadata, and may -also provide faster lookups for things like file status or directory listings. -S3A guard tables are created with a version marker which indicates -compatibility. +The S3Guard metastore used to store metadata in DynamoDB tables; +as such it had to maintain a compatibility strategy. +Now that S3Guard is removed, the tables are not needed. -###### Policy - -The S3A guard metadata schema SHALL be considered -[Private](./InterfaceClassification.html#Private) and -[Unstable](./InterfaceClassification.html#Unstable). Any incompatible change -to the schema MUST result in the version number of the schema being incremented. +Applications configured to use an S3A metadata store other than +the "null" store will fail. ##### YARN Resource Manager State Store diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/DownstreamDev.md b/hadoop-common-project/hadoop-common/src/site/markdown/DownstreamDev.md index b04bc2488f8ae..e38dfc4c88bc7 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/DownstreamDev.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/DownstreamDev.md @@ -300,7 +300,7 @@ that conflicts with a property defined by Hadoop can lead to unexpected and undesirable results. Users are encouraged to avoid using custom configuration property names that conflict with the namespace of Hadoop-defined properties and thus should avoid using any prefixes used by Hadoop, -e.g. hadoop, io, ipc, fs, net, file, ftp, kfs, ha, file, dfs, mapred, +e.g. hadoop, io, ipc, fs, net, ftp, ha, file, dfs, mapred, mapreduce, and yarn. ### Logging Configuration Files diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md index 9f634b645d0a1..9a690a8c5ccdd 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md @@ -177,7 +177,7 @@ Returns 0 on success and -1 on error. cp ---- -Usage: `hadoop fs -cp [-f] [-p | -p[topax]] URI [URI ...] ` +Usage: `hadoop fs -cp [-f] [-p | -p[topax]] [-t ] [-q ] URI [URI ...] ` Copy files from source to destination. This command allows multiple sources as well in which case the destination must be a directory. @@ -185,13 +185,18 @@ Copy files from source to destination. This command allows multiple sources as w Options: -* The -f option will overwrite the destination if it already exists. -* The -p option will preserve file attributes [topx] (timestamps, ownership, permission, ACL, XAttr). If -p is specified with no *arg*, then preserves timestamps, ownership, permission. If -pa is specified, then preserves permission also because ACL is a super-set of permission. Determination of whether raw namespace extended attributes are preserved is independent of the -p flag. +* `-f` : Overwrite the destination if it already exists. +* `-d` : Skip creation of temporary file with the suffix `._COPYING_`. +* `-p` : Preserve file attributes [topx] (timestamps, ownership, permission, ACL, XAttr). If -p is specified with no *arg*, then preserves timestamps, ownership, permission. If -pa is specified, then preserves permission also because ACL is a super-set of permission. Determination of whether raw namespace extended attributes are preserved is independent of the -p flag. +* `-t ` : Number of threads to be used, default is 1. Useful when copying directories containing more than 1 file. +* `-q ` : Thread pool queue size to be used, default is 1024. It takes effect only when thread count greater than 1. Example: * `hadoop fs -cp /user/hadoop/file1 /user/hadoop/file2` * `hadoop fs -cp /user/hadoop/file1 /user/hadoop/file2 /user/hadoop/dir` +* `hadoop fs -cp -t 5 /user/hadoop/file1 /user/hadoop/file2 /user/hadoop/dir` +* `hadoop fs -cp -t 10 -q 2048 /user/hadoop/file1 /user/hadoop/file2 /user/hadoop/dir` Exit Code: @@ -323,27 +328,33 @@ Returns 0 on success and -1 on error. get --- -Usage: `hadoop fs -get [-ignorecrc] [-crc] [-p] [-f] ` +Usage: `hadoop fs -get [-ignoreCrc] [-crc] [-p] [-f] [-t ] [-q ] ... ` + +Copy files to the local file system. Files that fail the CRC check may be copied with the -ignoreCrc option. Files and CRCs may be copied using the -crc option. + +Options: -Copy files to the local file system. Files that fail the CRC check may be copied with the -ignorecrc option. Files and CRCs may be copied using the -crc option. +* `-p` : Preserves access and modification times, ownership and the permissions. + (assuming the permissions can be propagated across filesystems) +* `-f` : Overwrites the destination if it already exists. +* `-ignoreCrc` : Skip CRC checks on the file(s) downloaded. +* `-crc`: write CRC checksums for the files downloaded. +* `-t ` : Number of threads to be used, default is 1. + Useful when downloading directories containing more than 1 file. +* `-q ` : Thread pool queue size to be used, default is 1024. + It takes effect only when thread count greater than 1. Example: * `hadoop fs -get /user/hadoop/file localfile` * `hadoop fs -get hdfs://nn.example.com/user/hadoop/file localfile` +* `hadoop fs -get -t 10 hdfs://nn.example.com/user/hadoop/dir1 localdir` +* `hadoop fs -get -t 10 -q 2048 hdfs://nn.example.com/user/hadoop/dir* localdir` Exit Code: Returns 0 on success and -1 on error. -Options: - -* `-p` : Preserves access and modification times, ownership and the permissions. -(assuming the permissions can be propagated across filesystems) -* `-f` : Overwrites the destination if it already exists. -* `-ignorecrc` : Skip CRC checks on the file(s) downloaded. -* `-crc`: write CRC checksums for the files downloaded. - getfacl ------- @@ -525,7 +536,7 @@ Returns 0 on success and -1 on error. put --- -Usage: `hadoop fs -put [-f] [-p] [-l] [-d] [-t ] [-q ] [ - | .. ]. ` +Usage: `hadoop fs -put [-f] [-p] [-l] [-d] [-t ] [-q ] [ - | ...] ` Copy single src, or multiple srcs from local file system to the destination file system. Also reads input from stdin and writes to destination file system if the source is set to "-" @@ -537,12 +548,13 @@ Options: * `-p` : Preserves access and modification times, ownership and the permissions. (assuming the permissions can be propagated across filesystems) * `-f` : Overwrites the destination if it already exists. -* `-t ` : Number of threads to be used, default is 1. Useful - when uploading a directory containing more than 1 file. * `-l` : Allow DataNode to lazily persist the file to disk, Forces a replication factor of 1. This flag will result in reduced durability. Use with care. * `-d` : Skip creation of temporary file with the suffix `._COPYING_`. -* `-q ` : ThreadPool queue size to be used, default is 1024. +* `-t ` : Number of threads to be used, default is 1. + Useful when uploading directories containing more than 1 file. +* `-q ` : Thread pool queue size to be used, default is 1024. + It takes effect only when thread count greater than 1. Examples: @@ -551,7 +563,8 @@ Examples: * `hadoop fs -put -f localfile1 localfile2 /user/hadoop/hadoopdir` * `hadoop fs -put -d localfile hdfs://nn.example.com/hadoop/hadoopfile` * `hadoop fs -put - hdfs://nn.example.com/hadoop/hadoopfile` Reads the input from stdin. -* `hadoop fs -put -q 500 localfile3 hdfs://nn.example.com/hadoop/hadoopfile3` +* `hadoop fs -put -t 5 localdir hdfs://nn.example.com/hadoop/hadoopdir` +* `hadoop fs -put -t 10 -q 2048 localdir1 localdir2 hdfs://nn.example.com/hadoop/hadoopdir` Exit Code: diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md index 0db6e1a58e47b..47786e473a52f 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md @@ -83,6 +83,7 @@ The default timeunit used for RPC metrics is milliseconds (as per the below desc | `RpcAuthorizationFailures` | Total number of authorization failures | | `RpcAuthorizationSuccesses` | Total number of authorization successes | | `NumOpenConnections` | Current number of open connections | +| `NumInProcessHandler` | Current number of handlers on working | | `CallQueueLength` | Current length of the call queue | | `numDroppedConnections` | Total number of dropped connections | | `rpcQueueTime`*num*`sNumOps` | Shows total number of RPC calls (*num* seconds granularity) if `rpc.metrics.quantile.enable` is set to true. *num* is specified by `rpc.metrics.percentiles.intervals`. | @@ -298,6 +299,7 @@ Each metrics record contains tags such as HAState and Hostname as additional inf | `FSN(Read/Write)Lock`*OperationName*`NanosAvgTime` | Average time of holding the lock by operations in nanoseconds | | `FSN(Read/Write)LockOverallNanosNumOps` | Total number of acquiring lock by all operations | | `FSN(Read/Write)LockOverallNanosAvgTime` | Average time of holding the lock by all operations in nanoseconds | +| `PendingSPSPaths` | The number of paths to be processed by storage policy satisfier | JournalNode ----------- @@ -479,6 +481,8 @@ Each metrics record contains tags such as SessionId and Hostname as additional i | `PacketsSlowWriteToMirror` | Total number of packets whose write to other Datanodes in the pipeline takes more than a certain time (300ms by default) | | `PacketsSlowWriteToDisk` | Total number of packets whose write to disk takes more than a certain time (300ms by default) | | `PacketsSlowWriteToOsCache` | Total number of packets whose write to os cache takes more than a certain time (300ms by default) | +| `slowFlushOrSyncCount` | Total number of packets whose sync/flush takes more than a certain time (300ms by default) | +| `slowAckToUpstreamCount` | Total number of packets whose upstream ack takes more than a certain time (300ms by default) | FsVolume -------- diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/UnixShellGuide.md b/hadoop-common-project/hadoop-common/src/site/markdown/UnixShellGuide.md index ffe2aec96ab7a..ca32fd8ee2f95 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/UnixShellGuide.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/UnixShellGuide.md @@ -134,7 +134,7 @@ Apache Hadoop's shell code has a [function library](./UnixShellAPI.html) that is The shell code allows for core functions to be overridden. However, not all functions can be or are safe to be replaced. If a function is not safe to replace, it will have an attribute of Replaceable: No. If a function is safe to replace, it will have the attribute of Replaceable: Yes. -In order to replace a function, create a file called `hadoop-user-functions.sh` in the `${HADOOP_CONF_DIR}` directory. Simply define the new, replacement function in this file and the system will pick it up automatically. There may be as many replacement functions as needed in this file. Examples of function replacement are in the `hadoop-user-functions.sh.examples` file. +In order to replace a function, create a file called `hadoop-user-functions.sh` in the `${HADOOP_CONF_DIR}` directory. Simply define the new, replacement function in this file and the system will pick it up automatically. There may be as many replacement functions as needed in this file. Examples of function replacement are in the `hadoop-user-functions.sh.example` file. Functions that are marked Public and Stable are safe to use in shell profiles as-is. Other functions may change in a minor release. diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md index 2eb0bc07196d6..004220c4bedaa 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md @@ -453,6 +453,26 @@ The function `getLocatedFileStatus(FS, d)` is as defined in The atomicity and consistency constraints are as for `listStatus(Path, PathFilter)`. + +### `ContentSummary getContentSummary(Path path)` + +Given a path return its content summary. + +`getContentSummary()` first checks if the given path is a file and if yes, it returns 0 for directory count +and 1 for file count. + +#### Preconditions + + exists(FS, path) else raise FileNotFoundException + +#### Postconditions + +Returns a `ContentSummary` object with information such as directory count +and file count for a given path. + +The atomicity and consistency constraints are as for +`listStatus(Path, PathFilter)`. + ### `BlockLocation[] getFileBlockLocations(FileStatus f, int s, int l)` #### Preconditions @@ -794,97 +814,11 @@ exists in the metadata, but no copies of any its blocks can be located; ### `FSDataInputStreamBuilder openFile(Path path)` -Creates a [`FSDataInputStreamBuilder`](fsdatainputstreambuilder.html) -to construct a operation to open the file at `path` for reading. - -When `build()` is invoked on the returned `FSDataInputStreamBuilder` instance, -the builder parameters are verified and -`openFileWithOptions(Path, OpenFileParameters)` invoked. - -This (protected) operation returns a `CompletableFuture` -which, when its `get()` method is called, either returns an input -stream of the contents of opened file, or raises an exception. - -The base implementation of the `openFileWithOptions(PathHandle, OpenFileParameters)` -ultimately invokes `open(Path, int)`. - -Thus the chain `openFile(path).build().get()` has the same preconditions -and postconditions as `open(Path p, int bufferSize)` +See [openFile()](openfile.html). -However, there is one difference which implementations are free to -take advantage of: - -The returned stream MAY implement a lazy open where file non-existence or -access permission failures may not surface until the first `read()` of the -actual data. - -The `openFile()` operation may check the state of the filesystem during its -invocation, but as the state of the filesystem may change betwen this call and -the actual `build()` and `get()` operations, this file-specific -preconditions (file exists, file is readable, etc) MUST NOT be checked here. - -FileSystem implementations which do not implement `open(Path, int)` -MAY postpone raising an `UnsupportedOperationException` until either the -`FSDataInputStreamBuilder.build()` or the subsequent `get()` call, -else they MAY fail fast in the `openFile()` call. - -### Implementors notes - -The base implementation of `openFileWithOptions()` actually executes -the `open(path)` operation synchronously, yet still returns the result -or any failures in the `CompletableFuture<>`, so as to ensure that users -code expecting this. - -Any filesystem where the time to open a file may be significant SHOULD -execute it asynchronously by submitting the operation in some executor/thread -pool. This is particularly recommended for object stores and other filesystems -likely to be accessed over long-haul connections. - -Arbitrary filesystem-specific options MAY be supported; these MUST -be prefixed with either the filesystem schema, e.g. `hdfs.` -or in the "fs.SCHEMA" format as normal configuration settings `fs.hdfs`). The -latter style allows the same configuration option to be used for both -filesystem configuration and file-specific configuration. - -It SHOULD be possible to always open a file without specifying any options, -so as to present a consistent model to users. However, an implementation MAY -opt to require one or more mandatory options to be set. - -The returned stream may perform "lazy" evaluation of file access. This is -relevant for object stores where the probes for existence are expensive, and, -even with an asynchronous open, may be considered needless. - ### `FSDataInputStreamBuilder openFile(PathHandle)` -Creates a `FSDataInputStreamBuilder` to build an operation to open a file. -Creates a [`FSDataInputStreamBuilder`](fsdatainputstreambuilder.html) -to construct a operation to open the file identified by the given `PathHandle` for reading. - -When `build()` is invoked on the returned `FSDataInputStreamBuilder` instance, -the builder parameters are verified and -`openFileWithOptions(PathHandle, OpenFileParameters)` invoked. - -This (protected) operation returns a `CompletableFuture` -which, when its `get()` method is called, either returns an input -stream of the contents of opened file, or raises an exception. - -The base implementation of the `openFileWithOptions(PathHandle, OpenFileParameters)` method -returns a future which invokes `open(Path, int)`. - -Thus the chain `openFile(pathhandle).build().get()` has the same preconditions -and postconditions as `open(Pathhandle, int)` - -As with `FSDataInputStreamBuilder openFile(PathHandle)`, the `openFile()` -call must not be where path-specific preconditions are checked -that -is postponed to the `build()` and `get()` calls. - -FileSystem implementations which do not implement `open(PathHandle handle, int bufferSize)` -MAY postpone raising an `UnsupportedOperationException` until either the -`FSDataInputStreamBuilder.build()` or the subsequent `get()` call, -else they MAY fail fast in the `openFile()` call. - -The base implementation raises this exception in the `build()` operation; -other implementations SHOULD copy this. +See [openFile()](openfile.html). ### `PathHandle getPathHandle(FileStatus stat, HandleOpt... options)` @@ -1240,7 +1174,7 @@ Renaming a file where the destination is a directory moves the file as a child FS' where: not exists(FS', src) and exists(FS', dest) - and data(FS', dest) == data (FS, dest) + and data(FS', dest) == data (FS, source) result = True @@ -1698,3 +1632,92 @@ in:readahead | READAHEAD | CanSetReadahead | Set the readahead on the input st dropbehind | DROPBEHIND | CanSetDropBehind | Drop the cache. in:unbuffer | UNBUFFER | CanUnbuffer | Reduce the buffering on the input stream. +## Etag probes through the interface `EtagSource` + +FileSystem implementations MAY support querying HTTP etags from `FileStatus` +entries. If so, the requirements are as follows + +### Etag support MUST BE across all list/`getFileStatus()` calls. + +That is: when adding etag support, all operations which return `FileStatus` or `ListLocatedStatus` +entries MUST return subclasses which are instances of `EtagSource`. + +### FileStatus instances MUST have etags whenever the remote store provides them. + +To support etags, they MUST BE to be provided in both `getFileStatus()` +and list calls. + +Implementors note: the core APIs which MUST BE overridden to achieve this are as follows: + +```java +FileStatus getFileStatus(Path) +FileStatus[] listStatus(Path) +RemoteIterator listStatusIterator(Path) +RemoteIterator listFiles([Path, boolean) +``` + + +### Etags of files MUST BE Consistent across all list/getFileStatus operations. + +The value of `EtagSource.getEtag()` MUST be the same for list* queries which return etags for calls of `getFileStatus()` for the specific object. + +```java +((EtagSource)getFileStatus(path)).getEtag() == ((EtagSource)listStatus(path)[0]).getEtag() +``` + +Similarly, the same value MUST BE returned for `listFiles()`, `listStatusIncremental()` of the path and +when listing the parent path, of all files in the listing. + +### Etags MUST BE different for different file contents. + +Two different arrays of data written to the same path MUST have different etag values when probed. +This is a requirement of the HTTP specification. + +### Etags of files SHOULD BE preserved across rename operations + +After a file is renamed, the value of `((EtagSource)getFileStatus(dest)).getEtag()` +SHOULD be the same as the value of `((EtagSource)getFileStatus(source)).getEtag()` +was before the rename took place. + +This is an implementation detail of the store; it does not hold for AWS S3. + +If and only if the store consistently meets this requirement, the filesystem SHOULD +declare in `hasPathCapability()` that it supports +`fs.capability.etags.preserved.in.rename` + +### Directories MAY have etags + +Directory entries MAY return etags in listing/probe operations; these entries MAY be preserved across renames. + +Equally, directory entries MAY NOT provide such entries, MAY NOT preserve them acrosss renames, +and MAY NOT guarantee consistency over time. + +Note: special mention of the root path "/". +As that isn't a real "directory", nobody should expect it to have an etag. + +### All etag-aware `FileStatus` subclass MUST BE `Serializable`; MAY BE `Writable` + +The base `FileStatus` class implements `Serializable` and `Writable` and marshalls its fields appropriately. + +Subclasses MUST support java serialization (Some Apache Spark applications use it), preserving the etag. +This is a matter of making the etag field non-static and adding a `serialVersionUID`. + +The `Writable` support was used for marshalling status data over Hadoop IPC calls; +in Hadoop 3 that is implemented through `org/apache/hadoop/fs/protocolPB/PBHelper.java`and the methods deprecated. +Subclasses MAY override the deprecated methods to add etag marshalling. +However -but there is no expectation of this and such marshalling is unlikely to ever take place. + +### Appropriate etag Path Capabilities SHOULD BE declared + +1. `hasPathCapability(path, "fs.capability.etags.available")` MUST return true iff + the filesystem returns valid (non-empty etags) on file status/listing operations. +2. `hasPathCapability(path, "fs.capability.etags.consistent.across.rename")` MUST return + true if and only if etags are preserved across renames. + +### Non-requirements of etag support + +* There is no requirement/expectation that `FileSystem.getFileChecksum(Path)` returns + a checksum value related to the etag of an object, if any value is returned. +* If the same data is uploaded to the twice to the same or a different path, + the etag of the second upload MAY NOT match that of the first upload. + diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdatainputstreambuilder.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdatainputstreambuilder.md index eadba174fc1a6..db630e05c22d4 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdatainputstreambuilder.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdatainputstreambuilder.md @@ -13,10 +13,10 @@ --> - + -# class `org.apache.hadoop.fs.FSDataInputStreamBuilder` +# class `org.apache.hadoop.fs.FutureDataInputStreamBuilder` @@ -27,7 +27,7 @@ file for reading. ## Invariants -The `FSDataInputStreamBuilder` interface does not require parameters or +The `FutureDataInputStreamBuilder` interface does not require parameters or or the state of `FileSystem` until [`build()`](#build) is invoked and/or during the asynchronous open operation itself. @@ -39,11 +39,11 @@ path validation. ## Implementation-agnostic parameters. -### `FSDataInputStreamBuilder bufferSize(int bufSize)` +### `FutureDataInputStreamBuilder bufferSize(int bufSize)` Set the size of the buffer to be used. -### `FSDataInputStreamBuilder withFileStatus(FileStatus status)` +### `FutureDataInputStreamBuilder withFileStatus(FileStatus status)` A `FileStatus` instance which refers to the file being opened. @@ -53,7 +53,7 @@ So potentially saving on remote calls especially to object stores. Requirements: * `status != null` -* `status.getPath()` == the resolved path of the file being opened. +* `status.getPath().getName()` == the name of the file being opened. The path validation MUST take place if the store uses the `FileStatus` when it opens files, and MAY be performed otherwise. The validation @@ -65,27 +65,85 @@ If a filesystem implementation extends the `FileStatus` returned in its implementation MAY use this information when opening the file. This is relevant with those stores which return version/etag information, -including the S3A and ABFS connectors -they MAY use this to guarantee that -the file they opened is exactly the one returned in the listing. +-they MAY use this to guarantee that the file they opened +is exactly the one returned in the listing. + + +The final `status.getPath().getName()` element of the supplied status MUST equal +the name value of the path supplied to the `openFile(path)` call. + +Filesystems MUST NOT validate the rest of the path. +This is needed to support viewfs and other mount-point wrapper filesystems +where schemas and paths are different. These often create their own FileStatus results + +Preconditions + +```python +status == null or status.getPath().getName() == path.getName() + +``` + +Filesystems MUST NOT require the class of `status` to equal +that of any specific subclass their implementation returns in filestatus/list +operations. This is to support wrapper filesystems and serialization/deserialization +of the status. + ### Set optional or mandatory parameters - FSDataInputStreamBuilder opt(String key, ...) - FSDataInputStreamBuilder must(String key, ...) + FutureDataInputStreamBuilder opt(String key, ...) + FutureDataInputStreamBuilder must(String key, ...) Set optional or mandatory parameters to the builder. Using `opt()` or `must()`, client can specify FS-specific parameters without inspecting the concrete type of `FileSystem`. +Example: + ```java out = fs.openFile(path) - .opt("fs.s3a.experimental.input.fadvise", "random") - .must("fs.s3a.readahead.range", 256 * 1024) + .must("fs.option.openfile.read.policy", "random") + .opt("fs.http.connection.timeout", 30_000L) .withFileStatus(statusFromListing) .build() .get(); ``` +Here the read policy of `random` has been specified, +with the requirement that the filesystem implementation must understand the option. +An http-specific option has been supplied which may be interpreted by any store; +If the filesystem opening the file does not recognize the option, it can safely be +ignored. + +### When to use `opt()` versus `must()` + +The difference between `opt()` versus `must()` is how the FileSystem opening +the file must react to an option which it does not recognize. + +```python + +def must(name, value): + if not name in known_keys: + raise IllegalArgumentException + if not name in supported_keys: + raise UnsupportedException + + +def opt(name, value): + if not name in known_keys: + # ignore option + +``` + +For any known key, the validation of the `value` argument MUST be the same +irrespective of how the (key, value) pair was declared. + +1. For a filesystem-specific option, it is the choice of the implementation + how to validate the entry. +1. For standard options, the specification of what is a valid `value` is + defined in this filesystem specification, validated through contract + tests. + #### Implementation Notes Checking for supported options must be performed in the `build()` operation. @@ -93,9 +151,9 @@ Checking for supported options must be performed in the `build()` operation. 1. If a mandatory parameter declared via `must(key, value)`) is not recognized, `IllegalArgumentException` MUST be thrown. -1. If a mandatory parameter declared via `must(key, value)`) relies on +1. If a mandatory parameter declared via `must(key, value)` relies on a feature which is recognized but not supported in the specific -Filesystem/FileContext instance `UnsupportedException` MUST be thrown. +`FileSystem`/`FileContext` instance `UnsupportedException` MUST be thrown. The behavior of resolving the conflicts between the parameters set by builder methods (i.e., `bufferSize()`) and `opt()`/`must()` is as follows: @@ -110,13 +168,18 @@ custom subclasses. This is critical to ensure safe use of the feature: directory listing/ status serialization/deserialization can result result in the `withFileStatus()` -argumennt not being the custom subclass returned by the Filesystem instance's +argument not being the custom subclass returned by the Filesystem instance's own `getFileStatus()`, `listFiles()`, `listLocatedStatus()` calls, etc. In such a situation the implementations must: -1. Validate the path (always). -1. Use the status/convert to the custom type, *or* simply discard it. +1. Verify that `status.getPath().getName()` matches the current `path.getName()` + value. The rest of the path MUST NOT be validated. +1. Use any status fields as desired -for example the file length. + +Even if not values of the status are used, the presence of the argument +can be interpreted as the caller declaring that they believe the file +to be present and of the given size. ## Builder interface @@ -128,26 +191,499 @@ completed, returns an input stream which can read data from the filesystem. The `build()` operation MAY perform the validation of the file's existence, its kind, so rejecting attempts to read from a directory or non-existent -file. **Alternatively**, the `build()` operation may delay all checks -until an asynchronous operation whose outcome is provided by the `Future` +file. Alternatively +* file existence/status checks MAY be performed asynchronously within the returned + `CompletableFuture<>`. +* file existence/status checks MAY be postponed until the first byte is read in + any of the read such as `read()` or `PositionedRead`. That is, the precondition `exists(FS, path)` and `isFile(FS, path)` are -only guaranteed to have been met after the `get()` on the returned future is successful. +only guaranteed to have been met after the `get()` called on returned future +and an attempt has been made to read the stream. -Thus, if even a file does not exist, the following call will still succeed, returning -a future to be evaluated. +Thus, if even when file does not exist, or is a directory rather than a file, +the following call MUST succeed, returning a `CompletableFuture` to be evaluated. ```java Path p = new Path("file://tmp/file-which-does-not-exist"); CompletableFuture future = p.getFileSystem(conf) .openFile(p) - .build; + .build(); ``` -The preconditions for opening the file are checked during the asynchronous -evaluation, and so will surface when the future is completed: +The inability to access/read a file MUST raise an `IOException`or subclass +in either the future's `get()` call, or, for late binding operations, +when an operation to read data is invoked. + +Therefore the following sequence SHALL fail when invoked on the +`future` returned by the previous example. ```java -FSDataInputStream in = future.get(); + future.get().read(); ``` + +Access permission checks have the same visibility requirements: permission failures +MUST be delayed until the `get()` call and MAY be delayed into subsequent operations. + +Note: some operations on the input stream, such as `seek()` may not attempt any IO +at all. Such operations MAY NOT raise exceotions when interacting with +nonexistent/unreadable files. + +## Standard `openFile()` options since Hadoop 3.3.3 + +These are options which `FileSystem` and `FileContext` implementation +MUST recognise and MAY support by changing the behavior of +their input streams as appropriate. + +Hadoop 3.3.0 added the `openFile()` API; these standard options were defined in +a later release. Therefore, although they are "well known", unless confident that +the application will only be executed against releases of Hadoop which knows of +the options -applications SHOULD set the options via `opt()` calls rather than `must()`. + +When opening a file through the `openFile()` builder API, callers MAY use +both `.opt(key, value)` and `.must(key, value)` calls to set standard and +filesystem-specific options. + +If set as an `opt()` parameter, unsupported "standard" options MUST be ignored, +as MUST unrecognized standard options. + +If set as a `must()` parameter, unsupported "standard" options MUST be ignored. +unrecognized standard options MUST be rejected. + +The standard `openFile()` options are defined +in `org.apache.hadoop.fs.OpenFileOptions`; they all SHALL start +with `fs.option.openfile.`. + +Note that while all `FileSystem`/`FileContext` instances SHALL support these +options to the extent that `must()` declarations SHALL NOT fail, the +implementations MAY support them to the extent of interpreting the values. This +means that it is not a requirement for the stores to actually read the read +policy or file length values and use them when opening files. + +Unless otherwise stated, they SHOULD be viewed as hints. + +Note: if a standard option is added such that if set but not +supported would be an error, then implementations SHALL reject it. For example, +the S3A filesystem client supports the ability to push down SQL commands. If +something like that were ever standardized, then the use of the option, either +in `opt()` or `must()` argument MUST be rejected for filesystems which don't +support the feature. + +### Option: `fs.option.openfile.buffer.size` + +Read buffer size in bytes. + +This overrides the default value set in the configuration with the option +`io.file.buffer.size`. + +It is supported by all filesystem clients which allow for stream-specific buffer +sizes to be set via `FileSystem.open(path, buffersize)`. + +### Option: `fs.option.openfile.read.policy` + +Declare the read policy of the input stream. This is a hint as to what the +expected read pattern of an input stream will be. This MAY control readahead, +buffering and other optimizations. + +Sequential reads may be optimized with prefetching data and/or reading data in +larger blocks. Some applications (e.g. distCp) perform sequential IO even over +columnar data. + +In contrast, random IO reads data in different parts of the file using a +sequence of `seek()/read()` +or via the `PositionedReadable` or `ByteBufferPositionedReadable` APIs. + +Random IO performance may be best if little/no prefetching takes place, along +with other possible optimizations + +Queries over columnar formats such as Apache ORC and Apache Parquet perform such +random IO; other data formats may be best read with sequential or whole-file +policies. + +What is key is that optimizing reads for seqential reads may impair random +performance -and vice versa. + +1. The seek policy is a hint; even if declared as a `must()` option, the + filesystem MAY ignore it. +1. The interpretation/implementation of a policy is a filesystem specific + behavior -and it may change with Hadoop releases and/or specific storage + subsystems. +1. If a policy is not recognized, the filesystem client MUST ignore it. + +| Policy | Meaning | +|--------------|----------------------------------------------------------| +| `adaptive` | Any adaptive policy implemented by the store. | +| `default` | The default policy for this store. Generally "adaptive". | +| `random` | Optimize for random access. | +| `sequential` | Optimize for sequential access. | +| `vector` | The Vectored IO API is intended to be used. | +| `whole-file` | The whole file will be read. | + +Choosing the wrong read policy for an input source may be inefficient. + +A list of read policies MAY be supplied; the first one recognized/supported by +the filesystem SHALL be the one used. This allows for custom policies to be +supported, for example an `hbase-hfile` policy optimized for HBase HFiles. + +The S3A and ABFS input streams both implement +the [IOStatisticsSource](iostatistics.html) API, and can be queried for their IO +Performance. + +*Tip:* log the `toString()` value of input streams at `DEBUG`. The S3A and ABFS +Input Streams log read statistics, which can provide insight about whether reads +are being performed efficiently or not. + +_Futher reading_ + +* [Linux fadvise()](https://linux.die.net/man/2/fadvise). +* [Windows `CreateFile()`](https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilea#caching-behavior) + +#### Read Policy `adaptive` + +Try to adapt the seek policy to the read pattern of the application. + +The `normal` policy of the S3A client and the sole policy supported by +the `wasb:` client are both adaptive -they assume sequential IO, but once a +backwards seek/positioned read call is made the stream switches to random IO. + +Other filesystem implementations may wish to adopt similar strategies, and/or +extend the algorithms to detect forward seeks and/or switch from random to +sequential IO if that is considered more efficient. + +Adaptive read policies are the absence of the ability to +declare the seek policy in the `open()` API, so requiring it to be declared, if +configurable, in the cluster/application configuration. However, the switch from +sequential to random seek policies may be exensive. + +When applications explicitly set the `fs.option.openfile.read.policy` option, if +they know their read plan, they SHOULD declare which policy is most appropriate. + +#### Read Policy `` + +The default policy for the filesystem instance. +Implementation/installation-specific. + +#### Read Policy `sequential` + +Expect sequential reads from the first byte read to the end of the file/until +the stream is closed. + +#### Read Policy `random` + +Expect `seek()/read()` sequences, or use of `PositionedReadable` +or `ByteBufferPositionedReadable` APIs. + + +#### Read Policy `vector` + +This declares that the caller intends to use the Vectored read API of +[HADOOP-11867](https://issues.apache.org/jira/browse/HADOOP-11867) +_Add a high-performance vectored read API_. + +This is a hint: it is not a requirement when using the API. +It does inform the implemenations that the stream should be +configured for optimal vectored IO performance, if such a +feature has been implemented. + +It is *not* exclusive: the same stream may still be used for +classic `InputStream` and `PositionedRead` API calls. +Implementations SHOULD use the `random` read policy +with these operations. + +#### Read Policy `whole-file` + + +This declares that the whole file is to be read end-to-end; the file system client is free to enable +whatever strategies maximise performance for this. In particular, larger ranged reads/GETs can +deliver high bandwidth by reducing socket/TLS setup costs and providing a connection long-lived +enough for TCP flow control to determine the optimal download rate. + +Strategies can include: + +* Initiate an HTTP GET of the entire file in `openFile()` operation. +* Prefech data in large blocks, possibly in parallel read operations. + +Applications which know that the entire file is to be read from an opened stream SHOULD declare this +read policy. + +### Option: `fs.option.openfile.length` + +Declare the length of a file. + +This can be used by clients to skip querying a remote store for the size +of/existence of a file when opening it, similar to declaring a file status +through the `withFileStatus()` option. + +If supported by a filesystem connector, this option MUST be interpreted as +declaring the minimum length of the file: + +1. If the value is negative, the option SHALL be considered unset. +2. It SHALL NOT be an error if the actual length of the file is greater than + this value. +3. `read()`, `seek()` and positioned read calls MAY use a position across/beyond + this length but below the actual length of the file. Implementations MAY + raise `EOFExceptions` in such cases, or they MAY return data. + +If this option is used by the FileSystem implementation + +*Implementor's Notes* + +* A value of `fs.option.openfile.length` < 0 MUST be rejected. +* If a file status is supplied along with a value in `fs.opt.openfile.length`; + the file status values take precedence. + +### Options: `fs.option.openfile.split.start` and `fs.option.openfile.split.end` + +Declare the start and end of the split when a file has been split for processing +in pieces. + +1. If a value is negative, the option SHALL be considered unset. +1. Filesystems MAY assume that the length of the file is greater than or equal + to the value of `fs.option.openfile.split.end`. +1. And that they MAY raise an exception if the client application reads past the + value set in `fs.option.openfile.split.end`. +1. The pair of options MAY be used to optimise the read plan, such as setting + the content range for GET requests, or using the split end as an implicit + declaration of the guaranteed minimum length of the file. +1. If both options are set, and the split start is declared as greater than the + split end, then the split start SHOULD just be reset to zero, rather than + rejecting the operation. + +The split end value can provide a hint as to the end of the input stream. The +split start can be used to optimize any initial read offset for filesystem +clients. + +*Note for implementors: applications will read past the end of a split when they +need to read to the end of a record/line which begins before the end of the +split. + +Therefore clients MUST be allowed to `seek()`/`read()` past the length +set in `fs.option.openfile.split.end` if the file is actually longer +than that value. + +## S3A-specific options + +The S3A Connector supports custom options for readahead and seek policy. + +| Name | Type | Meaning | +|--------------------------------------|----------|-------------------------------------------------------------| +| `fs.s3a.readahead.range` | `long` | readahead range in bytes | +| `fs.s3a.input.async.drain.threshold` | `long` | threshold to switch to asynchronous draining of the stream | +| `fs.s3a.experimental.input.fadvise` | `String` | seek policy. Superceded by `fs.option.openfile.read.policy` | + +If the option set contains a SQL statement in the `fs.s3a.select.sql` statement, +then the file is opened as an S3 Select query. +Consult the S3A documentation for more details. + +## ABFS-specific options + +The ABFS Connector supports custom input stream options. + +| Name | Type | Meaning | +|-----------------------------------|-----------|----------------------------------------------------| +| `fs.azure.buffered.pread.disable` | `boolean` | disable caching on the positioned read operations. | + + +Disables caching on data read through the [PositionedReadable](fsdatainputstream.html#PositionedReadable) +APIs. + +Consult the ABFS Documentation for more details. + +## Examples + +#### Declaring seek policy and split limits when opening a file. + +Here is an example from a proof of +concept `org.apache.parquet.hadoop.util.HadoopInputFile` +reader which uses a (nullable) file status and a split start/end. + +The `FileStatus` value is always passed in -but if it is null, then the split +end is used to declare the length of the file. + +```java +protected SeekableInputStream newStream(Path path, FileStatus stat, + long splitStart, long splitEnd) + throws IOException { + + FutureDataInputStreamBuilder builder = fs.openFile(path) + .opt("fs.option.openfile.read.policy", "vector, random") + .withFileStatus(stat); + + builder.opt("fs.option.openfile.split.start", splitStart); + builder.opt("fs.option.openfile.split.end", splitEnd); + CompletableFuture streamF = builder.build(); + return HadoopStreams.wrap(FutureIO.awaitFuture(streamF)); +} +``` + +As a result, whether driven directly by a file listing, or when opening a file +from a query plan of `(path, splitStart, splitEnd)`, there is no need to probe +the remote store for the length of the file. When working with remote object +stores, this can save tens to hundreds of milliseconds, even if such a probe is +done asynchronously. + +If both the file length and the split end is set, then the file length MUST be +considered "more" authoritative, that is it really SHOULD be defining the file +length. If the split end is set, the caller MAY ot read past it. + +The `CompressedSplitLineReader` can read past the end of a split if it is +partway through processing a compressed record. That is: it assumes an +incomplete record read means that the file length is greater than the split +length, and that it MUST read the entirety of the partially read record. Other +readers may behave similarly. + +Therefore + +1. File length as supplied in a `FileStatus` or in `fs.option.openfile.length` + SHALL set the strict upper limit on the length of a file +2. The split end as set in `fs.option.openfile.split.end` MUST be viewed as a + hint, rather than the strict end of the file. + +### Opening a file with both standard and non-standard options + +Standard and non-standard options MAY be combined in the same `openFile()` +operation. + +```java +Future f = openFile(path) + .must("fs.option.openfile.read.policy", "random, adaptive") + .opt("fs.s3a.readahead.range", 1024 * 1024) + .build(); + +FSDataInputStream is = f.get(); +``` + +The option set in `must()` MUST be understood, or at least recognized and +ignored by all filesystems. In this example, S3A-specific option MAY be +ignored by all other filesystem clients. + +### Opening a file with older releases + +Not all hadoop releases recognize the `fs.option.openfile.read.policy` option. + +The option can be safely used in application code if it is added via the `opt()` +builder argument, as it will be treated as an unknown optional key which can +then be discarded. + +```java +Future f = openFile(path) + .opt("fs.option.openfile.read.policy", "vector, random, adaptive") + .build(); + +FSDataInputStream is = f.get(); +``` + +*Note 1* if the option name is set by a reference to a constant in +`org.apache.hadoop.fs.Options.OpenFileOptions`, then the program will not link +against versions of Hadoop without the specific option. Therefore for resilient +linking against older releases -use a copy of the value. + +*Note 2* as option validation is performed in the FileSystem connector, +a third-party connector designed to work with multiple hadoop versions +MAY NOT support the option. + +### Passing options in to MapReduce + +Hadoop MapReduce will automatically read MR Job Options with the prefixes +`mapreduce.job.input.file.option.` and `mapreduce.job.input.file.must.` +prefixes, and apply these values as `.opt()` and `must()` respectively, after +remove the mapreduce-specific prefixes. + +This makes passing options in to MR jobs straightforward. For example, to +declare that a job should read its data using random IO: + +```java +JobConf jobConf = (JobConf) job.getConfiguration() +jobConf.set( + "mapreduce.job.input.file.option.fs.option.openfile.read.policy", + "random"); +``` + +### MapReduce input format propagating options + +An example of a record reader passing in options to the file it opens. + +```java + public void initialize(InputSplit genericSplit, + TaskAttemptContext context) throws IOException { + FileSplit split = (FileSplit)genericSplit; + Configuration job = context.getConfiguration(); + start = split.getStart(); + end = start + split.getLength(); + Path file = split.getPath(); + + // open the file and seek to the start of the split + FutureDataInputStreamBuilder builder = + file.getFileSystem(job).openFile(file); + // the start and end of the split may be used to build + // an input strategy. + builder.opt("fs.option.openfile.split.start", start); + builder.opt("fs.option.openfile.split.end", end); + FutureIO.propagateOptions(builder, job, + "mapreduce.job.input.file.option", + "mapreduce.job.input.file.must"); + + fileIn = FutureIO.awaitFuture(builder.build()); + fileIn.seek(start) + /* Rest of the operation on the opened stream */ + } +``` + +### `FileContext.openFile` + +From `org.apache.hadoop.fs.AvroFSInput`; a file is opened with sequential input. +Because the file length has already been probed for, the length is passd down + +```java + public AvroFSInput(FileContext fc, Path p) throws IOException { + FileStatus status = fc.getFileStatus(p); + this.len = status.getLen(); + this.stream = awaitFuture(fc.openFile(p) + .opt("fs.option.openfile.read.policy", + "sequential") + .opt("fs.option.openfile.length", + Long.toString(status.getLen())) + .build()); + fc.open(p); + } +``` + +In this example, the length is passed down as a string (via `Long.toString()`) +rather than directly as a long. This is to ensure that the input format will +link against versions of $Hadoop which do not have the +`opt(String, long)` and `must(String, long)` builder parameters. Similarly, the +values are passed as optional, so that if unrecognized the application will +still succeed. + +### Example: reading a whole file + +This is from `org.apache.hadoop.util.JsonSerialization`. + +Its `load(FileSystem, Path, FileStatus)` method +* declares the whole file is to be read end to end. +* passes down the file status + +```java +public T load(FileSystem fs, + Path path, + status) + throws IOException { + + try (FSDataInputStream dataInputStream = + awaitFuture(fs.openFile(path) + .opt("fs.option.openfile.read.policy", "whole-file") + .withFileStatus(status) + .build())) { + return fromJsonStream(dataInputStream); + } catch (JsonProcessingException e) { + throw new PathIOException(path.toString(), + "Failed to read JSON file " + e, e); + } +} +``` + +*Note:* : in Hadoop 3.3.2 and earlier, the `withFileStatus(status)` call +required a non-null parameter; this has since been relaxed. +For maximum compatibility across versions, only invoke the method +when the file status is known to be non-null. \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/index.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/index.md index a4aa136033a0c..e18f4c3bf4ab2 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/index.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/index.md @@ -41,3 +41,4 @@ HDFS as these are commonly expected by Hadoop client applications. 2. [Extending the specification and its tests](extending.html) 1. [Uploading a file using Multiple Parts](multipartuploader.html) 1. [IOStatistics](iostatistics.html) +1. [openFile()](openfile.html). diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/introduction.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/introduction.md index 37191a5b2a69a..903d2bb90ff56 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/introduction.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/introduction.md @@ -343,7 +343,7 @@ stores pretend that they are a FileSystem, a FileSystem with the same features and operations as HDFS. This is —ultimately—a pretence: they have different characteristics and occasionally the illusion fails. -1. **Consistency**. Object stores are generally *Eventually Consistent*: it +1. **Consistency**. Object may be *Eventually Consistent*: it can take time for changes to objects —creation, deletion and updates— to become visible to all callers. Indeed, there is no guarantee a change is immediately visible to the client which just made the change. As an example, @@ -447,10 +447,6 @@ Object stores have an even vaguer view of time, which can be summarized as * The timestamp is likely to be in UTC or the TZ of the object store. If the client is in a different timezone, the timestamp of objects may be ahead or behind that of the client. - * Object stores with cached metadata databases (for example: AWS S3 with - an in-memory or a DynamoDB metadata store) may have timestamps generated - from the local system clock, rather than that of the service. - This is an optimization to avoid round-trip calls to the object stores. + A file's modification time is often the same as its creation time. + The `FileSystem.setTimes()` operation to set file timestamps *may* be ignored. * `FileSystem.chmod()` may update modification times (example: Azure `wasb://`). diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/openfile.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/openfile.md new file mode 100644 index 0000000000000..afb3245c5105f --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/openfile.md @@ -0,0 +1,122 @@ + + +# `FileSystem.openFile()`/`FileContext.openFile()` + +This is a method provided by both FileSystem and FileContext for +advanced file opening options and, where implemented, +an asynchrounous/lazy opening of a file. + +Creates a builder to open a file, supporting options +both standard and filesystem specific. The return +value of the `build()` call is a `Future`, +which must be waited on. The file opening may be +asynchronous, and it may actually be postponed (including +permission/existence checks) until reads are actually +performed. + +This API call was added to `FileSystem` and `FileContext` in +Hadoop 3.3.0; it was tuned in Hadoop 3.3.1 as follows. + +* Added `opt(key, long)` and `must(key, long)`. +* Declared that `withFileStatus(null)` is allowed. +* Declared that `withFileStatus(status)` only checks + the filename of the path, not the full path. + This is needed to support passthrough/mounted filesystems. +* Added standard option keys. + +### `FutureDataInputStreamBuilder openFile(Path path)` + +Creates a [`FutureDataInputStreamBuilder`](fsdatainputstreambuilder.html) +to construct a operation to open the file at `path` for reading. + +When `build()` is invoked on the returned `FutureDataInputStreamBuilder` instance, +the builder parameters are verified and +`FileSystem.openFileWithOptions(Path, OpenFileParameters)` or +`AbstractFileSystem.openFileWithOptions(Path, OpenFileParameters)` invoked. + +These protected methods returns a `CompletableFuture` +which, when its `get()` method is called, either returns an input +stream of the contents of opened file, or raises an exception. + +The base implementation of the `FileSystem.openFileWithOptions(PathHandle, OpenFileParameters)` +ultimately invokes `FileSystem.open(Path, int)`. + +Thus the chain `FileSystem.openFile(path).build().get()` has the same preconditions +and postconditions as `FileSystem.open(Path p, int bufferSize)` + +However, there is one difference which implementations are free to +take advantage of: + +The returned stream MAY implement a lazy open where file non-existence or +access permission failures may not surface until the first `read()` of the +actual data. + +This saves network IO on object stores. + +The `openFile()` operation MAY check the state of the filesystem during its +invocation, but as the state of the filesystem may change between this call and +the actual `build()` and `get()` operations, this file-specific +preconditions (file exists, file is readable, etc) MUST NOT be checked here. + +FileSystem implementations which do not implement `open(Path, int)` +MAY postpone raising an `UnsupportedOperationException` until either the +`FutureDataInputStreamBuilder.build()` or the subsequent `get()` call, +else they MAY fail fast in the `openFile()` call. + +Consult [`FutureDataInputStreamBuilder`](fsdatainputstreambuilder.html) for details +on how to use the builder, and for standard options which may be passed in. + +### `FutureDataInputStreamBuilder openFile(PathHandle)` + +Creates a [`FutureDataInputStreamBuilder`](fsdatainputstreambuilder.html) +to construct a operation to open the file identified by the given `PathHandle` for reading. + +If implemented by a filesystem, the semantics of [`openFile(Path)`](#openfile_path_) +Thus the chain `openFile(pathhandle).build().get()` has the same preconditions and postconditions +as `open(Pathhandle, int)` + +FileSystem implementations which do not implement `open(PathHandle handle, int bufferSize)` +MAY postpone raising an `UnsupportedOperationException` until either the +`FutureDataInputStreamBuilder.build()` or the subsequent `get()` call, else they MAY fail fast in +the `openFile(PathHandle)` call. + +The base implementation raises this exception in the `build()` operation; other implementations +SHOULD copy this. + +### Implementors notes + +The base implementation of `openFileWithOptions()` actually executes +the `open(path)` operation synchronously, yet still returns the result +or any failures in the `CompletableFuture<>`, so as to provide a consistent +lifecycle across all filesystems. + +Any filesystem client where the time to open a file may be significant SHOULD +execute it asynchronously by submitting the operation in some executor/thread +pool. This is particularly recommended for object stores and other filesystems +likely to be accessed over long-haul connections. + +Arbitrary filesystem-specific options MAY be supported; these MUST +be prefixed with either the filesystem schema, e.g. `hdfs.` +or in the `fs.SCHEMA` format as normal configuration settings `fs.hdfs`. The +latter style allows the same configuration option to be used for both +filesystem configuration and file-specific configuration. + +It SHOULD be possible to always open a file without specifying any options, +so as to present a consistent model to users. However, an implementation MAY +opt to require one or more mandatory options to be set. + +The returned stream may perform "lazy" evaluation of file access. This is +relevant for object stores where the probes for existence are expensive, and, +even with an asynchronous open, may be considered needless. diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/outputstream.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/outputstream.md index 8d0d4c4354f0b..1498d8db2e23b 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/outputstream.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/outputstream.md @@ -42,7 +42,7 @@ or `FSDataOutputStreamBuilder.build()`. These all return instances of `FSDataOutputStream`, through which data can be written through various `write()` methods. After a stream's `close()` method is called, all data written to the -stream MUST BE persisted to the fileysystem and visible to oll other +stream MUST BE persisted to the filesystem and visible to oll other clients attempting to read data from that path via `FileSystem.open()`. As well as operations to write the data, Hadoop's `OutputStream` implementations @@ -126,7 +126,7 @@ consistent with the data at the filesystem. The output stream returned from a call of `FileSystem.append(path, buffersize, progress)` within a filesystem `FS`, -can be modelled as a stream whose `buffer` is intialized to that of +can be modelled as a stream whose `buffer` is initialized to that of the original file: ```python @@ -184,7 +184,7 @@ This document covers the requirements of such implementations. HDFS's `FileSystem` implementation, `DistributedFileSystem`, returns an instance of `HdfsDataOutputStream`. This implementation has at least two behaviors -which are not explicitly declared by the base Java implmentation +which are not explicitly declared by the base Java implementation 1. Writes are synchronized: more than one thread can write to the same output stream. This is a use pattern which HBase relies on. @@ -648,7 +648,7 @@ in production. 1. `OutputStream.write()` MAY persist the data, synchronously or asynchronously 1. `OutputStream.flush()` flushes data to the destination. There are no strict persistence requirements. -1. `Syncable.hflush()` synchronously sends all outstaning data to the destination +1. `Syncable.hflush()` synchronously sends all outstanding data to the destination filesystem. After returning to the caller, the data MUST be visible to other readers, it MAY be durable. That is: it does not have to be persisted, merely guaranteed to be consistently visible to all clients attempting to open a new stream reading @@ -678,7 +678,7 @@ public void hflush() throws IOException { ``` This is perfectly acceptable as an implementation: the semantics of `hflush()` -are satisifed. +are satisfied. What is not acceptable is downgrading `hsync()` to `hflush()`, as the durability guarantee is no longer met. @@ -863,7 +863,7 @@ local data as can be written to full checksummed blocks of data. That is, the hsync/hflush operations are not guaranteed to write all the pending data until the file is finally closed. -For this reason, the local fileystem accessed via `file://` URLs +For this reason, the local filesystem accessed via `file://` URLs does not support `Syncable` unless `setWriteChecksum(false)` was called on that FileSystem instance so as to disable checksum creation. After which, obviously, checksums are not generated for any file. diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.2/CHANGELOG.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.2/CHANGELOG.3.2.2.md similarity index 100% rename from hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.2/CHANGELOG.md rename to hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.2/CHANGELOG.3.2.2.md diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.2/RELEASENOTES.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.2/RELEASENOTES.3.2.2.md similarity index 100% rename from hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.2/RELEASENOTES.md rename to hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.2/RELEASENOTES.3.2.2.md diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.3/CHANGELOG.3.2.3.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.3/CHANGELOG.3.2.3.md new file mode 100644 index 0000000000000..34928bf54e50d --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.3/CHANGELOG.3.2.3.md @@ -0,0 +1,386 @@ + + +# Apache Hadoop Changelog + +## Release 3.2.3 - 2022-03-02 + + + +### NEW FEATURES: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HADOOP-15691](https://issues.apache.org/jira/browse/HADOOP-15691) | Add PathCapabilities to FS and FC to complement StreamCapabilities | Major | . | Steve Loughran | Steve Loughran | +| [HDFS-15711](https://issues.apache.org/jira/browse/HDFS-15711) | Add Metrics to HttpFS Server | Major | httpfs | Ahmed Hussein | Ahmed Hussein | +| [HDFS-15759](https://issues.apache.org/jira/browse/HDFS-15759) | EC: Verify EC reconstruction correctness on DataNode | Major | datanode, ec, erasure-coding | Toshihiko Uchida | Toshihiko Uchida | +| [HDFS-16337](https://issues.apache.org/jira/browse/HDFS-16337) | Show start time of Datanode on Web | Minor | . | tomscut | tomscut | + + +### IMPROVEMENTS: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HADOOP-16052](https://issues.apache.org/jira/browse/HADOOP-16052) | Remove Subversion and Forrest from Dockerfile | Minor | build | Akira Ajisaka | Xieming Li | +| [YARN-9783](https://issues.apache.org/jira/browse/YARN-9783) | Remove low-level zookeeper test to be able to build Hadoop against zookeeper 3.5.5 | Major | test | Mate Szalay-Beko | Mate Szalay-Beko | +| [HADOOP-16717](https://issues.apache.org/jira/browse/HADOOP-16717) | Remove GenericsUtil isLog4jLogger dependency on Log4jLoggerAdapter | Major | . | David Mollitor | Xieming Li | +| [YARN-10036](https://issues.apache.org/jira/browse/YARN-10036) | Install yarnpkg and upgrade nodejs in Dockerfile | Major | buid, yarn-ui-v2 | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-16811](https://issues.apache.org/jira/browse/HADOOP-16811) | Use JUnit TemporaryFolder Rule in TestFileUtils | Minor | common, test | David Mollitor | David Mollitor | +| [HDFS-15075](https://issues.apache.org/jira/browse/HDFS-15075) | Remove process command timing from BPServiceActor | Major | . | Íñigo Goiri | Xiaoqiao He | +| [HADOOP-16054](https://issues.apache.org/jira/browse/HADOOP-16054) | Update Dockerfile to use Bionic | Major | build, test | Akira Ajisaka | Akira Ajisaka | +| [HDFS-15574](https://issues.apache.org/jira/browse/HDFS-15574) | Remove unnecessary sort of block list in DirectoryScanner | Major | . | Stephen O'Donnell | Stephen O'Donnell | +| [HDFS-15583](https://issues.apache.org/jira/browse/HDFS-15583) | Backport DirectoryScanner improvements HDFS-14476, HDFS-14751 and HDFS-15048 to branch 3.2 and 3.1 | Major | datanode | Stephen O'Donnell | Stephen O'Donnell | +| [HDFS-15567](https://issues.apache.org/jira/browse/HDFS-15567) | [SBN Read] HDFS should expose msync() API to allow downstream applications call it explicitly. | Major | ha, hdfs-client | Konstantin Shvachko | Konstantin Shvachko | +| [HDFS-15633](https://issues.apache.org/jira/browse/HDFS-15633) | Avoid redundant RPC calls for getDiskStatus | Major | dfsclient | Ayush Saxena | Ayush Saxena | +| [YARN-10450](https://issues.apache.org/jira/browse/YARN-10450) | Add cpu and memory utilization per node and cluster-wide metrics | Minor | yarn | Jim Brennan | Jim Brennan | +| [HDFS-15652](https://issues.apache.org/jira/browse/HDFS-15652) | Make block size from NNThroughputBenchmark configurable | Minor | benchmarks | Hui Fei | Hui Fei | +| [YARN-10475](https://issues.apache.org/jira/browse/YARN-10475) | Scale RM-NM heartbeat interval based on node utilization | Minor | yarn | Jim Brennan | Jim Brennan | +| [HDFS-15665](https://issues.apache.org/jira/browse/HDFS-15665) | Balancer logging improvement | Major | balancer & mover | Konstantin Shvachko | Konstantin Shvachko | +| [HADOOP-17342](https://issues.apache.org/jira/browse/HADOOP-17342) | Creating a token identifier should not do kerberos name resolution | Major | common | Jim Brennan | Jim Brennan | +| [YARN-10479](https://issues.apache.org/jira/browse/YARN-10479) | RMProxy should retry on SocketTimeout Exceptions | Major | yarn | Jim Brennan | Jim Brennan | +| [HDFS-15623](https://issues.apache.org/jira/browse/HDFS-15623) | Respect configured values of rpc.engine | Major | hdfs | Hector Sandoval Chaverri | Hector Sandoval Chaverri | +| [HDFS-14395](https://issues.apache.org/jira/browse/HDFS-14395) | Remove WARN Logging From Interrupts in DataStreamer | Minor | hdfs-client | David Mollitor | David Mollitor | +| [HADOOP-17367](https://issues.apache.org/jira/browse/HADOOP-17367) | Add InetAddress api to ProxyUsers.authorize | Major | performance, security | Ahmed Hussein | Ahmed Hussein | +| [HDFS-15694](https://issues.apache.org/jira/browse/HDFS-15694) | Avoid calling UpdateHeartBeatState inside DataNodeDescriptor | Major | . | Ahmed Hussein | Ahmed Hussein | +| [HDFS-15703](https://issues.apache.org/jira/browse/HDFS-15703) | Don't generate edits for set operations that are no-op | Major | namenode | Ahmed Hussein | Ahmed Hussein | +| [HADOOP-17392](https://issues.apache.org/jira/browse/HADOOP-17392) | Remote exception messages should not include the exception class | Major | ipc | Ahmed Hussein | Ahmed Hussein | +| [HDFS-15706](https://issues.apache.org/jira/browse/HDFS-15706) | HttpFS: Log more information on request failures | Major | httpfs | Ahmed Hussein | Ahmed Hussein | +| [HADOOP-17389](https://issues.apache.org/jira/browse/HADOOP-17389) | KMS should log full UGI principal | Major | . | Ahmed Hussein | Ahmed Hussein | +| [HDFS-15720](https://issues.apache.org/jira/browse/HDFS-15720) | namenode audit async logger should add some log4j config | Minor | hdfs | Max Xie | | +| [HDFS-15704](https://issues.apache.org/jira/browse/HDFS-15704) | Mitigate lease monitor's rapid infinite loop | Major | namenode | Ahmed Hussein | Ahmed Hussein | +| [HDFS-15751](https://issues.apache.org/jira/browse/HDFS-15751) | Add documentation for msync() API to filesystem.md | Major | documentation | Konstantin Shvachko | Konstantin Shvachko | +| [YARN-10538](https://issues.apache.org/jira/browse/YARN-10538) | Add recommissioning nodes to the list of updated nodes returned to the AM | Major | . | Srinivas S T | Srinivas S T | +| [YARN-4589](https://issues.apache.org/jira/browse/YARN-4589) | Diagnostics for localization timeouts is lacking | Major | . | Chang Li | Chang Li | +| [YARN-10562](https://issues.apache.org/jira/browse/YARN-10562) | Follow up changes for YARN-9833 | Major | yarn | Jim Brennan | Jim Brennan | +| [HDFS-15783](https://issues.apache.org/jira/browse/HDFS-15783) | Speed up BlockPlacementPolicyRackFaultTolerant#verifyBlockPlacement | Major | block placement | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-17478](https://issues.apache.org/jira/browse/HADOOP-17478) | Improve the description of hadoop.http.authentication.signature.secret.file | Minor | documentation | Akira Ajisaka | Akira Ajisaka | +| [HDFS-15789](https://issues.apache.org/jira/browse/HDFS-15789) | Lease renewal does not require namesystem lock | Major | hdfs | Jim Brennan | Jim Brennan | +| [HADOOP-17501](https://issues.apache.org/jira/browse/HADOOP-17501) | Fix logging typo in ShutdownHookManager | Major | common | Konstantin Shvachko | Fengnan Li | +| [HADOOP-17354](https://issues.apache.org/jira/browse/HADOOP-17354) | Move Jenkinsfile outside of the root directory | Major | build | Akira Ajisaka | Akira Ajisaka | +| [HDFS-15799](https://issues.apache.org/jira/browse/HDFS-15799) | Make DisallowedDatanodeException terse | Minor | hdfs | Richard | Richard | +| [HDFS-15813](https://issues.apache.org/jira/browse/HDFS-15813) | DataStreamer: keep sending heartbeat packets while streaming | Major | hdfs | Jim Brennan | Jim Brennan | +| [MAPREDUCE-7319](https://issues.apache.org/jira/browse/MAPREDUCE-7319) | Log list of mappers at trace level in ShuffleHandler audit log | Minor | yarn | Jim Brennan | Jim Brennan | +| [HDFS-15821](https://issues.apache.org/jira/browse/HDFS-15821) | Add metrics for in-service datanodes | Minor | . | Zehao Chen | Zehao Chen | +| [YARN-10626](https://issues.apache.org/jira/browse/YARN-10626) | Log resource allocation in NM log at container start time | Major | . | Eric Badger | Eric Badger | +| [HDFS-15815](https://issues.apache.org/jira/browse/HDFS-15815) | if required storageType are unavailable, log the failed reason during choosing Datanode | Minor | block placement | Yang Yun | Yang Yun | +| [HDFS-15826](https://issues.apache.org/jira/browse/HDFS-15826) | Solve the problem of incorrect progress of delegation tokens when loading FsImage | Major | . | JiangHua Zhu | JiangHua Zhu | +| [HDFS-15734](https://issues.apache.org/jira/browse/HDFS-15734) | [READ] DirectoryScanner#scan need not check StorageType.PROVIDED | Minor | datanode | Yuxuan Wang | Yuxuan Wang | +| [HADOOP-17538](https://issues.apache.org/jira/browse/HADOOP-17538) | Add kms-default.xml and httpfs-default.xml to site index | Minor | documentation | Masatake Iwasaki | Masatake Iwasaki | +| [YARN-10613](https://issues.apache.org/jira/browse/YARN-10613) | Config to allow Intra- and Inter-queue preemption to enable/disable conservativeDRF | Minor | capacity scheduler, scheduler preemption | Eric Payne | Eric Payne | +| [YARN-10653](https://issues.apache.org/jira/browse/YARN-10653) | Fixed the findbugs issues introduced by YARN-10647. | Major | . | Qi Zhu | Qi Zhu | +| [MAPREDUCE-7324](https://issues.apache.org/jira/browse/MAPREDUCE-7324) | ClientHSSecurityInfo class is in wrong META-INF file | Major | . | Eric Badger | Eric Badger | +| [HADOOP-17546](https://issues.apache.org/jira/browse/HADOOP-17546) | Update Description of hadoop-http-auth-signature-secret in HttpAuthentication.md | Minor | . | Ravuri Sushma sree | Ravuri Sushma sree | +| [YARN-10664](https://issues.apache.org/jira/browse/YARN-10664) | Allow parameter expansion in NM\_ADMIN\_USER\_ENV | Major | yarn | Jim Brennan | Jim Brennan | +| [HADOOP-17570](https://issues.apache.org/jira/browse/HADOOP-17570) | Apply YETUS-1102 to re-enable GitHub comments | Major | build | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-17594](https://issues.apache.org/jira/browse/HADOOP-17594) | DistCp: Expose the JobId for applications executing through run method | Major | . | Ayush Saxena | Ayush Saxena | +| [HDFS-15911](https://issues.apache.org/jira/browse/HDFS-15911) | Provide blocks moved count in Balancer iteration result | Major | balancer & mover | Viraj Jasani | Viraj Jasani | +| [HDFS-15919](https://issues.apache.org/jira/browse/HDFS-15919) | BlockPoolManager should log stack trace if unable to get Namenode addresses | Major | datanode | Stephen O'Donnell | Stephen O'Donnell | +| [HADOOP-16870](https://issues.apache.org/jira/browse/HADOOP-16870) | Use spotbugs-maven-plugin instead of findbugs-maven-plugin | Major | build | Akira Ajisaka | Akira Ajisaka | +| [HDFS-15932](https://issues.apache.org/jira/browse/HDFS-15932) | Improve the balancer error message when process exits abnormally. | Major | . | Renukaprasad C | Renukaprasad C | +| [HDFS-15931](https://issues.apache.org/jira/browse/HDFS-15931) | Fix non-static inner classes for better memory management | Major | . | Viraj Jasani | Viraj Jasani | +| [HDFS-15942](https://issues.apache.org/jira/browse/HDFS-15942) | Increase Quota initialization threads | Major | namenode | Stephen O'Donnell | Stephen O'Donnell | +| [HDFS-15937](https://issues.apache.org/jira/browse/HDFS-15937) | Reduce memory used during datanode layout upgrade | Major | datanode | Stephen O'Donnell | Stephen O'Donnell | +| [HADOOP-17569](https://issues.apache.org/jira/browse/HADOOP-17569) | Building native code fails on Fedora 33 | Major | build, common | Kengo Seki | Masatake Iwasaki | +| [HADOOP-17633](https://issues.apache.org/jira/browse/HADOOP-17633) | Bump json-smart to 2.4.2 and nimbus-jose-jwt to 9.8 due to CVEs | Major | auth, build | helen huang | Viraj Jasani | +| [HADOOP-16822](https://issues.apache.org/jira/browse/HADOOP-16822) | Provide source artifacts for hadoop-client-api | Major | . | Karel Kolman | Karel Kolman | +| [HADOOP-17680](https://issues.apache.org/jira/browse/HADOOP-17680) | Allow ProtobufRpcEngine to be extensible | Major | common | Hector Sandoval Chaverri | Hector Sandoval Chaverri | +| [YARN-10123](https://issues.apache.org/jira/browse/YARN-10123) | Error message around yarn app -stop/start can be improved to highlight that an implementation at framework level is needed for the stop/start functionality to work | Minor | client, documentation | Siddharth Ahuja | Siddharth Ahuja | +| [HADOOP-17756](https://issues.apache.org/jira/browse/HADOOP-17756) | Increase precommit job timeout from 20 hours to 24 hours. | Major | build | Takanobu Asanuma | Takanobu Asanuma | +| [HDFS-16073](https://issues.apache.org/jira/browse/HDFS-16073) | Remove redundant RPC requests for getFileLinkInfo in ClientNamenodeProtocolTranslatorPB | Minor | . | lei w | lei w | +| [HDFS-16074](https://issues.apache.org/jira/browse/HDFS-16074) | Remove an expensive debug string concatenation | Major | . | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HDFS-15150](https://issues.apache.org/jira/browse/HDFS-15150) | Introduce read write lock to Datanode | Major | datanode | Stephen O'Donnell | Stephen O'Donnell | +| [YARN-10834](https://issues.apache.org/jira/browse/YARN-10834) | Intra-queue preemption: apps that don't use defined custom resource won't be preempted. | Major | . | Eric Payne | Eric Payne | +| [HADOOP-17749](https://issues.apache.org/jira/browse/HADOOP-17749) | Remove lock contention in SelectorPool of SocketIOWithTimeout | Major | common | Xuesen Liang | Xuesen Liang | +| [HADOOP-17775](https://issues.apache.org/jira/browse/HADOOP-17775) | Remove JavaScript package from Docker environment | Major | build | Masatake Iwasaki | Masatake Iwasaki | +| [HADOOP-17794](https://issues.apache.org/jira/browse/HADOOP-17794) | Add a sample configuration to use ZKDelegationTokenSecretManager in Hadoop KMS | Major | documentation, kms, security | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-12665](https://issues.apache.org/jira/browse/HADOOP-12665) | Document hadoop.security.token.service.use\_ip | Major | documentation | Arpit Agarwal | Akira Ajisaka | +| [YARN-10456](https://issues.apache.org/jira/browse/YARN-10456) | RM PartitionQueueMetrics records are named QueueMetrics in Simon metrics registry | Major | resourcemanager | Eric Payne | Eric Payne | +| [HDFS-15650](https://issues.apache.org/jira/browse/HDFS-15650) | Make the socket timeout for computing checksum of striped blocks configurable | Minor | datanode, ec, erasure-coding | Yushi Hayasaka | Yushi Hayasaka | +| [YARN-10858](https://issues.apache.org/jira/browse/YARN-10858) | [UI2] YARN-10826 breaks Queue view | Major | yarn-ui-v2 | Andras Gyori | Masatake Iwasaki | +| [YARN-10860](https://issues.apache.org/jira/browse/YARN-10860) | Make max container per heartbeat configs refreshable | Major | . | Eric Badger | Eric Badger | +| [HADOOP-17813](https://issues.apache.org/jira/browse/HADOOP-17813) | Checkstyle - Allow line length: 100 | Major | . | Akira Ajisaka | Viraj Jasani | +| [HADOOP-17819](https://issues.apache.org/jira/browse/HADOOP-17819) | Add extensions to ProtobufRpcEngine RequestHeaderProto | Major | common | Hector Sandoval Chaverri | Hector Sandoval Chaverri | +| [HDFS-16153](https://issues.apache.org/jira/browse/HDFS-16153) | Avoid evaluation of LOG.debug statement in QuorumJournalManager | Trivial | . | wangzhaohui | wangzhaohui | +| [HDFS-16154](https://issues.apache.org/jira/browse/HDFS-16154) | TestMiniJournalCluster failing intermittently because of not reseting UserGroupInformation completely | Minor | . | wangzhaohui | wangzhaohui | +| [HADOOP-17849](https://issues.apache.org/jira/browse/HADOOP-17849) | Exclude spotbugs-annotations from transitive dependencies on branch-3.2 | Major | . | Masatake Iwasaki | Masatake Iwasaki | +| [HDFS-16173](https://issues.apache.org/jira/browse/HDFS-16173) | Improve CopyCommands#Put#executor queue configurability | Major | fs | JiangHua Zhu | JiangHua Zhu | +| [HDFS-15160](https://issues.apache.org/jira/browse/HDFS-15160) | ReplicaMap, Disk Balancer, Directory Scanner and various FsDatasetImpl methods should use datanode readlock | Major | datanode | Stephen O'Donnell | Stephen O'Donnell | +| [HDFS-14997](https://issues.apache.org/jira/browse/HDFS-14997) | BPServiceActor processes commands from NameNode asynchronously | Major | datanode | Xiaoqiao He | Xiaoqiao He | +| [HDFS-16241](https://issues.apache.org/jira/browse/HDFS-16241) | Standby close reconstruction thread | Major | . | zhanghuazong | zhanghuazong | +| [HDFS-16286](https://issues.apache.org/jira/browse/HDFS-16286) | Debug tool to verify the correctness of erasure coding on file | Minor | erasure-coding, tools | daimin | daimin | +| [HADOOP-17998](https://issues.apache.org/jira/browse/HADOOP-17998) | Allow get command to run with multi threads. | Major | fs | Chengwei Wang | Chengwei Wang | +| [HADOOP-18023](https://issues.apache.org/jira/browse/HADOOP-18023) | Allow cp command to run with multi threads. | Major | fs | Chengwei Wang | Chengwei Wang | +| [HADOOP-17643](https://issues.apache.org/jira/browse/HADOOP-17643) | WASB : Make metadata checks case insensitive | Major | . | Anoop Sam John | Anoop Sam John | +| [HDFS-16386](https://issues.apache.org/jira/browse/HDFS-16386) | Reduce DataNode load when FsDatasetAsyncDiskService is working | Major | datanode | JiangHua Zhu | JiangHua Zhu | +| [HDFS-16430](https://issues.apache.org/jira/browse/HDFS-16430) | Validate maximum blocks in EC group when adding an EC policy | Minor | ec, erasure-coding | daimin | daimin | +| [HDFS-16403](https://issues.apache.org/jira/browse/HDFS-16403) | Improve FUSE IO performance by supporting FUSE parameter max\_background | Minor | fuse-dfs | daimin | daimin | +| [HADOOP-18093](https://issues.apache.org/jira/browse/HADOOP-18093) | Better exception handling for testFileStatusOnMountLink() in ViewFsBaseTest.java | Trivial | . | Xing Lin | Xing Lin | +| [HADOOP-18155](https://issues.apache.org/jira/browse/HADOOP-18155) | Refactor tests in TestFileUtil | Trivial | common | Gautham Banasandra | Gautham Banasandra | + + +### BUG FIXES: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HADOOP-15939](https://issues.apache.org/jira/browse/HADOOP-15939) | Filter overlapping objenesis class in hadoop-client-minicluster | Minor | build | Xiaoyu Yao | Xiaoyu Yao | +| [YARN-8936](https://issues.apache.org/jira/browse/YARN-8936) | Bump up Atsv2 hbase versions | Major | . | Rohith Sharma K S | Vrushali C | +| [HDFS-14189](https://issues.apache.org/jira/browse/HDFS-14189) | Fix intermittent failure of TestNameNodeMetrics | Major | . | Ayush Saxena | Ayush Saxena | +| [YARN-9246](https://issues.apache.org/jira/browse/YARN-9246) | NPE when executing a command yarn node -status or -states without additional arguments | Minor | client | Masahiro Tanaka | Masahiro Tanaka | +| [YARN-7266](https://issues.apache.org/jira/browse/YARN-7266) | Timeline Server event handler threads locked | Major | ATSv2, timelineserver | Venkata Puneet Ravuri | Prabhu Joseph | +| [YARN-9990](https://issues.apache.org/jira/browse/YARN-9990) | Testcase fails with "Insufficient configured threads: required=16 \< max=10" | Major | . | Prabhu Joseph | Prabhu Joseph | +| [YARN-10020](https://issues.apache.org/jira/browse/YARN-10020) | Fix build instruction of hadoop-yarn-ui | Minor | yarn-ui-v2 | Masatake Iwasaki | Masatake Iwasaki | +| [YARN-10037](https://issues.apache.org/jira/browse/YARN-10037) | Upgrade build tools for YARN Web UI v2 | Major | build, security, yarn-ui-v2 | Akira Ajisaka | Masatake Iwasaki | +| [HDFS-15187](https://issues.apache.org/jira/browse/HDFS-15187) | CORRUPT replica mismatch between namenodes after failover | Critical | . | Ayush Saxena | Ayush Saxena | +| [HDFS-15200](https://issues.apache.org/jira/browse/HDFS-15200) | Delete Corrupt Replica Immediately Irrespective of Replicas On Stale Storage | Critical | . | Ayush Saxena | Ayush Saxena | +| [HDFS-15113](https://issues.apache.org/jira/browse/HDFS-15113) | Missing IBR when NameNode restart if open processCommand async feature | Blocker | datanode | Xiaoqiao He | Xiaoqiao He | +| [HDFS-15210](https://issues.apache.org/jira/browse/HDFS-15210) | EC : File write hanged when DN is shutdown by admin command. | Major | ec | Surendra Singh Lilhore | Surendra Singh Lilhore | +| [HADOOP-16768](https://issues.apache.org/jira/browse/HADOOP-16768) | SnappyCompressor test cases wrongly assume that the compressed data is always smaller than the input data | Major | io, test | zhao bo | Akira Ajisaka | +| [HDFS-11041](https://issues.apache.org/jira/browse/HDFS-11041) | Unable to unregister FsDatasetState MBean if DataNode is shutdown twice | Trivial | datanode | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HADOOP-17068](https://issues.apache.org/jira/browse/HADOOP-17068) | client fails forever when namenode ipaddr changed | Major | hdfs-client | Sean Chow | Sean Chow | +| [HDFS-15378](https://issues.apache.org/jira/browse/HDFS-15378) | TestReconstructStripedFile#testErasureCodingWorkerXmitsWeight is failing on trunk | Major | . | Hemanth Boyina | Hemanth Boyina | +| [YARN-10331](https://issues.apache.org/jira/browse/YARN-10331) | Upgrade node.js to 10.21.0 | Critical | build, yarn-ui-v2 | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-17119](https://issues.apache.org/jira/browse/HADOOP-17119) | Jetty upgrade to 9.4.x causes MR app fail with IOException | Major | . | Bilwa S T | Bilwa S T | +| [HADOOP-17138](https://issues.apache.org/jira/browse/HADOOP-17138) | Fix spotbugs warnings surfaced after upgrade to 4.0.6 | Minor | . | Masatake Iwasaki | Masatake Iwasaki | +| [HDFS-15439](https://issues.apache.org/jira/browse/HDFS-15439) | Setting dfs.mover.retry.max.attempts to negative value will retry forever. | Major | balancer & mover | AMC-team | AMC-team | +| [YARN-10430](https://issues.apache.org/jira/browse/YARN-10430) | Log improvements in NodeStatusUpdaterImpl | Minor | nodemanager | Bilwa S T | Bilwa S T | +| [HDFS-15438](https://issues.apache.org/jira/browse/HDFS-15438) | Setting dfs.disk.balancer.max.disk.errors = 0 will fail the block copy | Major | balancer & mover | AMC-team | AMC-team | +| [YARN-10438](https://issues.apache.org/jira/browse/YARN-10438) | Handle null containerId in ClientRMService#getContainerReport() | Major | resourcemanager | Raghvendra Singh | Shubham Gupta | +| [HDFS-15628](https://issues.apache.org/jira/browse/HDFS-15628) | HttpFS server throws NPE if a file is a symlink | Major | fs, httpfs | Ahmed Hussein | Ahmed Hussein | +| [HDFS-15627](https://issues.apache.org/jira/browse/HDFS-15627) | Audit log deletes before collecting blocks | Major | logging, namenode | Ahmed Hussein | Ahmed Hussein | +| [HADOOP-17309](https://issues.apache.org/jira/browse/HADOOP-17309) | Javadoc warnings and errors are ignored in the precommit jobs | Major | build, documentation | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-17310](https://issues.apache.org/jira/browse/HADOOP-17310) | Touch command with -c option is broken | Major | . | Ayush Saxena | Ayush Saxena | +| [HDFS-15639](https://issues.apache.org/jira/browse/HDFS-15639) | [JDK 11] Fix Javadoc errors in hadoop-hdfs-client | Major | . | Takanobu Asanuma | Takanobu Asanuma | +| [HDFS-15622](https://issues.apache.org/jira/browse/HDFS-15622) | Deleted blocks linger in the replications queue | Major | hdfs | Ahmed Hussein | Ahmed Hussein | +| [HDFS-15641](https://issues.apache.org/jira/browse/HDFS-15641) | DataNode could meet deadlock if invoke refreshNameNode | Critical | . | Hongbing Wang | Hongbing Wang | +| [MAPREDUCE-7302](https://issues.apache.org/jira/browse/MAPREDUCE-7302) | Upgrading to JUnit 4.13 causes testcase TestFetcher.testCorruptedIFile() to fail | Major | test | Peter Bacsko | Peter Bacsko | +| [HDFS-15644](https://issues.apache.org/jira/browse/HDFS-15644) | Failed volumes can cause DNs to stop block reporting | Major | block placement, datanode | Ahmed Hussein | Ahmed Hussein | +| [HADOOP-17236](https://issues.apache.org/jira/browse/HADOOP-17236) | Bump up snakeyaml to 1.26 to mitigate CVE-2017-18640 | Major | . | Brahma Reddy Battula | Brahma Reddy Battula | +| [YARN-10467](https://issues.apache.org/jira/browse/YARN-10467) | ContainerIdPBImpl objects can be leaked in RMNodeImpl.completedContainers | Major | resourcemanager | Haibo Chen | Haibo Chen | +| [HADOOP-17329](https://issues.apache.org/jira/browse/HADOOP-17329) | mvn site commands fails due to MetricsSystemImpl changes | Major | . | Xiaoqiao He | Xiaoqiao He | +| [HDFS-15651](https://issues.apache.org/jira/browse/HDFS-15651) | Client could not obtain block when DN CommandProcessingThread exit | Major | . | Yiqun Lin | Mingxiang Li | +| [HADOOP-17340](https://issues.apache.org/jira/browse/HADOOP-17340) | TestLdapGroupsMapping failing -string mismatch in exception validation | Major | test | Steve Loughran | Steve Loughran | +| [HADOOP-17352](https://issues.apache.org/jira/browse/HADOOP-17352) | Update PATCH\_NAMING\_RULE in the personality file | Minor | build | Akira Ajisaka | Akira Ajisaka | +| [HDFS-15485](https://issues.apache.org/jira/browse/HDFS-15485) | Fix outdated properties of JournalNode when performing rollback | Minor | . | Deegue | Deegue | +| [HADOOP-17358](https://issues.apache.org/jira/browse/HADOOP-17358) | Improve excessive reloading of Configurations | Major | conf | Ahmed Hussein | Ahmed Hussein | +| [HDFS-15538](https://issues.apache.org/jira/browse/HDFS-15538) | Fix the documentation for dfs.namenode.replication.max-streams in hdfs-default.xml | Major | . | Xieming Li | Xieming Li | +| [HADOOP-17362](https://issues.apache.org/jira/browse/HADOOP-17362) | Doing hadoop ls on Har file triggers too many RPC calls | Major | fs | Ahmed Hussein | Ahmed Hussein | +| [YARN-10485](https://issues.apache.org/jira/browse/YARN-10485) | TimelineConnector swallows InterruptedException | Major | . | Ahmed Hussein | Ahmed Hussein | +| [HADOOP-17360](https://issues.apache.org/jira/browse/HADOOP-17360) | Log the remote address for authentication success | Minor | ipc | Ahmed Hussein | Ahmed Hussein | +| [HADOOP-17346](https://issues.apache.org/jira/browse/HADOOP-17346) | Fair call queue is defeated by abusive service principals | Major | common, ipc | Ahmed Hussein | Ahmed Hussein | +| [YARN-10470](https://issues.apache.org/jira/browse/YARN-10470) | When building new web ui with root user, the bower install should support it. | Major | build, yarn-ui-v2 | Qi Zhu | Qi Zhu | +| [YARN-10498](https://issues.apache.org/jira/browse/YARN-10498) | Fix Yarn CapacityScheduler Markdown document | Trivial | documentation | zhaoshengjie | zhaoshengjie | +| [HDFS-15695](https://issues.apache.org/jira/browse/HDFS-15695) | NN should not let the balancer run in safemode | Major | namenode | Ahmed Hussein | Ahmed Hussein | +| [YARN-10511](https://issues.apache.org/jira/browse/YARN-10511) | Update yarn.nodemanager.env-whitelist value in docs | Minor | documentation | Andrea Scarpino | Andrea Scarpino | +| [HDFS-15707](https://issues.apache.org/jira/browse/HDFS-15707) | NNTop counts don't add up as expected | Major | hdfs, metrics, namenode | Ahmed Hussein | Ahmed Hussein | +| [HDFS-15709](https://issues.apache.org/jira/browse/HDFS-15709) | EC: Socket file descriptor leak in StripedBlockChecksumReconstructor | Major | datanode, ec, erasure-coding | Yushi Hayasaka | Yushi Hayasaka | +| [YARN-10491](https://issues.apache.org/jira/browse/YARN-10491) | Fix deprecation warnings in SLSWebApp.java | Minor | build | Akira Ajisaka | Ankit Kumar | +| [HADOOP-13571](https://issues.apache.org/jira/browse/HADOOP-13571) | ServerSocketUtil.getPort() should use loopback address, not 0.0.0.0 | Major | . | Eric Badger | Eric Badger | +| [HDFS-15725](https://issues.apache.org/jira/browse/HDFS-15725) | Lease Recovery never completes for a committed block which the DNs never finalize | Major | namenode | Stephen O'Donnell | Stephen O'Donnell | +| [HDFS-15170](https://issues.apache.org/jira/browse/HDFS-15170) | EC: Block gets marked as CORRUPT in case of failover and pipeline recovery | Critical | erasure-coding | Ayush Saxena | Ayush Saxena | +| [HDFS-15719](https://issues.apache.org/jira/browse/HDFS-15719) | [Hadoop 3] Both NameNodes can crash simultaneously due to the short JN socket timeout | Critical | . | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [YARN-10560](https://issues.apache.org/jira/browse/YARN-10560) | Upgrade node.js to 10.23.1 and yarn to 1.22.5 in Web UI v2 | Major | webapp, yarn-ui-v2 | Akira Ajisaka | Akira Ajisaka | +| [YARN-10528](https://issues.apache.org/jira/browse/YARN-10528) | maxAMShare should only be accepted for leaf queues, not parent queues | Major | . | Siddharth Ahuja | Siddharth Ahuja | +| [HADOOP-17438](https://issues.apache.org/jira/browse/HADOOP-17438) | Increase docker memory limit in Jenkins | Major | build, scripts, test, yetus | Ahmed Hussein | Ahmed Hussein | +| [MAPREDUCE-7310](https://issues.apache.org/jira/browse/MAPREDUCE-7310) | Clear the fileMap in JHEventHandlerForSigtermTest | Minor | test | Zhengxi Li | Zhengxi Li | +| [HADOOP-16947](https://issues.apache.org/jira/browse/HADOOP-16947) | Stale record should be remove when MutableRollingAverages generating aggregate data. | Major | . | Haibin Huang | Haibin Huang | +| [HDFS-15632](https://issues.apache.org/jira/browse/HDFS-15632) | AbstractContractDeleteTest should set recursive parameter to true for recursive test cases. | Major | . | Konstantin Shvachko | Anton Kutuzov | +| [HDFS-10498](https://issues.apache.org/jira/browse/HDFS-10498) | Intermittent test failure org.apache.hadoop.hdfs.server.namenode.snapshot.TestSnapshotFileLength.testSnapshotfileLength | Major | hdfs, snapshots | Hanisha Koneru | Jim Brennan | +| [HADOOP-17506](https://issues.apache.org/jira/browse/HADOOP-17506) | Fix typo in BUILDING.txt | Trivial | documentation | Gautham Banasandra | Gautham Banasandra | +| [HDFS-15795](https://issues.apache.org/jira/browse/HDFS-15795) | EC: Wrong checksum when reconstruction was failed by exception | Major | datanode, ec, erasure-coding | Yushi Hayasaka | Yushi Hayasaka | +| [HDFS-15779](https://issues.apache.org/jira/browse/HDFS-15779) | EC: fix NPE caused by StripedWriter.clearBuffers during reconstruct block | Major | . | Hongbing Wang | Hongbing Wang | +| [HDFS-15798](https://issues.apache.org/jira/browse/HDFS-15798) | EC: Reconstruct task failed, and It would be XmitsInProgress of DN has negative number | Major | . | Haiyang Hu | Haiyang Hu | +| [YARN-10428](https://issues.apache.org/jira/browse/YARN-10428) | Zombie applications in the YARN queue using FAIR + sizebasedweight | Critical | capacityscheduler | Guang Yang | Andras Gyori | +| [YARN-10607](https://issues.apache.org/jira/browse/YARN-10607) | User environment is unable to prepend PATH when mapreduce.admin.user.env also sets PATH | Major | . | Eric Badger | Eric Badger | +| [HADOOP-17516](https://issues.apache.org/jira/browse/HADOOP-17516) | Upgrade ant to 1.10.9 | Major | . | Akira Ajisaka | Akira Ajisaka | +| [YARN-10500](https://issues.apache.org/jira/browse/YARN-10500) | TestDelegationTokenRenewer fails intermittently | Major | test | Akira Ajisaka | Masatake Iwasaki | +| [HADOOP-17534](https://issues.apache.org/jira/browse/HADOOP-17534) | Upgrade Jackson databind to 2.10.5.1 | Major | build | Adam Roberts | Akira Ajisaka | +| [MAPREDUCE-7323](https://issues.apache.org/jira/browse/MAPREDUCE-7323) | Remove job\_history\_summary.py | Major | . | Akira Ajisaka | Akira Ajisaka | +| [YARN-10647](https://issues.apache.org/jira/browse/YARN-10647) | Fix TestRMNodeLabelsManager failed after YARN-10501. | Major | . | Qi Zhu | Qi Zhu | +| [HADOOP-17510](https://issues.apache.org/jira/browse/HADOOP-17510) | Hadoop prints sensitive Cookie information. | Major | . | Renukaprasad C | Renukaprasad C | +| [HDFS-15422](https://issues.apache.org/jira/browse/HDFS-15422) | Reported IBR is partially replaced with stored info when queuing. | Critical | namenode | Kihwal Lee | Stephen O'Donnell | +| [YARN-10651](https://issues.apache.org/jira/browse/YARN-10651) | CapacityScheduler crashed with NPE in AbstractYarnScheduler.updateNodeResource() | Major | . | Haibo Chen | Haibo Chen | +| [MAPREDUCE-7320](https://issues.apache.org/jira/browse/MAPREDUCE-7320) | ClusterMapReduceTestCase does not clean directories | Major | . | Ahmed Hussein | Ahmed Hussein | +| [HDFS-14013](https://issues.apache.org/jira/browse/HDFS-14013) | Skip any credentials stored in HDFS when starting ZKFC | Major | hdfs | Krzysztof Adamski | Stephen O'Donnell | +| [HDFS-15849](https://issues.apache.org/jira/browse/HDFS-15849) | ExpiredHeartbeats metric should be of Type.COUNTER | Major | metrics | Konstantin Shvachko | Qi Zhu | +| [YARN-10672](https://issues.apache.org/jira/browse/YARN-10672) | All testcases in TestReservations are flaky | Major | . | Szilard Nemeth | Szilard Nemeth | +| [HADOOP-17557](https://issues.apache.org/jira/browse/HADOOP-17557) | skip-dir option is not processed by Yetus | Major | build, precommit, yetus | Ahmed Hussein | Ahmed Hussein | +| [HDFS-15875](https://issues.apache.org/jira/browse/HDFS-15875) | Check whether file is being truncated before truncate | Major | . | Hui Fei | Hui Fei | +| [HADOOP-17582](https://issues.apache.org/jira/browse/HADOOP-17582) | Replace GitHub App Token with GitHub OAuth token | Major | build | Akira Ajisaka | Akira Ajisaka | +| [YARN-10687](https://issues.apache.org/jira/browse/YARN-10687) | Add option to disable/enable free disk space checking and percentage checking for full and not-full disks | Major | nodemanager | Qi Zhu | Qi Zhu | +| [HADOOP-17586](https://issues.apache.org/jira/browse/HADOOP-17586) | Upgrade org.codehaus.woodstox:stax2-api to 4.2.1 | Major | . | Ayush Saxena | Ayush Saxena | +| [HADOOP-17585](https://issues.apache.org/jira/browse/HADOOP-17585) | Correct timestamp format in the docs for the touch command | Major | . | Stephen O'Donnell | Stephen O'Donnell | +| [YARN-10588](https://issues.apache.org/jira/browse/YARN-10588) | Percentage of queue and cluster is zero in WebUI | Major | . | Bilwa S T | Bilwa S T | +| [MAPREDUCE-7322](https://issues.apache.org/jira/browse/MAPREDUCE-7322) | revisiting TestMRIntermediateDataEncryption | Major | job submission, security, test | Ahmed Hussein | Ahmed Hussein | +| [HADOOP-17592](https://issues.apache.org/jira/browse/HADOOP-17592) | Fix the wrong CIDR range example in Proxy User documentation | Minor | documentation | Kwangsun Noh | Kwangsun Noh | +| [YARN-10706](https://issues.apache.org/jira/browse/YARN-10706) | Upgrade com.github.eirslett:frontend-maven-plugin to 1.11.2 | Major | buid | Mingliang Liu | Mingliang Liu | +| [MAPREDUCE-7325](https://issues.apache.org/jira/browse/MAPREDUCE-7325) | Intermediate data encryption is broken in LocalJobRunner | Major | job submission, security | Ahmed Hussein | Ahmed Hussein | +| [YARN-10697](https://issues.apache.org/jira/browse/YARN-10697) | Resources are displayed in bytes in UI for schedulers other than capacity | Major | . | Bilwa S T | Bilwa S T | +| [HADOOP-17602](https://issues.apache.org/jira/browse/HADOOP-17602) | Upgrade JUnit to 4.13.1 | Major | build, security, test | Ahmed Hussein | Ahmed Hussein | +| [HDFS-15900](https://issues.apache.org/jira/browse/HDFS-15900) | RBF: empty blockpool id on dfsrouter caused by UNAVAILABLE NameNode | Major | rbf | Harunobu Daikoku | Harunobu Daikoku | +| [YARN-10501](https://issues.apache.org/jira/browse/YARN-10501) | Can't remove all node labels after add node label without nodemanager port | Critical | yarn | caozhiqiang | caozhiqiang | +| [YARN-10716](https://issues.apache.org/jira/browse/YARN-10716) | Fix typo in ContainerRuntime | Trivial | documentation | Wanqiang Ji | xishuhai | +| [HDFS-15950](https://issues.apache.org/jira/browse/HDFS-15950) | Remove unused hdfs.proto import | Major | hdfs-client | Gautham Banasandra | Gautham Banasandra | +| [HDFS-15949](https://issues.apache.org/jira/browse/HDFS-15949) | Fix integer overflow | Major | libhdfs++ | Gautham Banasandra | Gautham Banasandra | +| [HDFS-15948](https://issues.apache.org/jira/browse/HDFS-15948) | Fix test4tests for libhdfspp | Critical | build, libhdfs++ | Gautham Banasandra | Gautham Banasandra | +| [HADOOP-17608](https://issues.apache.org/jira/browse/HADOOP-17608) | Fix TestKMS failure | Major | kms | Akira Ajisaka | Akira Ajisaka | +| [YARN-10460](https://issues.apache.org/jira/browse/YARN-10460) | Upgrading to JUnit 4.13 causes tests in TestNodeStatusUpdater to fail | Major | nodemanager, test | Peter Bacsko | Peter Bacsko | +| [HADOOP-17641](https://issues.apache.org/jira/browse/HADOOP-17641) | ITestWasbUriAndConfiguration.testCanonicalServiceName() failing now mockaccount exists | Minor | fs/azure, test | Steve Loughran | Steve Loughran | +| [HADOOP-17655](https://issues.apache.org/jira/browse/HADOOP-17655) | Upgrade Jetty to 9.4.40 | Blocker | . | Akira Ajisaka | Akira Ajisaka | +| [YARN-10749](https://issues.apache.org/jira/browse/YARN-10749) | Can't remove all node labels after add node label without nodemanager port, broken by YARN-10647 | Major | . | D M Murali Krishna Reddy | D M Murali Krishna Reddy | +| [HDFS-15621](https://issues.apache.org/jira/browse/HDFS-15621) | Datanode DirectoryScanner uses excessive memory | Major | datanode | Stephen O'Donnell | Stephen O'Donnell | +| [YARN-10756](https://issues.apache.org/jira/browse/YARN-10756) | Remove additional junit 4.11 dependency from javadoc | Major | build, test, timelineservice | ANANDA G B | Akira Ajisaka | +| [YARN-10555](https://issues.apache.org/jira/browse/YARN-10555) | Missing access check before getAppAttempts | Critical | webapp | lujie | lujie | +| [HADOOP-17703](https://issues.apache.org/jira/browse/HADOOP-17703) | checkcompatibility.py errors out when specifying annotations | Major | . | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HADOOP-14922](https://issues.apache.org/jira/browse/HADOOP-14922) | Build of Mapreduce Native Task module fails with unknown opcode "bswap" | Major | . | Anup Halarnkar | Anup Halarnkar | +| [HADOOP-17718](https://issues.apache.org/jira/browse/HADOOP-17718) | Explicitly set locale in the Dockerfile | Blocker | build | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HADOOP-17700](https://issues.apache.org/jira/browse/HADOOP-17700) | ExitUtil#halt info log should log HaltException | Major | . | Viraj Jasani | Viraj Jasani | +| [YARN-10770](https://issues.apache.org/jira/browse/YARN-10770) | container-executor permission is wrong in SecureContainer.md | Major | documentation | Akira Ajisaka | Siddharth Ahuja | +| [HDFS-15915](https://issues.apache.org/jira/browse/HDFS-15915) | Race condition with async edits logging due to updating txId outside of the namesystem log | Major | hdfs, namenode | Konstantin Shvachko | Konstantin Shvachko | +| [HDFS-16040](https://issues.apache.org/jira/browse/HDFS-16040) | RpcQueueTime metric counts requeued calls as unique events. | Major | hdfs | Simbarashe Dzinamarira | Simbarashe Dzinamarira | +| [YARN-10809](https://issues.apache.org/jira/browse/YARN-10809) | testWithHbaseConfAtHdfsFileSystem consistently failing | Major | . | Viraj Jasani | Viraj Jasani | +| [HDFS-16055](https://issues.apache.org/jira/browse/HDFS-16055) | Quota is not preserved in snapshot INode | Major | hdfs | Siyao Meng | Siyao Meng | +| [HDFS-16068](https://issues.apache.org/jira/browse/HDFS-16068) | WebHdfsFileSystem has a possible connection leak in connection with HttpFS | Major | . | Takanobu Asanuma | Takanobu Asanuma | +| [YARN-10767](https://issues.apache.org/jira/browse/YARN-10767) | Yarn Logs Command retrying on Standby RM for 30 times | Major | . | D M Murali Krishna Reddy | D M Murali Krishna Reddy | +| [HDFS-15618](https://issues.apache.org/jira/browse/HDFS-15618) | Improve datanode shutdown latency | Major | datanode | Ahmed Hussein | Ahmed Hussein | +| [HADOOP-17760](https://issues.apache.org/jira/browse/HADOOP-17760) | Delete hadoop.ssl.enabled and dfs.https.enable from docs and core-default.xml | Major | documentation | Takanobu Asanuma | Takanobu Asanuma | +| [HDFS-13671](https://issues.apache.org/jira/browse/HDFS-13671) | Namenode deletes large dir slowly caused by FoldedTreeSet#removeAndGet | Major | . | Yiqun Lin | Haibin Huang | +| [HDFS-16061](https://issues.apache.org/jira/browse/HDFS-16061) | DFTestUtil.waitReplication can produce false positives | Major | hdfs | Ahmed Hussein | Ahmed Hussein | +| [HDFS-14575](https://issues.apache.org/jira/browse/HDFS-14575) | LeaseRenewer#daemon threads leak in DFSClient | Major | . | Tao Yang | Renukaprasad C | +| [YARN-10826](https://issues.apache.org/jira/browse/YARN-10826) | [UI2] Upgrade Node.js to at least v12.22.1 | Major | yarn-ui-v2 | Akira Ajisaka | Masatake Iwasaki | +| [YARN-10828](https://issues.apache.org/jira/browse/YARN-10828) | Backport YARN-9789 to branch-3.2 | Major | . | Tarun Parimi | Tarun Parimi | +| [HADOOP-17769](https://issues.apache.org/jira/browse/HADOOP-17769) | Upgrade JUnit to 4.13.2 | Major | . | Ahmed Hussein | Ahmed Hussein | +| [YARN-10824](https://issues.apache.org/jira/browse/YARN-10824) | Title not set for JHS and NM webpages | Major | . | Rajshree Mishra | Bilwa S T | +| [HDFS-16092](https://issues.apache.org/jira/browse/HDFS-16092) | Avoid creating LayoutFlags redundant objects | Major | . | Viraj Jasani | Viraj Jasani | +| [HDFS-16108](https://issues.apache.org/jira/browse/HDFS-16108) | Incorrect log placeholders used in JournalNodeSyncer | Minor | . | Viraj Jasani | Viraj Jasani | +| [MAPREDUCE-7353](https://issues.apache.org/jira/browse/MAPREDUCE-7353) | Mapreduce job fails when NM is stopped | Major | . | Bilwa S T | Bilwa S T | +| [HDFS-16121](https://issues.apache.org/jira/browse/HDFS-16121) | Iterative snapshot diff report can generate duplicate records for creates, deletes and Renames | Major | snapshots | Srinivasu Majeti | Shashikant Banerjee | +| [HDFS-15796](https://issues.apache.org/jira/browse/HDFS-15796) | ConcurrentModificationException error happens on NameNode occasionally | Critical | hdfs | Daniel Ma | Daniel Ma | +| [HADOOP-17793](https://issues.apache.org/jira/browse/HADOOP-17793) | Better token validation | Major | . | Artem Smotrakov | Artem Smotrakov | +| [HDFS-16042](https://issues.apache.org/jira/browse/HDFS-16042) | DatanodeAdminMonitor scan should be delay based | Major | datanode | Ahmed Hussein | Ahmed Hussein | +| [HDFS-16127](https://issues.apache.org/jira/browse/HDFS-16127) | Improper pipeline close recovery causes a permanent write failure or data loss. | Major | . | Kihwal Lee | Kihwal Lee | +| [HADOOP-17028](https://issues.apache.org/jira/browse/HADOOP-17028) | ViewFS should initialize target filesystems lazily | Major | client-mounts, fs, viewfs | Uma Maheswara Rao G | Abhishek Das | +| [HDFS-12920](https://issues.apache.org/jira/browse/HDFS-12920) | HDFS default value change (with adding time unit) breaks old version MR tarball work with Hadoop 3.x | Critical | configuration, hdfs | Junping Du | Akira Ajisaka | +| [YARN-10813](https://issues.apache.org/jira/browse/YARN-10813) | Set default capacity of root for node labels | Major | . | Andras Gyori | Andras Gyori | +| [YARN-9551](https://issues.apache.org/jira/browse/YARN-9551) | TestTimelineClientV2Impl.testSyncCall fails intermittently | Minor | ATSv2, test | Prabhu Joseph | Andras Gyori | +| [HDFS-15175](https://issues.apache.org/jira/browse/HDFS-15175) | Multiple CloseOp shared block instance causes the standby namenode to crash when rolling editlog | Critical | . | Yicong Cai | Wan Chang | +| [YARN-10789](https://issues.apache.org/jira/browse/YARN-10789) | RM HA startup can fail due to race conditions in ZKConfigurationStore | Major | . | Tarun Parimi | Tarun Parimi | +| [YARN-6221](https://issues.apache.org/jira/browse/YARN-6221) | Entities missing from ATS when summary log file info got returned to the ATS before the domain log | Critical | yarn | Sushmitha Sreenivasan | Xiaomin Zhang | +| [MAPREDUCE-7258](https://issues.apache.org/jira/browse/MAPREDUCE-7258) | HistoryServerRest.html#Task\_Counters\_API, modify the jobTaskCounters's itemName from "taskcounterGroup" to "taskCounterGroup". | Minor | documentation | jenny | jenny | +| [YARN-8990](https://issues.apache.org/jira/browse/YARN-8990) | Fix fair scheduler race condition in app submit and queue cleanup | Blocker | fairscheduler | Wilfred Spiegelenburg | Wilfred Spiegelenburg | +| [YARN-8992](https://issues.apache.org/jira/browse/YARN-8992) | Fair scheduler can delete a dynamic queue while an application attempt is being added to the queue | Major | fairscheduler | Haibo Chen | Wilfred Spiegelenburg | +| [HADOOP-17370](https://issues.apache.org/jira/browse/HADOOP-17370) | Upgrade commons-compress to 1.21 | Major | common | Dongjoon Hyun | Akira Ajisaka | +| [HADOOP-17844](https://issues.apache.org/jira/browse/HADOOP-17844) | Upgrade JSON smart to 2.4.7 | Major | . | Renukaprasad C | Renukaprasad C | +| [HADOOP-17850](https://issues.apache.org/jira/browse/HADOOP-17850) | Upgrade ZooKeeper to 3.4.14 in branch-3.2 | Major | . | Akira Ajisaka | Masatake Iwasaki | +| [HDFS-16177](https://issues.apache.org/jira/browse/HDFS-16177) | Bug fix for Util#receiveFile | Minor | . | tomscut | tomscut | +| [YARN-10814](https://issues.apache.org/jira/browse/YARN-10814) | YARN shouldn't start with empty hadoop.http.authentication.signature.secret.file | Major | . | Benjamin Teke | Tamas Domok | +| [HADOOP-17858](https://issues.apache.org/jira/browse/HADOOP-17858) | Avoid possible class loading deadlock with VerifierNone initialization | Major | . | Viraj Jasani | Viraj Jasani | +| [HADOOP-17886](https://issues.apache.org/jira/browse/HADOOP-17886) | Upgrade ant to 1.10.11 | Major | . | Ahmed Hussein | Ahmed Hussein | +| [YARN-10901](https://issues.apache.org/jira/browse/YARN-10901) | Permission checking error on an existing directory in LogAggregationFileController#verifyAndCreateRemoteLogDir | Major | nodemanager | Tamas Domok | Tamas Domok | +| [HDFS-16187](https://issues.apache.org/jira/browse/HDFS-16187) | SnapshotDiff behaviour with Xattrs and Acls is not consistent across NN restarts with checkpointing | Major | snapshots | Srinivasu Majeti | Shashikant Banerjee | +| [HDFS-16198](https://issues.apache.org/jira/browse/HDFS-16198) | Short circuit read leaks Slot objects when InvalidToken exception is thrown | Major | . | Eungsop Yoo | Eungsop Yoo | +| [HADOOP-17917](https://issues.apache.org/jira/browse/HADOOP-17917) | Backport HADOOP-15993 to branch-3.2 which address CVE-2014-4611 | Major | . | Brahma Reddy Battula | Brahma Reddy Battula | +| [HDFS-16233](https://issues.apache.org/jira/browse/HDFS-16233) | Do not use exception handler to implement copy-on-write for EnumCounters | Major | namenode | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HDFS-16235](https://issues.apache.org/jira/browse/HDFS-16235) | Deadlock in LeaseRenewer for static remove method | Major | hdfs | angerszhu | angerszhu | +| [HADOOP-17940](https://issues.apache.org/jira/browse/HADOOP-17940) | Upgrade Kafka to 2.8.1 | Major | . | Takanobu Asanuma | Takanobu Asanuma | +| [HDFS-16272](https://issues.apache.org/jira/browse/HDFS-16272) | Int overflow in computing safe length during EC block recovery | Critical | 3.1.1 | daimin | daimin | +| [HADOOP-17971](https://issues.apache.org/jira/browse/HADOOP-17971) | Exclude IBM Java security classes from being shaded/relocated | Major | build | Nicholas Marion | Nicholas Marion | +| [HADOOP-17972](https://issues.apache.org/jira/browse/HADOOP-17972) | Backport HADOOP-17683 for branch-3.2 | Major | security | Ananya Singh | Ananya Singh | +| [HADOOP-17993](https://issues.apache.org/jira/browse/HADOOP-17993) | Disable JIRA plugin for YETUS on Hadoop | Critical | build | Gautham Banasandra | Gautham Banasandra | +| [HDFS-16182](https://issues.apache.org/jira/browse/HDFS-16182) | numOfReplicas is given the wrong value in BlockPlacementPolicyDefault$chooseTarget can cause DataStreamer to fail with Heterogeneous Storage | Major | namanode | Max Xie | Max Xie | +| [HDFS-16350](https://issues.apache.org/jira/browse/HDFS-16350) | Datanode start time should be set after RPC server starts successfully | Minor | . | Viraj Jasani | Viraj Jasani | +| [HADOOP-13500](https://issues.apache.org/jira/browse/HADOOP-13500) | Synchronizing iteration of Configuration properties object | Major | conf | Jason Darrell Lowe | Dhananjay Badaya | +| [HDFS-16317](https://issues.apache.org/jira/browse/HDFS-16317) | Backport HDFS-14729 for branch-3.2 | Major | security | Ananya Singh | Ananya Singh | +| [HDFS-14099](https://issues.apache.org/jira/browse/HDFS-14099) | Unknown frame descriptor when decompressing multiple frames in ZStandardDecompressor | Major | . | xuzq | xuzq | +| [HDFS-16410](https://issues.apache.org/jira/browse/HDFS-16410) | Insecure Xml parsing in OfflineEditsXmlLoader | Minor | . | Ashutosh Gupta | Ashutosh Gupta | +| [HDFS-16420](https://issues.apache.org/jira/browse/HDFS-16420) | Avoid deleting unique data blocks when deleting redundancy striped blocks | Critical | ec, erasure-coding | qinyuren | Jackson Wang | +| [HDFS-16428](https://issues.apache.org/jira/browse/HDFS-16428) | Source path with storagePolicy cause wrong typeConsumed while rename | Major | hdfs, namenode | lei w | lei w | +| [HDFS-16437](https://issues.apache.org/jira/browse/HDFS-16437) | ReverseXML processor doesn't accept XML files without the SnapshotDiffSection. | Critical | hdfs | yanbin.zhang | yanbin.zhang | +| [HDFS-16422](https://issues.apache.org/jira/browse/HDFS-16422) | Fix thread safety of EC decoding during concurrent preads | Critical | dfsclient, ec, erasure-coding | daimin | daimin | + + +### TESTS: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [YARN-9338](https://issues.apache.org/jira/browse/YARN-9338) | Timeline related testcases are failing | Major | . | Prabhu Joseph | Abhishek Modi | +| [HDFS-15092](https://issues.apache.org/jira/browse/HDFS-15092) | TestRedudantBlocks#testProcessOverReplicatedAndRedudantBlock sometimes fails | Minor | test | Hui Fei | Hui Fei | + + +### SUB-TASKS: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HADOOP-15775](https://issues.apache.org/jira/browse/HADOOP-15775) | [JDK9] Add missing javax.activation-api dependency | Critical | test | Akira Ajisaka | Akira Ajisaka | +| [YARN-9875](https://issues.apache.org/jira/browse/YARN-9875) | FSSchedulerConfigurationStore fails to update with hdfs path | Major | capacityscheduler | Prabhu Joseph | Prabhu Joseph | +| [HADOOP-16764](https://issues.apache.org/jira/browse/HADOOP-16764) | Rewrite Python example codes using Python3 | Minor | documentation | Kengo Seki | Kengo Seki | +| [HADOOP-16905](https://issues.apache.org/jira/browse/HADOOP-16905) | Update jackson-databind to 2.10.3 to relieve us from the endless CVE patches | Major | . | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [YARN-10337](https://issues.apache.org/jira/browse/YARN-10337) | TestRMHATimelineCollectors fails on hadoop trunk | Major | test, yarn | Ahmed Hussein | Bilwa S T | +| [HDFS-15464](https://issues.apache.org/jira/browse/HDFS-15464) | ViewFsOverloadScheme should work when -fs option pointing to remote cluster without mount links | Major | viewfsOverloadScheme | Uma Maheswara Rao G | Uma Maheswara Rao G | +| [HDFS-15478](https://issues.apache.org/jira/browse/HDFS-15478) | When Empty mount points, we are assigning fallback link to self. But it should not use full URI for target fs. | Major | . | Uma Maheswara Rao G | Uma Maheswara Rao G | +| [HDFS-15459](https://issues.apache.org/jira/browse/HDFS-15459) | TestBlockTokenWithDFSStriped fails intermittently | Major | hdfs | Ahmed Hussein | Ahmed Hussein | +| [HDFS-15461](https://issues.apache.org/jira/browse/HDFS-15461) | TestDFSClientRetries#testGetFileChecksum fails intermittently | Major | dfsclient, test | Ahmed Hussein | Ahmed Hussein | +| [HDFS-9776](https://issues.apache.org/jira/browse/HDFS-9776) | TestHAAppend#testMultipleAppendsDuringCatchupTailing is flaky | Major | . | Vinayakumar B | Ahmed Hussein | +| [HDFS-15457](https://issues.apache.org/jira/browse/HDFS-15457) | TestFsDatasetImpl fails intermittently | Major | hdfs | Ahmed Hussein | Ahmed Hussein | +| [HADOOP-17330](https://issues.apache.org/jira/browse/HADOOP-17330) | Backport HADOOP-16005-"NativeAzureFileSystem does not support setXAttr" to branch-3.2 | Major | fs/azure | Sally Zuo | Sally Zuo | +| [HDFS-15643](https://issues.apache.org/jira/browse/HDFS-15643) | EC: Fix checksum computation in case of native encoders | Blocker | . | Ahmed Hussein | Ayush Saxena | +| [HADOOP-17325](https://issues.apache.org/jira/browse/HADOOP-17325) | WASB: Test failures | Major | fs/azure, test | Sneha Vijayarajan | Steve Loughran | +| [HDFS-15716](https://issues.apache.org/jira/browse/HDFS-15716) | TestUpgradeDomainBlockPlacementPolicy flaky | Major | namenode, test | Ahmed Hussein | Ahmed Hussein | +| [HDFS-15762](https://issues.apache.org/jira/browse/HDFS-15762) | TestMultipleNNPortQOP#testMultipleNNPortOverwriteDownStream fails intermittently | Minor | . | Toshihiko Uchida | Toshihiko Uchida | +| [HDFS-15672](https://issues.apache.org/jira/browse/HDFS-15672) | TestBalancerWithMultipleNameNodes#testBalancingBlockpoolsWithBlockPoolPolicy fails on trunk | Major | . | Ahmed Hussein | Masatake Iwasaki | +| [HDFS-15818](https://issues.apache.org/jira/browse/HDFS-15818) | Fix TestFsDatasetImpl.testReadLockCanBeDisabledByConfig | Minor | test | Leon Gao | Leon Gao | +| [HADOOP-16748](https://issues.apache.org/jira/browse/HADOOP-16748) | Migrate to Python 3 and upgrade Yetus to 0.13.0 | Major | . | Akira Ajisaka | Akira Ajisaka | +| [HDFS-15890](https://issues.apache.org/jira/browse/HDFS-15890) | Improve the Logs for File Concat Operation | Minor | namenode | Bhavik Patel | Bhavik Patel | +| [HDFS-13975](https://issues.apache.org/jira/browse/HDFS-13975) | TestBalancer#testMaxIterationTime fails sporadically | Major | . | Jason Darrell Lowe | Toshihiko Uchida | +| [YARN-10688](https://issues.apache.org/jira/browse/YARN-10688) | ClusterMetrics should support GPU capacity related metrics. | Major | metrics, resourcemanager | Qi Zhu | Qi Zhu | +| [HDFS-15902](https://issues.apache.org/jira/browse/HDFS-15902) | Improve the log for HTTPFS server operation | Minor | httpfs | Bhavik Patel | Bhavik Patel | +| [HDFS-15940](https://issues.apache.org/jira/browse/HDFS-15940) | Some tests in TestBlockRecovery are consistently failing | Major | . | Viraj Jasani | Viraj Jasani | +| [YARN-10702](https://issues.apache.org/jira/browse/YARN-10702) | Add cluster metric for amount of CPU used by RM Event Processor | Minor | yarn | Jim Brennan | Jim Brennan | +| [HADOOP-17630](https://issues.apache.org/jira/browse/HADOOP-17630) | [JDK 15] TestPrintableString fails due to Unicode 13.0 support | Major | . | Akira Ajisaka | Akira Ajisaka | +| [YARN-10723](https://issues.apache.org/jira/browse/YARN-10723) | Change CS nodes page in UI to support custom resource. | Major | . | Qi Zhu | Qi Zhu | +| [HADOOP-17112](https://issues.apache.org/jira/browse/HADOOP-17112) | whitespace not allowed in paths when saving files to s3a via committer | Blocker | fs/s3 | Krzysztof Adamski | Krzysztof Adamski | +| [HADOOP-17661](https://issues.apache.org/jira/browse/HADOOP-17661) | mvn versions:set fails to parse pom.xml | Blocker | build | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [YARN-10642](https://issues.apache.org/jira/browse/YARN-10642) | Race condition: AsyncDispatcher can get stuck by the changes introduced in YARN-8995 | Critical | resourcemanager | zhengchenyu | zhengchenyu | +| [HDFS-15659](https://issues.apache.org/jira/browse/HDFS-15659) | Set dfs.namenode.redundancy.considerLoad to false in MiniDFSCluster | Major | test | Akira Ajisaka | Ahmed Hussein | +| [HADOOP-17840](https://issues.apache.org/jira/browse/HADOOP-17840) | Backport HADOOP-17837 to branch-3.2 | Minor | . | Bryan Beaudreault | Bryan Beaudreault | +| [HADOOP-17126](https://issues.apache.org/jira/browse/HADOOP-17126) | implement non-guava Precondition checkNotNull | Major | . | Ahmed Hussein | Ahmed Hussein | + + +### OTHER: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HDFS-15870](https://issues.apache.org/jira/browse/HDFS-15870) | Remove unused configuration dfs.namenode.stripe.min | Minor | . | tomscut | tomscut | +| [HDFS-15808](https://issues.apache.org/jira/browse/HDFS-15808) | Add metrics for FSNamesystem read/write lock hold long time | Major | hdfs | tomscut | tomscut | +| [HDFS-15873](https://issues.apache.org/jira/browse/HDFS-15873) | Add namenode address in logs for block report | Minor | datanode, hdfs | tomscut | tomscut | +| [HDFS-15906](https://issues.apache.org/jira/browse/HDFS-15906) | Close FSImage and FSNamesystem after formatting is complete | Minor | . | tomscut | tomscut | +| [HDFS-15892](https://issues.apache.org/jira/browse/HDFS-15892) | Add metric for editPendingQ in FSEditLogAsync | Minor | . | tomscut | tomscut | +| [HDFS-16078](https://issues.apache.org/jira/browse/HDFS-16078) | Remove unused parameters for DatanodeManager.handleLifeline() | Minor | . | tomscut | tomscut | +| [YARN-10278](https://issues.apache.org/jira/browse/YARN-10278) | CapacityScheduler test framework ProportionalCapacityPreemptionPolicyMockFramework need some review | Major | . | Gergely Pollák | Szilard Nemeth | +| [HDFS-15731](https://issues.apache.org/jira/browse/HDFS-15731) | Reduce threadCount for unit tests to reduce the memory usage | Major | build, test | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-17571](https://issues.apache.org/jira/browse/HADOOP-17571) | Upgrade com.fasterxml.woodstox:woodstox-core for security reasons | Major | . | Viraj Jasani | Viraj Jasani | +| [HDFS-15895](https://issues.apache.org/jira/browse/HDFS-15895) | DFSAdmin#printOpenFiles has redundant String#format usage | Minor | . | Viraj Jasani | Viraj Jasani | +| [HADOOP-17614](https://issues.apache.org/jira/browse/HADOOP-17614) | Bump netty to the latest 4.1.61 | Blocker | . | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HADOOP-17627](https://issues.apache.org/jira/browse/HADOOP-17627) | Backport to branch-3.2 HADOOP-17371, HADOOP-17621, HADOOP-17625 to update Jetty to 9.4.39 | Major | . | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HDFS-15989](https://issues.apache.org/jira/browse/HDFS-15989) | Split TestBalancer into two classes | Major | . | Viraj Jasani | Viraj Jasani | +| [HADOOP-17808](https://issues.apache.org/jira/browse/HADOOP-17808) | ipc.Client not setting interrupt flag after catching InterruptedException | Minor | . | Viraj Jasani | Viraj Jasani | +| [HADOOP-17834](https://issues.apache.org/jira/browse/HADOOP-17834) | Bump aliyun-sdk-oss to 3.13.0 | Major | . | Siyao Meng | Siyao Meng | +| [HADOOP-17955](https://issues.apache.org/jira/browse/HADOOP-17955) | Bump netty to the latest 4.1.68 | Major | . | Takanobu Asanuma | Takanobu Asanuma | +| [HADOOP-18061](https://issues.apache.org/jira/browse/HADOOP-18061) | Update the year to 2022 | Major | . | Ayush Saxena | Ayush Saxena | +| [HADOOP-18125](https://issues.apache.org/jira/browse/HADOOP-18125) | Utility to identify git commit / Jira fixVersion discrepancies for RC preparation | Major | . | Viraj Jasani | Viraj Jasani | + + diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.3/RELEASENOTES.3.2.3.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.3/RELEASENOTES.3.2.3.md new file mode 100644 index 0000000000000..5c53bb4cb876b --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.3/RELEASENOTES.3.2.3.md @@ -0,0 +1,71 @@ + + +# Apache Hadoop 3.2.3 Release Notes + +These release notes cover new developer and user-facing incompatibilities, important issues, features, and major improvements. + + +--- + +* [YARN-10036](https://issues.apache.org/jira/browse/YARN-10036) | *Major* | **Install yarnpkg and upgrade nodejs in Dockerfile** + +In the Dockerfile, nodejs is upgraded to 8.17.0 and yarn 1.12.1 is installed. + + +--- + +* [HADOOP-16054](https://issues.apache.org/jira/browse/HADOOP-16054) | *Major* | **Update Dockerfile to use Bionic** + +The build image has been upgraded to Bionic. + + +--- + +* [HDFS-15719](https://issues.apache.org/jira/browse/HDFS-15719) | *Critical* | **[Hadoop 3] Both NameNodes can crash simultaneously due to the short JN socket timeout** + +The default value of the configuration hadoop.http.idle\_timeout.ms (how long does Jetty disconnect an idle connection) is changed from 10000 to 60000. +This property is inlined during compile time, so an application that references this property must be recompiled in order for it to take effect. + + +--- + +* [HADOOP-16748](https://issues.apache.org/jira/browse/HADOOP-16748) | *Major* | **Migrate to Python 3 and upgrade Yetus to 0.13.0** + + +- Upgraded Yetus to 0.13.0. +- Removed determine-flaky-tests-hadoop.py. +- Temporarily disabled shelldocs check in the Jenkins jobs due to YETUS-1099. + + +--- + +* [HADOOP-16870](https://issues.apache.org/jira/browse/HADOOP-16870) | *Major* | **Use spotbugs-maven-plugin instead of findbugs-maven-plugin** + +Removed findbugs from the hadoop build images and added spotbugs instead. +Upgraded SpotBugs to 4.2.2 and spotbugs-maven-plugin to 4.2.0. + + +--- + +* [HDFS-15942](https://issues.apache.org/jira/browse/HDFS-15942) | *Major* | **Increase Quota initialization threads** + +The default quota initialization thread count during the NameNode startup process (dfs.namenode.quota.init-threads) is increased from 4 to 12. + + + diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.2/CHANGELOG.3.3.2.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.2/CHANGELOG.3.3.2.md new file mode 100644 index 0000000000000..162f9928489ee --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.2/CHANGELOG.3.3.2.md @@ -0,0 +1,350 @@ + + +# Apache Hadoop Changelog + +## Release 3.3.2 - 2022-02-21 + + + +### IMPORTANT ISSUES: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HDFS-15814](https://issues.apache.org/jira/browse/HDFS-15814) | Make some parameters configurable for DataNodeDiskMetrics | Major | hdfs | tomscut | tomscut | + + +### NEW FEATURES: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HDFS-15288](https://issues.apache.org/jira/browse/HDFS-15288) | Add Available Space Rack Fault Tolerant BPP | Major | . | Ayush Saxena | Ayush Saxena | +| [HDFS-16048](https://issues.apache.org/jira/browse/HDFS-16048) | RBF: Print network topology on the router web | Minor | . | tomscut | tomscut | +| [HDFS-16337](https://issues.apache.org/jira/browse/HDFS-16337) | Show start time of Datanode on Web | Minor | . | tomscut | tomscut | +| [HADOOP-17979](https://issues.apache.org/jira/browse/HADOOP-17979) | Interface EtagSource to allow FileStatus subclasses to provide etags | Major | fs, fs/azure, fs/s3 | Steve Loughran | Steve Loughran | + + +### IMPROVEMENTS: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [YARN-10123](https://issues.apache.org/jira/browse/YARN-10123) | Error message around yarn app -stop/start can be improved to highlight that an implementation at framework level is needed for the stop/start functionality to work | Minor | client, documentation | Siddharth Ahuja | Siddharth Ahuja | +| [HADOOP-17756](https://issues.apache.org/jira/browse/HADOOP-17756) | Increase precommit job timeout from 20 hours to 24 hours. | Major | build | Takanobu Asanuma | Takanobu Asanuma | +| [HDFS-16073](https://issues.apache.org/jira/browse/HDFS-16073) | Remove redundant RPC requests for getFileLinkInfo in ClientNamenodeProtocolTranslatorPB | Minor | . | lei w | lei w | +| [HDFS-16074](https://issues.apache.org/jira/browse/HDFS-16074) | Remove an expensive debug string concatenation | Major | . | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HDFS-16080](https://issues.apache.org/jira/browse/HDFS-16080) | RBF: Invoking method in all locations should break the loop after successful result | Minor | . | Viraj Jasani | Viraj Jasani | +| [HDFS-16075](https://issues.apache.org/jira/browse/HDFS-16075) | Use empty array constants present in StorageType and DatanodeInfo to avoid creating redundant objects | Major | . | Viraj Jasani | Viraj Jasani | +| [MAPREDUCE-7354](https://issues.apache.org/jira/browse/MAPREDUCE-7354) | Use empty array constants present in TaskCompletionEvent to avoid creating redundant objects | Minor | . | Viraj Jasani | Viraj Jasani | +| [HDFS-16082](https://issues.apache.org/jira/browse/HDFS-16082) | Avoid non-atomic operations on exceptionsSinceLastBalance and failedTimesSinceLastSuccessfulBalance in Balancer | Major | . | Viraj Jasani | Viraj Jasani | +| [HDFS-16076](https://issues.apache.org/jira/browse/HDFS-16076) | Avoid using slow DataNodes for reading by sorting locations | Major | hdfs | tomscut | tomscut | +| [HDFS-16085](https://issues.apache.org/jira/browse/HDFS-16085) | Move the getPermissionChecker out of the read lock | Minor | . | tomscut | tomscut | +| [YARN-10834](https://issues.apache.org/jira/browse/YARN-10834) | Intra-queue preemption: apps that don't use defined custom resource won't be preempted. | Major | . | Eric Payne | Eric Payne | +| [HADOOP-17777](https://issues.apache.org/jira/browse/HADOOP-17777) | Update clover-maven-plugin version from 3.3.0 to 4.4.1 | Major | . | Wanqiang Ji | Wanqiang Ji | +| [HDFS-16090](https://issues.apache.org/jira/browse/HDFS-16090) | Fine grained locking for datanodeNetworkCounts | Major | . | Viraj Jasani | Viraj Jasani | +| [HADOOP-17749](https://issues.apache.org/jira/browse/HADOOP-17749) | Remove lock contention in SelectorPool of SocketIOWithTimeout | Major | common | Xuesen Liang | Xuesen Liang | +| [HADOOP-17775](https://issues.apache.org/jira/browse/HADOOP-17775) | Remove JavaScript package from Docker environment | Major | build | Masatake Iwasaki | Masatake Iwasaki | +| [HADOOP-17402](https://issues.apache.org/jira/browse/HADOOP-17402) | Add GCS FS impl reference to core-default.xml | Major | fs | Rafal Wojdyla | Rafal Wojdyla | +| [HADOOP-17794](https://issues.apache.org/jira/browse/HADOOP-17794) | Add a sample configuration to use ZKDelegationTokenSecretManager in Hadoop KMS | Major | documentation, kms, security | Akira Ajisaka | Akira Ajisaka | +| [HDFS-16122](https://issues.apache.org/jira/browse/HDFS-16122) | Fix DistCpContext#toString() | Minor | . | tomscut | tomscut | +| [HADOOP-12665](https://issues.apache.org/jira/browse/HADOOP-12665) | Document hadoop.security.token.service.use\_ip | Major | documentation | Arpit Agarwal | Akira Ajisaka | +| [YARN-10456](https://issues.apache.org/jira/browse/YARN-10456) | RM PartitionQueueMetrics records are named QueueMetrics in Simon metrics registry | Major | resourcemanager | Eric Payne | Eric Payne | +| [HDFS-15650](https://issues.apache.org/jira/browse/HDFS-15650) | Make the socket timeout for computing checksum of striped blocks configurable | Minor | datanode, ec, erasure-coding | Yushi Hayasaka | Yushi Hayasaka | +| [YARN-10858](https://issues.apache.org/jira/browse/YARN-10858) | [UI2] YARN-10826 breaks Queue view | Major | yarn-ui-v2 | Andras Gyori | Masatake Iwasaki | +| [HADOOP-16290](https://issues.apache.org/jira/browse/HADOOP-16290) | Enable RpcMetrics units to be configurable | Major | ipc, metrics | Erik Krogen | Viraj Jasani | +| [YARN-10860](https://issues.apache.org/jira/browse/YARN-10860) | Make max container per heartbeat configs refreshable | Major | . | Eric Badger | Eric Badger | +| [HADOOP-17813](https://issues.apache.org/jira/browse/HADOOP-17813) | Checkstyle - Allow line length: 100 | Major | . | Akira Ajisaka | Viraj Jasani | +| [HADOOP-17811](https://issues.apache.org/jira/browse/HADOOP-17811) | ABFS ExponentialRetryPolicy doesn't pick up configuration values | Minor | documentation, fs/azure | Brian Frank Loss | Brian Frank Loss | +| [HADOOP-17819](https://issues.apache.org/jira/browse/HADOOP-17819) | Add extensions to ProtobufRpcEngine RequestHeaderProto | Major | common | Hector Sandoval Chaverri | Hector Sandoval Chaverri | +| [HDFS-15936](https://issues.apache.org/jira/browse/HDFS-15936) | Solve BlockSender#sendPacket() does not record SocketTimeout exception | Minor | . | JiangHua Zhu | JiangHua Zhu | +| [HDFS-16153](https://issues.apache.org/jira/browse/HDFS-16153) | Avoid evaluation of LOG.debug statement in QuorumJournalManager | Trivial | . | wangzhaohui | wangzhaohui | +| [HDFS-16154](https://issues.apache.org/jira/browse/HDFS-16154) | TestMiniJournalCluster failing intermittently because of not reseting UserGroupInformation completely | Minor | . | wangzhaohui | wangzhaohui | +| [HADOOP-17837](https://issues.apache.org/jira/browse/HADOOP-17837) | Make it easier to debug UnknownHostExceptions from NetUtils.connect | Minor | . | Bryan Beaudreault | Bryan Beaudreault | +| [HDFS-16175](https://issues.apache.org/jira/browse/HDFS-16175) | Improve the configurable value of Server #PURGE\_INTERVAL\_NANOS | Major | ipc | JiangHua Zhu | JiangHua Zhu | +| [HDFS-16173](https://issues.apache.org/jira/browse/HDFS-16173) | Improve CopyCommands#Put#executor queue configurability | Major | fs | JiangHua Zhu | JiangHua Zhu | +| [HADOOP-17897](https://issues.apache.org/jira/browse/HADOOP-17897) | Allow nested blocks in switch case in checkstyle settings | Minor | build | Masatake Iwasaki | Masatake Iwasaki | +| [HADOOP-17857](https://issues.apache.org/jira/browse/HADOOP-17857) | Check real user ACLs in addition to proxied user ACLs | Major | . | Eric Payne | Eric Payne | +| [HDFS-16210](https://issues.apache.org/jira/browse/HDFS-16210) | RBF: Add the option of refreshCallQueue to RouterAdmin | Major | . | Janus Chow | Janus Chow | +| [HDFS-16221](https://issues.apache.org/jira/browse/HDFS-16221) | RBF: Add usage of refreshCallQueue for Router | Major | . | Janus Chow | Janus Chow | +| [HDFS-16223](https://issues.apache.org/jira/browse/HDFS-16223) | AvailableSpaceRackFaultTolerantBlockPlacementPolicy should use chooseRandomWithStorageTypeTwoTrial() for better performance. | Major | . | Ayush Saxena | Ayush Saxena | +| [HADOOP-17893](https://issues.apache.org/jira/browse/HADOOP-17893) | Improve PrometheusSink for Namenode TopMetrics | Major | metrics | Max Xie | Max Xie | +| [HADOOP-17926](https://issues.apache.org/jira/browse/HADOOP-17926) | Maven-eclipse-plugin is no longer needed since Eclipse can import Maven projects by itself. | Minor | documentation | Rintaro Ikeda | Rintaro Ikeda | +| [YARN-10935](https://issues.apache.org/jira/browse/YARN-10935) | AM Total Queue Limit goes below per-user AM Limit if parent is full. | Major | capacity scheduler, capacityscheduler | Eric Payne | Eric Payne | +| [HADOOP-17939](https://issues.apache.org/jira/browse/HADOOP-17939) | Support building on Apple Silicon | Major | build, common | Dongjoon Hyun | Dongjoon Hyun | +| [HADOOP-17941](https://issues.apache.org/jira/browse/HADOOP-17941) | Update xerces to 2.12.1 | Minor | . | Zhongwei Zhu | Zhongwei Zhu | +| [HDFS-16246](https://issues.apache.org/jira/browse/HDFS-16246) | Print lockWarningThreshold in InstrumentedLock#logWarning and InstrumentedLock#logWaitWarning | Minor | . | tomscut | tomscut | +| [HDFS-16252](https://issues.apache.org/jira/browse/HDFS-16252) | Correct docs for dfs.http.client.retry.policy.spec | Major | . | Stephen O'Donnell | Stephen O'Donnell | +| [HDFS-16241](https://issues.apache.org/jira/browse/HDFS-16241) | Standby close reconstruction thread | Major | . | zhanghuazong | zhanghuazong | +| [HADOOP-17974](https://issues.apache.org/jira/browse/HADOOP-17974) | Fix the import statements in hadoop-aws module | Minor | build, fs/azure | Tamas Domok | | +| [HDFS-16277](https://issues.apache.org/jira/browse/HDFS-16277) | Improve decision in AvailableSpaceBlockPlacementPolicy | Major | block placement | guophilipse | guophilipse | +| [HADOOP-17770](https://issues.apache.org/jira/browse/HADOOP-17770) | WASB : Support disabling buffered reads in positional reads | Major | . | Anoop Sam John | Anoop Sam John | +| [HDFS-16282](https://issues.apache.org/jira/browse/HDFS-16282) | Duplicate generic usage information to hdfs debug command | Minor | tools | daimin | daimin | +| [YARN-1115](https://issues.apache.org/jira/browse/YARN-1115) | Provide optional means for a scheduler to check real user ACLs | Major | capacity scheduler, scheduler | Eric Payne | | +| [HDFS-16279](https://issues.apache.org/jira/browse/HDFS-16279) | Print detail datanode info when process first storage report | Minor | . | tomscut | tomscut | +| [HDFS-16286](https://issues.apache.org/jira/browse/HDFS-16286) | Debug tool to verify the correctness of erasure coding on file | Minor | erasure-coding, tools | daimin | daimin | +| [HDFS-16294](https://issues.apache.org/jira/browse/HDFS-16294) | Remove invalid DataNode#CONFIG\_PROPERTY\_SIMULATED | Major | datanode | JiangHua Zhu | JiangHua Zhu | +| [HDFS-16299](https://issues.apache.org/jira/browse/HDFS-16299) | Fix bug for TestDataNodeVolumeMetrics#verifyDataNodeVolumeMetrics | Minor | . | tomscut | tomscut | +| [HDFS-16301](https://issues.apache.org/jira/browse/HDFS-16301) | Improve BenchmarkThroughput#SIZE naming standardization | Minor | benchmarks, test | JiangHua Zhu | JiangHua Zhu | +| [HDFS-16287](https://issues.apache.org/jira/browse/HDFS-16287) | Support to make dfs.namenode.avoid.read.slow.datanode reconfigurable | Major | . | Haiyang Hu | Haiyang Hu | +| [HDFS-16321](https://issues.apache.org/jira/browse/HDFS-16321) | Fix invalid config in TestAvailableSpaceRackFaultTolerantBPP | Minor | test | guophilipse | guophilipse | +| [HDFS-16315](https://issues.apache.org/jira/browse/HDFS-16315) | Add metrics related to Transfer and NativeCopy for DataNode | Major | . | tomscut | tomscut | +| [HADOOP-17998](https://issues.apache.org/jira/browse/HADOOP-17998) | Allow get command to run with multi threads. | Major | fs | Chengwei Wang | Chengwei Wang | +| [HDFS-16344](https://issues.apache.org/jira/browse/HDFS-16344) | Improve DirectoryScanner.Stats#toString | Major | . | tomscut | tomscut | +| [HADOOP-18023](https://issues.apache.org/jira/browse/HADOOP-18023) | Allow cp command to run with multi threads. | Major | fs | Chengwei Wang | Chengwei Wang | +| [HDFS-16314](https://issues.apache.org/jira/browse/HDFS-16314) | Support to make dfs.namenode.block-placement-policy.exclude-slow-nodes.enabled reconfigurable | Major | . | Haiyang Hu | Haiyang Hu | +| [HADOOP-18026](https://issues.apache.org/jira/browse/HADOOP-18026) | Fix default value of Magic committer | Minor | common | guophilipse | guophilipse | +| [HDFS-16345](https://issues.apache.org/jira/browse/HDFS-16345) | Fix test cases fail in TestBlockStoragePolicy | Major | build | guophilipse | guophilipse | +| [HADOOP-18040](https://issues.apache.org/jira/browse/HADOOP-18040) | Use maven.test.failure.ignore instead of ignoreTestFailure | Major | build | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-17643](https://issues.apache.org/jira/browse/HADOOP-17643) | WASB : Make metadata checks case insensitive | Major | . | Anoop Sam John | Anoop Sam John | +| [HADOOP-18033](https://issues.apache.org/jira/browse/HADOOP-18033) | Upgrade fasterxml Jackson to 2.13.0 | Major | build | Akira Ajisaka | Viraj Jasani | +| [HDFS-16327](https://issues.apache.org/jira/browse/HDFS-16327) | Make dfs.namenode.max.slowpeer.collect.nodes reconfigurable | Major | . | tomscut | tomscut | +| [HDFS-16375](https://issues.apache.org/jira/browse/HDFS-16375) | The FBR lease ID should be exposed to the log | Major | . | tomscut | tomscut | +| [HDFS-16386](https://issues.apache.org/jira/browse/HDFS-16386) | Reduce DataNode load when FsDatasetAsyncDiskService is working | Major | datanode | JiangHua Zhu | JiangHua Zhu | +| [HDFS-16391](https://issues.apache.org/jira/browse/HDFS-16391) | Avoid evaluation of LOG.debug statement in NameNodeHeartbeatService | Trivial | . | wangzhaohui | wangzhaohui | +| [YARN-8234](https://issues.apache.org/jira/browse/YARN-8234) | Improve RM system metrics publisher's performance by pushing events to timeline server in batch | Critical | resourcemanager, timelineserver | Hu Ziqian | Ashutosh Gupta | +| [HADOOP-18052](https://issues.apache.org/jira/browse/HADOOP-18052) | Support Apple Silicon in start-build-env.sh | Major | build | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-18056](https://issues.apache.org/jira/browse/HADOOP-18056) | DistCp: Filter duplicates in the source paths | Major | . | Ayush Saxena | Ayush Saxena | +| [HADOOP-18065](https://issues.apache.org/jira/browse/HADOOP-18065) | ExecutorHelper.logThrowableFromAfterExecute() is too noisy. | Minor | . | Mukund Thakur | Mukund Thakur | +| [HDFS-16043](https://issues.apache.org/jira/browse/HDFS-16043) | Add markedDeleteBlockScrubberThread to delete blocks asynchronously | Major | hdfs, namanode | Xiangyi Zhu | Xiangyi Zhu | +| [HADOOP-18094](https://issues.apache.org/jira/browse/HADOOP-18094) | Disable S3A auditing by default. | Blocker | fs/s3 | Steve Loughran | Steve Loughran | + + +### BUG FIXES: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [YARN-10438](https://issues.apache.org/jira/browse/YARN-10438) | Handle null containerId in ClientRMService#getContainerReport() | Major | resourcemanager | Raghvendra Singh | Shubham Gupta | +| [YARN-10428](https://issues.apache.org/jira/browse/YARN-10428) | Zombie applications in the YARN queue using FAIR + sizebasedweight | Critical | capacityscheduler | Guang Yang | Andras Gyori | +| [HDFS-15916](https://issues.apache.org/jira/browse/HDFS-15916) | DistCp: Backward compatibility: Distcp fails from Hadoop 3 to Hadoop 2 for snapshotdiff | Major | distcp | Srinivasu Majeti | Ayush Saxena | +| [HDFS-15977](https://issues.apache.org/jira/browse/HDFS-15977) | Call explicit\_bzero only if it is available | Major | libhdfs++ | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-14922](https://issues.apache.org/jira/browse/HADOOP-14922) | Build of Mapreduce Native Task module fails with unknown opcode "bswap" | Major | . | Anup Halarnkar | Anup Halarnkar | +| [HADOOP-17700](https://issues.apache.org/jira/browse/HADOOP-17700) | ExitUtil#halt info log should log HaltException | Major | . | Viraj Jasani | Viraj Jasani | +| [YARN-10770](https://issues.apache.org/jira/browse/YARN-10770) | container-executor permission is wrong in SecureContainer.md | Major | documentation | Akira Ajisaka | Siddharth Ahuja | +| [YARN-10691](https://issues.apache.org/jira/browse/YARN-10691) | DominantResourceCalculator isInvalidDivisor should consider only countable resource types | Major | . | Bilwa S T | Bilwa S T | +| [HDFS-16031](https://issues.apache.org/jira/browse/HDFS-16031) | Possible Resource Leak in org.apache.hadoop.hdfs.server.aliasmap#InMemoryAliasMap | Major | . | Narges Shadab | Narges Shadab | +| [MAPREDUCE-7348](https://issues.apache.org/jira/browse/MAPREDUCE-7348) | TestFrameworkUploader#testNativeIO fails | Major | test | Akira Ajisaka | Akira Ajisaka | +| [HDFS-15915](https://issues.apache.org/jira/browse/HDFS-15915) | Race condition with async edits logging due to updating txId outside of the namesystem log | Major | hdfs, namenode | Konstantin Shvachko | Konstantin Shvachko | +| [HDFS-16040](https://issues.apache.org/jira/browse/HDFS-16040) | RpcQueueTime metric counts requeued calls as unique events. | Major | hdfs | Simbarashe Dzinamarira | Simbarashe Dzinamarira | +| [MAPREDUCE-7287](https://issues.apache.org/jira/browse/MAPREDUCE-7287) | Distcp will delete existing file , If we use "-delete and -update" options and distcp file. | Major | distcp | zhengchenyu | zhengchenyu | +| [HDFS-15998](https://issues.apache.org/jira/browse/HDFS-15998) | Fix NullPointException In listOpenFiles | Major | . | Haiyang Hu | Haiyang Hu | +| [HDFS-16050](https://issues.apache.org/jira/browse/HDFS-16050) | Some dynamometer tests fail | Major | test | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-17631](https://issues.apache.org/jira/browse/HADOOP-17631) | Configuration ${env.VAR:-FALLBACK} should eval FALLBACK when restrictSystemProps=true | Minor | common | Steve Loughran | Steve Loughran | +| [YARN-10809](https://issues.apache.org/jira/browse/YARN-10809) | testWithHbaseConfAtHdfsFileSystem consistently failing | Major | . | Viraj Jasani | Viraj Jasani | +| [YARN-10803](https://issues.apache.org/jira/browse/YARN-10803) | [JDK 11] TestRMFailoverProxyProvider and TestNoHaRMFailoverProxyProvider fails by ClassCastException | Major | test | Akira Ajisaka | Akira Ajisaka | +| [HDFS-16057](https://issues.apache.org/jira/browse/HDFS-16057) | Make sure the order for location in ENTERING\_MAINTENANCE state | Minor | . | tomscut | tomscut | +| [HDFS-16055](https://issues.apache.org/jira/browse/HDFS-16055) | Quota is not preserved in snapshot INode | Major | hdfs | Siyao Meng | Siyao Meng | +| [HDFS-16068](https://issues.apache.org/jira/browse/HDFS-16068) | WebHdfsFileSystem has a possible connection leak in connection with HttpFS | Major | . | Takanobu Asanuma | Takanobu Asanuma | +| [YARN-10767](https://issues.apache.org/jira/browse/YARN-10767) | Yarn Logs Command retrying on Standby RM for 30 times | Major | . | D M Murali Krishna Reddy | D M Murali Krishna Reddy | +| [HADOOP-17760](https://issues.apache.org/jira/browse/HADOOP-17760) | Delete hadoop.ssl.enabled and dfs.https.enable from docs and core-default.xml | Major | documentation | Takanobu Asanuma | Takanobu Asanuma | +| [HDFS-13671](https://issues.apache.org/jira/browse/HDFS-13671) | Namenode deletes large dir slowly caused by FoldedTreeSet#removeAndGet | Major | . | Yiqun Lin | Haibin Huang | +| [HDFS-16061](https://issues.apache.org/jira/browse/HDFS-16061) | DFTestUtil.waitReplication can produce false positives | Major | hdfs | Ahmed Hussein | Ahmed Hussein | +| [HDFS-14575](https://issues.apache.org/jira/browse/HDFS-14575) | LeaseRenewer#daemon threads leak in DFSClient | Major | . | Tao Yang | Renukaprasad C | +| [YARN-10826](https://issues.apache.org/jira/browse/YARN-10826) | [UI2] Upgrade Node.js to at least v12.22.1 | Major | yarn-ui-v2 | Akira Ajisaka | Masatake Iwasaki | +| [HADOOP-17769](https://issues.apache.org/jira/browse/HADOOP-17769) | Upgrade JUnit to 4.13.2 | Major | . | Ahmed Hussein | Ahmed Hussein | +| [YARN-10824](https://issues.apache.org/jira/browse/YARN-10824) | Title not set for JHS and NM webpages | Major | . | Rajshree Mishra | Bilwa S T | +| [HDFS-16092](https://issues.apache.org/jira/browse/HDFS-16092) | Avoid creating LayoutFlags redundant objects | Major | . | Viraj Jasani | Viraj Jasani | +| [HADOOP-17764](https://issues.apache.org/jira/browse/HADOOP-17764) | S3AInputStream read does not re-open the input stream on the second read retry attempt | Major | fs/s3 | Zamil Majdy | Zamil Majdy | +| [HDFS-16109](https://issues.apache.org/jira/browse/HDFS-16109) | Fix flaky some unit tests since they offen timeout | Minor | test | tomscut | tomscut | +| [HDFS-16108](https://issues.apache.org/jira/browse/HDFS-16108) | Incorrect log placeholders used in JournalNodeSyncer | Minor | . | Viraj Jasani | Viraj Jasani | +| [MAPREDUCE-7353](https://issues.apache.org/jira/browse/MAPREDUCE-7353) | Mapreduce job fails when NM is stopped | Major | . | Bilwa S T | Bilwa S T | +| [HDFS-16121](https://issues.apache.org/jira/browse/HDFS-16121) | Iterative snapshot diff report can generate duplicate records for creates, deletes and Renames | Major | snapshots | Srinivasu Majeti | Shashikant Banerjee | +| [HDFS-15796](https://issues.apache.org/jira/browse/HDFS-15796) | ConcurrentModificationException error happens on NameNode occasionally | Critical | hdfs | Daniel Ma | Daniel Ma | +| [HADOOP-17793](https://issues.apache.org/jira/browse/HADOOP-17793) | Better token validation | Major | . | Artem Smotrakov | Artem Smotrakov | +| [HDFS-16042](https://issues.apache.org/jira/browse/HDFS-16042) | DatanodeAdminMonitor scan should be delay based | Major | datanode | Ahmed Hussein | Ahmed Hussein | +| [HADOOP-17803](https://issues.apache.org/jira/browse/HADOOP-17803) | Remove WARN logging from LoggingAuditor when executing a request outside an audit span | Major | fs/s3 | Mehakmeet Singh | Mehakmeet Singh | +| [HDFS-16127](https://issues.apache.org/jira/browse/HDFS-16127) | Improper pipeline close recovery causes a permanent write failure or data loss. | Major | . | Kihwal Lee | Kihwal Lee | +| [HADOOP-17028](https://issues.apache.org/jira/browse/HADOOP-17028) | ViewFS should initialize target filesystems lazily | Major | client-mounts, fs, viewfs | Uma Maheswara Rao G | Abhishek Das | +| [HADOOP-17801](https://issues.apache.org/jira/browse/HADOOP-17801) | No error message reported when bucket doesn't exist in S3AFS | Major | fs/s3 | Mehakmeet Singh | Mehakmeet Singh | +| [HADOOP-17796](https://issues.apache.org/jira/browse/HADOOP-17796) | Upgrade jetty version to 9.4.43 | Major | . | Wei-Chiu Chuang | Renukaprasad C | +| [HDFS-12920](https://issues.apache.org/jira/browse/HDFS-12920) | HDFS default value change (with adding time unit) breaks old version MR tarball work with Hadoop 3.x | Critical | configuration, hdfs | Junping Du | Akira Ajisaka | +| [HDFS-16145](https://issues.apache.org/jira/browse/HDFS-16145) | CopyListing fails with FNF exception with snapshot diff | Major | distcp | Shashikant Banerjee | Shashikant Banerjee | +| [YARN-10813](https://issues.apache.org/jira/browse/YARN-10813) | Set default capacity of root for node labels | Major | . | Andras Gyori | Andras Gyori | +| [HDFS-16144](https://issues.apache.org/jira/browse/HDFS-16144) | Revert HDFS-15372 (Files in snapshots no longer see attribute provider permissions) | Major | . | Stephen O'Donnell | Stephen O'Donnell | +| [HADOOP-17817](https://issues.apache.org/jira/browse/HADOOP-17817) | HADOOP-17817. S3A to raise IOE if both S3-CSE and S3Guard enabled | Major | fs/s3 | Mehakmeet Singh | Mehakmeet Singh | +| [YARN-9551](https://issues.apache.org/jira/browse/YARN-9551) | TestTimelineClientV2Impl.testSyncCall fails intermittently | Minor | ATSv2, test | Prabhu Joseph | Andras Gyori | +| [HDFS-15175](https://issues.apache.org/jira/browse/HDFS-15175) | Multiple CloseOp shared block instance causes the standby namenode to crash when rolling editlog | Critical | . | Yicong Cai | Wan Chang | +| [YARN-10869](https://issues.apache.org/jira/browse/YARN-10869) | CS considers only the default maximum-allocation-mb/vcore property as a maximum when it creates dynamic queues | Major | capacity scheduler | Benjamin Teke | Benjamin Teke | +| [YARN-10789](https://issues.apache.org/jira/browse/YARN-10789) | RM HA startup can fail due to race conditions in ZKConfigurationStore | Major | . | Tarun Parimi | Tarun Parimi | +| [HADOOP-17812](https://issues.apache.org/jira/browse/HADOOP-17812) | NPE in S3AInputStream read() after failure to reconnect to store | Major | fs/s3 | Bobby Wang | Bobby Wang | +| [YARN-6221](https://issues.apache.org/jira/browse/YARN-6221) | Entities missing from ATS when summary log file info got returned to the ATS before the domain log | Critical | yarn | Sushmitha Sreenivasan | Xiaomin Zhang | +| [MAPREDUCE-7258](https://issues.apache.org/jira/browse/MAPREDUCE-7258) | HistoryServerRest.html#Task\_Counters\_API, modify the jobTaskCounters's itemName from "taskcounterGroup" to "taskCounterGroup". | Minor | documentation | jenny | jenny | +| [HADOOP-17370](https://issues.apache.org/jira/browse/HADOOP-17370) | Upgrade commons-compress to 1.21 | Major | common | Dongjoon Hyun | Akira Ajisaka | +| [HDFS-16151](https://issues.apache.org/jira/browse/HDFS-16151) | Improve the parameter comments related to ProtobufRpcEngine2#Server() | Minor | documentation | JiangHua Zhu | JiangHua Zhu | +| [HADOOP-17844](https://issues.apache.org/jira/browse/HADOOP-17844) | Upgrade JSON smart to 2.4.7 | Major | . | Renukaprasad C | Renukaprasad C | +| [HDFS-16177](https://issues.apache.org/jira/browse/HDFS-16177) | Bug fix for Util#receiveFile | Minor | . | tomscut | tomscut | +| [YARN-10814](https://issues.apache.org/jira/browse/YARN-10814) | YARN shouldn't start with empty hadoop.http.authentication.signature.secret.file | Major | . | Benjamin Teke | Tamas Domok | +| [HADOOP-17858](https://issues.apache.org/jira/browse/HADOOP-17858) | Avoid possible class loading deadlock with VerifierNone initialization | Major | . | Viraj Jasani | Viraj Jasani | +| [HADOOP-17869](https://issues.apache.org/jira/browse/HADOOP-17869) | fs.s3a.connection.maximum should be bigger than fs.s3a.threads.max | Major | common | Dongjoon Hyun | Dongjoon Hyun | +| [HADOOP-17886](https://issues.apache.org/jira/browse/HADOOP-17886) | Upgrade ant to 1.10.11 | Major | . | Ahmed Hussein | Ahmed Hussein | +| [HADOOP-17874](https://issues.apache.org/jira/browse/HADOOP-17874) | ExceptionsHandler to add terse/suppressed Exceptions in thread-safe manner | Major | . | Viraj Jasani | Viraj Jasani | +| [HADOOP-15129](https://issues.apache.org/jira/browse/HADOOP-15129) | Datanode caches namenode DNS lookup failure and cannot startup | Minor | ipc | Karthik Palaniappan | Chris Nauroth | +| [HADOOP-17870](https://issues.apache.org/jira/browse/HADOOP-17870) | HTTP Filesystem to qualify paths in open()/getFileStatus() | Minor | fs | VinothKumar Raman | VinothKumar Raman | +| [HADOOP-17899](https://issues.apache.org/jira/browse/HADOOP-17899) | Avoid using implicit dependency on junit-jupiter-api | Major | test | Masatake Iwasaki | Masatake Iwasaki | +| [YARN-10901](https://issues.apache.org/jira/browse/YARN-10901) | Permission checking error on an existing directory in LogAggregationFileController#verifyAndCreateRemoteLogDir | Major | nodemanager | Tamas Domok | Tamas Domok | +| [HADOOP-17804](https://issues.apache.org/jira/browse/HADOOP-17804) | Prometheus metrics only include the last set of labels | Major | common | Adam Binford | Adam Binford | +| [HDFS-16207](https://issues.apache.org/jira/browse/HDFS-16207) | Remove NN logs stack trace for non-existent xattr query | Major | namenode | Ahmed Hussein | Ahmed Hussein | +| [HDFS-16187](https://issues.apache.org/jira/browse/HDFS-16187) | SnapshotDiff behaviour with Xattrs and Acls is not consistent across NN restarts with checkpointing | Major | snapshots | Srinivasu Majeti | Shashikant Banerjee | +| [HDFS-16198](https://issues.apache.org/jira/browse/HDFS-16198) | Short circuit read leaks Slot objects when InvalidToken exception is thrown | Major | . | Eungsop Yoo | Eungsop Yoo | +| [YARN-10870](https://issues.apache.org/jira/browse/YARN-10870) | Missing user filtering check -\> yarn.webapp.filter-entity-list-by-user for RM Scheduler page | Major | yarn | Siddharth Ahuja | Gergely Pollák | +| [HADOOP-17891](https://issues.apache.org/jira/browse/HADOOP-17891) | lz4-java and snappy-java should be excluded from relocation in shaded Hadoop libraries | Major | . | L. C. Hsieh | L. C. Hsieh | +| [HADOOP-17919](https://issues.apache.org/jira/browse/HADOOP-17919) | Fix command line example in Hadoop Cluster Setup documentation | Minor | documentation | Rintaro Ikeda | Rintaro Ikeda | +| [YARN-9606](https://issues.apache.org/jira/browse/YARN-9606) | Set sslfactory for AuthenticatedURL() while creating LogsCLI#webServiceClient | Major | . | Bilwa S T | Bilwa S T | +| [HDFS-16233](https://issues.apache.org/jira/browse/HDFS-16233) | Do not use exception handler to implement copy-on-write for EnumCounters | Major | namenode | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HDFS-16235](https://issues.apache.org/jira/browse/HDFS-16235) | Deadlock in LeaseRenewer for static remove method | Major | hdfs | angerszhu | angerszhu | +| [HADOOP-17940](https://issues.apache.org/jira/browse/HADOOP-17940) | Upgrade Kafka to 2.8.1 | Major | . | Takanobu Asanuma | Takanobu Asanuma | +| [YARN-10970](https://issues.apache.org/jira/browse/YARN-10970) | Standby RM should expose prom endpoint | Major | resourcemanager | Max Xie | Max Xie | +| [HADOOP-17934](https://issues.apache.org/jira/browse/HADOOP-17934) | NullPointerException when no HTTP response set on AbfsRestOperation | Major | fs/azure | Josh Elser | Josh Elser | +| [HDFS-16181](https://issues.apache.org/jira/browse/HDFS-16181) | [SBN Read] Fix metric of RpcRequestCacheMissAmount can't display when tailEditLog form JN | Critical | . | wangzhaohui | wangzhaohui | +| [HADOOP-17922](https://issues.apache.org/jira/browse/HADOOP-17922) | Lookup old S3 encryption configs for JCEKS | Major | fs/s3 | Mehakmeet Singh | Mehakmeet Singh | +| [HADOOP-17925](https://issues.apache.org/jira/browse/HADOOP-17925) | BUILDING.txt should not encourage to activate docs profile on building binary artifacts | Minor | documentation | Rintaro Ikeda | Masatake Iwasaki | +| [HADOOP-16532](https://issues.apache.org/jira/browse/HADOOP-16532) | Fix TestViewFsTrash to use the correct homeDir. | Minor | test, viewfs | Steve Loughran | Xing Lin | +| [HDFS-16268](https://issues.apache.org/jira/browse/HDFS-16268) | Balancer stuck when moving striped blocks due to NPE | Major | balancer & mover, erasure-coding | Leon Gao | Leon Gao | +| [HDFS-16271](https://issues.apache.org/jira/browse/HDFS-16271) | RBF: NullPointerException when setQuota through routers with quota disabled | Major | . | Chengwei Wang | Chengwei Wang | +| [YARN-10976](https://issues.apache.org/jira/browse/YARN-10976) | Fix resource leak due to Files.walk | Minor | . | lujie | lujie | +| [HADOOP-17932](https://issues.apache.org/jira/browse/HADOOP-17932) | Distcp file length comparison have no effect | Major | common, tools, tools/distcp | yinan zhan | yinan zhan | +| [HDFS-16272](https://issues.apache.org/jira/browse/HDFS-16272) | Int overflow in computing safe length during EC block recovery | Critical | 3.1.1 | daimin | daimin | +| [HADOOP-17953](https://issues.apache.org/jira/browse/HADOOP-17953) | S3A: ITestS3AFileContextStatistics test to lookup global or per-bucket configuration for encryption algorithm | Minor | fs/s3 | Mehakmeet Singh | Mehakmeet Singh | +| [HADOOP-17971](https://issues.apache.org/jira/browse/HADOOP-17971) | Exclude IBM Java security classes from being shaded/relocated | Major | build | Nicholas Marion | Nicholas Marion | +| [HDFS-7612](https://issues.apache.org/jira/browse/HDFS-7612) | TestOfflineEditsViewer.testStored() uses incorrect default value for cacheDir | Major | test | Konstantin Shvachko | Michael Kuchenbecker | +| [HDFS-16269](https://issues.apache.org/jira/browse/HDFS-16269) | [Fix] Improve NNThroughputBenchmark#blockReport operation | Major | benchmarks, namenode | JiangHua Zhu | JiangHua Zhu | +| [HADOOP-17945](https://issues.apache.org/jira/browse/HADOOP-17945) | JsonSerialization raises EOFException reading JSON data stored on google GCS | Major | fs | Steve Loughran | Steve Loughran | +| [HDFS-16259](https://issues.apache.org/jira/browse/HDFS-16259) | Catch and re-throw sub-classes of AccessControlException thrown by any permission provider plugins (eg Ranger) | Major | namenode | Stephen O'Donnell | Stephen O'Donnell | +| [HADOOP-17988](https://issues.apache.org/jira/browse/HADOOP-17988) | Disable JIRA plugin for YETUS on Hadoop | Critical | build | Gautham Banasandra | Gautham Banasandra | +| [HDFS-16311](https://issues.apache.org/jira/browse/HDFS-16311) | Metric metadataOperationRate calculation error in DataNodeVolumeMetrics | Major | . | tomscut | tomscut | +| [HADOOP-18002](https://issues.apache.org/jira/browse/HADOOP-18002) | abfs rename idempotency broken -remove recovery | Major | fs/azure | Steve Loughran | Steve Loughran | +| [HDFS-16182](https://issues.apache.org/jira/browse/HDFS-16182) | numOfReplicas is given the wrong value in BlockPlacementPolicyDefault$chooseTarget can cause DataStreamer to fail with Heterogeneous Storage | Major | namanode | Max Xie | Max Xie | +| [HADOOP-17999](https://issues.apache.org/jira/browse/HADOOP-17999) | No-op implementation of setWriteChecksum and setVerifyChecksum in ViewFileSystem | Major | . | Abhishek Das | Abhishek Das | +| [HDFS-16329](https://issues.apache.org/jira/browse/HDFS-16329) | Fix log format for BlockManager | Minor | . | tomscut | tomscut | +| [HDFS-16330](https://issues.apache.org/jira/browse/HDFS-16330) | Fix incorrect placeholder for Exception logs in DiskBalancer | Major | . | Viraj Jasani | Viraj Jasani | +| [HDFS-16328](https://issues.apache.org/jira/browse/HDFS-16328) | Correct disk balancer param desc | Minor | documentation, hdfs | guophilipse | guophilipse | +| [HDFS-16334](https://issues.apache.org/jira/browse/HDFS-16334) | Correct NameNode ACL description | Minor | documentation | guophilipse | guophilipse | +| [HDFS-16343](https://issues.apache.org/jira/browse/HDFS-16343) | Add some debug logs when the dfsUsed are not used during Datanode startup | Major | datanode | Mukul Kumar Singh | Mukul Kumar Singh | +| [YARN-10991](https://issues.apache.org/jira/browse/YARN-10991) | Fix to ignore the grouping "[]" for resourcesStr in parseResourcesString method | Minor | distributed-shell | Ashutosh Gupta | Ashutosh Gupta | +| [HADOOP-17975](https://issues.apache.org/jira/browse/HADOOP-17975) | Fallback to simple auth does not work for a secondary DistributedFileSystem instance | Major | ipc | István Fajth | István Fajth | +| [HDFS-16350](https://issues.apache.org/jira/browse/HDFS-16350) | Datanode start time should be set after RPC server starts successfully | Minor | . | Viraj Jasani | Viraj Jasani | +| [YARN-11007](https://issues.apache.org/jira/browse/YARN-11007) | Correct words in YARN documents | Minor | documentation | guophilipse | guophilipse | +| [YARN-10975](https://issues.apache.org/jira/browse/YARN-10975) | EntityGroupFSTimelineStore#ActiveLogParser parses already processed files | Major | timelineserver | Prabhu Joseph | Ravuri Sushma sree | +| [HDFS-16332](https://issues.apache.org/jira/browse/HDFS-16332) | Expired block token causes slow read due to missing handling in sasl handshake | Major | datanode, dfs, dfsclient | Shinya Yoshida | Shinya Yoshida | +| [HDFS-16293](https://issues.apache.org/jira/browse/HDFS-16293) | Client sleeps and holds 'dataQueue' when DataNodes are congested | Major | hdfs-client | Yuanxin Zhu | Yuanxin Zhu | +| [YARN-9063](https://issues.apache.org/jira/browse/YARN-9063) | ATS 1.5 fails to start if RollingLevelDb files are corrupt or missing | Major | timelineserver, timelineservice | Tarun Parimi | Ashutosh Gupta | +| [HDFS-16333](https://issues.apache.org/jira/browse/HDFS-16333) | fix balancer bug when transfer an EC block | Major | balancer & mover, erasure-coding | qinyuren | qinyuren | +| [YARN-11020](https://issues.apache.org/jira/browse/YARN-11020) | [UI2] No container is found for an application attempt with a single AM container | Major | yarn-ui-v2 | Andras Gyori | Andras Gyori | +| [HDFS-16373](https://issues.apache.org/jira/browse/HDFS-16373) | Fix MiniDFSCluster restart in case of multiple namenodes | Major | . | Ayush Saxena | Ayush Saxena | +| [HADOOP-18048](https://issues.apache.org/jira/browse/HADOOP-18048) | [branch-3.3] Dockerfile\_aarch64 build fails with fatal error: Python.h: No such file or directory | Major | . | Siyao Meng | Siyao Meng | +| [HDFS-16377](https://issues.apache.org/jira/browse/HDFS-16377) | Should CheckNotNull before access FsDatasetSpi | Major | . | tomscut | tomscut | +| [YARN-6862](https://issues.apache.org/jira/browse/YARN-6862) | Nodemanager resource usage metrics sometimes are negative | Major | nodemanager | YunFan Zhou | Benjamin Teke | +| [HADOOP-13500](https://issues.apache.org/jira/browse/HADOOP-13500) | Synchronizing iteration of Configuration properties object | Major | conf | Jason Darrell Lowe | Dhananjay Badaya | +| [YARN-10178](https://issues.apache.org/jira/browse/YARN-10178) | Global Scheduler async thread crash caused by 'Comparison method violates its general contract | Major | capacity scheduler | tuyu | Andras Gyori | +| [YARN-11053](https://issues.apache.org/jira/browse/YARN-11053) | AuxService should not use class name as default system classes | Major | auxservices | Cheng Pan | Cheng Pan | +| [HDFS-16395](https://issues.apache.org/jira/browse/HDFS-16395) | Remove useless NNThroughputBenchmark#dummyActionNoSynch() | Major | benchmarks, namenode | JiangHua Zhu | JiangHua Zhu | +| [HADOOP-18045](https://issues.apache.org/jira/browse/HADOOP-18045) | Disable TestDynamometerInfra | Major | test | Akira Ajisaka | Akira Ajisaka | +| [HDFS-14099](https://issues.apache.org/jira/browse/HDFS-14099) | Unknown frame descriptor when decompressing multiple frames in ZStandardDecompressor | Major | . | xuzq | xuzq | +| [HADOOP-18063](https://issues.apache.org/jira/browse/HADOOP-18063) | Remove unused import AbstractJavaKeyStoreProvider in Shell class | Minor | . | JiangHua Zhu | JiangHua Zhu | +| [HDFS-16409](https://issues.apache.org/jira/browse/HDFS-16409) | Fix typo: testHasExeceptionsReturnsCorrectValue -\> testHasExceptionsReturnsCorrectValue | Trivial | . | Ashutosh Gupta | Ashutosh Gupta | +| [HDFS-16408](https://issues.apache.org/jira/browse/HDFS-16408) | Ensure LeaseRecheckIntervalMs is greater than zero | Major | namenode | Jingxuan Fu | Jingxuan Fu | +| [HDFS-16410](https://issues.apache.org/jira/browse/HDFS-16410) | Insecure Xml parsing in OfflineEditsXmlLoader | Minor | . | Ashutosh Gupta | Ashutosh Gupta | +| [HDFS-16420](https://issues.apache.org/jira/browse/HDFS-16420) | Avoid deleting unique data blocks when deleting redundancy striped blocks | Critical | ec, erasure-coding | qinyuren | Jackson Wang | +| [YARN-10561](https://issues.apache.org/jira/browse/YARN-10561) | Upgrade node.js to 12.22.1 and yarn to 1.22.5 in YARN application catalog webapp | Critical | webapp | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-18096](https://issues.apache.org/jira/browse/HADOOP-18096) | Distcp: Sync moves filtered file to home directory rather than deleting | Critical | . | Ayush Saxena | Ayush Saxena | + + +### TESTS: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [MAPREDUCE-7342](https://issues.apache.org/jira/browse/MAPREDUCE-7342) | Stop RMService in TestClientRedirect.testRedirect() | Minor | . | Zhengxi Li | Zhengxi Li | +| [MAPREDUCE-7311](https://issues.apache.org/jira/browse/MAPREDUCE-7311) | Fix non-idempotent test in TestTaskProgressReporter | Minor | . | Zhengxi Li | Zhengxi Li | +| [HADOOP-17936](https://issues.apache.org/jira/browse/HADOOP-17936) | TestLocalFSCopyFromLocal.testDestinationFileIsToParentDirectory failure after reverting HADOOP-16878 | Major | . | Chao Sun | Chao Sun | +| [HDFS-15862](https://issues.apache.org/jira/browse/HDFS-15862) | Make TestViewfsWithNfs3.testNfsRenameSingleNN() idempotent | Minor | nfs | Zhengxi Li | Zhengxi Li | + + +### SUB-TASKS: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [YARN-10337](https://issues.apache.org/jira/browse/YARN-10337) | TestRMHATimelineCollectors fails on hadoop trunk | Major | test, yarn | Ahmed Hussein | Bilwa S T | +| [HDFS-15457](https://issues.apache.org/jira/browse/HDFS-15457) | TestFsDatasetImpl fails intermittently | Major | hdfs | Ahmed Hussein | Ahmed Hussein | +| [HADOOP-17424](https://issues.apache.org/jira/browse/HADOOP-17424) | Replace HTrace with No-Op tracer | Major | . | Siyao Meng | Siyao Meng | +| [HADOOP-17705](https://issues.apache.org/jira/browse/HADOOP-17705) | S3A to add option fs.s3a.endpoint.region to set AWS region | Major | fs/s3 | Mehakmeet Singh | Mehakmeet Singh | +| [HADOOP-17670](https://issues.apache.org/jira/browse/HADOOP-17670) | S3AFS and ABFS to log IOStats at DEBUG mode or optionally at INFO level in close() | Minor | fs/azure, fs/s3 | Mehakmeet Singh | Mehakmeet Singh | +| [HADOOP-17511](https://issues.apache.org/jira/browse/HADOOP-17511) | Add an Audit plugin point for S3A auditing/context | Major | . | Steve Loughran | Steve Loughran | +| [HADOOP-17470](https://issues.apache.org/jira/browse/HADOOP-17470) | Collect more S3A IOStatistics | Major | fs/s3 | Steve Loughran | Steve Loughran | +| [HADOOP-17735](https://issues.apache.org/jira/browse/HADOOP-17735) | Upgrade aws-java-sdk to 1.11.1026 | Major | build, fs/s3 | Steve Loughran | Steve Loughran | +| [HADOOP-17547](https://issues.apache.org/jira/browse/HADOOP-17547) | Magic committer to downgrade abort in cleanup if list uploads fails with access denied | Major | fs/s3 | Steve Loughran | Bogdan Stolojan | +| [HADOOP-17771](https://issues.apache.org/jira/browse/HADOOP-17771) | S3AFS creation fails "Unable to find a region via the region provider chain." | Blocker | fs/s3 | Steve Loughran | Steve Loughran | +| [HDFS-15659](https://issues.apache.org/jira/browse/HDFS-15659) | Set dfs.namenode.redundancy.considerLoad to false in MiniDFSCluster | Major | test | Akira Ajisaka | Ahmed Hussein | +| [HADOOP-17774](https://issues.apache.org/jira/browse/HADOOP-17774) | bytesRead FS statistic showing twice the correct value in S3A | Major | fs/s3 | Mehakmeet Singh | Mehakmeet Singh | +| [HADOOP-17290](https://issues.apache.org/jira/browse/HADOOP-17290) | ABFS: Add Identifiers to Client Request Header | Major | fs/azure | Sumangala Patki | Sumangala Patki | +| [HADOOP-17250](https://issues.apache.org/jira/browse/HADOOP-17250) | ABFS: Random read perf improvement | Major | fs/azure | Sneha Vijayarajan | Mukund Thakur | +| [HADOOP-17596](https://issues.apache.org/jira/browse/HADOOP-17596) | ABFS: Change default Readahead Queue Depth from num(processors) to const | Major | fs/azure | Sumangala Patki | Sumangala Patki | +| [HADOOP-17715](https://issues.apache.org/jira/browse/HADOOP-17715) | ABFS: Append blob tests with non HNS accounts fail | Minor | . | Sneha Varma | Sneha Varma | +| [HADOOP-17714](https://issues.apache.org/jira/browse/HADOOP-17714) | ABFS: testBlobBackCompatibility, testRandomRead & WasbAbfsCompatibility tests fail when triggered with default configs | Minor | test | Sneha Varma | Sneha Varma | +| [HDFS-16140](https://issues.apache.org/jira/browse/HDFS-16140) | TestBootstrapAliasmap fails by BindException | Major | test | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-13887](https://issues.apache.org/jira/browse/HADOOP-13887) | Encrypt S3A data client-side with AWS SDK (S3-CSE) | Minor | fs/s3 | Jeeyoung Kim | Mehakmeet Singh | +| [HADOOP-17458](https://issues.apache.org/jira/browse/HADOOP-17458) | S3A to treat "SdkClientException: Data read has a different length than the expected" as EOFException | Minor | fs/s3 | Steve Loughran | Bogdan Stolojan | +| [HADOOP-17628](https://issues.apache.org/jira/browse/HADOOP-17628) | Distcp contract test is really slow with ABFS and S3A; timing out | Minor | fs/azure, fs/s3, test, tools/distcp | Bilahari T H | Steve Loughran | +| [HADOOP-17822](https://issues.apache.org/jira/browse/HADOOP-17822) | fs.s3a.acl.default not working after S3A Audit feature added | Major | fs/s3 | Steve Loughran | Steve Loughran | +| [HADOOP-17139](https://issues.apache.org/jira/browse/HADOOP-17139) | Re-enable optimized copyFromLocal implementation in S3AFileSystem | Minor | fs/s3 | Sahil Takiar | Bogdan Stolojan | +| [HADOOP-17823](https://issues.apache.org/jira/browse/HADOOP-17823) | S3A Tests to skip if S3Guard and S3-CSE are enabled. | Major | build, fs/s3 | Mehakmeet Singh | Mehakmeet Singh | +| [HDFS-16184](https://issues.apache.org/jira/browse/HDFS-16184) | De-flake TestBlockScanner#testSkipRecentAccessFile | Major | . | Viraj Jasani | Viraj Jasani | +| [HADOOP-17677](https://issues.apache.org/jira/browse/HADOOP-17677) | Distcp is unable to determine region with S3 PrivateLink endpoints | Major | fs/s3, tools/distcp | KJ | | +| [HDFS-16192](https://issues.apache.org/jira/browse/HDFS-16192) | ViewDistributedFileSystem#rename wrongly using src in the place of dst. | Major | . | Uma Maheswara Rao G | Uma Maheswara Rao G | +| [HADOOP-17156](https://issues.apache.org/jira/browse/HADOOP-17156) | Clear abfs readahead requests on stream close | Major | fs/azure | Rajesh Balamohan | Mukund Thakur | +| [HADOOP-17618](https://issues.apache.org/jira/browse/HADOOP-17618) | ABFS: Partially obfuscate SAS object IDs in Logs | Major | fs/azure | Sumangala Patki | Sumangala Patki | +| [HADOOP-17894](https://issues.apache.org/jira/browse/HADOOP-17894) | CredentialProviderFactory.getProviders() recursion loading JCEKS file from s3a | Major | conf, fs/s3 | Steve Loughran | Steve Loughran | +| [HADOOP-17126](https://issues.apache.org/jira/browse/HADOOP-17126) | implement non-guava Precondition checkNotNull | Major | . | Ahmed Hussein | Ahmed Hussein | +| [HADOOP-17195](https://issues.apache.org/jira/browse/HADOOP-17195) | Intermittent OutOfMemory error while performing hdfs CopyFromLocal to abfs | Major | fs/azure | Mehakmeet Singh | Mehakmeet Singh | +| [HADOOP-17929](https://issues.apache.org/jira/browse/HADOOP-17929) | implement non-guava Precondition checkArgument | Major | . | Ahmed Hussein | Ahmed Hussein | +| [HADOOP-17198](https://issues.apache.org/jira/browse/HADOOP-17198) | Support S3 Access Points | Major | fs/s3 | Steve Loughran | Bogdan Stolojan | +| [HADOOP-17871](https://issues.apache.org/jira/browse/HADOOP-17871) | S3A CSE: minor tuning | Minor | fs/s3 | Steve Loughran | Mehakmeet Singh | +| [HADOOP-17947](https://issues.apache.org/jira/browse/HADOOP-17947) | Provide alternative to Guava VisibleForTesting | Major | . | Viraj Jasani | Viraj Jasani | +| [HADOOP-17930](https://issues.apache.org/jira/browse/HADOOP-17930) | implement non-guava Precondition checkState | Major | . | Ahmed Hussein | Ahmed Hussein | +| [HADOOP-17374](https://issues.apache.org/jira/browse/HADOOP-17374) | AliyunOSS: support ListObjectsV2 | Major | fs/oss | wujinhu | wujinhu | +| [HADOOP-17863](https://issues.apache.org/jira/browse/HADOOP-17863) | ABFS: Fix compiler deprecation warning in TextFileBasedIdentityHandler | Minor | fs/azure | Sumangala Patki | Sumangala Patki | +| [HADOOP-17928](https://issues.apache.org/jira/browse/HADOOP-17928) | s3a: set fs.s3a.downgrade.syncable.exceptions = true by default | Major | fs/s3 | Steve Loughran | Steve Loughran | +| [HDFS-16336](https://issues.apache.org/jira/browse/HDFS-16336) | De-flake TestRollingUpgrade#testRollback | Minor | hdfs, test | Kevin Wikant | Viraj Jasani | +| [HDFS-16171](https://issues.apache.org/jira/browse/HDFS-16171) | De-flake testDecommissionStatus | Major | . | Viraj Jasani | Viraj Jasani | +| [HADOOP-17226](https://issues.apache.org/jira/browse/HADOOP-17226) | Failure of ITestAssumeRole.testRestrictedCommitActions | Minor | fs/s3, test | Steve Loughran | Steve Loughran | +| [HADOOP-14334](https://issues.apache.org/jira/browse/HADOOP-14334) | S3 SSEC tests to downgrade when running against a mandatory encryption object store | Minor | fs/s3, test | Steve Loughran | Monthon Klongklaew | +| [HADOOP-16223](https://issues.apache.org/jira/browse/HADOOP-16223) | remove misleading fs.s3a.delegation.tokens.enabled prompt | Minor | fs/s3 | Steve Loughran | | + + +### OTHER: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HDFS-16078](https://issues.apache.org/jira/browse/HDFS-16078) | Remove unused parameters for DatanodeManager.handleLifeline() | Minor | . | tomscut | tomscut | +| [HDFS-16079](https://issues.apache.org/jira/browse/HDFS-16079) | Improve the block state change log | Minor | . | tomscut | tomscut | +| [HDFS-16089](https://issues.apache.org/jira/browse/HDFS-16089) | EC: Add metric EcReconstructionValidateTimeMillis for StripedBlockReconstructor | Minor | . | tomscut | tomscut | +| [HDFS-16298](https://issues.apache.org/jira/browse/HDFS-16298) | Improve error msg for BlockMissingException | Minor | . | tomscut | tomscut | +| [HDFS-16312](https://issues.apache.org/jira/browse/HDFS-16312) | Fix typo for DataNodeVolumeMetrics and ProfilingFileIoEvents | Minor | . | tomscut | tomscut | +| [HADOOP-18005](https://issues.apache.org/jira/browse/HADOOP-18005) | Correct log format for LdapGroupsMapping | Minor | . | tomscut | tomscut | +| [HDFS-16319](https://issues.apache.org/jira/browse/HDFS-16319) | Add metrics doc for ReadLockLongHoldCount and WriteLockLongHoldCount | Minor | . | tomscut | tomscut | +| [HDFS-16326](https://issues.apache.org/jira/browse/HDFS-16326) | Simplify the code for DiskBalancer | Minor | . | tomscut | tomscut | +| [HDFS-16335](https://issues.apache.org/jira/browse/HDFS-16335) | Fix HDFSCommands.md | Minor | . | tomscut | tomscut | +| [HDFS-16339](https://issues.apache.org/jira/browse/HDFS-16339) | Show the threshold when mover threads quota is exceeded | Minor | . | tomscut | tomscut | +| [YARN-10820](https://issues.apache.org/jira/browse/YARN-10820) | Make GetClusterNodesRequestPBImpl thread safe | Major | client | Prabhu Joseph | SwathiChandrashekar | +| [HADOOP-17808](https://issues.apache.org/jira/browse/HADOOP-17808) | ipc.Client not setting interrupt flag after catching InterruptedException | Minor | . | Viraj Jasani | Viraj Jasani | +| [HADOOP-17834](https://issues.apache.org/jira/browse/HADOOP-17834) | Bump aliyun-sdk-oss to 3.13.0 | Major | . | Siyao Meng | Siyao Meng | +| [HADOOP-17950](https://issues.apache.org/jira/browse/HADOOP-17950) | Provide replacement for deprecated APIs of commons-io IOUtils | Major | . | Viraj Jasani | Viraj Jasani | +| [HADOOP-17955](https://issues.apache.org/jira/browse/HADOOP-17955) | Bump netty to the latest 4.1.68 | Major | . | Takanobu Asanuma | Takanobu Asanuma | +| [HADOOP-17946](https://issues.apache.org/jira/browse/HADOOP-17946) | Update commons-lang to latest 3.x | Minor | . | Sean Busbey | Renukaprasad C | +| [HDFS-16323](https://issues.apache.org/jira/browse/HDFS-16323) | DatanodeHttpServer doesn't require handler state map while retrieving filter handlers | Minor | . | Viraj Jasani | Viraj Jasani | +| [HADOOP-13464](https://issues.apache.org/jira/browse/HADOOP-13464) | update GSON to 2.7+ | Minor | build | Sean Busbey | Igor Dvorzhak | +| [HADOOP-18061](https://issues.apache.org/jira/browse/HADOOP-18061) | Update the year to 2022 | Major | . | Ayush Saxena | Ayush Saxena | + + diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.2/RELEASENOTES.3.3.2.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.2/RELEASENOTES.3.3.2.md new file mode 100644 index 0000000000000..9948d8ff3222c --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.2/RELEASENOTES.3.3.2.md @@ -0,0 +1,93 @@ + + +# Apache Hadoop 3.3.2 Release Notes + +These release notes cover new developer and user-facing incompatibilities, important issues, features, and major improvements. + + +--- + +* [HDFS-15288](https://issues.apache.org/jira/browse/HDFS-15288) | *Major* | **Add Available Space Rack Fault Tolerant BPP** + +Added a new BlockPlacementPolicy: "AvailableSpaceRackFaultTolerantBlockPlacementPolicy" which uses the same optimization logic as the AvailableSpaceBlockPlacementPolicy along with spreading the replicas across maximum number of racks, similar to BlockPlacementPolicyRackFaultTolerant. +The BPP can be configured by setting the blockplacement policy class as org.apache.hadoop.hdfs.server.blockmanagement.AvailableSpaceRackFaultTolerantBlockPlacementPolicy + + +--- + +* [HADOOP-17424](https://issues.apache.org/jira/browse/HADOOP-17424) | *Major* | **Replace HTrace with No-Op tracer** + +Dependency on HTrace and TraceAdmin protocol/utility were removed. Tracing functionality is no-op until alternative tracer implementation is added. + + +--- + +* [HDFS-15814](https://issues.apache.org/jira/browse/HDFS-15814) | *Major* | **Make some parameters configurable for DataNodeDiskMetrics** + +**WARNING: No release note provided for this change.** + + +--- + +* [YARN-10820](https://issues.apache.org/jira/browse/YARN-10820) | *Major* | **Make GetClusterNodesRequestPBImpl thread safe** + +Added syncronization so that the "yarn node list" command does not fail intermittently + + +--- + +* [HADOOP-13887](https://issues.apache.org/jira/browse/HADOOP-13887) | *Minor* | **Encrypt S3A data client-side with AWS SDK (S3-CSE)** + +Adds support for client side encryption in AWS S3, +with keys managed by AWS-KMS. + +Read the documentation in encryption.md very, very carefully before +use and consider it unstable. + +S3-CSE is enabled in the existing configuration option +"fs.s3a.server-side-encryption-algorithm": + +fs.s3a.server-side-encryption-algorithm=CSE-KMS +fs.s3a.server-side-encryption.key=\ + +You cannot enable CSE and SSE in the same client, although +you can still enable a default SSE option in the S3 console. + +\* Not compatible with S3Guard. +\* Filesystem list/get status operations subtract 16 bytes from the length + of all files \>= 16 bytes long to compensate for the padding which CSE + adds. +\* The SDK always warns about the specific algorithm chosen being + deprecated. It is critical to use this algorithm for ranged + GET requests to work (i.e. random IO). Ignore. +\* Unencrypted files CANNOT BE READ. + The entire bucket SHOULD be encrypted with S3-CSE. +\* Uploading files may be a bit slower as blocks are now + written sequentially. +\* The Multipart Upload API is disabled when S3-CSE is active. + + +--- + +* [YARN-8234](https://issues.apache.org/jira/browse/YARN-8234) | *Critical* | **Improve RM system metrics publisher's performance by pushing events to timeline server in batch** + +When Timeline Service V1 or V1.5 is used, if "yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.enable-batch" is set to true, ResourceManager sends timeline events in batch. The default value is false. If this functionality is enabled, the maximum number that events published in batch is configured by "yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.batch-size". The default value is 1000. The interval of publishing events can be configured by "yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.interval-seconds". By default, it is set to 60 seconds. + + + diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java index 3748eed12246b..b3487ef309fc9 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java @@ -38,6 +38,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.ConcurrentModificationException; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -45,6 +46,7 @@ import java.util.Properties; import java.util.Random; import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.regex.Pattern; import static java.util.concurrent.TimeUnit.*; @@ -444,6 +446,17 @@ public void testVariableSubstitution() throws IOException { assertTrue(mock.getInt("my.int", -1) == 42); } + /** + * Checks if variable substitution is accessible via a public API. + */ + @Test + public void testCommonVariableSubstitution() { + conf.set("intvar", String.valueOf(42)); + String intVar = conf.substituteCommonVariables("${intvar}"); + + assertEquals("42", intVar); + } + @Test public void testEnvDefault() throws IOException { Configuration mock = Mockito.spy(conf); @@ -2676,4 +2689,31 @@ private static Configuration checkCDATA(byte[] bytes) { assertEquals(" prefix >cdata\nsuffix ", conf.get("cdata-whitespace")); return conf; } + + @Test + public void testConcurrentModificationDuringIteration() throws InterruptedException { + Configuration configuration = new Configuration(); + new Thread(() -> { + while (true) { + configuration.set(String.valueOf(Math.random()), String.valueOf(Math.random())); + } + }).start(); + + AtomicBoolean exceptionOccurred = new AtomicBoolean(false); + + new Thread(() -> { + while (true) { + try { + configuration.iterator(); + } catch (final ConcurrentModificationException e) { + exceptionOccurred.set(true); + break; + } + } + }).start(); + + Thread.sleep(1000); //give enough time for threads to run + + assertFalse("ConcurrentModificationException occurred", exceptionOccurred.get()); + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java index 3a88b7352f1a6..c884e223365c9 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.fs; +import static org.apache.hadoop.test.LambdaTestUtils.intercept; import static org.apache.hadoop.test.PlatformAssumptions.assumeNotWindows; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; @@ -41,27 +42,31 @@ import java.net.URL; import java.net.UnknownHostException; import java.nio.charset.StandardCharsets; -import java.nio.file.FileSystems; import java.nio.file.Files; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.jar.Attributes; import java.util.jar.JarFile; import java.util.jar.Manifest; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.test.LambdaTestUtils; import org.apache.hadoop.util.StringUtils; import org.apache.tools.tar.TarEntry; import org.apache.tools.tar.TarOutputStream; + +import org.assertj.core.api.Assertions; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -157,13 +162,12 @@ public void setup() throws IOException { FileUtils.forceMkdir(dir1); FileUtils.forceMkdir(dir2); - new File(del, FILE).createNewFile(); - File tmpFile = new File(tmp, FILE); - tmpFile.createNewFile(); + Verify.createNewFile(new File(del, FILE)); + File tmpFile = Verify.createNewFile(new File(tmp, FILE)); // create files - new File(dir1, FILE).createNewFile(); - new File(dir2, FILE).createNewFile(); + Verify.createNewFile(new File(dir1, FILE)); + Verify.createNewFile(new File(dir2, FILE)); // create a symlink to file File link = new File(del, LINK); @@ -172,7 +176,7 @@ public void setup() throws IOException { // create a symlink to dir File linkDir = new File(del, "tmpDir"); FileUtil.symLink(tmp.toString(), linkDir.toString()); - Assert.assertEquals(5, del.listFiles().length); + Assert.assertEquals(5, Objects.requireNonNull(del.listFiles()).length); // create files in partitioned directories createFile(partitioned, "part-r-00000", "foo"); @@ -199,13 +203,9 @@ public void tearDown() throws IOException { private File createFile(File directory, String name, String contents) throws IOException { File newFile = new File(directory, name); - PrintWriter pw = new PrintWriter(newFile); - try { + try (PrintWriter pw = new PrintWriter(newFile)) { pw.println(contents); } - finally { - pw.close(); - } return newFile; } @@ -217,11 +217,11 @@ public void testListFiles() throws IOException { //Test existing directory with no files case File newDir = new File(tmp.getPath(),"test"); - newDir.mkdir(); + Verify.mkdir(newDir); Assert.assertTrue("Failed to create test dir", newDir.exists()); files = FileUtil.listFiles(newDir); Assert.assertEquals(0, files.length); - newDir.delete(); + assertTrue(newDir.delete()); Assert.assertFalse("Failed to delete test dir", newDir.exists()); //Test non-existing directory case, this throws @@ -243,11 +243,11 @@ public void testListAPI() throws IOException { //Test existing directory with no files case File newDir = new File(tmp.getPath(),"test"); - newDir.mkdir(); + Verify.mkdir(newDir); Assert.assertTrue("Failed to create test dir", newDir.exists()); files = FileUtil.list(newDir); Assert.assertEquals("New directory unexpectedly contains files", 0, files.length); - newDir.delete(); + assertTrue(newDir.delete()); Assert.assertFalse("Failed to delete test dir", newDir.exists()); //Test non-existing directory case, this throws @@ -265,7 +265,7 @@ public void testListAPI() throws IOException { public void testFullyDelete() throws IOException { boolean ret = FileUtil.fullyDelete(del); Assert.assertTrue(ret); - Assert.assertFalse(del.exists()); + Verify.notExists(del); validateTmpDir(); } @@ -278,13 +278,13 @@ public void testFullyDelete() throws IOException { @Test (timeout = 30000) public void testFullyDeleteSymlinks() throws IOException { File link = new File(del, LINK); - Assert.assertEquals(5, del.list().length); + assertDelListLength(5); // Since tmpDir is symlink to tmp, fullyDelete(tmpDir) should not // delete contents of tmp. See setupDirs for details. boolean ret = FileUtil.fullyDelete(link); Assert.assertTrue(ret); - Assert.assertFalse(link.exists()); - Assert.assertEquals(4, del.list().length); + Verify.notExists(link); + assertDelListLength(4); validateTmpDir(); File linkDir = new File(del, "tmpDir"); @@ -292,8 +292,8 @@ public void testFullyDeleteSymlinks() throws IOException { // delete contents of tmp. See setupDirs for details. ret = FileUtil.fullyDelete(linkDir); Assert.assertTrue(ret); - Assert.assertFalse(linkDir.exists()); - Assert.assertEquals(3, del.list().length); + Verify.notExists(linkDir); + assertDelListLength(3); validateTmpDir(); } @@ -309,16 +309,16 @@ public void testFullyDeleteDanglingSymlinks() throws IOException { // to make y as a dangling link to file tmp/x boolean ret = FileUtil.fullyDelete(tmp); Assert.assertTrue(ret); - Assert.assertFalse(tmp.exists()); + Verify.notExists(tmp); // dangling symlink to file File link = new File(del, LINK); - Assert.assertEquals(5, del.list().length); + assertDelListLength(5); // Even though 'y' is dangling symlink to file tmp/x, fullyDelete(y) // should delete 'y' properly. ret = FileUtil.fullyDelete(link); Assert.assertTrue(ret); - Assert.assertEquals(4, del.list().length); + assertDelListLength(4); // dangling symlink to directory File linkDir = new File(del, "tmpDir"); @@ -326,22 +326,22 @@ public void testFullyDeleteDanglingSymlinks() throws IOException { // delete tmpDir properly. ret = FileUtil.fullyDelete(linkDir); Assert.assertTrue(ret); - Assert.assertEquals(3, del.list().length); + assertDelListLength(3); } @Test (timeout = 30000) public void testFullyDeleteContents() throws IOException { boolean ret = FileUtil.fullyDeleteContents(del); Assert.assertTrue(ret); - Assert.assertTrue(del.exists()); - Assert.assertEquals(0, del.listFiles().length); + Verify.exists(del); + Assert.assertEquals(0, Objects.requireNonNull(del.listFiles()).length); validateTmpDir(); } private void validateTmpDir() { - Assert.assertTrue(tmp.exists()); - Assert.assertEquals(1, tmp.listFiles().length); - Assert.assertTrue(new File(tmp, FILE).exists()); + Verify.exists(tmp); + Assert.assertEquals(1, Objects.requireNonNull(tmp.listFiles()).length); + Verify.exists(new File(tmp, FILE)); } /** @@ -365,15 +365,15 @@ private void validateTmpDir() { * @throws IOException */ private void setupDirsAndNonWritablePermissions() throws IOException { - new MyFile(del, FILE_1_NAME).createNewFile(); + Verify.createNewFile(new MyFile(del, FILE_1_NAME)); // "file1" is non-deletable by default, see MyFile.delete(). - xSubDir.mkdirs(); - file2.createNewFile(); + Verify.mkdirs(xSubDir); + Verify.createNewFile(file2); - xSubSubDir.mkdirs(); - file22.createNewFile(); + Verify.mkdirs(xSubSubDir); + Verify.createNewFile(file22); revokePermissions(file22); revokePermissions(xSubSubDir); @@ -381,8 +381,8 @@ private void setupDirsAndNonWritablePermissions() throws IOException { revokePermissions(file2); revokePermissions(xSubDir); - ySubDir.mkdirs(); - file3.createNewFile(); + Verify.mkdirs(ySubDir); + Verify.createNewFile(file3); File tmpFile = new File(tmp, FILE); tmpFile.createNewFile(); @@ -461,8 +461,8 @@ public void testFailFullyDeleteDirSymlinks() throws IOException { boolean ret = FileUtil.fullyDelete(linkDir); // fail symlink deletion Assert.assertFalse(ret); - Assert.assertTrue(linkDir.exists()); - Assert.assertEquals(5, del.list().length); + Verify.exists(linkDir); + assertDelListLength(5); // tmp dir should exist validateTmpDir(); // simulate disk recovers and turns good @@ -470,12 +470,94 @@ public void testFailFullyDeleteDirSymlinks() throws IOException { ret = FileUtil.fullyDelete(linkDir); // success symlink deletion Assert.assertTrue(ret); - Assert.assertFalse(linkDir.exists()); - Assert.assertEquals(4, del.list().length); + Verify.notExists(linkDir); + assertDelListLength(4); // tmp dir should exist validateTmpDir(); } + /** + * Asserts if the {@link TestFileUtil#del} meets the given expected length. + * + * @param expectedLength The expected length of the {@link TestFileUtil#del}. + */ + private void assertDelListLength(int expectedLength) { + Assertions.assertThat(del.list()).describedAs("del list").isNotNull().hasSize(expectedLength); + } + + /** + * Helper class to perform {@link File} operation and also verify them. + */ + public static class Verify { + /** + * Invokes {@link File#createNewFile()} on the given {@link File} instance. + * + * @param file The file to call {@link File#createNewFile()} on. + * @return The result of {@link File#createNewFile()}. + * @throws IOException As per {@link File#createNewFile()}. + */ + public static File createNewFile(File file) throws IOException { + assertTrue("Unable to create new file " + file, file.createNewFile()); + return file; + } + + /** + * Invokes {@link File#mkdir()} on the given {@link File} instance. + * + * @param file The file to call {@link File#mkdir()} on. + * @return The result of {@link File#mkdir()}. + */ + public static File mkdir(File file) { + assertTrue("Unable to mkdir for " + file, file.mkdir()); + return file; + } + + /** + * Invokes {@link File#mkdirs()} on the given {@link File} instance. + * + * @param file The file to call {@link File#mkdirs()} on. + * @return The result of {@link File#mkdirs()}. + */ + public static File mkdirs(File file) { + assertTrue("Unable to mkdirs for " + file, file.mkdirs()); + return file; + } + + /** + * Invokes {@link File#delete()} on the given {@link File} instance. + * + * @param file The file to call {@link File#delete()} on. + * @return The result of {@link File#delete()}. + */ + public static File delete(File file) { + assertTrue("Unable to delete " + file, file.delete()); + return file; + } + + /** + * Invokes {@link File#exists()} on the given {@link File} instance. + * + * @param file The file to call {@link File#exists()} on. + * @return The result of {@link File#exists()}. + */ + public static File exists(File file) { + assertTrue("Expected file " + file + " doesn't exist", file.exists()); + return file; + } + + /** + * Invokes {@link File#exists()} on the given {@link File} instance to check if the + * {@link File} doesn't exists. + * + * @param file The file to call {@link File#exists()} on. + * @return The negation of the result of {@link File#exists()}. + */ + public static File notExists(File file) { + assertFalse("Expected file " + file + " must not exist", file.exists()); + return file; + } + } + /** * Extend {@link File}. Same as {@link File} except for two things: (1) This * treats file1Name as a very special file which is not delete-able @@ -608,14 +690,13 @@ public void testGetDU() throws Exception { FileUtil.chmod(partitioned.getAbsolutePath(), "0777", true/*recursive*/); } } - + @Test (timeout = 30000) - public void testUnTar() throws IOException { + public void testUnTar() throws Exception { // make a simple tar: final File simpleTar = new File(del, FILE); - OutputStream os = new FileOutputStream(simpleTar); - TarOutputStream tos = new TarOutputStream(os); - try { + OutputStream os = new FileOutputStream(simpleTar); + try (TarOutputStream tos = new TarOutputStream(os)) { TarEntry te = new TarEntry("/bar/foo"); byte[] data = "some-content".getBytes("UTF-8"); te.setSize(data.length); @@ -624,55 +705,42 @@ public void testUnTar() throws IOException { tos.closeEntry(); tos.flush(); tos.finish(); - } finally { - tos.close(); } // successfully untar it into an existing dir: FileUtil.unTar(simpleTar, tmp); // check result: - assertTrue(new File(tmp, "/bar/foo").exists()); + Verify.exists(new File(tmp, "/bar/foo")); assertEquals(12, new File(tmp, "/bar/foo").length()); - - final File regularFile = new File(tmp, "QuickBrownFoxJumpsOverTheLazyDog"); - regularFile.createNewFile(); - assertTrue(regularFile.exists()); - try { - FileUtil.unTar(simpleTar, regularFile); - assertTrue("An IOException expected.", false); - } catch (IOException ioe) { - // okay - } + + final File regularFile = + Verify.createNewFile(new File(tmp, "QuickBrownFoxJumpsOverTheLazyDog")); + LambdaTestUtils.intercept(IOException.class, () -> FileUtil.unTar(simpleTar, regularFile)); } @Test (timeout = 30000) public void testReplaceFile() throws IOException { - final File srcFile = new File(tmp, "src"); - // src exists, and target does not exist: - srcFile.createNewFile(); - assertTrue(srcFile.exists()); + final File srcFile = Verify.createNewFile(new File(tmp, "src")); final File targetFile = new File(tmp, "target"); - assertTrue(!targetFile.exists()); + Verify.notExists(targetFile); FileUtil.replaceFile(srcFile, targetFile); - assertTrue(!srcFile.exists()); - assertTrue(targetFile.exists()); + Verify.notExists(srcFile); + Verify.exists(targetFile); // src exists and target is a regular file: - srcFile.createNewFile(); - assertTrue(srcFile.exists()); + Verify.createNewFile(srcFile); + Verify.exists(srcFile); FileUtil.replaceFile(srcFile, targetFile); - assertTrue(!srcFile.exists()); - assertTrue(targetFile.exists()); + Verify.notExists(srcFile); + Verify.exists(targetFile); // src exists, and target is a non-empty directory: - srcFile.createNewFile(); - assertTrue(srcFile.exists()); - targetFile.delete(); - targetFile.mkdirs(); - File obstacle = new File(targetFile, "obstacle"); - obstacle.createNewFile(); - assertTrue(obstacle.exists()); + Verify.createNewFile(srcFile); + Verify.exists(srcFile); + Verify.delete(targetFile); + Verify.mkdirs(targetFile); + File obstacle = Verify.createNewFile(new File(targetFile, "obstacle")); assertTrue(targetFile.exists() && targetFile.isDirectory()); try { FileUtil.replaceFile(srcFile, targetFile); @@ -681,9 +749,9 @@ public void testReplaceFile() throws IOException { // okay } // check up the post-condition: nothing is deleted: - assertTrue(srcFile.exists()); + Verify.exists(srcFile); assertTrue(targetFile.exists() && targetFile.isDirectory()); - assertTrue(obstacle.exists()); + Verify.exists(obstacle); } @Test (timeout = 30000) @@ -696,45 +764,84 @@ public void testCreateLocalTempFile() throws IOException { assertTrue(tmp1.exists() && tmp2.exists()); assertTrue(tmp1.canWrite() && tmp2.canWrite()); assertTrue(tmp1.canRead() && tmp2.canRead()); - tmp1.delete(); - tmp2.delete(); + Verify.delete(tmp1); + Verify.delete(tmp2); assertTrue(!tmp1.exists() && !tmp2.exists()); } @Test (timeout = 30000) - public void testUnZip() throws IOException { + public void testUnZip() throws Exception { // make sa simple zip final File simpleZip = new File(del, FILE); - OutputStream os = new FileOutputStream(simpleZip); - ZipOutputStream tos = new ZipOutputStream(os); - try { - ZipEntry ze = new ZipEntry("foo"); - byte[] data = "some-content".getBytes("UTF-8"); - ze.setSize(data.length); - tos.putNextEntry(ze); - tos.write(data); - tos.closeEntry(); + try (OutputStream os = new FileOutputStream(simpleZip); + ZipArchiveOutputStream tos = new ZipArchiveOutputStream(os)) { + List ZipArchiveList = new ArrayList<>(7); + int count = 0; + // create 7 files to verify permissions + for (int i = 0; i < 7; i++) { + ZipArchiveList.add(new ZipArchiveEntry("foo_" + i)); + ZipArchiveEntry archiveEntry = ZipArchiveList.get(i); + archiveEntry.setUnixMode(count += 0100); + byte[] data = "some-content".getBytes("UTF-8"); + archiveEntry.setSize(data.length); + tos.putArchiveEntry(archiveEntry); + tos.write(data); + } + tos.closeArchiveEntry(); tos.flush(); tos.finish(); - } finally { - tos.close(); } - + // successfully unzip it into an existing dir: FileUtil.unZip(simpleZip, tmp); + File foo0 = new File(tmp, "foo_0"); + File foo1 = new File(tmp, "foo_1"); + File foo2 = new File(tmp, "foo_2"); + File foo3 = new File(tmp, "foo_3"); + File foo4 = new File(tmp, "foo_4"); + File foo5 = new File(tmp, "foo_5"); + File foo6 = new File(tmp, "foo_6"); // check result: - assertTrue(new File(tmp, "foo").exists()); - assertEquals(12, new File(tmp, "foo").length()); - - final File regularFile = new File(tmp, "QuickBrownFoxJumpsOverTheLazyDog"); - regularFile.createNewFile(); - assertTrue(regularFile.exists()); - try { - FileUtil.unZip(simpleZip, regularFile); - assertTrue("An IOException expected.", false); - } catch (IOException ioe) { - // okay - } + assertTrue(foo0.exists()); + assertTrue(foo1.exists()); + assertTrue(foo2.exists()); + assertTrue(foo3.exists()); + assertTrue(foo4.exists()); + assertTrue(foo5.exists()); + assertTrue(foo6.exists()); + assertEquals(12, foo0.length()); + // tests whether file foo_0 has executable permissions + assertTrue("file lacks execute permissions", foo0.canExecute()); + assertFalse("file has write permissions", foo0.canWrite()); + assertFalse("file has read permissions", foo0.canRead()); + // tests whether file foo_1 has writable permissions + assertFalse("file has execute permissions", foo1.canExecute()); + assertTrue("file lacks write permissions", foo1.canWrite()); + assertFalse("file has read permissions", foo1.canRead()); + // tests whether file foo_2 has executable and writable permissions + assertTrue("file lacks execute permissions", foo2.canExecute()); + assertTrue("file lacks write permissions", foo2.canWrite()); + assertFalse("file has read permissions", foo2.canRead()); + // tests whether file foo_3 has readable permissions + assertFalse("file has execute permissions", foo3.canExecute()); + assertFalse("file has write permissions", foo3.canWrite()); + assertTrue("file lacks read permissions", foo3.canRead()); + // tests whether file foo_4 has readable and executable permissions + assertTrue("file lacks execute permissions", foo4.canExecute()); + assertFalse("file has write permissions", foo4.canWrite()); + assertTrue("file lacks read permissions", foo4.canRead()); + // tests whether file foo_5 has readable and writable permissions + assertFalse("file has execute permissions", foo5.canExecute()); + assertTrue("file lacks write permissions", foo5.canWrite()); + assertTrue("file lacks read permissions", foo5.canRead()); + // tests whether file foo_6 has readable, writable and executable permissions + assertTrue("file lacks execute permissions", foo6.canExecute()); + assertTrue("file lacks write permissions", foo6.canWrite()); + assertTrue("file lacks read permissions", foo6.canRead()); + + final File regularFile = + Verify.createNewFile(new File(tmp, "QuickBrownFoxJumpsOverTheLazyDog")); + LambdaTestUtils.intercept(IOException.class, () -> FileUtil.unZip(simpleZip, regularFile)); } @Test (timeout = 30000) @@ -742,14 +849,14 @@ public void testUnZip2() throws IOException { // make a simple zip final File simpleZip = new File(del, FILE); OutputStream os = new FileOutputStream(simpleZip); - try (ZipOutputStream tos = new ZipOutputStream(os)) { + try (ZipArchiveOutputStream tos = new ZipArchiveOutputStream(os)) { // Add an entry that contains invalid filename - ZipEntry ze = new ZipEntry("../foo"); + ZipArchiveEntry ze = new ZipArchiveEntry("../foo"); byte[] data = "some-content".getBytes(StandardCharsets.UTF_8); ze.setSize(data.length); - tos.putNextEntry(ze); + tos.putArchiveEntry(ze); tos.write(data); - tos.closeEntry(); + tos.closeArchiveEntry(); tos.flush(); tos.finish(); } @@ -780,24 +887,24 @@ public void testCopy5() throws IOException { final File dest = new File(del, "dest"); boolean result = FileUtil.copy(fs, srcPath, dest, false, conf); assertTrue(result); - assertTrue(dest.exists()); + Verify.exists(dest); assertEquals(content.getBytes().length + System.getProperty("line.separator").getBytes().length, dest.length()); - assertTrue(srcFile.exists()); // should not be deleted + Verify.exists(srcFile); // should not be deleted // copy regular file, delete src: - dest.delete(); - assertTrue(!dest.exists()); + Verify.delete(dest); + Verify.notExists(dest); result = FileUtil.copy(fs, srcPath, dest, true, conf); assertTrue(result); - assertTrue(dest.exists()); + Verify.exists(dest); assertEquals(content.getBytes().length + System.getProperty("line.separator").getBytes().length, dest.length()); - assertTrue(!srcFile.exists()); // should be deleted + Verify.notExists(srcFile); // should be deleted // copy a dir: - dest.delete(); - assertTrue(!dest.exists()); + Verify.delete(dest); + Verify.notExists(dest); srcPath = new Path(partitioned.toURI()); result = FileUtil.copy(fs, srcPath, dest, true, conf); assertTrue(result); @@ -809,7 +916,7 @@ public void testCopy5() throws IOException { assertEquals(3 + System.getProperty("line.separator").getBytes().length, f.length()); } - assertTrue(!partitioned.exists()); // should be deleted + Verify.notExists(partitioned); // should be deleted } @Test (timeout = 30000) @@ -897,8 +1004,8 @@ public void testSymlinkRenameTo() throws Exception { // create the symlink FileUtil.symLink(file.getAbsolutePath(), link.getAbsolutePath()); - Assert.assertTrue(file.exists()); - Assert.assertTrue(link.exists()); + Verify.exists(file); + Verify.exists(link); File link2 = new File(del, "_link2"); @@ -908,10 +1015,10 @@ public void testSymlinkRenameTo() throws Exception { // Make sure the file still exists // (NOTE: this would fail on Java6 on Windows if we didn't // copy the file in FileUtil#symlink) - Assert.assertTrue(file.exists()); + Verify.exists(file); - Assert.assertTrue(link2.exists()); - Assert.assertFalse(link.exists()); + Verify.exists(link2); + Verify.notExists(link); } /** @@ -926,13 +1033,13 @@ public void testSymlinkDelete() throws Exception { // create the symlink FileUtil.symLink(file.getAbsolutePath(), link.getAbsolutePath()); - Assert.assertTrue(file.exists()); - Assert.assertTrue(link.exists()); + Verify.exists(file); + Verify.exists(link); // make sure that deleting a symlink works properly - Assert.assertTrue(link.delete()); - Assert.assertFalse(link.exists()); - Assert.assertTrue(file.exists()); + Verify.delete(link); + Verify.notExists(link); + Verify.exists(file); } /** @@ -959,13 +1066,13 @@ public void testSymlinkLength() throws Exception { Assert.assertEquals(data.length, file.length()); Assert.assertEquals(data.length, link.length()); - file.delete(); - Assert.assertFalse(file.exists()); + Verify.delete(file); + Verify.notExists(file); Assert.assertEquals(0, link.length()); - link.delete(); - Assert.assertFalse(link.exists()); + Verify.delete(link); + Verify.notExists(link); } /** @@ -1031,7 +1138,7 @@ public void testSymlinkFileAlreadyExists() throws IOException { public void testSymlinkSameFile() throws IOException { File file = new File(del, FILE); - file.delete(); + Verify.delete(file); // Create a symbolic link // The operation should succeed @@ -1104,21 +1211,21 @@ private void doUntarAndVerify(File tarFile, File untarDir) String parentDir = untarDir.getCanonicalPath() + Path.SEPARATOR + "name"; File testFile = new File(parentDir + Path.SEPARATOR + "version"); - Assert.assertTrue(testFile.exists()); + Verify.exists(testFile); Assert.assertTrue(testFile.length() == 0); String imageDir = parentDir + Path.SEPARATOR + "image"; testFile = new File(imageDir + Path.SEPARATOR + "fsimage"); - Assert.assertTrue(testFile.exists()); + Verify.exists(testFile); Assert.assertTrue(testFile.length() == 157); String currentDir = parentDir + Path.SEPARATOR + "current"; testFile = new File(currentDir + Path.SEPARATOR + "fsimage"); - Assert.assertTrue(testFile.exists()); + Verify.exists(testFile); Assert.assertTrue(testFile.length() == 4331); testFile = new File(currentDir + Path.SEPARATOR + "edits"); - Assert.assertTrue(testFile.exists()); + Verify.exists(testFile); Assert.assertTrue(testFile.length() == 1033); testFile = new File(currentDir + Path.SEPARATOR + "fstime"); - Assert.assertTrue(testFile.exists()); + Verify.exists(testFile); Assert.assertTrue(testFile.length() == 8); } @@ -1135,6 +1242,38 @@ public void testUntar() throws IOException { doUntarAndVerify(new File(tarFileName), untarDir); } + /** + * Verify we can't unTar a file which isn't there. + * This will test different codepaths on Windows from unix, + * but both MUST throw an IOE of some kind. + */ + @Test(timeout = 30000) + public void testUntarMissingFile() throws Throwable { + File dataDir = GenericTestUtils.getTestDir(); + File tarFile = new File(dataDir, "missing; true"); + File untarDir = new File(dataDir, "untarDir"); + intercept(IOException.class, () -> + FileUtil.unTar(tarFile, untarDir)); + } + + /** + * Verify we can't unTar a file which isn't there + * through the java untar code. + * This is how {@code FileUtil.unTar(File, File} + * will behave on Windows, + */ + @Test(timeout = 30000) + public void testUntarMissingFileThroughJava() throws Throwable { + File dataDir = GenericTestUtils.getTestDir(); + File tarFile = new File(dataDir, "missing; true"); + File untarDir = new File(dataDir, "untarDir"); + // java8 on unix throws java.nio.file.NoSuchFileException here; + // leaving as an IOE intercept in case windows throws something + // else. + intercept(IOException.class, () -> + FileUtil.unTarUsingJava(tarFile, untarDir, false)); + } + @Test (timeout = 30000) public void testCreateJarWithClassPath() throws Exception { // create files expected to match a wildcard @@ -1147,9 +1286,9 @@ public void testCreateJarWithClassPath() throws Exception { } // create non-jar files, which we expect to not be included in the classpath - Assert.assertTrue(new File(tmp, "text.txt").createNewFile()); - Assert.assertTrue(new File(tmp, "executable.exe").createNewFile()); - Assert.assertTrue(new File(tmp, "README").createNewFile()); + Verify.createNewFile(new File(tmp, "text.txt")); + Verify.createNewFile(new File(tmp, "executable.exe")); + Verify.createNewFile(new File(tmp, "README")); // create classpath jar String wildcardPath = tmp.getCanonicalPath() + File.separator + "*"; @@ -1235,9 +1374,9 @@ public void testGetJarsInDirectory() throws Exception { } // create non-jar files, which we expect to not be included in the result - assertTrue(new File(tmp, "text.txt").createNewFile()); - assertTrue(new File(tmp, "executable.exe").createNewFile()); - assertTrue(new File(tmp, "README").createNewFile()); + Verify.createNewFile(new File(tmp, "text.txt")); + Verify.createNewFile(new File(tmp, "executable.exe")); + Verify.createNewFile(new File(tmp, "README")); // pass in the directory String directory = tmp.getCanonicalPath(); @@ -1271,7 +1410,7 @@ public void setupCompareFs() { uri4 = new URI(uris4); uri5 = new URI(uris5); uri6 = new URI(uris6); - } catch (URISyntaxException use) { + } catch (URISyntaxException ignored) { } // Set up InetAddress inet1 = mock(InetAddress.class); @@ -1294,7 +1433,7 @@ public void setupCompareFs() { when(InetAddress.getByName(uris3)).thenReturn(inet3); when(InetAddress.getByName(uris4)).thenReturn(inet4); when(InetAddress.getByName(uris5)).thenReturn(inet5); - } catch (UnknownHostException ue) { + } catch (UnknownHostException ignored) { } fs1 = mock(FileSystem.class); @@ -1314,62 +1453,87 @@ public void setupCompareFs() { @Test public void testCompareFsNull() throws Exception { setupCompareFs(); - assertEquals(FileUtil.compareFs(null,fs1),false); - assertEquals(FileUtil.compareFs(fs1,null),false); + assertFalse(FileUtil.compareFs(null, fs1)); + assertFalse(FileUtil.compareFs(fs1, null)); } @Test public void testCompareFsDirectories() throws Exception { setupCompareFs(); - assertEquals(FileUtil.compareFs(fs1,fs1),true); - assertEquals(FileUtil.compareFs(fs1,fs2),false); - assertEquals(FileUtil.compareFs(fs1,fs5),false); - assertEquals(FileUtil.compareFs(fs3,fs4),true); - assertEquals(FileUtil.compareFs(fs1,fs6),false); + assertTrue(FileUtil.compareFs(fs1, fs1)); + assertFalse(FileUtil.compareFs(fs1, fs2)); + assertFalse(FileUtil.compareFs(fs1, fs5)); + assertTrue(FileUtil.compareFs(fs3, fs4)); + assertFalse(FileUtil.compareFs(fs1, fs6)); } @Test(timeout = 8000) public void testCreateSymbolicLinkUsingJava() throws IOException { final File simpleTar = new File(del, FILE); OutputStream os = new FileOutputStream(simpleTar); - TarArchiveOutputStream tos = new TarArchiveOutputStream(os); - File untarFile = null; - try { + try (TarArchiveOutputStream tos = new TarArchiveOutputStream(os)) { // Files to tar final String tmpDir = "tmp/test"; File tmpDir1 = new File(tmpDir, "dir1/"); File tmpDir2 = new File(tmpDir, "dir2/"); - // Delete the directories if they already exist - tmpDir1.mkdirs(); - tmpDir2.mkdirs(); + Verify.mkdirs(tmpDir1); + Verify.mkdirs(tmpDir2); - java.nio.file.Path symLink = FileSystems - .getDefault().getPath(tmpDir1.getPath() + "/sl"); + java.nio.file.Path symLink = Paths.get(tmpDir1.getPath(), "sl"); // Create Symbolic Link - Files.createSymbolicLink(symLink, - FileSystems.getDefault().getPath(tmpDir2.getPath())).toString(); + Files.createSymbolicLink(symLink, Paths.get(tmpDir2.getPath())); assertTrue(Files.isSymbolicLink(symLink.toAbsolutePath())); - // put entries in tar file + // Put entries in tar file putEntriesInTar(tos, tmpDir1.getParentFile()); tos.close(); - untarFile = new File(tmpDir, "2"); - // Untar using java + File untarFile = new File(tmpDir, "2"); + // Untar using Java FileUtil.unTarUsingJava(simpleTar, untarFile, false); // Check symbolic link and other directories are there in untar file assertTrue(Files.exists(untarFile.toPath())); - assertTrue(Files.exists(FileSystems.getDefault().getPath(untarFile - .getPath(), tmpDir))); - assertTrue(Files.isSymbolicLink(FileSystems.getDefault().getPath(untarFile - .getPath().toString(), symLink.toString()))); - + assertTrue(Files.exists(Paths.get(untarFile.getPath(), tmpDir))); + assertTrue(Files.isSymbolicLink(Paths.get(untarFile.getPath(), symLink.toString()))); } finally { FileUtils.deleteDirectory(new File("tmp")); - tos.close(); } + } + + @Test(expected = IOException.class) + public void testCreateArbitrarySymlinkUsingJava() throws IOException { + final File simpleTar = new File(del, FILE); + OutputStream os = new FileOutputStream(simpleTar); + + File rootDir = new File("tmp"); + try (TarArchiveOutputStream tos = new TarArchiveOutputStream(os)) { + tos.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU); + + // Create arbitrary dir + File arbitraryDir = new File(rootDir, "arbitrary-dir/"); + Verify.mkdirs(arbitraryDir); + + // We will tar from the tar-root lineage + File tarRoot = new File(rootDir, "tar-root/"); + File symlinkRoot = new File(tarRoot, "dir1/"); + Verify.mkdirs(symlinkRoot); + + // Create Symbolic Link to an arbitrary dir + java.nio.file.Path symLink = Paths.get(symlinkRoot.getPath(), "sl"); + Files.createSymbolicLink(symLink, arbitraryDir.toPath().toAbsolutePath()); + + // Put entries in tar file + putEntriesInTar(tos, tarRoot); + putEntriesInTar(tos, new File(symLink.toFile(), "dir-outside-tar-root/")); + tos.close(); + // Untar using Java + File untarFile = new File(rootDir, "extracted"); + FileUtil.unTarUsingJava(simpleTar, untarFile, false); + } finally { + FileUtils.deleteDirectory(rootDir); + } } private void putEntriesInTar(TarArchiveOutputStream tos, File f) @@ -1433,6 +1597,23 @@ public void testReadSymlink() throws IOException { Assert.assertEquals(file.getAbsolutePath(), result); } + @Test + public void testRegularFile() throws IOException { + byte[] data = "testRegularData".getBytes(); + File tmpFile = new File(del, "reg1"); + + // write some data to the file + FileOutputStream os = new FileOutputStream(tmpFile); + os.write(data); + os.close(); + assertTrue(FileUtil.isRegularFile(tmpFile)); + + // create a symlink to file + File link = new File(del, "reg2"); + FileUtil.symLink(tmpFile.toString(), link.toString()); + assertFalse(FileUtil.isRegularFile(link, false)); + } + /** * This test validates the correctness of {@link FileUtil#readLink(File)} when * it gets a file in input. @@ -1446,7 +1627,7 @@ public void testReadSymlinkWithAFileAsInput() throws IOException { String result = FileUtil.readLink(file); Assert.assertEquals("", result); - file.delete(); + Verify.delete(file); } /** diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellTouch.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellTouch.java index 49bbd5af04f25..c2bd5b2133d47 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellTouch.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellTouch.java @@ -200,6 +200,78 @@ public void testTouch() throws Exception { FileStatus fileStatus = lfs.getFileStatus(newFile); assertThat(fileStatus.getAccessTime()).isEqualTo(dateObj.getTime()); assertThat(fileStatus.getModificationTime()).isEqualTo(dateObj.getTime()); + + lfs.delete(newFile, true); + assertThat(lfs.exists(newFile)).isFalse(); + + } + + @Test + public void testTouchDir() throws Exception { + String strTime; + final String newFileName = "dir3/newFile3"; + Date dateObj; + final Path newFile = new Path(newFileName); + FileStatus fstatus; + Path dirPath = new Path("dir3"); + lfs.delete(dirPath, true); + lfs.mkdirs(dirPath); + lfs.delete(newFile, true); + assertThat(lfs.exists(newFile)).isFalse(); + + strTime = formatTimestamp(System.currentTimeMillis()); + dateObj = parseTimestamp(strTime); + + assertThat(shellRun("-touch", "-t", strTime, newFileName)).as( + "Expected successful touch on a new file with a specified timestamp").isEqualTo(0); + FileStatus newStatus = lfs.getFileStatus(newFile); + assertThat(newStatus.getAccessTime()).isEqualTo(dateObj.getTime()); + assertThat(newStatus.getModificationTime()).isEqualTo(dateObj.getTime()); + + Thread.sleep(500); + strTime = formatTimestamp(System.currentTimeMillis()); + dateObj = parseTimestamp(strTime); + + assertThat(shellRun("-touch", "-m", "-a", "-t", strTime, "dir3")).as( + "Expected successful touch with a specified modification time").isEqualTo(0); + + newStatus = lfs.getFileStatus(dirPath); + // Verify if both modification and access times are recorded correctly + assertThat(newStatus.getAccessTime()).isEqualTo(dateObj.getTime()); + assertThat(newStatus.getModificationTime()).isEqualTo(dateObj.getTime()); + + fstatus = lfs.getFileStatus(dirPath); + Thread.sleep(500); + strTime = formatTimestamp(System.currentTimeMillis()); + dateObj = parseTimestamp(strTime); + + assertThat(shellRun("-touch", "-m", "-t", strTime, "dir3")).as( + "Expected successful touch with a specified modification time").isEqualTo(0); + + newStatus = lfs.getFileStatus(dirPath); + // Verify if modification time is recorded correctly (and access time + // remains unchanged). + assertThat(newStatus.getAccessTime()).isEqualTo(fstatus.getAccessTime()); + assertThat(newStatus.getModificationTime()).isEqualTo(dateObj.getTime()); + + fstatus = lfs.getFileStatus(dirPath); + Thread.sleep(500); + strTime = formatTimestamp(System.currentTimeMillis()); + dateObj = parseTimestamp(strTime); + + assertThat(shellRun("-touch", "-a", "-t", strTime, "dir3")).as( + "Expected successful touch with a specified modification time").isEqualTo(0); + + newStatus = lfs.getFileStatus(dirPath); + // Verify if access time is recorded correctly (and modification time + // remains unchanged). + assertThat(newStatus.getAccessTime()).isEqualTo(dateObj.getTime()); + assertThat(newStatus.getModificationTime()).isEqualTo(fstatus.getModificationTime()); + + lfs.delete(newFile, true); + lfs.delete(dirPath, true); + assertThat(lfs.exists(newFile)).isFalse(); + assertThat(lfs.exists(dirPath)).isFalse(); } private String formatTimestamp(long timeInMillis) { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/audit/TestCommonAuditContext.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/audit/TestCommonAuditContext.java index 798841a2d6905..9782eb276d306 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/audit/TestCommonAuditContext.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/audit/TestCommonAuditContext.java @@ -132,6 +132,15 @@ private AbstractStringAssert assertContextValue(final String key) { .describedAs("Value of context element %s", key) .isNotBlank(); } + /** + * Assert a context value is null. + * @param key key to look up + */ + private void assertContextValueIsNull(final String key) { + assertThat(context.get(key)) + .describedAs("Value of context element %s", key) + .isNull(); + } @Test public void testNoteEntryPoint() throws Throwable { @@ -158,4 +167,13 @@ private AbstractStringAssert assertGlobalEntry(final String key) { return anAssert; } + @Test + public void testAddRemove() throws Throwable { + final String key = "testAddRemove"; + assertContextValueIsNull(key); + context.put(key, key); + assertContextValue(key).isEqualTo(key); + context.remove(key); + assertContextValueIsNull(key); + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractContentSummaryTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractContentSummaryTest.java new file mode 100644 index 0000000000000..5e5c917395413 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractContentSummaryTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract; + +import org.apache.hadoop.fs.ContentSummary; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +import org.assertj.core.api.Assertions; +import org.junit.Test; + +import java.io.FileNotFoundException; + +import static org.apache.hadoop.fs.contract.ContractTestUtils.touch; +import static org.apache.hadoop.test.LambdaTestUtils.intercept; + +public abstract class AbstractContractContentSummaryTest extends AbstractFSContractTestBase { + + @Test + public void testGetContentSummary() throws Throwable { + FileSystem fs = getFileSystem(); + + Path parent = path("parent"); + Path nested = path(parent + "/a/b/c"); + Path filePath = path(nested + "file.txt"); + + fs.mkdirs(parent); + fs.mkdirs(nested); + touch(getFileSystem(), filePath); + + ContentSummary summary = fs.getContentSummary(parent); + + Assertions.assertThat(summary.getDirectoryCount()).as("Summary " + summary).isEqualTo(4); + + Assertions.assertThat(summary.getFileCount()).as("Summary " + summary).isEqualTo(1); + } + + @Test + public void testGetContentSummaryIncorrectPath() throws Throwable { + FileSystem fs = getFileSystem(); + + Path parent = path("parent"); + Path nested = path(parent + "/a"); + + fs.mkdirs(parent); + + intercept(FileNotFoundException.class, () -> fs.getContentSummary(nested)); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractEtagTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractEtagTest.java new file mode 100644 index 0000000000000..e7a121b704677 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractEtagTest.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract; + +import java.nio.charset.StandardCharsets; + +import org.assertj.core.api.Assertions; +import org.junit.Assume; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.fs.EtagSource; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.Path; + +import static org.apache.hadoop.fs.CommonPathCapabilities.ETAGS_AVAILABLE; +import static org.apache.hadoop.fs.CommonPathCapabilities.ETAGS_PRESERVED_IN_RENAME; + +/** + * For filesystems which support etags, validate correctness + * of their implementation. + */ +public abstract class AbstractContractEtagTest extends + AbstractFSContractTestBase { + + private static final Logger LOG = + LoggerFactory.getLogger(AbstractContractEtagTest.class); + + /** + * basic consistency across operations, as well as being non-empty. + */ + @Test + public void testEtagConsistencyAcrossListAndHead() throws Throwable { + describe("Etag values must be non-empty and consistent across LIST and HEAD Calls."); + final Path path = methodPath(); + final FileSystem fs = getFileSystem(); + + Assertions.assertThat(fs.hasPathCapability(path, ETAGS_AVAILABLE)) + .describedAs("path capability %s of %s", + ETAGS_AVAILABLE, path) + .isTrue(); + + ContractTestUtils.touch(fs, path); + + final FileStatus st = fs.getFileStatus(path); + final String etag = etagFromStatus(st); + LOG.info("etag of empty file is \"{}\"", etag); + + final FileStatus[] statuses = fs.listStatus(path); + Assertions.assertThat(statuses) + .describedAs("List(%s)", path) + .hasSize(1); + final FileStatus lsStatus = statuses[0]; + Assertions.assertThat(etagFromStatus(lsStatus)) + .describedAs("etag of list status (%s) compared to HEAD value of %s", lsStatus, st) + .isEqualTo(etag); + } + + /** + * Get an etag from a FileStatus which MUST BE + * an implementation of EtagSource and + * whose etag MUST NOT BE null/empty. + * @param st the status + * @return the etag + */ + String etagFromStatus(FileStatus st) { + Assertions.assertThat(st) + .describedAs("FileStatus %s", st) + .isInstanceOf(EtagSource.class); + final String etag = ((EtagSource) st).getEtag(); + Assertions.assertThat(etag) + .describedAs("Etag of %s", st) + .isNotBlank(); + return etag; + } + + /** + * Overwritten data has different etags. + */ + @Test + public void testEtagsOfDifferentDataDifferent() throws Throwable { + describe("Verify that two different blocks of data written have different tags"); + + final Path path = methodPath(); + final FileSystem fs = getFileSystem(); + Path src = new Path(path, "src"); + + ContractTestUtils.createFile(fs, src, true, + "data1234".getBytes(StandardCharsets.UTF_8)); + final FileStatus srcStatus = fs.getFileStatus(src); + final String srcTag = etagFromStatus(srcStatus); + LOG.info("etag of file 1 is \"{}\"", srcTag); + + // now overwrite with data of same length + // (ensure that path or length aren't used exclusively as tag) + ContractTestUtils.createFile(fs, src, true, + "1234data".getBytes(StandardCharsets.UTF_8)); + + // validate + final String tag2 = etagFromStatus(fs.getFileStatus(src)); + LOG.info("etag of file 2 is \"{}\"", tag2); + + Assertions.assertThat(tag2) + .describedAs("etag of updated file") + .isNotEqualTo(srcTag); + } + + /** + * If supported, rename preserves etags. + */ + @Test + public void testEtagConsistencyAcrossRename() throws Throwable { + describe("Verify that when a file is renamed, the etag remains unchanged"); + final Path path = methodPath(); + final FileSystem fs = getFileSystem(); + Assume.assumeTrue( + "Filesystem does not declare that etags are preserved across renames", + fs.hasPathCapability(path, ETAGS_PRESERVED_IN_RENAME)); + Path src = new Path(path, "src"); + Path dest = new Path(path, "dest"); + + ContractTestUtils.createFile(fs, src, true, + "sample data".getBytes(StandardCharsets.UTF_8)); + final FileStatus srcStatus = fs.getFileStatus(src); + LOG.info("located file status string value " + srcStatus); + + final String srcTag = etagFromStatus(srcStatus); + LOG.info("etag of short file is \"{}\"", srcTag); + + Assertions.assertThat(srcTag) + .describedAs("Etag of %s", srcStatus) + .isNotBlank(); + + // rename + fs.rename(src, dest); + + // validate + FileStatus destStatus = fs.getFileStatus(dest); + final String destTag = etagFromStatus(destStatus); + Assertions.assertThat(destTag) + .describedAs("etag of list status (%s) compared to HEAD value of %s", + destStatus, srcStatus) + .isEqualTo(srcTag); + } + + /** + * For effective use of etags, listLocatedStatus SHOULD return status entries + * with consistent values. + * This ensures that listing during query planning can collect and use the etags. + */ + @Test + public void testLocatedStatusAlsoHasEtag() throws Throwable { + describe("verify that listLocatedStatus() and listFiles() are etag sources"); + final Path path = methodPath(); + final FileSystem fs = getFileSystem(); + Path src = new Path(path, "src"); + ContractTestUtils.createFile(fs, src, true, + "sample data".getBytes(StandardCharsets.UTF_8)); + final FileStatus srcStatus = fs.getFileStatus(src); + final String srcTag = etagFromStatus(srcStatus); + final LocatedFileStatus entry = fs.listLocatedStatus(path).next(); + LOG.info("located file status string value " + entry); + final String listTag = etagFromStatus(entry); + Assertions.assertThat(listTag) + .describedAs("etag of listLocatedStatus (%s) compared to HEAD value of %s", + entry, srcStatus) + .isEqualTo(srcTag); + + final LocatedFileStatus entry2 = fs.listFiles(path, false).next(); + Assertions.assertThat(etagFromStatus(entry2)) + .describedAs("etag of listFiles (%s) compared to HEAD value of %s", + entry, srcStatus) + .isEqualTo(srcTag); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java index 3e754e4578de8..c395afdb3779b 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java @@ -50,11 +50,11 @@ import org.apache.hadoop.util.DurationInfo; import static org.apache.hadoop.fs.contract.ContractTestUtils.verifyPathExists; -import static org.apache.hadoop.fs.impl.FutureIOSupport.awaitFuture; import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.ioStatisticsSourceToString; import static org.apache.hadoop.io.IOUtils.cleanupWithLogger; import static org.apache.hadoop.test.LambdaTestUtils.eventually; import static org.apache.hadoop.test.LambdaTestUtils.intercept; +import static org.apache.hadoop.util.functional.FutureIO.awaitFuture; /** * Tests of multipart uploads. diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java index a43053180fbf8..25bfe082b01f6 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java @@ -30,14 +30,18 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FutureDataInputStreamBuilder; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.impl.FutureIOSupport; import org.apache.hadoop.io.IOUtils; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_BUFFER_SIZE; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_LENGTH; +import static org.apache.hadoop.fs.contract.ContractTestUtils.compareByteArrays; import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile; import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; import static org.apache.hadoop.fs.contract.ContractTestUtils.touch; import static org.apache.hadoop.test.LambdaTestUtils.intercept; import static org.apache.hadoop.test.LambdaTestUtils.interceptFuture; +import static org.apache.hadoop.util.functional.FutureIO.awaitFuture; import org.junit.Test; @@ -232,7 +236,7 @@ public void testAwaitFutureFailToFNFE() throws Throwable { getFileSystem().openFile(path("testAwaitFutureFailToFNFE")) .opt("fs.test.something", true); intercept(FileNotFoundException.class, - () -> FutureIOSupport.awaitFuture(builder.build())); + () -> awaitFuture(builder.build())); } @Test @@ -242,7 +246,7 @@ public void testAwaitFutureTimeoutFailToFNFE() throws Throwable { getFileSystem().openFile(path("testAwaitFutureFailToFNFE")) .opt("fs.test.something", true); intercept(FileNotFoundException.class, - () -> FutureIOSupport.awaitFuture(builder.build(), + () -> awaitFuture(builder.build(), 10, TimeUnit.DAYS)); } @@ -250,7 +254,7 @@ public void testAwaitFutureTimeoutFailToFNFE() throws Throwable { public void testOpenFileExceptionallyTranslating() throws Throwable { describe("openFile missing file chains into exceptionally()"); CompletableFuture f = getFileSystem() - .openFile(path("testOpenFileUnknownOption")).build(); + .openFile(path("testOpenFileExceptionallyTranslating")).build(); interceptFuture(RuntimeException.class, "exceptionally", f.exceptionally(ex -> { @@ -262,11 +266,12 @@ public void testOpenFileExceptionallyTranslating() throws Throwable { public void testChainedFailureAwaitFuture() throws Throwable { describe("await Future handles chained failures"); CompletableFuture f = getFileSystem() - .openFile(path("testOpenFileUnknownOption")) + .openFile(path("testChainedFailureAwaitFuture")) + .withFileStatus(null) .build(); intercept(RuntimeException.class, "exceptionally", - () -> FutureIOSupport.awaitFuture( + () -> awaitFuture( f.exceptionally(ex -> { throw new RuntimeException("exceptionally", ex); }))); @@ -280,13 +285,34 @@ public void testOpenFileApplyRead() throws Throwable { int len = 4096; createFile(fs, path, true, dataset(len, 0x40, 0x80)); + FileStatus st = fs.getFileStatus(path); CompletableFuture readAllBytes = fs.openFile(path) - .withFileStatus(fs.getFileStatus(path)) + .withFileStatus(st) .build() .thenApply(ContractTestUtils::readStream); assertEquals("Wrong number of bytes read value", len, (long) readAllBytes.get()); + // now reattempt with a new FileStatus and a different path + // other than the final name element + // implementations MUST use path in openFile() call + FileStatus st2 = new FileStatus( + len, false, + st.getReplication(), + st.getBlockSize(), + st.getModificationTime(), + st.getAccessTime(), + st.getPermission(), + st.getOwner(), + st.getGroup(), + new Path("gopher:///localhost:/" + path.getName())); + assertEquals("Wrong number of bytes read value", + len, + (long) fs.openFile(path) + .withFileStatus(st2) + .build() + .thenApply(ContractTestUtils::readStream) + .get()); } @Test @@ -298,17 +324,47 @@ public void testOpenFileApplyAsyncRead() throws Throwable { dataset(4, 0x40, 0x80)); CompletableFuture future = fs.openFile(path).build(); AtomicBoolean accepted = new AtomicBoolean(false); - future.thenAcceptAsync(i -> accepted.set(true)).get(); + future.thenApply(stream -> { + accepted.set(true); + return stream; + }).get().close(); assertTrue("async accept operation not invoked", accepted.get()); } + /** + * Open a file with a null status, and the length + * passed in as an opt() option (along with sequential IO). + * The file is opened, the data read, and it must match + * the source data. + * opt() is used so that integration testing with external + * filesystem connectors will downgrade if the option is not + * recognized. + */ @Test - public void testOpenFileNullStatus() throws Throwable { - describe("use openFile() with a null status"); + public void testOpenFileNullStatusButFileLength() throws Throwable { + describe("use openFile() with a null status and expect the status to be" + + " ignored. block size, fadvise and length are passed in as" + + " opt() options"); Path path = path("testOpenFileNullStatus"); - intercept(NullPointerException.class, - () -> getFileSystem().openFile(path).withFileStatus(null)); + FileSystem fs = getFileSystem(); + int len = 4; + byte[] result = new byte[len]; + byte[] dataset = dataset(len, 0x40, 0x80); + createFile(fs, path, true, + dataset); + CompletableFuture future = fs.openFile(path) + .withFileStatus(null) + .opt(FS_OPTION_OPENFILE_READ_POLICY, + "unknown, sequential, random") + .opt(FS_OPTION_OPENFILE_BUFFER_SIZE, 32768) + .opt(FS_OPTION_OPENFILE_LENGTH, len) + .build(); + + try (FSDataInputStream in = future.get()) { + in.readFully(result); + } + compareByteArrays(dataset, result, len); } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java index e13a49ca10e70..eb56d957d9a1a 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java @@ -1642,17 +1642,22 @@ public static int read(InputStream in) { /** * Read a whole stream; downgrades an IOE to a runtime exception. + * Closes the stream afterwards. * @param in input * @return the number of bytes read. * @throws AssertionError on any IOException */ public static long readStream(InputStream in) { - long count = 0; + try { + long count = 0; - while (read(in) >= 0) { - count++; + while (read(in) >= 0) { + count++; + } + return count; + } finally { + IOUtils.cleanupWithLogger(LOG, in); } - return count; } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractContentSummary.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractContentSummary.java new file mode 100644 index 0000000000000..7555cf85158f9 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractContentSummary.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.localfs; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractContentSummaryTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +public class TestLocalFSContractContentSummary extends AbstractContractContentSummaryTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new LocalFSContract(conf); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyFromLocal.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyFromLocal.java index e7f36fc85013b..757c588104ea1 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyFromLocal.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyFromLocal.java @@ -17,23 +17,25 @@ */ package org.apache.hadoop.fs.shell; +import java.io.IOException; +import java.util.LinkedList; +import java.util.concurrent.ThreadPoolExecutor; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.RandomUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.FileSystemTestHelper; -import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.LocalFileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.shell.CopyCommands.CopyFromLocal; -import org.junit.BeforeClass; -import org.junit.AfterClass; -import org.junit.Test; -import org.junit.Assert; - -import java.io.IOException; -import java.util.LinkedList; -import java.util.concurrent.ThreadPoolExecutor; import static org.junit.Assert.assertEquals; @@ -48,6 +50,9 @@ public class TestCopyFromLocal { private static Path testDir; private static Configuration conf; + private Path dir = null; + private int numFiles = 0; + public static int initialize(Path dir) throws Exception { fs.mkdirs(dir); Path fromDirPath = new Path(dir, FROM_DIR_NAME); @@ -66,7 +71,7 @@ public static int initialize(Path dir) throws Exception { Path subFile = new Path(subDirPath, "file" + fileCount); fs.createNewFile(subFile); FSDataOutputStream output = fs.create(subFile, true); - for(int i = 0; i < 100; ++i) { + for (int i = 0; i < 100; ++i) { output.writeInt(i); output.writeChar('\n'); } @@ -96,48 +101,36 @@ public static void cleanup() throws Exception { fs.close(); } + @Before + public void initDirectory() throws Exception { + dir = new Path("dir" + RandomStringUtils.randomNumeric(4)); + numFiles = initialize(dir); + } + + private void run(CommandWithDestination cmd, String... args) { cmd.setConf(conf); assertEquals(0, cmd.run(args)); } @Test(timeout = 10000) - public void testCopyFromLocal() throws Exception { - Path dir = new Path("dir" + RandomStringUtils.randomNumeric(4)); - TestCopyFromLocal.initialize(dir); + public void testCopyFromLocal() { run(new TestMultiThreadedCopy(1, 0), new Path(dir, FROM_DIR_NAME).toString(), new Path(dir, TO_DIR_NAME).toString()); } @Test(timeout = 10000) - public void testCopyFromLocalWithThreads() throws Exception { - Path dir = new Path("dir" + RandomStringUtils.randomNumeric(4)); - int numFiles = TestCopyFromLocal.initialize(dir); - int maxThreads = Runtime.getRuntime().availableProcessors() * 2; - int randThreads = RandomUtils.nextInt(0, maxThreads - 1) + 1; - String numThreads = Integer.toString(randThreads); - run(new TestMultiThreadedCopy(randThreads, - randThreads == 1 ? 0 : numFiles), "-t", numThreads, - new Path(dir, FROM_DIR_NAME).toString(), - new Path(dir, TO_DIR_NAME).toString()); - } - - @Test(timeout = 10000) - public void testCopyFromLocalWithThreadWrong() throws Exception { - Path dir = new Path("dir" + RandomStringUtils.randomNumeric(4)); - int numFiles = TestCopyFromLocal.initialize(dir); - int maxThreads = Runtime.getRuntime().availableProcessors() * 2; - String numThreads = Integer.toString(maxThreads * 2); - run(new TestMultiThreadedCopy(maxThreads, numFiles), "-t", numThreads, + public void testCopyFromLocalWithThreads(){ + int threads = Runtime.getRuntime().availableProcessors() * 2 + 1; + run(new TestMultiThreadedCopy(threads, numFiles), + "-t", Integer.toString(threads), new Path(dir, FROM_DIR_NAME).toString(), new Path(dir, TO_DIR_NAME).toString()); } @Test(timeout = 10000) - public void testCopyFromLocalWithZeroThreads() throws Exception { - Path dir = new Path("dir" + RandomStringUtils.randomNumeric(4)); - TestCopyFromLocal.initialize(dir); + public void testCopyFromLocalWithThreadWrong(){ run(new TestMultiThreadedCopy(1, 0), "-t", "0", new Path(dir, FROM_DIR_NAME).toString(), new Path(dir, TO_DIR_NAME).toString()); @@ -148,8 +141,7 @@ private class TestMultiThreadedCopy extends CopyFromLocal { private int expectedThreads; private int expectedCompletedTaskCount; - TestMultiThreadedCopy(int expectedThreads, - int expectedCompletedTaskCount) { + TestMultiThreadedCopy(int expectedThreads, int expectedCompletedTaskCount) { this.expectedThreads = expectedThreads; this.expectedCompletedTaskCount = expectedCompletedTaskCount; } @@ -158,17 +150,22 @@ private class TestMultiThreadedCopy extends CopyFromLocal { protected void processArguments(LinkedList args) throws IOException { // Check if the correct number of threads are spawned - Assert.assertEquals(expectedThreads, getNumThreads()); + Assert.assertEquals(expectedThreads, getThreadCount()); super.processArguments(args); - // Once the copy is complete, check following - // 1) number of completed tasks are same as expected - // 2) There are no active tasks in the executor - // 3) Executor has shutdown correctly - ThreadPoolExecutor executor = getExecutor(); - Assert.assertEquals(expectedCompletedTaskCount, - executor.getCompletedTaskCount()); - Assert.assertEquals(0, executor.getActiveCount()); - Assert.assertTrue(executor.isTerminated()); + + if (isMultiThreadNecessary(args)) { + // Once the copy is complete, check following + // 1) number of completed tasks are same as expected + // 2) There are no active tasks in the executor + // 3) Executor has shutdown correctly + ThreadPoolExecutor executor = getExecutor(); + Assert.assertEquals(expectedCompletedTaskCount, + executor.getCompletedTaskCount()); + Assert.assertEquals(0, executor.getActiveCount()); + Assert.assertTrue(executor.isTerminated()); + } else { + assert getExecutor() == null; + } } } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyPreserveFlag.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyPreserveFlag.java index 0f0ddcc4ee498..b68be243c956e 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyPreserveFlag.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyPreserveFlag.java @@ -17,11 +17,12 @@ */ package org.apache.hadoop.fs.shell; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; - import java.io.IOException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; @@ -31,13 +32,13 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.fs.shell.CopyCommands.CopyFromLocal; import org.apache.hadoop.fs.shell.CopyCommands.Cp; import org.apache.hadoop.fs.shell.CopyCommands.Get; import org.apache.hadoop.fs.shell.CopyCommands.Put; -import org.apache.hadoop.fs.shell.CopyCommands.CopyFromLocal; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; public class TestCopyPreserveFlag { private static final int MODIFICATION_TIME = 12345000; @@ -176,6 +177,34 @@ public void testGetWithoutP() throws Exception { assertAttributesChanged(TO); } + @Test(timeout = 10000) + public void testGetWithPQ() throws Exception { + Get get = new Get(); + run(get, "-p", "-q", "100", FROM.toString(), TO.toString()); + assertEquals(get.getThreadPoolQueueSize(), 100); + assertAttributesPreserved(TO); + } + + @Test(timeout = 10000) + public void testGetWithQ() throws Exception { + Get get = new Get(); + run(get, "-q", "100", FROM.toString(), TO.toString()); + assertEquals(get.getThreadPoolQueueSize(), 100); + assertAttributesChanged(TO); + } + + @Test(timeout = 10000) + public void testGetWithThreads() throws Exception { + run(new Get(), "-t", "10", FROM.toString(), TO.toString()); + assertAttributesChanged(TO); + } + + @Test(timeout = 10000) + public void testGetWithThreadsPreserve() throws Exception { + run(new Get(), "-p", "-t", "10", FROM.toString(), TO.toString()); + assertAttributesPreserved(TO); + } + @Test(timeout = 10000) public void testCpWithP() throws Exception { run(new Cp(), "-p", FROM.toString(), TO.toString()); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyToLocal.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyToLocal.java new file mode 100644 index 0000000000000..202b81912c104 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyToLocal.java @@ -0,0 +1,210 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.shell; + +import java.io.IOException; +import java.util.LinkedList; +import java.util.concurrent.ThreadPoolExecutor; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.RandomUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileSystemTestHelper; +import org.apache.hadoop.fs.LocalFileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.shell.CopyCommands.CopyToLocal; + +import static org.apache.hadoop.fs.shell.CopyCommandWithMultiThread.DEFAULT_QUEUE_SIZE; +import static org.junit.Assert.assertEquals; + +public class TestCopyToLocal { + + private static final String FROM_DIR_NAME = "fromDir"; + private static final String TO_DIR_NAME = "toDir"; + + private static FileSystem fs; + private static Path testDir; + private static Configuration conf; + + private Path dir = null; + private int numFiles = 0; + + private static int initialize(Path dir) throws Exception { + fs.mkdirs(dir); + Path fromDirPath = new Path(dir, FROM_DIR_NAME); + fs.mkdirs(fromDirPath); + Path toDirPath = new Path(dir, TO_DIR_NAME); + fs.mkdirs(toDirPath); + + int numTotalFiles = 0; + int numDirs = RandomUtils.nextInt(0, 5); + for (int dirCount = 0; dirCount < numDirs; ++dirCount) { + Path subDirPath = new Path(fromDirPath, "subdir" + dirCount); + fs.mkdirs(subDirPath); + int numFiles = RandomUtils.nextInt(0, 10); + for (int fileCount = 0; fileCount < numFiles; ++fileCount) { + numTotalFiles++; + Path subFile = new Path(subDirPath, "file" + fileCount); + fs.createNewFile(subFile); + FSDataOutputStream output = fs.create(subFile, true); + for (int i = 0; i < 100; ++i) { + output.writeInt(i); + output.writeChar('\n'); + } + output.close(); + } + } + + return numTotalFiles; + } + + @BeforeClass + public static void init() throws Exception { + conf = new Configuration(false); + conf.set("fs.file.impl", LocalFileSystem.class.getName()); + fs = FileSystem.getLocal(conf); + testDir = new FileSystemTestHelper().getTestRootPath(fs); + // don't want scheme on the path, just an absolute path + testDir = new Path(fs.makeQualified(testDir).toUri().getPath()); + + FileSystem.setDefaultUri(conf, fs.getUri()); + fs.setWorkingDirectory(testDir); + } + + @AfterClass + public static void cleanup() throws Exception { + fs.delete(testDir, true); + fs.close(); + } + + private void run(CopyCommandWithMultiThread cmd, String... args) { + cmd.setConf(conf); + assertEquals(0, cmd.run(args)); + } + + @Before + public void initDirectory() throws Exception { + dir = new Path("dir" + RandomStringUtils.randomNumeric(4)); + numFiles = initialize(dir); + } + + @Test(timeout = 10000) + public void testCopy() throws Exception { + MultiThreadedCopy copy = new MultiThreadedCopy(1, DEFAULT_QUEUE_SIZE, 0); + run(copy, new Path(dir, FROM_DIR_NAME).toString(), + new Path(dir, TO_DIR_NAME).toString()); + assert copy.getExecutor() == null; + } + + @Test(timeout = 10000) + public void testCopyWithThreads() { + run(new MultiThreadedCopy(5, DEFAULT_QUEUE_SIZE, numFiles), "-t", "5", + new Path(dir, FROM_DIR_NAME).toString(), + new Path(dir, TO_DIR_NAME).toString()); + } + + @Test(timeout = 10000) + public void testCopyWithThreadWrong() { + run(new MultiThreadedCopy(1, DEFAULT_QUEUE_SIZE, 0), "-t", "0", + new Path(dir, FROM_DIR_NAME).toString(), + new Path(dir, TO_DIR_NAME).toString()); + } + + @Test(timeout = 10000) + public void testCopyWithThreadsAndQueueSize() { + int queueSize = 256; + run(new MultiThreadedCopy(5, queueSize, numFiles), "-t", "5", "-q", + Integer.toString(queueSize), + new Path(dir, FROM_DIR_NAME).toString(), + new Path(dir, TO_DIR_NAME).toString()); + } + + @Test(timeout = 10000) + public void testCopyWithThreadsAndQueueSizeWrong() { + int queueSize = 0; + run(new MultiThreadedCopy(5, DEFAULT_QUEUE_SIZE, numFiles), "-t", "5", "-q", + Integer.toString(queueSize), + new Path(dir, FROM_DIR_NAME).toString(), + new Path(dir, TO_DIR_NAME).toString()); + } + + @Test(timeout = 10000) + public void testCopySingleFile() throws Exception { + Path fromDirPath = new Path(dir, FROM_DIR_NAME); + Path subFile = new Path(fromDirPath, "file0"); + fs.createNewFile(subFile); + FSDataOutputStream output = fs.create(subFile, true); + for (int i = 0; i < 100; ++i) { + output.writeInt(i); + output.writeChar('\n'); + } + output.close(); + + MultiThreadedCopy copy = new MultiThreadedCopy(5, DEFAULT_QUEUE_SIZE, 0); + run(copy, "-t", "5", subFile.toString(), + new Path(dir, TO_DIR_NAME).toString()); + assert copy.getExecutor() == null; + } + + private static class MultiThreadedCopy extends CopyToLocal { + public static final String NAME = "multiThreadCopy"; + private final int expectedThreads; + private final int expectedQueuePoolSize; + private final int expectedCompletedTaskCount; + + MultiThreadedCopy(int expectedThreads, int expectedQueuePoolSize, + int expectedCompletedTaskCount) { + this.expectedThreads = expectedThreads; + this.expectedQueuePoolSize = expectedQueuePoolSize; + this.expectedCompletedTaskCount = expectedCompletedTaskCount; + } + + @Override + protected void processArguments(LinkedList args) + throws IOException { + // Check if the number of threads are same as expected + Assert.assertEquals(expectedThreads, getThreadCount()); + // Check if the queue pool size of executor is same as expected + Assert.assertEquals(expectedQueuePoolSize, getThreadPoolQueueSize()); + + super.processArguments(args); + + if (isMultiThreadNecessary(args)) { + // Once the copy is complete, check following + // 1) number of completed tasks are same as expected + // 2) There are no active tasks in the executor + // 3) Executor has shutdown correctly + ThreadPoolExecutor executor = getExecutor(); + Assert.assertEquals(expectedCompletedTaskCount, + executor.getCompletedTaskCount()); + Assert.assertEquals(0, executor.getActiveCount()); + Assert.assertTrue(executor.isTerminated()); + } else { + assert getExecutor() == null; + } + } + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCpCommand.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCpCommand.java new file mode 100644 index 0000000000000..214f1a0686cd9 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCpCommand.java @@ -0,0 +1,210 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.shell; + +import java.io.IOException; +import java.util.LinkedList; +import java.util.concurrent.ThreadPoolExecutor; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.RandomUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileSystemTestHelper; +import org.apache.hadoop.fs.LocalFileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.shell.CopyCommands.Cp; + +import static org.apache.hadoop.fs.shell.CopyCommandWithMultiThread.DEFAULT_QUEUE_SIZE; +import static org.junit.Assert.assertEquals; + +public class TestCpCommand { + + private static final String FROM_DIR_NAME = "fromDir"; + private static final String TO_DIR_NAME = "toDir"; + + private static FileSystem fs; + private static Path testDir; + private static Configuration conf; + + private Path dir = null; + private int numFiles = 0; + + private static int initialize(Path dir) throws Exception { + fs.mkdirs(dir); + Path fromDirPath = new Path(dir, FROM_DIR_NAME); + fs.mkdirs(fromDirPath); + Path toDirPath = new Path(dir, TO_DIR_NAME); + fs.mkdirs(toDirPath); + + int numTotalFiles = 0; + int numDirs = RandomUtils.nextInt(0, 5); + for (int dirCount = 0; dirCount < numDirs; ++dirCount) { + Path subDirPath = new Path(fromDirPath, "subdir" + dirCount); + fs.mkdirs(subDirPath); + int numFiles = RandomUtils.nextInt(0, 10); + for (int fileCount = 0; fileCount < numFiles; ++fileCount) { + numTotalFiles++; + Path subFile = new Path(subDirPath, "file" + fileCount); + fs.createNewFile(subFile); + FSDataOutputStream output = fs.create(subFile, true); + for (int i = 0; i < 100; ++i) { + output.writeInt(i); + output.writeChar('\n'); + } + output.close(); + } + } + + return numTotalFiles; + } + + @BeforeClass + public static void init() throws Exception { + conf = new Configuration(false); + conf.set("fs.file.impl", LocalFileSystem.class.getName()); + fs = FileSystem.getLocal(conf); + testDir = new FileSystemTestHelper().getTestRootPath(fs); + // don't want scheme on the path, just an absolute path + testDir = new Path(fs.makeQualified(testDir).toUri().getPath()); + + FileSystem.setDefaultUri(conf, fs.getUri()); + fs.setWorkingDirectory(testDir); + } + + @AfterClass + public static void cleanup() throws Exception { + fs.delete(testDir, true); + fs.close(); + } + + private void run(CopyCommandWithMultiThread cmd, String... args) { + cmd.setConf(conf); + assertEquals(0, cmd.run(args)); + } + + @Before + public void initDirectory() throws Exception { + dir = new Path("dir" + RandomStringUtils.randomNumeric(4)); + numFiles = initialize(dir); + } + + @Test(timeout = 10000) + public void testCp() throws Exception { + MultiThreadedCp copy = new MultiThreadedCp(1, DEFAULT_QUEUE_SIZE, 0); + run(copy, new Path(dir, FROM_DIR_NAME).toString(), + new Path(dir, TO_DIR_NAME).toString()); + assert copy.getExecutor() == null; + } + + @Test(timeout = 10000) + public void testCpWithThreads() { + run(new MultiThreadedCp(5, DEFAULT_QUEUE_SIZE, numFiles), "-t", "5", + new Path(dir, FROM_DIR_NAME).toString(), + new Path(dir, TO_DIR_NAME).toString()); + } + + @Test(timeout = 10000) + public void testCpWithThreadWrong() { + run(new MultiThreadedCp(1, DEFAULT_QUEUE_SIZE, 0), "-t", "0", + new Path(dir, FROM_DIR_NAME).toString(), + new Path(dir, TO_DIR_NAME).toString()); + } + + @Test(timeout = 10000) + public void testCpWithThreadsAndQueueSize() { + int queueSize = 256; + run(new MultiThreadedCp(5, queueSize, numFiles), "-t", "5", "-q", + Integer.toString(queueSize), + new Path(dir, FROM_DIR_NAME).toString(), + new Path(dir, TO_DIR_NAME).toString()); + } + + @Test(timeout = 10000) + public void testCpWithThreadsAndQueueSizeWrong() { + int queueSize = 0; + run(new MultiThreadedCp(5, DEFAULT_QUEUE_SIZE, numFiles), "-t", "5", "-q", + Integer.toString(queueSize), + new Path(dir, FROM_DIR_NAME).toString(), + new Path(dir, TO_DIR_NAME).toString()); + } + + @Test(timeout = 10000) + public void testCpSingleFile() throws Exception { + Path fromDirPath = new Path(dir, FROM_DIR_NAME); + Path subFile = new Path(fromDirPath, "file0"); + fs.createNewFile(subFile); + FSDataOutputStream output = fs.create(subFile, true); + for (int i = 0; i < 100; ++i) { + output.writeInt(i); + output.writeChar('\n'); + } + output.close(); + + MultiThreadedCp copy = new MultiThreadedCp(5, DEFAULT_QUEUE_SIZE, 0); + run(copy, "-t", "5", subFile.toString(), + new Path(dir, TO_DIR_NAME).toString()); + assert copy.getExecutor() == null; + } + + private static class MultiThreadedCp extends Cp { + public static final String NAME = "multiThreadCp"; + private final int expectedThreads; + private final int expectedQueuePoolSize; + private final int expectedCompletedTaskCount; + + MultiThreadedCp(int expectedThreads, int expectedQueuePoolSize, + int expectedCompletedTaskCount) { + this.expectedThreads = expectedThreads; + this.expectedQueuePoolSize = expectedQueuePoolSize; + this.expectedCompletedTaskCount = expectedCompletedTaskCount; + } + + @Override + protected void processArguments(LinkedList args) + throws IOException { + // Check if the number of threads are same as expected + Assert.assertEquals(expectedThreads, getThreadCount()); + // Check if the queue pool size of executor is same as expected + Assert.assertEquals(expectedQueuePoolSize, getThreadPoolQueueSize()); + + super.processArguments(args); + + if (isMultiThreadNecessary(args)) { + // Once the copy is complete, check following + // 1) number of completed tasks are same as expected + // 2) There are no active tasks in the executor + // 3) Executor has shutdown correctly + ThreadPoolExecutor executor = getExecutor(); + Assert.assertEquals(expectedCompletedTaskCount, + executor.getCompletedTaskCount()); + Assert.assertEquals(0, executor.getActiveCount()); + Assert.assertTrue(executor.isTerminated()); + } else { + assert getExecutor() == null; + } + } + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/statistics/IOStatisticAssertions.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/statistics/IOStatisticAssertions.java index 22f6c33d2e260..755599f0c390c 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/statistics/IOStatisticAssertions.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/statistics/IOStatisticAssertions.java @@ -36,6 +36,8 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import static org.apache.hadoop.fs.statistics.StoreStatisticNames.SUFFIX_MAX; +import static org.apache.hadoop.fs.statistics.StoreStatisticNames.SUFFIX_MIN; import static org.assertj.core.api.Assertions.assertThat; /** @@ -347,6 +349,24 @@ public static AbstractLongAssert assertThatStatisticMaximum( verifyStatisticsNotNull(stats).maximums()); } + /** + * Assert that a duration is within a given minimum/maximum range. + * @param stats statistics source + * @param key statistic key without any suffix + * @param min minimum statistic must be equal to or greater than this. + * @param max maximum statistic must be equal to or less than this. + */ + public static void assertDurationRange( + final IOStatistics stats, + final String key, + final long min, + final long max) { + assertThatStatisticMinimum(stats, key + SUFFIX_MIN) + .isGreaterThanOrEqualTo(min); + assertThatStatisticMaximum(stats, key + SUFFIX_MAX) + .isLessThanOrEqualTo(max); + } + /** * Start an assertion chain on * a required mean statistic. diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/statistics/TestDurationTracking.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/statistics/TestDurationTracking.java index 8258b62c1f759..cfde1583e2c21 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/statistics/TestDurationTracking.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/statistics/TestDurationTracking.java @@ -30,7 +30,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hadoop.fs.impl.FutureIOSupport; import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore; import org.apache.hadoop.test.AbstractHadoopTestBase; import org.apache.hadoop.util.functional.FunctionRaisingIOE; @@ -276,7 +275,7 @@ public void testCallableIOEFailureDuration() throws Throwable { */ @Test public void testDurationThroughEval() throws Throwable { - CompletableFuture eval = FutureIOSupport.eval( + CompletableFuture eval = FutureIO.eval( trackDurationOfOperation(stats, REQUESTS, () -> { sleepf(100); throw new FileNotFoundException("oops"); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemLocalFileSystem.java index 808d8b06c35ba..adc5db87e7725 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemLocalFileSystem.java @@ -33,6 +33,8 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import static org.apache.hadoop.fs.FileSystem.TRASH_PREFIX; +import org.apache.hadoop.security.UserGroupInformation; import org.junit.After; import org.junit.Before; @@ -61,6 +63,13 @@ public void setUp() throws Exception { } + @Override + Path getTrashRootInFallBackFS() throws IOException { + return new Path( + "/" + TRASH_PREFIX + "/" + UserGroupInformation.getCurrentUser() + .getShortUserName()); + } + @Test public void testNflyWriteSimple() throws IOException { LOG.info("Starting testNflyWriteSimple"); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemWithAuthorityLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemWithAuthorityLocalFileSystem.java index 877c2228c1eea..9223338f34bf5 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemWithAuthorityLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemWithAuthorityLocalFileSystem.java @@ -25,6 +25,9 @@ import org.apache.hadoop.fs.FileSystemTestHelper; import org.apache.hadoop.fs.FsConstants; import org.apache.hadoop.fs.Path; +import static org.apache.hadoop.fs.FileSystem.TRASH_PREFIX; +import org.apache.hadoop.security.UserGroupInformation; +import java.io.IOException; import org.junit.After; import org.junit.Assert; @@ -63,6 +66,13 @@ public void tearDown() throws Exception { super.tearDown(); } + @Override + Path getTrashRootInFallBackFS() throws IOException { + return new Path( + "/" + TRASH_PREFIX + "/" + UserGroupInformation.getCurrentUser() + .getShortUserName()); + } + @Override @Test public void testBasicPaths() { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java index 434eff0bef32e..6eb0570f9c60b 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java @@ -69,6 +69,8 @@ import static org.apache.hadoop.fs.FileSystemTestHelper.*; import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE; import static org.apache.hadoop.fs.viewfs.Constants.PERMISSION_555; +import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT; +import static org.apache.hadoop.fs.FileSystem.TRASH_PREFIX; import org.junit.After; import org.junit.Assert; @@ -1102,6 +1104,176 @@ public void testTrashRoot() throws IOException { Assert.assertTrue("", fsView.getTrashRoots(true).size() > 0); } + // Default implementation of getTrashRoot for a fallback FS mounted at root: + // e.g., fallbackFS.uri.getPath = '/' + Path getTrashRootInFallBackFS() throws IOException { + return new Path(fsTarget.getHomeDirectory().toUri().getPath(), + TRASH_PREFIX); + } + + /** + * Test TRASH_FORCE_INSIDE_MOUNT_POINT feature for getTrashRoot. + */ + @Test + public void testTrashRootForceInsideMountPoint() throws IOException { + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + Configuration conf2 = new Configuration(conf); + conf2.setBoolean(CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT, true); + ConfigUtil.addLinkFallback(conf2, targetTestRoot.toUri()); + FileSystem fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, conf2); + + // Case 1: path p in the /data mount point. + // Return a trash root within the /data mount point. + Path dataTestPath = new Path("/data/dir/file"); + Path dataTrashRoot = fsView2.makeQualified( + new Path("/data/" + TRASH_PREFIX + "/" + ugi.getShortUserName())); + Assert.assertEquals(dataTrashRoot, fsView2.getTrashRoot(dataTestPath)); + + // Case 2: path p not found in mount table. + // Return a trash root in fallback FS. + Path nonExistentPath = new Path("/nonExistentDir/nonExistentFile"); + Path expectedTrash = + fsView2.makeQualified(getTrashRootInFallBackFS()); + Assert.assertEquals(expectedTrash, fsView2.getTrashRoot(nonExistentPath)); + + // Case 3: turn off the CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT flag. + // Return a trash root in user home dir. + conf2.setBoolean(CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT, false); + fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, conf2); + Path targetFSUserHomeTrashRoot = fsTarget.makeQualified( + new Path(fsTarget.getHomeDirectory(), TRASH_PREFIX)); + Assert.assertEquals(targetFSUserHomeTrashRoot, + fsView2.getTrashRoot(dataTestPath)); + + // Case 4: viewFS without fallback. Expect exception for a nonExistent path + conf2 = new Configuration(conf); + fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, conf2); + try { + fsView2.getTrashRoot(nonExistentPath); + } catch (NotInMountpointException ignored) { + } + } + + /** + * A mocked FileSystem which returns a deep trash dir. + */ + static class DeepTrashRootMockFS extends MockFileSystem { + public static final Path TRASH = + new Path("/vol/very/deep/deep/trash/dir/.Trash"); + + @Override + public Path getTrashRoot(Path path) { + return TRASH; + } + } + + /** + * Test getTrashRoot that is very deep inside a mount point. + */ + @Test + public void testTrashRootDeepTrashDir() throws IOException { + + Configuration conf2 = ViewFileSystemTestSetup.createConfig(); + conf2.setBoolean(CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT, true); + conf2.setClass("fs.mocktrashfs.impl", DeepTrashRootMockFS.class, + FileSystem.class); + ConfigUtil.addLink(conf2, "/mnt/datavol1", + URI.create("mocktrashfs://localhost/vol")); + Path testPath = new Path("/mnt/datavol1/projs/proj"); + FileSystem fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, conf2); + Path expectedTrash = fsView2.makeQualified( + new Path("/mnt/datavol1/very/deep/deep/trash/dir/.Trash")); + Assert.assertEquals(expectedTrash, fsView2.getTrashRoot(testPath)); + } + + /** + * Test getTrashRoots() for all users. + */ + @Test + public void testTrashRootsAllUsers() throws IOException { + Configuration conf2 = new Configuration(conf); + conf2.setBoolean(CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT, true); + FileSystem fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, conf2); + + // Case 1: verify correct trash roots from fsView and fsView2 + int beforeTrashRootNum = fsView.getTrashRoots(true).size(); + int beforeTrashRootNum2 = fsView2.getTrashRoots(true).size(); + Assert.assertEquals(beforeTrashRootNum, beforeTrashRootNum2); + + fsView.mkdirs(new Path("/data/" + TRASH_PREFIX + "/user1")); + fsView.mkdirs(new Path("/data/" + TRASH_PREFIX + "/user2")); + fsView.mkdirs(new Path("/user/" + TRASH_PREFIX + "/user3")); + fsView.mkdirs(new Path("/user/" + TRASH_PREFIX + "/user4")); + fsView.mkdirs(new Path("/user2/" + TRASH_PREFIX + "/user5")); + int afterTrashRootsNum = fsView.getTrashRoots(true).size(); + int afterTrashRootsNum2 = fsView2.getTrashRoots(true).size(); + Assert.assertEquals(beforeTrashRootNum, afterTrashRootsNum); + Assert.assertEquals(beforeTrashRootNum2 + 5, afterTrashRootsNum2); + + // Case 2: per-user mount point + fsTarget.mkdirs(new Path(targetTestRoot, "Users/userA/.Trash/userA")); + Configuration conf3 = new Configuration(conf2); + ConfigUtil.addLink(conf3, "/Users/userA", + new Path(targetTestRoot, "Users/userA").toUri()); + FileSystem fsView3 = FileSystem.get(FsConstants.VIEWFS_URI, conf3); + int trashRootsNum3 = fsView3.getTrashRoots(true).size(); + Assert.assertEquals(afterTrashRootsNum2 + 1, trashRootsNum3); + + // Case 3: single /Users mount point for all users + fsTarget.mkdirs(new Path(targetTestRoot, "Users/.Trash/user1")); + fsTarget.mkdirs(new Path(targetTestRoot, "Users/.Trash/user2")); + Configuration conf4 = new Configuration(conf2); + ConfigUtil.addLink(conf4, "/Users", + new Path(targetTestRoot, "Users").toUri()); + FileSystem fsView4 = FileSystem.get(FsConstants.VIEWFS_URI, conf4); + int trashRootsNum4 = fsView4.getTrashRoots(true).size(); + Assert.assertEquals(afterTrashRootsNum2 + 2, trashRootsNum4); + + // Case 4: test trash roots in fallback FS + fsTarget.mkdirs(new Path(targetTestRoot, ".Trash/user10")); + fsTarget.mkdirs(new Path(targetTestRoot, ".Trash/user11")); + fsTarget.mkdirs(new Path(targetTestRoot, ".Trash/user12")); + Configuration conf5 = new Configuration(conf2); + ConfigUtil.addLinkFallback(conf5, targetTestRoot.toUri()); + FileSystem fsView5 = FileSystem.get(FsConstants.VIEWFS_URI, conf5); + int trashRootsNum5 = fsView5.getTrashRoots(true).size(); + Assert.assertEquals(afterTrashRootsNum2 + 3, trashRootsNum5); + } + + /** + * Test getTrashRoots() for current user. + */ + @Test + public void testTrashRootsCurrentUser() throws IOException { + String currentUser = + UserGroupInformation.getCurrentUser().getShortUserName(); + Configuration conf2 = new Configuration(conf); + conf2.setBoolean(CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT, true); + FileSystem fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, conf2); + + int beforeTrashRootNum = fsView.getTrashRoots(false).size(); + int beforeTrashRootNum2 = fsView2.getTrashRoots(false).size(); + Assert.assertEquals(beforeTrashRootNum, beforeTrashRootNum2); + + fsView.mkdirs(new Path("/data/" + TRASH_PREFIX + "/" + currentUser)); + fsView.mkdirs(new Path("/data/" + TRASH_PREFIX + "/user2")); + fsView.mkdirs(new Path("/user/" + TRASH_PREFIX + "/" + currentUser)); + fsView.mkdirs(new Path("/user/" + TRASH_PREFIX + "/user4")); + fsView.mkdirs(new Path("/user2/" + TRASH_PREFIX + "/user5")); + int afterTrashRootsNum = fsView.getTrashRoots(false).size(); + int afterTrashRootsNum2 = fsView2.getTrashRoots(false).size(); + Assert.assertEquals(beforeTrashRootNum, afterTrashRootsNum); + Assert.assertEquals(beforeTrashRootNum2 + 2, afterTrashRootsNum2); + + // Test trash roots in fallback FS + Configuration conf3 = new Configuration(conf2); + fsTarget.mkdirs(new Path(targetTestRoot, TRASH_PREFIX + "/" + currentUser)); + ConfigUtil.addLinkFallback(conf3, targetTestRoot.toUri()); + FileSystem fsView3 = FileSystem.get(FsConstants.VIEWFS_URI, conf3); + int trashRootsNum3 = fsView3.getTrashRoots(false).size(); + Assert.assertEquals(afterTrashRootsNum2 + 1, trashRootsNum3); + } + @Test(expected = NotInMountpointException.class) public void testViewFileSystemUtil() throws Exception { Configuration newConf = new Configuration(conf); @@ -1515,4 +1687,22 @@ public void testTargetFileSystemLazyInitializationForChecksumMethods() // viewfs inner cache is disabled assertEquals(cacheSize + 1, TestFileUtil.getCacheSize()); } + + @Test + public void testInvalidMountPoints() throws Exception { + final String clusterName = "cluster" + new Random().nextInt(); + Configuration config = new Configuration(conf); + config.set(ConfigUtil.getConfigViewFsPrefix(clusterName) + "." + + Constants.CONFIG_VIEWFS_LINK + "." + "/invalidPath", + "othermockfs:|mockauth/mockpath"); + + try { + FileSystem viewFs = FileSystem.get( + new URI("viewfs://" + clusterName + "/"), config); + fail("FileSystem should not initialize. Should fail with IOException"); + } catch (IOException ex) { + assertTrue("Should get URISyntax Exception", + ex.getMessage().startsWith("URISyntax exception")); + } + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java index 73f6265eee72c..1d855ab442600 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java @@ -522,7 +522,7 @@ public void testListOnInternalDirsOfMountTable() throws IOException { Assert.assertTrue("A mount should appear as symlink", fs.isSymlink()); } - @Test + @Test(expected = FileNotFoundException.class) public void testFileStatusOnMountLink() throws IOException { Assert.assertTrue("Slash should appear as dir", fcView.getFileStatus(new Path("/")).isDirectory()); @@ -534,12 +534,7 @@ public void testFileStatusOnMountLink() throws IOException { checkFileStatus(fcView, "/internalDir/internalDir2/linkToDir3", fileType.isDir); checkFileStatus(fcView, "/linkToAFile", fileType.isFile); - try { - fcView.getFileStatus(new Path("/danglingLink")); - Assert.fail("Excepted a not found exception here"); - } catch ( FileNotFoundException e) { - // as excepted - } + fcView.getFileStatus(new Path("/danglingLink")); } @Test diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestDisabledProfileServlet.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestDisabledProfileServlet.java new file mode 100644 index 0000000000000..ce068bb6f1cf6 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestDisabledProfileServlet.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.http; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import javax.servlet.http.HttpServletResponse; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Small test to cover default disabled prof endpoint. + */ +public class TestDisabledProfileServlet extends HttpServerFunctionalTest { + + private static HttpServer2 server; + private static URL baseUrl; + + @BeforeClass + public static void setup() throws Exception { + server = createTestServer(); + server.start(); + baseUrl = getServerURL(server); + } + + @AfterClass + public static void cleanup() throws Exception { + server.stop(); + } + + @Test + public void testQuery() throws Exception { + try { + readOutput(new URL(baseUrl, "/prof")); + throw new IllegalStateException("Should not reach here"); + } catch (IOException e) { + assertTrue(e.getMessage() + .contains(HttpServletResponse.SC_INTERNAL_SERVER_ERROR + " for URL: " + baseUrl)); + } + + // CORS headers + HttpURLConnection conn = + (HttpURLConnection) new URL(baseUrl, "/prof").openConnection(); + assertEquals("GET", conn.getHeaderField(ProfileServlet.ACCESS_CONTROL_ALLOW_METHODS)); + assertNotNull(conn.getHeaderField(ProfileServlet.ACCESS_CONTROL_ALLOW_ORIGIN)); + conn.disconnect(); + } + + @Test + public void testRequestMethods() throws IOException { + HttpURLConnection connection = getConnection("PUT"); + assertEquals("Unexpected response code", HttpServletResponse.SC_METHOD_NOT_ALLOWED, + connection.getResponseCode()); + connection.disconnect(); + connection = getConnection("POST"); + assertEquals("Unexpected response code", HttpServletResponse.SC_METHOD_NOT_ALLOWED, + connection.getResponseCode()); + connection.disconnect(); + connection = getConnection("DELETE"); + assertEquals("Unexpected response code", HttpServletResponse.SC_METHOD_NOT_ALLOWED, + connection.getResponseCode()); + connection.disconnect(); + connection = getConnection("GET"); + assertEquals("Unexpected response code", HttpServletResponse.SC_INTERNAL_SERVER_ERROR, + connection.getResponseCode()); + connection.disconnect(); + } + + private HttpURLConnection getConnection(final String method) throws IOException { + URL url = new URL(baseUrl, "/prof"); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod(method); + return conn; + } + +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpRequestLog.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpRequestLog.java index d0123e32039c9..58721c4baa8f9 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpRequestLog.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpRequestLog.java @@ -17,32 +17,25 @@ */ package org.apache.hadoop.http; -import org.apache.log4j.Logger; +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + import org.eclipse.jetty.server.CustomRequestLog; import org.eclipse.jetty.server.RequestLog; +import org.eclipse.jetty.server.Slf4jRequestLogWriter; import org.junit.Test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; - public class TestHttpRequestLog { - @Test - public void testAppenderUndefined() { - RequestLog requestLog = HttpRequestLog.getRequestLog("test"); - assertNull("RequestLog should be null", requestLog); - } - @Test public void testAppenderDefined() { - HttpRequestLogAppender requestLogAppender = new HttpRequestLogAppender(); - requestLogAppender.setName("testrequestlog"); - Logger.getLogger("http.requests.test").addAppender(requestLogAppender); RequestLog requestLog = HttpRequestLog.getRequestLog("test"); - Logger.getLogger("http.requests.test").removeAppender(requestLogAppender); assertNotNull("RequestLog should not be null", requestLog); - assertEquals("Class mismatch", - CustomRequestLog.class, requestLog.getClass()); + assertThat(requestLog, instanceOf(CustomRequestLog.class)); + CustomRequestLog crl = (CustomRequestLog) requestLog; + assertThat(crl.getWriter(), instanceOf(Slf4jRequestLogWriter.class)); + assertEquals(CustomRequestLog.EXTENDED_NCSA_FORMAT, crl.getFormatString()); } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerLifecycle.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerLifecycle.java index 40f1b3df08c6e..757ea0c05e7c0 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerLifecycle.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerLifecycle.java @@ -68,27 +68,6 @@ public void testStartedServerIsAlive() throws Throwable { stop(server); } - /** - * Test that the server with request logging enabled - * - * @throws Throwable on failure - */ - @Test - public void testStartedServerWithRequestLog() throws Throwable { - HttpRequestLogAppender requestLogAppender = new HttpRequestLogAppender(); - requestLogAppender.setName("httprequestlog"); - requestLogAppender.setFilename( - GenericTestUtils.getTempPath("jetty-name-yyyy_mm_dd.log")); - Logger.getLogger(HttpServer2.class.getName() + ".test").addAppender(requestLogAppender); - HttpServer2 server = null; - server = createTestServer(); - assertNotLive(server); - server.start(); - assertAlive(server); - stop(server); - Logger.getLogger(HttpServer2.class.getName() + ".test").removeAppender(requestLogAppender); - } - /** * Assert that the result of {@link HttpServer2#toString()} contains the specific text * @param server server to examine diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestProfileServlet.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestProfileServlet.java new file mode 100644 index 0000000000000..5c87451a49e6c --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestProfileServlet.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.http; + +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.UUID; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Test coverage for async profiler servlets: ProfileServlet and ProfileOutputServlet. + */ +public class TestProfileServlet extends HttpServerFunctionalTest { + + private static HttpServer2 server; + private static URL baseUrl; + + private static final Logger LOG = LoggerFactory.getLogger(TestProfileServlet.class); + + @BeforeClass + public static void setup() throws Exception { + ProfileServlet.setIsTestRun(true); + System.setProperty("async.profiler.home", UUID.randomUUID().toString()); + server = createTestServer(); + server.start(); + baseUrl = getServerURL(server); + } + + @AfterClass + public static void cleanup() throws Exception { + ProfileServlet.setIsTestRun(false); + System.clearProperty("async.profiler.home"); + server.stop(); + } + + @Test + public void testQuery() throws Exception { + String output = readOutput(new URL(baseUrl, "/prof")); + LOG.info("/prof output: {}", output); + assertTrue(output.startsWith( + "Started [cpu] profiling. This page will automatically redirect to /prof-output-hadoop/")); + assertTrue(output.contains( + "If empty diagram and Linux 4.6+, see 'Basic Usage' section on the Async Profiler Home" + + " Page, https://github.com/jvm-profiling-tools/async-profiler.")); + + HttpURLConnection conn = + (HttpURLConnection) new URL(baseUrl, "/prof").openConnection(); + assertEquals("GET", conn.getHeaderField(ProfileServlet.ACCESS_CONTROL_ALLOW_METHODS)); + assertEquals(HttpURLConnection.HTTP_ACCEPTED, conn.getResponseCode()); + assertNotNull(conn.getHeaderField(ProfileServlet.ACCESS_CONTROL_ALLOW_ORIGIN)); + assertTrue(conn.getHeaderField("Refresh").startsWith("10;/prof-output-hadoop/async-prof-pid")); + + String redirectOutput = readOutput(new URL(baseUrl, "/prof-output-hadoop")); + LOG.info("/prof-output-hadoop output: {}", redirectOutput); + + HttpURLConnection redirectedConn = + (HttpURLConnection) new URL(baseUrl, "/prof-output-hadoop").openConnection(); + assertEquals(HttpURLConnection.HTTP_OK, redirectedConn.getResponseCode()); + + redirectedConn.disconnect(); + conn.disconnect(); + } + +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/AvroTestUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/AvroTestUtil.java index ec76ea008077d..9c9b75fa76e6c 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/AvroTestUtil.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/AvroTestUtil.java @@ -41,7 +41,7 @@ public static void testReflect(Object value, Type type, String schema) // check that schema matches expected Schema s = ReflectData.get().getSchema(type); - assertEquals(Schema.parse(schema), s); + assertEquals(new Schema.Parser().parse(schema), s); // check that value is serialized correctly ReflectDatumWriter writer = new ReflectDatumWriter(s); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestEnumSetWritable.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestEnumSetWritable.java index 5e71601742f3c..11459261f5b74 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestEnumSetWritable.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestEnumSetWritable.java @@ -119,7 +119,7 @@ public void testSerializeAndDeserializeNull() throws IOException { public void testAvroReflect() throws Exception { String schema = "{\"type\":\"array\",\"items\":{\"type\":\"enum\"," + "\"name\":\"TestEnumSet\"," - + "\"namespace\":\"org.apache.hadoop.io.TestEnumSetWritable$\"," + + "\"namespace\":\"org.apache.hadoop.io.TestEnumSetWritable\"," + "\"symbols\":[\"CREATE\",\"OVERWRITE\",\"APPEND\"]}," + "\"java-class\":\"org.apache.hadoop.io.EnumSetWritable\"}"; Type type = diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java index edab634a0b877..7461ea36f59a3 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java @@ -29,6 +29,7 @@ import org.junit.Test; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.fail; public class TestCodecFactory { @@ -45,8 +46,8 @@ public Configuration getConf() { } @Override - public CompressionOutputStream createOutputStream(OutputStream out) - throws IOException { + public CompressionOutputStream createOutputStream(OutputStream out) + throws IOException { return null; } @@ -62,21 +63,21 @@ public Compressor createCompressor() { @Override public CompressionInputStream createInputStream(InputStream in, - Decompressor decompressor) - throws IOException { + Decompressor decompressor) + throws IOException { return null; } @Override - public CompressionInputStream createInputStream(InputStream in) - throws IOException { + public CompressionInputStream createInputStream(InputStream in) + throws IOException { return null; } @Override public CompressionOutputStream createOutputStream(OutputStream out, - Compressor compressor) - throws IOException { + Compressor compressor) + throws IOException { return null; } @@ -125,7 +126,7 @@ public String getDefaultExtension() { } /** - * Returns a factory for a given set of codecs + * Returns a factory for a given set of codecs. * @param classes the codec classes to include * @return a new factory */ @@ -137,15 +138,21 @@ private static CompressionCodecFactory setClasses(Class[] classes) { private static void checkCodec(String msg, Class expected, CompressionCodec actual) { - assertEquals(msg + " unexpected codec found", - expected.getName(), - actual.getClass().getName()); + if (expected == null) { + assertNull(msg, actual); + } else if (actual == null) { + fail(msg + " result was null"); + } else { + assertEquals(msg + " unexpected codec found", + expected.getName(), + actual.getClass().getName()); + } } @Test public void testFinding() { CompressionCodecFactory factory = - new CompressionCodecFactory(new Configuration()); + new CompressionCodecFactory(new Configuration()); CompressionCodec codec = factory.getCodec(new Path("/tmp/foo.bar")); assertEquals("default factory foo codec", null, codec); codec = factory.getCodecByClassName(BarCodec.class.getCanonicalName()); @@ -153,6 +160,8 @@ public void testFinding() { codec = factory.getCodec(new Path("/tmp/foo.gz")); checkCodec("default factory for .gz", GzipCodec.class, codec); + codec = factory.getCodec(new Path("/tmp/foo.GZ")); + checkCodec("default factory for .GZ", GzipCodec.class, codec); codec = factory.getCodecByClassName(GzipCodec.class.getCanonicalName()); checkCodec("default factory for gzip codec", GzipCodec.class, codec); codec = factory.getCodecByName("gzip"); @@ -168,6 +177,8 @@ public void testFinding() { codec = factory.getCodec(new Path("/tmp/foo.bz2")); checkCodec("default factory for .bz2", BZip2Codec.class, codec); + codec = factory.getCodec(new Path("/tmp/foo.BZ2")); + checkCodec("default factory for .BZ2", BZip2Codec.class, codec); codec = factory.getCodecByClassName(BZip2Codec.class.getCanonicalName()); checkCodec("default factory for bzip2 codec", BZip2Codec.class, codec); codec = factory.getCodecByName("bzip2"); @@ -221,16 +232,22 @@ public void testFinding() { FooBarCodec.class}); codec = factory.getCodec(new Path("/tmp/.foo.bar.gz")); checkCodec("full factory gz codec", GzipCodec.class, codec); + codec = factory.getCodec(new Path("/tmp/.foo.bar.GZ")); + checkCodec("full factory GZ codec", GzipCodec.class, codec); codec = factory.getCodecByClassName(GzipCodec.class.getCanonicalName()); checkCodec("full codec gz codec", GzipCodec.class, codec); codec = factory.getCodec(new Path("/tmp/foo.bz2")); checkCodec("full factory for .bz2", BZip2Codec.class, codec); + codec = factory.getCodec(new Path("/tmp/foo.BZ2")); + checkCodec("full factory for .BZ2", BZip2Codec.class, codec); codec = factory.getCodecByClassName(BZip2Codec.class.getCanonicalName()); checkCodec("full codec bzip2 codec", BZip2Codec.class, codec); codec = factory.getCodec(new Path("/tmp/foo.bar")); checkCodec("full factory bar codec", BarCodec.class, codec); + codec = factory.getCodec(new Path("/tmp/foo.BAR")); + checkCodec("full factory BAR codec", BarCodec.class, codec); codec = factory.getCodecByClassName(BarCodec.class.getCanonicalName()); checkCodec("full factory bar codec", BarCodec.class, codec); codec = factory.getCodecByName("bar"); @@ -240,6 +257,8 @@ public void testFinding() { codec = factory.getCodec(new Path("/tmp/foo/baz.foo.bar")); checkCodec("full factory foo bar codec", FooBarCodec.class, codec); + codec = factory.getCodec(new Path("/tmp/foo/baz.FOO.bar")); + checkCodec("full factory FOO bar codec", FooBarCodec.class, codec); codec = factory.getCodecByClassName(FooBarCodec.class.getCanonicalName()); checkCodec("full factory foo bar codec", FooBarCodec.class, codec); codec = factory.getCodecByName("foobar"); @@ -249,6 +268,8 @@ public void testFinding() { codec = factory.getCodec(new Path("/tmp/foo.foo")); checkCodec("full factory foo codec", FooCodec.class, codec); + codec = factory.getCodec(new Path("/tmp/FOO.FOO")); + checkCodec("full factory FOO codec", FooCodec.class, codec); codec = factory.getCodecByClassName(FooCodec.class.getCanonicalName()); checkCodec("full factory foo codec", FooCodec.class, codec); codec = factory.getCodecByName("foo"); @@ -259,6 +280,8 @@ public void testFinding() { factory = setClasses(new Class[]{NewGzipCodec.class}); codec = factory.getCodec(new Path("/tmp/foo.gz")); checkCodec("overridden factory for .gz", NewGzipCodec.class, codec); + codec = factory.getCodec(new Path("/tmp/foo.GZ")); + checkCodec("overridden factory for .GZ", NewGzipCodec.class, codec); codec = factory.getCodecByClassName(NewGzipCodec.class.getCanonicalName()); checkCodec("overridden factory for gzip codec", NewGzipCodec.class, codec); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zstd/TestZStandardCompressorDecompressor.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zstd/TestZStandardCompressorDecompressor.java index f12226d897d59..d4c0718220a20 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zstd/TestZStandardCompressorDecompressor.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zstd/TestZStandardCompressorDecompressor.java @@ -231,6 +231,65 @@ public void testCompressorDecompressorLogicWithCompressionStreams() } } + /** + * Verify decompressor logic with some finish operation in compress. + */ + @Test + public void testCompressorDecompressorWithFinish() throws Exception { + DataOutputStream deflateOut = null; + DataInputStream inflateIn = null; + int byteSize = 1024 * 100; + byte[] bytes = generate(byteSize); + int firstLength = 1024 * 30; + + int bufferSize = IO_FILE_BUFFER_SIZE_DEFAULT; + try { + DataOutputBuffer compressedDataBuffer = new DataOutputBuffer(); + CompressionOutputStream deflateFilter = + new CompressorStream(compressedDataBuffer, new ZStandardCompressor(), + bufferSize); + + deflateOut = + new DataOutputStream(new BufferedOutputStream(deflateFilter)); + + // Write some data and finish. + deflateOut.write(bytes, 0, firstLength); + deflateFilter.finish(); + deflateOut.flush(); + + // ResetState then write some data and finish. + deflateFilter.resetState(); + deflateOut.write(bytes, firstLength, firstLength); + deflateFilter.finish(); + deflateOut.flush(); + + // ResetState then write some data and finish. + deflateFilter.resetState(); + deflateOut.write(bytes, firstLength * 2, byteSize - firstLength * 2); + deflateFilter.finish(); + deflateOut.flush(); + + DataInputBuffer deCompressedDataBuffer = new DataInputBuffer(); + deCompressedDataBuffer.reset(compressedDataBuffer.getData(), 0, + compressedDataBuffer.getLength()); + + CompressionInputStream inflateFilter = + new DecompressorStream(deCompressedDataBuffer, + new ZStandardDecompressor(bufferSize), bufferSize); + + inflateIn = new DataInputStream(new BufferedInputStream(inflateFilter)); + + byte[] result = new byte[byteSize]; + inflateIn.read(result); + assertArrayEquals( + "original array not equals compress/decompressed array", bytes, + result); + } finally { + IOUtils.closeStream(deflateOut); + IOUtils.closeStream(inflateIn); + } + } + @Test public void testZStandardCompressDecompressInMultiThreads() throws Exception { MultithreadedTestUtil.TestContext ctx = diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java index b78900b609e54..5fc9cb5410b30 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ipc; +import org.apache.hadoop.ipc.metrics.RpcMetrics; import org.apache.hadoop.thirdparty.protobuf.ServiceException; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.conf.Configuration; @@ -1107,6 +1108,37 @@ public TestRpcService run() { } } + @Test + public void testNumInProcessHandlerMetrics() throws Exception { + UserGroupInformation ugi = UserGroupInformation. + createUserForTesting("user123", new String[0]); + // use 1 handler so the callq can be plugged + final Server server = setupTestServer(conf, 1); + try { + RpcMetrics rpcMetrics = server.getRpcMetrics(); + assertEquals(0, rpcMetrics.getNumInProcessHandler()); + + ExternalCall call1 = newExtCall(ugi, () -> { + assertEquals(1, rpcMetrics.getNumInProcessHandler()); + return UserGroupInformation.getCurrentUser().getUserName(); + }); + ExternalCall call2 = newExtCall(ugi, () -> { + assertEquals(1, rpcMetrics.getNumInProcessHandler()); + return null; + }); + + server.queueCall(call1); + server.queueCall(call2); + + // Wait for call1 and call2 to enter the handler. + call1.get(); + call2.get(); + assertEquals(0, rpcMetrics.getNumInProcessHandler()); + } finally { + server.stop(); + } + } + /** * Test RPC backoff by queue full. */ diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java index 0962b50099c57..e9019e3d24eba 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java @@ -22,6 +22,7 @@ import org.apache.hadoop.thirdparty.protobuf.RpcController; import org.apache.hadoop.thirdparty.protobuf.ServiceException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import org.apache.hadoop.conf.Configuration; @@ -124,18 +125,19 @@ protected static RPC.Server setupTestServer( return server; } - protected static TestRpcService getClient(InetSocketAddress serverAddr, - Configuration clientConf) + protected static TestRpcService getClient(InetSocketAddress serverAddr, Configuration clientConf) throws ServiceException { - try { - return RPC.getProxy(TestRpcService.class, 0, serverAddr, clientConf); - } catch (IOException e) { - throw new ServiceException(e); - } + return getClient(serverAddr, clientConf, null); + } + + protected static TestRpcService getClient(InetSocketAddress serverAddr, + Configuration clientConf, RetryPolicy connectionRetryPolicy) throws ServiceException { + return getClient(serverAddr, clientConf, connectionRetryPolicy, null); } protected static TestRpcService getClient(InetSocketAddress serverAddr, - Configuration clientConf, final RetryPolicy connectionRetryPolicy) + Configuration clientConf, final RetryPolicy connectionRetryPolicy, + AtomicBoolean fallbackToSimpleAuth) throws ServiceException { try { return RPC.getProtocolProxy( @@ -146,7 +148,7 @@ protected static TestRpcService getClient(InetSocketAddress serverAddr, clientConf, NetUtils.getDefaultSocketFactory(clientConf), RPC.getRpcTimeout(clientConf), - connectionRetryPolicy, null).getProxy(); + connectionRetryPolicy, fallbackToSimpleAuth).getProxy(); } catch (IOException e) { throw new ServiceException(e); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java index 72085a19ec711..662faea599648 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java @@ -72,6 +72,7 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Pattern; @@ -569,6 +570,72 @@ public void testSimpleServer() throws Exception { assertAuthEquals(SIMPLE, getAuthMethod(KERBEROS, SIMPLE, UseToken.OTHER)); } + /** + * In DfsClient there is a fallback mechanism to simple auth, which passes in an atomic boolean + * to the ipc Client, which then sets it during setupIOStreams. + * SetupIOStreams were running only once per connection, so if two separate DfsClient was + * instantiated, then due to the connection caching inside the ipc client, the second DfsClient + * did not have the passed in atomic boolean set properly if the first client was not yet closed, + * as setupIOStreams was yielding to set up new streams as it has reused the already existing + * connection. + * This test mimics this behaviour, and asserts the fallback whether it is set correctly. + * @see HADOOP-17975 + */ + @Test + public void testClientFallbackToSimpleAuthForASecondClient() throws Exception { + Configuration serverConf = createConfForAuth(SIMPLE); + Server server = startServer(serverConf, + setupServerUgi(SIMPLE, serverConf), + createServerSecretManager(SIMPLE, new TestTokenSecretManager())); + final InetSocketAddress serverAddress = NetUtils.getConnectAddress(server); + + clientFallBackToSimpleAllowed = true; + Configuration clientConf = createConfForAuth(KERBEROS); + UserGroupInformation clientUgi = setupClientUgi(KERBEROS, clientConf); + + AtomicBoolean fallbackToSimpleAuth1 = new AtomicBoolean(); + AtomicBoolean fallbackToSimpleAuth2 = new AtomicBoolean(); + try { + LOG.info("trying ugi:"+ clientUgi +" tokens:"+ clientUgi.getTokens()); + clientUgi.doAs((PrivilegedExceptionAction) () -> { + TestRpcService proxy1 = null; + TestRpcService proxy2 = null; + try { + proxy1 = getClient(serverAddress, clientConf, null, fallbackToSimpleAuth1); + proxy1.ping(null, newEmptyRequest()); + // make sure the other side thinks we are who we said we are!!! + assertEquals(clientUgi.getUserName(), + proxy1.getAuthUser(null, newEmptyRequest()).getUser()); + AuthMethod authMethod = + convert(proxy1.getAuthMethod(null, newEmptyRequest())); + assertAuthEquals(SIMPLE, authMethod.toString()); + + proxy2 = getClient(serverAddress, clientConf, null, fallbackToSimpleAuth2); + proxy2.ping(null, newEmptyRequest()); + // make sure the other side thinks we are who we said we are!!! + assertEquals(clientUgi.getUserName(), + proxy2.getAuthUser(null, newEmptyRequest()).getUser()); + AuthMethod authMethod2 = + convert(proxy2.getAuthMethod(null, newEmptyRequest())); + assertAuthEquals(SIMPLE, authMethod2.toString()); + } finally { + if (proxy1 != null) { + RPC.stopProxy(proxy1); + } + if (proxy2 != null) { + RPC.stopProxy(proxy2); + } + } + return null; + }); + } finally { + server.stop(); + } + + assertTrue("First client does not set to fall back properly.", fallbackToSimpleAuth1.get()); + assertTrue("Second client does not set to fall back properly.", fallbackToSimpleAuth2.get()); + } + @Test public void testNoClientFallbackToSimple() throws Exception { @@ -815,22 +882,44 @@ private String getAuthMethod( return e.toString(); } } - + private String internalGetAuthMethod( final AuthMethod clientAuth, final AuthMethod serverAuth, final UseToken tokenType) throws Exception { - - final Configuration serverConf = new Configuration(conf); - serverConf.set(HADOOP_SECURITY_AUTHENTICATION, serverAuth.toString()); - UserGroupInformation.setConfiguration(serverConf); - - final UserGroupInformation serverUgi = (serverAuth == KERBEROS) - ? UserGroupInformation.createRemoteUser("server/localhost@NONE") - : UserGroupInformation.createRemoteUser("server"); - serverUgi.setAuthenticationMethod(serverAuth); final TestTokenSecretManager sm = new TestTokenSecretManager(); + + Configuration serverConf = createConfForAuth(serverAuth); + Server server = startServer( + serverConf, + setupServerUgi(serverAuth, serverConf), + createServerSecretManager(serverAuth, sm)); + final InetSocketAddress serverAddress = NetUtils.getConnectAddress(server); + + final Configuration clientConf = createConfForAuth(clientAuth); + final UserGroupInformation clientUgi = setupClientUgi(clientAuth, clientConf); + + setupTokenIfNeeded(tokenType, sm, clientUgi, serverAddress); + + try { + return createClientAndQueryAuthMethod(serverAddress, clientConf, clientUgi, null); + } finally { + server.stop(); + } + } + + private Configuration createConfForAuth(AuthMethod clientAuth) { + final Configuration clientConf = new Configuration(conf); + clientConf.set(HADOOP_SECURITY_AUTHENTICATION, clientAuth.toString()); + clientConf.setBoolean( + CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_KEY, + clientFallBackToSimpleAllowed); + return clientConf; + } + + private SecretManager createServerSecretManager( + AuthMethod serverAuth, TestTokenSecretManager sm) { boolean useSecretManager = (serverAuth != SIMPLE); if (enableSecretManager != null) { useSecretManager &= enableSecretManager; @@ -839,26 +928,43 @@ private String internalGetAuthMethod( useSecretManager |= forceSecretManager; } final SecretManager serverSm = useSecretManager ? sm : null; + return serverSm; + } + private Server startServer(Configuration serverConf, UserGroupInformation serverUgi, + SecretManager serverSm) throws IOException, InterruptedException { Server server = serverUgi.doAs(new PrivilegedExceptionAction() { @Override public Server run() throws IOException { return setupTestServer(serverConf, 5, serverSm); } }); + return server; + } - final Configuration clientConf = new Configuration(conf); - clientConf.set(HADOOP_SECURITY_AUTHENTICATION, clientAuth.toString()); - clientConf.setBoolean( - CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_KEY, - clientFallBackToSimpleAllowed); + private UserGroupInformation setupServerUgi(AuthMethod serverAuth, + Configuration serverConf) { + UserGroupInformation.setConfiguration(serverConf); + + final UserGroupInformation serverUgi = (serverAuth == KERBEROS) + ? UserGroupInformation.createRemoteUser("server/localhost@NONE") + : UserGroupInformation.createRemoteUser("server"); + serverUgi.setAuthenticationMethod(serverAuth); + return serverUgi; + } + + private UserGroupInformation setupClientUgi(AuthMethod clientAuth, + Configuration clientConf) { UserGroupInformation.setConfiguration(clientConf); - + final UserGroupInformation clientUgi = UserGroupInformation.createRemoteUser("client"); - clientUgi.setAuthenticationMethod(clientAuth); + clientUgi.setAuthenticationMethod(clientAuth); + return clientUgi; + } - final InetSocketAddress addr = NetUtils.getConnectAddress(server); + private void setupTokenIfNeeded(UseToken tokenType, TestTokenSecretManager sm, + UserGroupInformation clientUgi, InetSocketAddress addr) { if (tokenType != UseToken.NONE) { TestTokenIdentifier tokenId = new TestTokenIdentifier( new Text(clientUgi.getUserName())); @@ -881,44 +987,44 @@ public Server run() throws IOException { } clientUgi.addToken(token); } + } - try { - LOG.info("trying ugi:"+clientUgi+" tokens:"+clientUgi.getTokens()); - return clientUgi.doAs(new PrivilegedExceptionAction() { - @Override - public String run() throws IOException { - TestRpcService proxy = null; - try { - proxy = getClient(addr, clientConf); - - proxy.ping(null, newEmptyRequest()); - // make sure the other side thinks we are who we said we are!!! - assertEquals(clientUgi.getUserName(), - proxy.getAuthUser(null, newEmptyRequest()).getUser()); - AuthMethod authMethod = - convert(proxy.getAuthMethod(null, newEmptyRequest())); - // verify sasl completed with correct QOP - assertEquals((authMethod != SIMPLE) ? expectedQop.saslQop : null, - RPC.getConnectionIdForProxy(proxy).getSaslQop()); - return authMethod != null ? authMethod.toString() : null; - } catch (ServiceException se) { - if (se.getCause() instanceof RemoteException) { - throw (RemoteException) se.getCause(); - } else if (se.getCause() instanceof IOException) { - throw (IOException) se.getCause(); - } else { - throw new RuntimeException(se.getCause()); - } - } finally { - if (proxy != null) { - RPC.stopProxy(proxy); - } + private String createClientAndQueryAuthMethod(InetSocketAddress serverAddress, + Configuration clientConf, UserGroupInformation clientUgi, AtomicBoolean fallbackToSimpleAuth) + throws IOException, InterruptedException { + LOG.info("trying ugi:"+ clientUgi +" tokens:"+ clientUgi.getTokens()); + return clientUgi.doAs(new PrivilegedExceptionAction() { + @Override + public String run() throws IOException { + TestRpcService proxy = null; + try { + proxy = getClient(serverAddress, clientConf, null, fallbackToSimpleAuth); + + proxy.ping(null, newEmptyRequest()); + // make sure the other side thinks we are who we said we are!!! + assertEquals(clientUgi.getUserName(), + proxy.getAuthUser(null, newEmptyRequest()).getUser()); + AuthMethod authMethod = + convert(proxy.getAuthMethod(null, newEmptyRequest())); + // verify sasl completed with correct QOP + assertEquals((authMethod != SIMPLE) ? expectedQop.saslQop : null, + RPC.getConnectionIdForProxy(proxy).getSaslQop()); + return authMethod != null ? authMethod.toString() : null; + } catch (ServiceException se) { + if (se.getCause() instanceof RemoteException) { + throw (RemoteException) se.getCause(); + } else if (se.getCause() instanceof IOException) { + throw (IOException) se.getCause(); + } else { + throw new RuntimeException(se.getCause()); + } + } finally { + if (proxy != null) { + RPC.stopProxy(proxy); } } - }); - } finally { - server.stop(); - } + } + }); } private static void assertAuthEquals(AuthMethod expect, diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java index 328cf11c20fa6..57b620fde6c0e 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Arrays; +import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.math3.stat.inference.ChiSquareTest; import org.apache.hadoop.conf.Configuration; import org.junit.Assert; @@ -248,4 +249,41 @@ private NodeElement getNewNode(String name, String rackLocation) { node.setNetworkLocation(rackLocation); return node; } + + private NodeElement getNewNode(NetworkTopology cluster, + String name, String rackLocation) { + NodeElement node = getNewNode(name, rackLocation); + cluster.add(node); + return node; + } + + @Test + @SuppressWarnings("unchecked") + public void testWeights() { + // create the topology + NetworkTopology cluster = NetworkTopology.getInstance(new Configuration()); + NodeElement node1 = getNewNode(cluster, "node1", "/r1"); + NodeElement node2 = getNewNode(cluster, "node2", "/r1"); + NodeElement node3 = getNewNode(cluster, "node3", "/r2"); + for (Pair test: new Pair[]{Pair.of(0, node1), + Pair.of(2, node2), Pair.of(4, node3)}) { + int expect = test.getLeft(); + assertEquals(test.toString(), expect, cluster.getWeight(node1, test.getRight())); + assertEquals(test.toString(), expect, + cluster.getWeightUsingNetworkLocation(node1, test.getRight())); + } + // Reset so that we can have 2 levels + cluster = NetworkTopology.getInstance(new Configuration()); + NodeElement node5 = getNewNode(cluster, "node5", "/pod1/r1"); + NodeElement node6 = getNewNode(cluster, "node6", "/pod1/r1"); + NodeElement node7 = getNewNode(cluster, "node7", "/pod1/r2"); + NodeElement node8 = getNewNode(cluster, "node8", "/pod2/r3"); + for (Pair test: new Pair[]{Pair.of(0, node5), + Pair.of(2, node6), Pair.of(4, node7), Pair.of(6, node8)}) { + int expect = test.getLeft(); + assertEquals(test.toString(), expect, cluster.getWeight(node5, test.getRight())); + assertEquals(test.toString(), expect, + cluster.getWeightUsingNetworkLocation(node5, test.getRight())); + } + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestDelegationToken.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestDelegationToken.java index 8bc881ae5d1da..225cc658d39ba 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestDelegationToken.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestDelegationToken.java @@ -30,6 +30,13 @@ import java.util.List; import java.util.Map; +import java.util.concurrent.Callable; +import org.apache.hadoop.fs.statistics.IOStatisticAssertions; +import org.apache.hadoop.fs.statistics.MeanStatistic; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.metrics2.lib.MutableCounterLong; +import org.apache.hadoop.metrics2.lib.MutableRate; +import org.apache.hadoop.test.LambdaTestUtils; import org.junit.Assert; import org.apache.hadoop.io.DataInputBuffer; @@ -155,6 +162,55 @@ public DelegationKey getKey(TestDelegationTokenIdentifier id) { return allKeys.get(id.getMasterKeyId()); } } + + public static class TestFailureDelegationTokenSecretManager + extends TestDelegationTokenSecretManager { + private boolean throwError = false; + private long errorSleepMillis; + + public TestFailureDelegationTokenSecretManager(long errorSleepMillis) { + super(24*60*60*1000, 10*1000, 1*1000, 60*60*1000); + this.errorSleepMillis = errorSleepMillis; + } + + public void setThrowError(boolean throwError) { + this.throwError = throwError; + } + + private void sleepAndThrow() throws IOException { + try { + Thread.sleep(errorSleepMillis); + throw new IOException("Test exception"); + } catch (InterruptedException e) { + } + } + + @Override + protected void storeNewToken(TestDelegationTokenIdentifier ident, long renewDate) + throws IOException { + if (throwError) { + sleepAndThrow(); + } + super.storeNewToken(ident, renewDate); + } + + @Override + protected void removeStoredToken(TestDelegationTokenIdentifier ident) throws IOException { + if (throwError) { + sleepAndThrow(); + } + super.removeStoredToken(ident); + } + + @Override + protected void updateStoredToken(TestDelegationTokenIdentifier ident, long renewDate) + throws IOException { + if (throwError) { + sleepAndThrow(); + } + super.updateStoredToken(ident, renewDate); + } + } public static class TokenSelector extends AbstractDelegationTokenSelector{ @@ -579,4 +635,102 @@ public void testEmptyToken() throws IOException { assertEquals(token1, token2); assertEquals(token1.encodeToUrlString(), token2.encodeToUrlString()); } + + @Test + public void testMultipleDelegationTokenSecretManagerMetrics() { + TestDelegationTokenSecretManager dtSecretManager1 = + new TestDelegationTokenSecretManager(0, 0, 0, 0); + assertNotNull(dtSecretManager1.getMetrics()); + + TestDelegationTokenSecretManager dtSecretManager2 = + new TestDelegationTokenSecretManager(0, 0, 0, 0); + assertNotNull(dtSecretManager2.getMetrics()); + + DefaultMetricsSystem.instance().init("test"); + + TestDelegationTokenSecretManager dtSecretManager3 = + new TestDelegationTokenSecretManager(0, 0, 0, 0); + assertNotNull(dtSecretManager3.getMetrics()); + } + + @Test + public void testDelegationTokenSecretManagerMetrics() throws Exception { + TestDelegationTokenSecretManager dtSecretManager = + new TestDelegationTokenSecretManager(24*60*60*1000, + 10*1000, 1*1000, 60*60*1000); + try { + dtSecretManager.startThreads(); + + final Token token = callAndValidateMetrics( + dtSecretManager, dtSecretManager.getMetrics().getStoreToken(), "storeToken", + () -> generateDelegationToken(dtSecretManager, "SomeUser", "JobTracker")); + + callAndValidateMetrics(dtSecretManager, dtSecretManager.getMetrics().getUpdateToken(), + "updateToken", () -> dtSecretManager.renewToken(token, "JobTracker")); + + callAndValidateMetrics(dtSecretManager, dtSecretManager.getMetrics().getRemoveToken(), + "removeToken", () -> dtSecretManager.cancelToken(token, "JobTracker")); + } finally { + dtSecretManager.stopThreads(); + } + } + + @Test + public void testDelegationTokenSecretManagerMetricsFailures() throws Exception { + int errorSleepMillis = 200; + TestFailureDelegationTokenSecretManager dtSecretManager = + new TestFailureDelegationTokenSecretManager(errorSleepMillis); + + try { + dtSecretManager.startThreads(); + + final Token token = + generateDelegationToken(dtSecretManager, "SomeUser", "JobTracker"); + + dtSecretManager.setThrowError(true); + + callAndValidateFailureMetrics(dtSecretManager, "storeToken", false, + errorSleepMillis, + () -> generateDelegationToken(dtSecretManager, "SomeUser", "JobTracker")); + + callAndValidateFailureMetrics(dtSecretManager, "updateToken", true, + errorSleepMillis, () -> dtSecretManager.renewToken(token, "JobTracker")); + + callAndValidateFailureMetrics(dtSecretManager, "removeToken", true, + errorSleepMillis, () -> dtSecretManager.cancelToken(token, "JobTracker")); + } finally { + dtSecretManager.stopThreads(); + } + } + + private T callAndValidateMetrics(TestDelegationTokenSecretManager dtSecretManager, + MutableRate metric, String statName, Callable callable) + throws Exception { + MeanStatistic stat = IOStatisticAssertions.lookupMeanStatistic( + dtSecretManager.getMetrics().getIoStatistics(), statName + ".mean"); + long metricBefore = metric.lastStat().numSamples(); + long statBefore = stat.getSamples(); + T returnedObject = callable.call(); + assertEquals(metricBefore + 1, metric.lastStat().numSamples()); + assertEquals(statBefore + 1, stat.getSamples()); + return returnedObject; + } + + private void callAndValidateFailureMetrics(TestDelegationTokenSecretManager dtSecretManager, + String statName, boolean expectError, int errorSleepMillis, Callable callable) + throws Exception { + MutableCounterLong counter = dtSecretManager.getMetrics().getTokenFailure(); + MeanStatistic failureStat = IOStatisticAssertions.lookupMeanStatistic( + dtSecretManager.getMetrics().getIoStatistics(), statName + ".failures.mean"); + long counterBefore = counter.value(); + long statBefore = failureStat.getSamples(); + if (expectError) { + LambdaTestUtils.intercept(IOException.class, callable); + } else { + callable.call(); + } + assertEquals(counterBefore + 1, counter.value()); + assertEquals(statBefore + 1, failureStat.getSamples()); + assertTrue(failureStat.getSum() >= errorSleepMillis); + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestLists.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestLists.java index 537e3781edc0e..53241da695c63 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestLists.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestLists.java @@ -18,9 +18,11 @@ package org.apache.hadoop.util; +import org.assertj.core.api.Assertions; import org.junit.Assert; import org.junit.Test; +import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -79,6 +81,48 @@ public void testItrLinkedLists() { Assert.assertEquals(4, list.size()); } + @Test + public void testListsPartition() { + List list = new ArrayList<>(); + list.add("a"); + list.add("b"); + list.add("c"); + list.add("d"); + list.add("e"); + List> res = Lists. + partition(list, 2); + Assertions.assertThat(res) + .describedAs("Number of partitions post partition") + .hasSize(3); + Assertions.assertThat(res.get(0)) + .describedAs("Number of elements in first partition") + .hasSize(2); + Assertions.assertThat(res.get(2)) + .describedAs("Number of elements in last partition") + .hasSize(1); + + List> res2 = Lists. + partition(list, 1); + Assertions.assertThat(res2) + .describedAs("Number of partitions post partition") + .hasSize(5); + Assertions.assertThat(res2.get(0)) + .describedAs("Number of elements in first partition") + .hasSize(1); + Assertions.assertThat(res2.get(4)) + .describedAs("Number of elements in last partition") + .hasSize(1); + + List> res3 = Lists. + partition(list, 6); + Assertions.assertThat(res3) + .describedAs("Number of partitions post partition") + .hasSize(1); + Assertions.assertThat(res3.get(0)) + .describedAs("Number of elements in first partition") + .hasSize(5); + } + @Test public void testArrayListWithSize() { List list = Lists.newArrayListWithCapacity(3); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestWeakReferenceMap.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestWeakReferenceMap.java new file mode 100644 index 0000000000000..cf15743ea06a1 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestWeakReferenceMap.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.util; + +import java.util.ArrayList; +import java.util.List; + +import org.assertj.core.api.Assertions; +import org.junit.Before; +import org.junit.Test; + +import org.apache.hadoop.test.AbstractHadoopTestBase; + +/** + * Test {@link WeakReferenceMap}. + * There's no attempt to force GC here, so the tests are + * more about the basic behavior not the handling of empty references. + */ +public class TestWeakReferenceMap extends AbstractHadoopTestBase { + + public static final String FACTORY_STRING = "recreated %d"; + + /** + * The map to test. + */ + private WeakReferenceMap referenceMap; + + /** + * List of references notified of loss. + */ + private List lostReferences; + + @Before + public void setup() { + lostReferences = new ArrayList<>(); + referenceMap = new WeakReferenceMap<>( + this::factory, + this::referenceLost); + } + + /** + * Reference lost callback. + * @param key key lost + */ + private void referenceLost(Integer key) { + lostReferences.add(key); + } + + + /** + * Basic insertions and lookups of those values. + */ + @Test + public void testBasicOperationsWithValidReferences() { + + referenceMap.put(1, "1"); + referenceMap.put(2, "2"); + assertMapSize(2); + assertMapContainsKey(1); + assertMapEntryEquals(1, "1"); + assertMapEntryEquals(2, "2"); + // overwrite + referenceMap.put(1, "3"); + assertMapEntryEquals(1, "3"); + + // remove an entry + referenceMap.remove(1); + assertMapDoesNotContainKey(1); + assertMapSize(1); + + // clear the map + referenceMap.clear(); + assertMapSize(0); + } + + /** + * pruning removes null entries, leaves the others alone. + */ + @Test + public void testPruneNullEntries() { + referenceMap.put(1, "1"); + assertPruned(0); + referenceMap.put(2, null); + assertMapSize(2); + assertPruned(1); + assertMapSize(1); + assertMapDoesNotContainKey(2); + assertMapEntryEquals(1, "1"); + assertLostCount(1); + } + + /** + * Demand create entries. + */ + @Test + public void testDemandCreateEntries() { + + // ask for an unknown key and expect a generated value + assertMapEntryEquals(1, factory(1)); + assertMapSize(1); + assertMapContainsKey(1); + assertLostCount(0); + + // an empty ref has the same outcome + referenceMap.put(2, null); + assertMapEntryEquals(2, factory(2)); + // but the lost coun goes up + assertLostCount(1); + + } + + /** + * Assert that the value of a map entry is as expected. + * Will trigger entry creation if the key is absent. + * @param key key + * @param val expected valued + */ + private void assertMapEntryEquals(int key, String val) { + Assertions.assertThat(referenceMap.get(key)) + .describedAs("map enty of key %d", key) + .isEqualTo(val); + } + + /** + * Assert that a map entry is present. + * @param key key + */ + private void assertMapContainsKey(int key) { + Assertions.assertThat(referenceMap.containsKey(key)) + .describedAs("map enty of key %d should be present", key) + .isTrue(); + } + + /** + * Assert that a map entry is not present. + * @param key key + */ + private void assertMapDoesNotContainKey(int key) { + Assertions.assertThat(referenceMap.containsKey(key)) + .describedAs("map enty of key %d should be absent", key) + .isFalse(); + } + + /** + * Assert map size. + * @param size expected size. + */ + private void assertMapSize(int size) { + Assertions.assertThat(referenceMap.size()) + .describedAs("size of map %s", referenceMap) + .isEqualTo(size); + } + + /** + * Assert prune returned the given count. + * @param count expected count. + */ + private void assertPruned(int count) { + Assertions.assertThat(referenceMap.prune()) + .describedAs("number of entries pruned from map %s", referenceMap) + .isEqualTo(count); + } + + /** + * Assert number of entries lost matches expected count. + * @param count expected count. + */ + private void assertLostCount(int count) { + Assertions.assertThat(lostReferences) + .describedAs("number of entries lost from map %s", referenceMap) + .hasSize(count); + } + + /** + * Factory operation. + * @param key map key + * @return a string + */ + private String factory(Integer key) { + return String.format(FACTORY_STRING, key); + } + +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/functional/TestTaskPool.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/functional/TestTaskPool.java new file mode 100644 index 0000000000000..dfee6fc75dcb3 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/functional/TestTaskPool.java @@ -0,0 +1,585 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.util.functional; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.test.HadoopTestBase; + +import static org.apache.hadoop.test.LambdaTestUtils.intercept; + +/** + * Test Task Pool class. + * This is pulled straight out of the S3A version. + */ +@RunWith(Parameterized.class) +public class TestTaskPool extends HadoopTestBase { + + private static final Logger LOG = + LoggerFactory.getLogger(TestTaskPool.class); + + public static final int ITEM_COUNT = 16; + + private static final int FAILPOINT = 8; + + private final int numThreads; + + /** + * Thread pool for task execution. + */ + private ExecutorService threadPool; + + /** + * Task submitter bonded to the thread pool, or + * null for the 0-thread case. + */ + private TaskPool.Submitter submitter; + + private final CounterTask failingTask + = new CounterTask("failing committer", FAILPOINT, Item::commit); + + private final FailureCounter failures + = new FailureCounter("failures", 0, null); + + private final CounterTask reverter + = new CounterTask("reverter", 0, Item::revert); + + private final CounterTask aborter + = new CounterTask("aborter", 0, Item::abort); + + /** + * Test array for parameterized test runs: how many threads and + * to use. Threading makes some of the assertions brittle; there are + * more checks on single thread than parallel ops. + * @return a list of parameter tuples. + */ + @Parameterized.Parameters(name = "threads={0}") + public static Collection params() { + return Arrays.asList(new Object[][]{ + {0}, + {1}, + {3}, + {8}, + {16}, + }); + } + + private List items; + + /** + * Construct the parameterized test. + * @param numThreads number of threads + */ + public TestTaskPool(int numThreads) { + this.numThreads = numThreads; + } + + /** + * In a parallel test run there is more than one thread doing the execution. + * @return true if the threadpool size is >1 + */ + public boolean isParallel() { + return numThreads > 1; + } + + @Before + public void setup() { + items = IntStream.rangeClosed(1, ITEM_COUNT) + .mapToObj(i -> new Item(i, + String.format("With %d threads", numThreads))) + .collect(Collectors.toList()); + + if (numThreads > 0) { + threadPool = Executors.newFixedThreadPool(numThreads, + new ThreadFactoryBuilder() + .setDaemon(true) + .setNameFormat(getMethodName() + "-pool-%d") + .build()); + submitter = new PoolSubmitter(); + } else { + submitter = null; + } + + } + + @After + public void teardown() { + if (threadPool != null) { + threadPool.shutdown(); + threadPool = null; + } + } + + private class PoolSubmitter implements TaskPool.Submitter { + + @Override + public Future submit(final Runnable task) { + return threadPool.submit(task); + } + + } + + /** + * create the builder. + * @return pre-inited builder + */ + private TaskPool.Builder builder() { + return TaskPool.foreach(items).executeWith(submitter); + } + + private void assertRun(TaskPool.Builder builder, + CounterTask task) throws IOException { + boolean b = builder.run(task); + assertTrue("Run of " + task + " failed", b); + } + + private void assertFailed(TaskPool.Builder builder, + CounterTask task) throws IOException { + boolean b = builder.run(task); + assertFalse("Run of " + task + " unexpectedly succeeded", b); + } + + private String itemsToString() { + return "[" + items.stream().map(Item::toString) + .collect(Collectors.joining("\n")) + "]"; + } + + @Test + public void testSimpleInvocation() throws Throwable { + CounterTask t = new CounterTask("simple", 0, Item::commit); + assertRun(builder(), t); + t.assertInvoked("", ITEM_COUNT); + } + + @Test + public void testFailNoStoppingSuppressed() throws Throwable { + assertFailed(builder().suppressExceptions(), failingTask); + failingTask.assertInvoked("Continued through operations", ITEM_COUNT); + items.forEach(Item::assertCommittedOrFailed); + } + + @Test + public void testFailFastSuppressed() throws Throwable { + assertFailed(builder() + .suppressExceptions() + .stopOnFailure(), + failingTask); + if (isParallel()) { + failingTask.assertInvokedAtLeast("stop fast", FAILPOINT); + } else { + failingTask.assertInvoked("stop fast", FAILPOINT); + } + } + + @Test + public void testFailedCallAbortSuppressed() throws Throwable { + assertFailed(builder() + .stopOnFailure() + .suppressExceptions() + .abortWith(aborter), + failingTask); + failingTask.assertInvokedAtLeast("success", FAILPOINT); + if (!isParallel()) { + aborter.assertInvokedAtLeast("abort", 1); + // all uncommitted items were aborted + items.stream().filter(i -> !i.committed) + .map(Item::assertAborted); + items.stream().filter(i -> i.committed) + .forEach(i -> assertFalse(i.toString(), i.aborted)); + } + } + + @Test + public void testFailedCalledWhenNotStoppingSuppressed() throws Throwable { + assertFailed(builder() + .suppressExceptions() + .onFailure(failures), + failingTask); + failingTask.assertInvokedAtLeast("success", FAILPOINT); + // only one failure was triggered + failures.assertInvoked("failure event", 1); + } + + @Test + public void testFailFastCallRevertSuppressed() throws Throwable { + assertFailed(builder() + .stopOnFailure() + .revertWith(reverter) + .abortWith(aborter) + .suppressExceptions() + .onFailure(failures), + failingTask); + failingTask.assertInvokedAtLeast("success", FAILPOINT); + if (!isParallel()) { + aborter.assertInvokedAtLeast("abort", 1); + // all uncommitted items were aborted + items.stream().filter(i -> !i.committed) + .filter(i -> !i.failed) + .forEach(Item::assertAborted); + } + // all committed were reverted + items.stream().filter(i -> i.committed && !i.failed) + .forEach(Item::assertReverted); + // all reverted items are committed + items.stream().filter(i -> i.reverted) + .forEach(Item::assertCommitted); + + // only one failure was triggered + failures.assertInvoked("failure event", 1); + } + + @Test + public void testFailSlowCallRevertSuppressed() throws Throwable { + assertFailed(builder() + .suppressExceptions() + .revertWith(reverter) + .onFailure(failures), + failingTask); + failingTask.assertInvokedAtLeast("success", FAILPOINT); + // all committed were reverted + // identify which task failed from the set + int failing = failures.getItem().id; + items.stream() + .filter(i -> i.id != failing) + .filter(i -> i.committed) + .forEach(Item::assertReverted); + // all reverted items are committed + items.stream().filter(i -> i.reverted) + .forEach(Item::assertCommitted); + + // only one failure was triggered + failures.assertInvoked("failure event", 1); + } + + @Test + public void testFailFastExceptions() throws Throwable { + intercept(IOException.class, + () -> builder() + .stopOnFailure() + .run(failingTask)); + if (isParallel()) { + failingTask.assertInvokedAtLeast("stop fast", FAILPOINT); + } else { + failingTask.assertInvoked("stop fast", FAILPOINT); + } + } + + @Test + public void testFailSlowExceptions() throws Throwable { + intercept(IOException.class, + () -> builder() + .run(failingTask)); + failingTask.assertInvoked("continued through operations", ITEM_COUNT); + items.forEach(Item::assertCommittedOrFailed); + } + + @Test + public void testFailFastExceptionsWithAbortFailure() throws Throwable { + CounterTask failFirst = new CounterTask("task", 1, Item::commit); + CounterTask a = new CounterTask("aborter", 1, Item::abort); + intercept(IOException.class, + () -> builder() + .stopOnFailure() + .abortWith(a) + .run(failFirst)); + if (!isParallel()) { + // expect the other tasks to be aborted + a.assertInvokedAtLeast("abort", ITEM_COUNT - 1); + } + } + + @Test + public void testFailFastExceptionsWithAbortFailureStopped() throws Throwable { + CounterTask failFirst = new CounterTask("task", 1, Item::commit); + CounterTask a = new CounterTask("aborter", 1, Item::abort); + intercept(IOException.class, + () -> builder() + .stopOnFailure() + .stopAbortsOnFailure() + .abortWith(a) + .run(failFirst)); + if (!isParallel()) { + // expect the other tasks to be aborted + a.assertInvoked("abort", 1); + } + } + + /** + * Fail the last one committed, all the rest will be reverted. + * The actual ID of the last task has to be picke dup from the + * failure callback, as in the pool it may be one of any. + */ + @Test + public void testRevertAllSuppressed() throws Throwable { + CounterTask failLast = new CounterTask("task", ITEM_COUNT, Item::commit); + + assertFailed(builder() + .suppressExceptions() + .stopOnFailure() + .revertWith(reverter) + .abortWith(aborter) + .onFailure(failures), + failLast); + failLast.assertInvoked("success", ITEM_COUNT); + int abCount = aborter.getCount(); + int revCount = reverter.getCount(); + assertEquals(ITEM_COUNT, 1 + abCount + revCount); + // identify which task failed from the set + int failing = failures.getItem().id; + // all committed were reverted + items.stream() + .filter(i -> i.id != failing) + .filter(i -> i.committed) + .forEach(Item::assertReverted); + items.stream() + .filter(i -> i.id != failing) + .filter(i -> !i.committed) + .forEach(Item::assertAborted); + // all reverted items are committed + items.stream().filter(i -> i.reverted) + .forEach(Item::assertCommitted); + + // only one failure was triggered + failures.assertInvoked("failure event", 1); + } + + + /** + * The Item which tasks process. + */ + private final class Item { + + private final int id; + + private final String text; + + private volatile boolean committed, aborted, reverted, failed; + + private Item(int item, String text) { + this.id = item; + this.text = text; + } + + boolean commit() { + committed = true; + return true; + } + + boolean abort() { + aborted = true; + return true; + } + + boolean revert() { + reverted = true; + return true; + } + + boolean fail() { + failed = true; + return true; + } + + public Item assertCommitted() { + assertTrue(toString() + " was not committed in\n" + + itemsToString(), + committed); + return this; + } + + public Item assertCommittedOrFailed() { + assertTrue(toString() + " was not committed nor failed in\n" + + itemsToString(), + committed || failed); + return this; + } + + public Item assertAborted() { + assertTrue(toString() + " was not aborted in\n" + + itemsToString(), + aborted); + return this; + } + + public Item assertReverted() { + assertTrue(toString() + " was not reverted in\n" + + itemsToString(), + reverted); + return this; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("Item{"); + sb.append(String.format("[%02d]", id)); + sb.append(", committed=").append(committed); + sb.append(", aborted=").append(aborted); + sb.append(", reverted=").append(reverted); + sb.append(", failed=").append(failed); + sb.append(", text=").append(text); + sb.append('}'); + return sb.toString(); + } + } + + /** + * Class which can count invocations and, if limit > 0, will raise + * an exception on the specific invocation of {@link #note(Object)} + * whose count == limit. + */ + private class BaseCounter { + + private final AtomicInteger counter = new AtomicInteger(0); + + private final int limit; + + private final String name; + + private Item item; + + private final Optional> action; + + /** + * Base counter, tracks items. + * @param name name for string/exception/logs. + * @param limit limit at which an exception is raised, 0 == never + * @param action optional action to invoke after the increment, + * before limit check + */ + BaseCounter(String name, + int limit, + Function action) { + this.name = name; + this.limit = limit; + this.action = Optional.ofNullable(action); + } + + /** + * Apply the action to an item; log at info afterwards with both the + * before and after string values of the item. + * @param i item to process. + * @throws IOException failure in the action + */ + void process(Item i) throws IOException { + this.item = i; + int count = counter.incrementAndGet(); + if (limit == count) { + i.fail(); + LOG.info("{}: Failed {}", this, i); + throw new IOException(String.format("%s: Limit %d reached for %s", + this, limit, i)); + } + String before = i.toString(); + action.map(a -> a.apply(i)); + LOG.info("{}: {} -> {}", this, before, i); + } + + int getCount() { + return counter.get(); + } + + Item getItem() { + return item; + } + + void assertInvoked(String text, int expected) { + assertEquals(toString() + ": " + text, expected, getCount()); + } + + void assertInvokedAtLeast(String text, int expected) { + int actual = getCount(); + assertTrue(toString() + ": " + text + + "-expected " + expected + + " invocations, but got " + actual + + " in " + itemsToString(), + expected <= actual); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder( + "BaseCounter{"); + sb.append("name='").append(name).append('\''); + sb.append(", count=").append(counter.get()); + sb.append(", limit=").append(limit); + sb.append(", item=").append(item); + sb.append('}'); + return sb.toString(); + } + } + + private final class CounterTask + extends BaseCounter implements TaskPool.Task { + + private CounterTask(String name, int limit, + Function action) { + super(name, limit, action); + } + + @Override + public void run(Item item) throws IOException { + process(item); + } + + } + + private final class FailureCounter + extends BaseCounter + implements TaskPool.FailureTask { + + private Exception exception; + + private FailureCounter(String name, int limit, + Function action) { + super(name, limit, action); + } + + @Override + public void run(Item item, Exception ex) throws IOException { + process(item); + this.exception = ex; + } + + private Exception getException() { + return exception; + } + + } + +} diff --git a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml index fd6ee110c722f..574dfd2852277 100644 --- a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml +++ b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml @@ -162,38 +162,6 @@ - - help: help for get - - -help get - - - - - - - RegexpComparator - ^-get( )*\[-f\]( )*\[-p\]( )*\[-ignoreCrc\]( )*\[-crc\]( )*<src> \.\.\. <localdst> :\s* - - - RegexpComparator - \s*Copy files that match the file pattern <src> to the local name. <src> is kept.\s* - - - RegexpComparator - ^( |\t)*When copying multiple files, the destination must be a directory. Passing -f( )* - - - RegexpComparator - ^( |\t)*overwrites the destination if it already exists and -p preserves access and( )* - - - RegexpComparator - ^( |\t)*modification times, ownership and the mode.* - - - - help: help for du @@ -356,51 +324,72 @@ RegexpComparator - ^-cp \[-f\] \[-p \| -p\[topax\]\] \[-d\] <src> \.\.\. <dst> :\s* + ^-cp \[-f\] \[-p \| -p\[topax\]\] \[-d\] \[-t <thread count>\] \[-q <thread pool queue size>\] <src> \.\.\. <dst> :\s* RegexpComparator - ^\s*Copy files that match the file pattern <src> to a destination. When copying( )* + ^\s*Copy files that match the file pattern <src> to a destination. When copying( )* RegexpComparator - ^( |\t)*multiple files, the destination must be a directory.( )*Passing -p preserves status( )* + ^( |\t)*multiple files, the destination must be a directory.( )* RegexpComparator - ^( |\t)*\[topax\] \(timestamps, ownership, permission, ACLs, XAttr\). If -p is specified( )* + ^( |\t)*Flags :( )* RegexpComparator - ^( |\t)*with no <arg>, then preserves timestamps, ownership, permission. If -pa is( )* + ^( |\t)*-p\[topax\]\s+Preserve file attributes \[topx\] \(timestamps,( )* RegexpComparator - ^( |\t)*specified, then preserves permission also because ACL is a super-set of( )* + ^( |\t)*ownership, permission, ACL, XAttr\). If -p is( )* RegexpComparator - ^( |\t)*permission. Passing -f overwrites the destination if it already exists. raw( )* + ^( |\t)*specified with no arg, then preserves timestamps,( )* RegexpComparator - ^( |\t)*namespace extended attributes are preserved if \(1\) they are supported \(HDFS( )* + ^( |\t)*ownership, permission. If -pa is specified, then( )* RegexpComparator - ^( |\t)*only\) and, \(2\) all of the source and target pathnames are in the \/\.reserved\/raw( )* + ^( |\t)*preserves permission also because ACL is a( )* RegexpComparator - ^( |\t)*hierarchy. raw namespace xattr preservation is determined solely by the presence( )* + ^( |\t)*super-set of permission. Determination of whether( )* + - RegexpComparator - ^\s*\(or absence\) of the \/\.reserved\/raw prefix and not by the -p option\. Passing -d( )* + RegexpComparator + ^( |\t)*raw namespace extended attributes are preserved is( )* + + + RegexpComparator + ^( |\t)*independent of the -p flag.( )* RegexpComparator - ^\s*will skip creation of temporary file\(<dst>\._COPYING_\)\.( )* + ^\s*-f\s+Overwrite the destination if it already exists.( )* + + + RegexpComparator + ^\s*-d\s+Skip creation of temporary file\(<dst>\._COPYING_\).( )* + + + RegexpComparator + ^\s*-t <thread count>\s+Number of threads to be used, default is 1.( )* + + + RegexpComparator + ^\s*-q <thread pool queue size>\s+Thread pool queue size to be used, default is( )* + + + RegexpComparator + ^( |\t)*1024.\s* @@ -498,7 +487,7 @@ RegexpComparator RegexpComparator - ^-put \[-f\] \[-p\] \[-l\] \[-d\] \[-t <thread count>\] \[-q <threadPool queue size>\] <localsrc> \.\.\. <dst> :\s* + ^-put \[-f\] \[-p\] \[-l\] \[-d\] \[-t <thread count>\] \[-q <thread pool queue size>\] <localsrc> \.\.\. <dst> :\s* @@ -515,35 +504,39 @@ RegexpComparator - ^\s*-p Preserves timestamps, ownership and the mode.( )* + ^\s*-p\s+Preserves timestamps, ownership and the mode.( )* + + + RegexpComparator + ^\s*-f\s+Overwrites the destination if it already exists.( )* RegexpComparator - ^\s*-f Overwrites the destination if it already exists.( )* + ^\s*-t <thread count>\s+Number of threads to be used, default is 1.( )* RegexpComparator - ^\s*-t <thread count> Number of threads to be used, default is 1.( )* + ^\s*-q <thread pool queue size>\s+Thread pool queue size to be used, default is( )* RegexpComparator - ^\s*-q <threadPool size> ThreadPool queue size to be used, default is 1024.( )* + ^( |\t)*1024.\s* RegexpComparator - ^\s*-l Allow DataNode to lazily persist the file to disk. Forces( )* + ^\s*-l\s+Allow DataNode to lazily persist the file to disk.( )* RegexpComparator - ^\s*replication factor of 1. This flag will result in reduced( )* + ^\s*Forces replication factor of 1. This flag will( )* RegexpComparator - ^\s*durability. Use with care.( )* + ^\s*result in reduced durability. Use with care.( )* RegexpComparator - ^\s*-d Skip creation of temporary file\(<dst>\._COPYING_\).( )* + ^\s*-d\s+Skip creation of temporary file\(<dst>\._COPYING_\).( )* @@ -558,7 +551,7 @@ RegexpComparator - ^-copyFromLocal \[-f\] \[-p\] \[-l\] \[-d\] \[-t <thread count>\] \[-q <threadPool queue size>\] <localsrc> \.\.\. <dst> :\s* + ^-copyFromLocal \[-f\] \[-p\] \[-l\] \[-d\] \[-t <thread count>\] \[-q <thread pool queue size>\] <localsrc> \.\.\. <dst> :\s* RegexpComparator @@ -600,7 +593,7 @@ RegexpComparator - ^-get( )*\[-f\]( )*\[-p\]( )*\[-ignoreCrc\]( )*\[-crc\]( )*<src> \.\.\. <localdst> :\s* + ^-get \[-f\] \[-p\] \[-crc\] \[-ignoreCrc\] \[-t <thread count>\] \[-q <thread pool queue size>\] <src> \.\.\. <localdst> :\s* RegexpComparator @@ -608,15 +601,39 @@ RegexpComparator - ^( |\t)*When copying multiple files, the destination must be a directory. Passing -f( )* + ^( |\t)*When copying multiple files, the destination must be a directory.( )* + + + RegexpComparator + ^( |\t)*Flags:\s* + + + RegexpComparator + ^( |\t)*-p\s+Preserves timestamps, ownership and the mode.\s* + + + RegexpComparator + ^( |\t)*-f\s+Overwrites the destination if it already exists.\s* + + + RegexpComparator + ^( |\t)*-crc\s+ write CRC checksums for the files downloaded.\s* + + + RegexpComparator + ^( |\t)*-ignoreCrc\s+ Skip CRC checks on the file\(s\) downloaded.\s* + + + RegexpComparator + ^( |\t)*-t <thread count>\s+Number of threads to be used, default is 1.\s* RegexpComparator - ^( |\t)*overwrites the destination if it already exists and -p preserves access and( )* + ^( |\t)*-q <thread pool queue size>\s+Thread pool queue size to be used, default is\s* RegexpComparator - ^( |\t)*modification times, ownership and the mode.* + ^( |\t)*1024.\s* @@ -723,7 +740,7 @@ RegexpComparator - ^-copyToLocal \[-f\] \[-p\] \[-ignoreCrc\] \[-crc\] <src> \.\.\. <localdst> :\s* + ^-copyToLocal \[-f\] \[-p\] \[-crc\] \[-ignoreCrc\] \[-t <thread count>\] \[-q <thread pool queue size>\] <src> \.\.\. <localdst> :\s* RegexpComparator diff --git a/hadoop-common-project/hadoop-kms/pom.xml b/hadoop-common-project/hadoop-kms/pom.xml index 9de8b9caf6e68..96588a22b9419 100644 --- a/hadoop-common-project/hadoop-kms/pom.xml +++ b/hadoop-common-project/hadoop-kms/pom.xml @@ -186,7 +186,6 @@ org.apache.maven.plugins maven-surefire-plugin - ${ignoreTestFailure} 1 false 1 diff --git a/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm b/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm index 6ea21d5cf407d..09375d5aab528 100644 --- a/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm +++ b/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm @@ -1208,9 +1208,10 @@ Name | Description /logs | Display log files /stacks | Display JVM stacks /static/index.html | The static home page +/prof | Async Profiler endpoint To control the access to servlet `/conf`, `/jmx`, `/logLevel`, `/logs`, -and `/stacks`, configure the following properties in `kms-site.xml`: +`/stacks` and `/prof`, configure the following properties in `kms-site.xml`: ```xml @@ -1224,7 +1225,7 @@ and `/stacks`, configure the following properties in `kms-site.xml`: true Indicates if administrator ACLs are required to access - instrumentation servlets (JMX, METRICS, CONF, STACKS). + instrumentation servlets (JMX, METRICS, CONF, STACKS, PROF). diff --git a/hadoop-common-project/hadoop-registry/pom.xml b/hadoop-common-project/hadoop-registry/pom.xml index 171b7229035fb..725dda50f216b 100644 --- a/hadoop-common-project/hadoop-registry/pom.xml +++ b/hadoop-common-project/hadoop-registry/pom.xml @@ -231,7 +231,6 @@ org.apache.maven.plugins maven-surefire-plugin - ${ignoreTestFailure} false 900 -Xmx1024m -XX:+HeapDumpOnOutOfMemoryError diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml index d65e6030369b3..c4e65ef811dbf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml @@ -51,6 +51,10 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> log4j log4j + + org.slf4j + slf4j-ext + diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java index 50e7025f23f43..2377baa4fec25 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java @@ -69,6 +69,11 @@ public class ClientContext { */ private final String name; + /** + * The client conf used to initialize context. + */ + private final DfsClientConf dfsClientConf; + /** * String representation of the configuration. */ @@ -130,6 +135,17 @@ public class ClientContext { */ private volatile DeadNodeDetector deadNodeDetector = null; + /** + * The switch for the {@link LocatedBlocksRefresher}. + */ + private final boolean locatedBlocksRefresherEnabled; + + /** + * Periodically refresh the {@link org.apache.hadoop.hdfs.protocol.LocatedBlocks} backing + * registered {@link DFSInputStream}s, to take advantage of changes in block placement. + */ + private volatile LocatedBlocksRefresher locatedBlocksRefresher = null; + /** * Count the reference of ClientContext. */ @@ -146,6 +162,7 @@ private ClientContext(String name, DfsClientConf conf, final ShortCircuitConf scConf = conf.getShortCircuitConf(); this.name = name; + this.dfsClientConf = conf; this.confString = scConf.confAsString(); this.clientShortCircuitNum = conf.getClientShortCircuitNum(); this.shortCircuitCache = new ShortCircuitCache[this.clientShortCircuitNum]; @@ -164,6 +181,7 @@ private ClientContext(String name, DfsClientConf conf, this.byteArrayManager = ByteArrayManager.newInstance( conf.getWriteByteArrayManagerConf()); this.deadNodeDetectionEnabled = conf.isDeadNodeDetectionEnabled(); + this.locatedBlocksRefresherEnabled = conf.isLocatedBlocksRefresherEnabled(); initTopologyResolution(config); } @@ -301,6 +319,21 @@ public DeadNodeDetector getDeadNodeDetector() { return deadNodeDetector; } + /** + * If true, LocatedBlocksRefresher will be periodically refreshing LocatedBlocks + * of registered DFSInputStreams. + */ + public boolean isLocatedBlocksRefresherEnabled() { + return locatedBlocksRefresherEnabled; + } + + /** + * Obtain LocatedBlocksRefresher of the current client. + */ + public LocatedBlocksRefresher getLocatedBlocksRefresher() { + return locatedBlocksRefresher; + } + /** * Increment the counter. Start the dead node detector thread if there is no * reference. @@ -311,6 +344,10 @@ synchronized void reference() { deadNodeDetector = new DeadNodeDetector(name, configuration); deadNodeDetector.start(); } + if (locatedBlocksRefresherEnabled && locatedBlocksRefresher == null) { + locatedBlocksRefresher = new LocatedBlocksRefresher(name, configuration, dfsClientConf); + locatedBlocksRefresher.start(); + } } /** @@ -324,5 +361,10 @@ synchronized void unreference() { deadNodeDetector.shutdown(); deadNodeDetector = null; } + + if (counter == 0 && locatedBlocksRefresherEnabled && locatedBlocksRefresher != null) { + locatedBlocksRefresher.shutdown(); + locatedBlocksRefresher = null; + } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index b0b44b473217d..2682549cba126 100755 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -863,7 +863,7 @@ public void reportBadBlocks(LocatedBlock[] blocks) throws IOException { } public long getRefreshReadBlkLocationsInterval() { - return dfsClientConf.getRefreshReadBlockLocationsMS(); + return dfsClientConf.getLocatedBlocksRefresherInterval(); } /** @@ -3459,4 +3459,44 @@ private boolean isDeadNodeDetectionEnabled() { public DeadNodeDetector getDeadNodeDetector() { return clientContext.getDeadNodeDetector(); } + + /** + * Obtain LocatedBlocksRefresher of the current client. + */ + public LocatedBlocksRefresher getLocatedBlockRefresher() { + return clientContext.getLocatedBlocksRefresher(); + } + + /** + * Adds the {@link DFSInputStream} to the {@link LocatedBlocksRefresher}, so that + * the underlying {@link LocatedBlocks} is periodically refreshed. + */ + public void addLocatedBlocksRefresh(DFSInputStream dfsInputStream) { + if (isLocatedBlocksRefresherEnabled()) { + clientContext.getLocatedBlocksRefresher().addInputStream(dfsInputStream); + } + } + + /** + * Removes the {@link DFSInputStream} from the {@link LocatedBlocksRefresher}, so that + * the underlying {@link LocatedBlocks} is no longer periodically refreshed. + * @param dfsInputStream + */ + public void removeLocatedBlocksRefresh(DFSInputStream dfsInputStream) { + if (isLocatedBlocksRefresherEnabled()) { + clientContext.getLocatedBlocksRefresher().removeInputStream(dfsInputStream); + } + } + + private boolean isLocatedBlocksRefresherEnabled() { + return clientContext.isLocatedBlocksRefresherEnabled(); + } + + public DatanodeInfo[] slowDatanodeReport() throws IOException { + checkOpen(); + try (TraceScope ignored = tracer.newScope("slowDatanodeReport")) { + return namenode.getSlowDatanodeReport(); + } + } + } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClientFaultInjector.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClientFaultInjector.java index ecea795ee5f98..caf8aad32e343 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClientFaultInjector.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClientFaultInjector.java @@ -22,6 +22,7 @@ import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hdfs.protocol.LocatedBlock; /** * Used for injecting faults in DFSClient and DFSOutputStream tests. @@ -65,4 +66,7 @@ public boolean skipRollingRestartWait() { public void sleepBeforeHedgedGet() {} public void delayWhenRenewLeaseTimeout() {} + + public void onCreateBlockReader(LocatedBlock block, int chunkIndex, long offset, long length) {} + } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java index e65d04db385a5..7b664e4f31164 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java @@ -28,6 +28,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.EnumSet; +import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; import java.util.List; @@ -65,6 +66,7 @@ import org.apache.hadoop.hdfs.protocol.BlockType; import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.DatanodeInfoWithStorage; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; @@ -128,18 +130,18 @@ public class DFSInputStream extends FSInputStream private long lastBlockBeingWrittenLength = 0; private FileEncryptionInfo fileEncryptionInfo = null; protected CachingStrategy cachingStrategy; + // this is volatile because it will be polled outside the lock, + // but still only updated within the lock + private volatile long lastRefreshedBlocksAt = Time.monotonicNow(); //// + private AtomicBoolean refreshingBlockLocations = new AtomicBoolean(false); protected final ReadStatistics readStatistics = new ReadStatistics(); // lock for state shared between read and pread // Note: Never acquire a lock on with this lock held to avoid deadlocks // (it's OK to acquire this lock when the lock on is held) protected final Object infoLock = new Object(); - // refresh locatedBlocks periodically - private final long refreshReadBlockIntervals; - /** timeStamp of the last time a block location was refreshed. */ - private long locatedBlocksTimeStamp; /** * Track the ByteBuffers that we have handed out to readers. * @@ -156,10 +158,6 @@ public class DFSInputStream extends FSInputStream return extendedReadBuffers; } - private boolean isPeriodicRefreshEnabled() { - return (refreshReadBlockIntervals > 0L); - } - /** * This variable tracks the number of failures since the start of the * most recent user-facing operation. That is to say, it should be reset @@ -206,9 +204,6 @@ protected DFSClient getDFSClient() { DFSInputStream(DFSClient dfsClient, String src, boolean verifyChecksum, LocatedBlocks locatedBlocks) throws IOException { this.dfsClient = dfsClient; - this.refreshReadBlockIntervals = - this.dfsClient.getRefreshReadBlkLocationsInterval(); - setLocatedBlocksTimeStamp(); this.verifyChecksum = verifyChecksum; this.src = src; synchronized (infoLock) { @@ -228,19 +223,6 @@ boolean deadNodesContain(DatanodeInfo nodeInfo) { return deadNodes.containsKey(nodeInfo); } - @VisibleForTesting - void setReadTimeStampsForTesting(long timeStamp) { - setLocatedBlocksTimeStamp(timeStamp); - } - - private void setLocatedBlocksTimeStamp() { - setLocatedBlocksTimeStamp(Time.monotonicNow()); - } - - private void setLocatedBlocksTimeStamp(long timeStamp) { - this.locatedBlocksTimeStamp = timeStamp; - } - /** * Grab the open-file info from namenode * @param refreshLocatedBlocks whether to re-fetch locatedblocks @@ -248,33 +230,50 @@ private void setLocatedBlocksTimeStamp(long timeStamp) { void openInfo(boolean refreshLocatedBlocks) throws IOException { final DfsClientConf conf = dfsClient.getConf(); synchronized(infoLock) { - lastBlockBeingWrittenLength = - fetchLocatedBlocksAndGetLastBlockLength(refreshLocatedBlocks); int retriesForLastBlockLength = conf.getRetryTimesForGetLastBlockLength(); - while (retriesForLastBlockLength > 0) { + + while (true) { + LocatedBlocks newLocatedBlocks; + if (locatedBlocks == null || refreshLocatedBlocks) { + newLocatedBlocks = fetchAndCheckLocatedBlocks(locatedBlocks); + } else { + newLocatedBlocks = locatedBlocks; + } + + long lastBlockLength = getLastBlockLength(newLocatedBlocks); + if (lastBlockLength != -1) { + setLocatedBlocksFields(newLocatedBlocks, lastBlockLength); + return; + } + // Getting last block length as -1 is a special case. When cluster // restarts, DNs may not report immediately. At this time partial block // locations will not be available with NN for getting the length. Lets // retry for 3 times to get the length. - if (lastBlockBeingWrittenLength == -1) { - DFSClient.LOG.warn("Last block locations not available. " - + "Datanodes might not have reported blocks completely." - + " Will retry for " + retriesForLastBlockLength + " times"); - waitFor(conf.getRetryIntervalForGetLastBlockLength()); - lastBlockBeingWrittenLength = - fetchLocatedBlocksAndGetLastBlockLength(true); - } else { - break; + + if (retriesForLastBlockLength-- <= 0) { + throw new IOException("Could not obtain the last block locations."); } - retriesForLastBlockLength--; - } - if (lastBlockBeingWrittenLength == -1 - && retriesForLastBlockLength == 0) { - throw new IOException("Could not obtain the last block locations."); + + DFSClient.LOG.warn("Last block locations not available. " + + "Datanodes might not have reported blocks completely." + + " Will retry for " + retriesForLastBlockLength + " times"); + waitFor(conf.getRetryIntervalForGetLastBlockLength()); } } } + /** + * Set locatedBlocks and related fields, using the passed lastBlockLength. + * Should be called within infoLock. + */ + private void setLocatedBlocksFields(LocatedBlocks locatedBlocksToSet, long lastBlockLength) { + locatedBlocks = locatedBlocksToSet; + lastBlockBeingWrittenLength = lastBlockLength; + fileEncryptionInfo = locatedBlocks.getFileEncryptionInfo(); + setLastRefreshedBlocksAt(); + } + private void waitFor(int waitTime) throws IOException { try { Thread.sleep(waitTime); @@ -285,62 +284,18 @@ private void waitFor(int waitTime) throws IOException { } } - /** - * Checks whether the block locations timestamps have expired. - * In the case of expired timestamp: - * - clear list of deadNodes - * - call openInfo(true) which will re-fetch locatedblocks - * - update locatedBlocksTimeStamp - * @return true when the expiration feature is enabled and locatedblocks - * timestamp has expired. - * @throws IOException - */ - private boolean isLocatedBlocksExpired() { - if (!isPeriodicRefreshEnabled()) { - return false; - } - long now = Time.monotonicNow(); - long elapsed = now - locatedBlocksTimeStamp; - if (elapsed < refreshReadBlockIntervals) { - return false; - } - return true; - } - - /** - * Update the block locations timestamps if they have expired. - * In the case of expired timestamp: - * - clear list of deadNodes - * - call openInfo(true) which will re-fetch locatedblocks - * - update locatedBlocksTimeStamp - * @return true when the locatedblocks list is re-fetched from the namenode. - * @throws IOException - */ - private boolean updateBlockLocationsStamp() throws IOException { - if (!isLocatedBlocksExpired()) { - return false; - } - // clear dead nodes - deadNodes.clear(); - openInfo(true); - setLocatedBlocksTimeStamp(); - return true; - } - - private long fetchLocatedBlocksAndGetLastBlockLength(boolean refresh) + private LocatedBlocks fetchAndCheckLocatedBlocks(LocatedBlocks existing) throws IOException { - LocatedBlocks newInfo = locatedBlocks; - if (locatedBlocks == null || refresh) { - newInfo = dfsClient.getLocatedBlocks(src, 0); - } + LocatedBlocks newInfo = dfsClient.getLocatedBlocks(src, 0); + DFSClient.LOG.debug("newInfo = {}", newInfo); if (newInfo == null) { throw new IOException("Cannot open filename " + src); } - if (locatedBlocks != null) { + if (existing != null) { Iterator oldIter = - locatedBlocks.getLocatedBlocks().iterator(); + existing.getLocatedBlocks().iterator(); Iterator newIter = newInfo.getLocatedBlocks().iterator(); while (oldIter.hasNext() && newIter.hasNext()) { if (!oldIter.next().getBlock().equals(newIter.next().getBlock())) { @@ -348,17 +303,14 @@ private long fetchLocatedBlocksAndGetLastBlockLength(boolean refresh) } } } - locatedBlocks = newInfo; - long lastBlkBeingWrittenLength = getLastBlockLength(); - fileEncryptionInfo = locatedBlocks.getFileEncryptionInfo(); - return lastBlkBeingWrittenLength; + return newInfo; } - private long getLastBlockLength() throws IOException{ + private long getLastBlockLength(LocatedBlocks blocks) throws IOException{ long lastBlockBeingWrittenLength = 0; - if (!locatedBlocks.isLastBlockComplete()) { - final LocatedBlock last = locatedBlocks.getLastLocatedBlock(); + if (!blocks.isLastBlockComplete()) { + final LocatedBlock last = blocks.getLastLocatedBlock(); if (last != null) { if (last.getLocations().length == 0) { if (last.getBlockSize() == 0) { @@ -501,6 +453,14 @@ public List getAllBlocks() throws IOException { return getBlockRange(0, getFileLength()); } + protected String getSrc() { + return src; + } + + protected LocatedBlocks getLocatedBlocks() { + return locatedBlocks; + } + /** * Get block at the specified position. * Fetch it from the namenode if not cached. @@ -543,8 +503,8 @@ protected LocatedBlock fetchBlockAt(long offset) throws IOException { /** Fetch a block from namenode and cache it */ private LocatedBlock fetchBlockAt(long offset, long length, boolean useCache) throws IOException { + maybeRegisterBlockRefresh(); synchronized(infoLock) { - updateBlockLocationsStamp(); int targetBlockIdx = locatedBlocks.findBlock(offset); if (targetBlockIdx < 0) { // block is not cached targetBlockIdx = LocatedBlocks.getInsertIndex(targetBlockIdx); @@ -559,8 +519,7 @@ private LocatedBlock fetchBlockAt(long offset, long length, boolean useCache) } // Update the LastLocatedBlock, if offset is for last block. if (offset >= locatedBlocks.getFileLength()) { - locatedBlocks = newBlocks; - lastBlockBeingWrittenLength = getLastBlockLength(); + setLocatedBlocksFields(newBlocks, getLastBlockLength(newBlocks)); } else { locatedBlocks.insertRange(targetBlockIdx, newBlocks.getLocatedBlocks()); @@ -587,6 +546,7 @@ private List getBlockRange(long offset, throw new IOException("Offset: " + offset + " exceeds file length: " + getFileLength()); } + synchronized(infoLock) { final List blocks; final long lengthOfCompleteBlk = locatedBlocks.getFileLength(); @@ -644,6 +604,9 @@ private synchronized DatanodeInfo blockSeekTo(long target) if (target >= getFileLength()) { throw new IOException("Attempted to read past end of file"); } + + maybeRegisterBlockRefresh(); + // Will be getting a new BlockReader. closeCurrentBlockReaders(); @@ -657,9 +620,6 @@ private synchronized DatanodeInfo blockSeekTo(long target) boolean connectFailedOnce = false; while (true) { - // Re-fetch the locatedBlocks from NN if the timestamp has expired. - updateBlockLocationsStamp(); - // // Compute desired block // @@ -793,6 +753,7 @@ public void accept(ByteBuffer k, Object v) { * this dfsInputStream anymore. */ dfsClient.removeNodeFromDeadNodeDetector(this, locatedBlocks); + maybeDeRegisterBlockRefresh(); } } @@ -871,16 +832,16 @@ protected synchronized int readWithStrategy(ReaderStrategy strategy) int len = strategy.getTargetLength(); CorruptedBlocks corruptedBlocks = new CorruptedBlocks(); failures = 0; + + maybeRegisterBlockRefresh(); + if (pos < getFileLength()) { int retries = 2; while (retries > 0) { try { // currentNode can be left as null if previous read had a checksum // error on the same block. See HDFS-3067 - // currentNode needs to be updated if the blockLocations timestamp has - // expired. - if (pos > blockEnd || currentNode == null - || updateBlockLocationsStamp()) { + if (pos > blockEnd || currentNode == null) { currentNode = blockSeekTo(pos); } int realLen = (int) Math.min(len, (blockEnd - pos + 1L)); @@ -1958,4 +1919,153 @@ public boolean hasCapability(String capability) { return false; } } + + /** + * Many DFSInputStreams can be opened and closed in quick succession, in which case + * they would be registered/deregistered but never need to be refreshed. + * Defers registering with the located block refresher, in order to avoid an additional + * source of unnecessary synchronization for short-lived DFSInputStreams. + */ + protected void maybeRegisterBlockRefresh() { + if (!dfsClient.getConf().isRefreshReadBlockLocationsAutomatically() + || !dfsClient.getConf().isLocatedBlocksRefresherEnabled()) { + return; + } + + if (refreshingBlockLocations.get()) { + return; + } + + // not enough time elapsed to refresh + long timeSinceLastRefresh = Time.monotonicNow() - lastRefreshedBlocksAt; + if (timeSinceLastRefresh < dfsClient.getConf().getLocatedBlocksRefresherInterval()) { + return; + } + + if (!refreshingBlockLocations.getAndSet(true)) { + dfsClient.addLocatedBlocksRefresh(this); + } + } + + /** + * De-register periodic refresh of this inputstream, if it was added to begin with. + */ + private void maybeDeRegisterBlockRefresh() { + if (refreshingBlockLocations.get()) { + dfsClient.removeLocatedBlocksRefresh(this); + } + } + + /** + * Refresh blocks for the input stream, if necessary. + * + * @param addressCache optional map to use as a cache for resolving datanode InetSocketAddress + * @return whether a refresh was performed or not + */ + boolean refreshBlockLocations(Map addressCache) { + LocatedBlocks blocks; + synchronized (infoLock) { + blocks = getLocatedBlocks(); + } + + if (getLocalDeadNodes().isEmpty() && allBlocksLocal(blocks, addressCache)) { + return false; + } + + try { + DFSClient.LOG.debug("Refreshing {} for path {}", this, getSrc()); + LocatedBlocks newLocatedBlocks = fetchAndCheckLocatedBlocks(blocks); + long lastBlockLength = getLastBlockLength(newLocatedBlocks); + if (lastBlockLength == -1) { + DFSClient.LOG.debug( + "Discarding refreshed blocks for path {} because lastBlockLength was -1", + getSrc()); + return true; + } + + setRefreshedValues(newLocatedBlocks, lastBlockLength); + } catch (IOException e) { + DFSClient.LOG.debug("Failed to refresh DFSInputStream for path {}", getSrc(), e); + } + + return true; + } + + /** + * Once new LocatedBlocks have been fetched, sets them on the DFSInputStream and + * updates stateful read location within the necessary locks. + */ + private synchronized void setRefreshedValues(LocatedBlocks blocks, long lastBlockLength) + throws IOException { + synchronized (infoLock) { + setLocatedBlocksFields(blocks, lastBlockLength); + } + + getLocalDeadNodes().clear(); + + // if a stateful read has been initialized, refresh it + if (currentNode != null) { + currentNode = blockSeekTo(pos); + } + } + + private boolean allBlocksLocal(LocatedBlocks blocks, + Map addressCache) { + if (addressCache == null) { + addressCache = new HashMap<>(); + } + + // we only need to check the first location of each block, because the blocks are already + // sorted by distance from the current host + for (LocatedBlock lb : blocks.getLocatedBlocks()) { + if (lb.getLocations().length == 0) { + return false; + } + + DatanodeInfoWithStorage location = lb.getLocations()[0]; + if (location == null) { + return false; + } + + InetSocketAddress targetAddr = addressCache.computeIfAbsent( + location.getDatanodeUuid(), + unused -> { + String dnAddr = location.getXferAddr(dfsClient.getConf().isConnectToDnViaHostname()); + return NetUtils.createSocketAddr( + dnAddr, + -1, + null, + dfsClient.getConf().isUriCacheEnabled()); + }); + + if (!isResolveableAndLocal(targetAddr)) { + return false; + } + } + + return true; + } + + private boolean isResolveableAndLocal(InetSocketAddress targetAddr) { + try { + return DFSUtilClient.isLocalAddress(targetAddr); + } catch (IOException e) { + DFSClient.LOG.debug("Got an error checking if {} is local", targetAddr, e); + return false; + } + } + + @VisibleForTesting + void setLastRefreshedBlocksAtForTesting(long timestamp) { + lastRefreshedBlocksAt = timestamp; + } + + @VisibleForTesting + long getLastRefreshedBlocksAtForTesting() { + return lastRefreshedBlocksAt; + } + + private void setLastRefreshedBlocksAt() { + lastRefreshedBlocksAt = Time.monotonicNow(); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java index 95cce9d363e58..5ae517095931a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java @@ -141,14 +141,6 @@ protected ByteBuffer getCurStripeBuf() { return curStripeBuf; } - protected String getSrc() { - return src; - } - - protected LocatedBlocks getLocatedBlocks() { - return locatedBlocks; - } - protected ByteBufferPool getBufferPool() { return BUFFER_POOL; } @@ -166,6 +158,8 @@ synchronized void blockSeekTo(long target) throws IOException { throw new IOException("Attempted to read past end of file"); } + maybeRegisterBlockRefresh(); + // Will be getting a new BlockReader. closeCurrentBlockReaders(); @@ -238,7 +232,7 @@ private long getOffsetInBlockGroup(long pos) { boolean createBlockReader(LocatedBlock block, long offsetInBlock, LocatedBlock[] targetBlocks, BlockReaderInfo[] readerInfos, - int chunkIndex) throws IOException { + int chunkIndex, long readTo) throws IOException { BlockReader reader = null; final ReaderRetryPolicy retry = new ReaderRetryPolicy(); DFSInputStream.DNAddrPair dnInfo = @@ -256,9 +250,14 @@ boolean createBlockReader(LocatedBlock block, long offsetInBlock, if (dnInfo == null) { break; } + if (readTo < 0 || readTo > block.getBlockSize()) { + readTo = block.getBlockSize(); + } reader = getBlockReader(block, offsetInBlock, - block.getBlockSize() - offsetInBlock, + readTo - offsetInBlock, dnInfo.addr, dnInfo.storageType, dnInfo.info); + DFSClientFaultInjector.get().onCreateBlockReader(block, chunkIndex, offsetInBlock, + readTo - offsetInBlock); } catch (IOException e) { if (e instanceof InvalidEncryptionKeyException && retry.shouldRefetchEncryptionKey()) { @@ -491,11 +490,16 @@ protected void fetchBlockByteRange(LocatedBlock block, long start, final LocatedBlock[] blks = StripedBlockUtil.parseStripedBlockGroup( blockGroup, cellSize, dataBlkNum, parityBlkNum); final BlockReaderInfo[] preaderInfos = new BlockReaderInfo[groupSize]; + long readTo = -1; + for (AlignedStripe stripe : stripes) { + readTo = Math.max(readTo, stripe.getOffsetInBlock() + stripe.getSpanInBlock()); + } try { for (AlignedStripe stripe : stripes) { // Parse group to get chosen DN location StripeReader preader = new PositionStripeReader(stripe, ecPolicy, blks, preaderInfos, corruptedBlocks, decoder, this); + preader.setReadTo(readTo); try { preader.readStripe(); } finally { @@ -560,4 +564,5 @@ public synchronized void unbuffer() { parityBuf = null; } } + } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java index 7c71c64ec3657..1a1819dfa2c5a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java @@ -733,13 +733,13 @@ public static boolean isLocalAddress(InetSocketAddress targetAddr) InetAddress addr = targetAddr.getAddress(); Boolean cached = localAddrMap.get(addr.getHostAddress()); if (cached != null) { - LOG.trace("Address {} is {} local", targetAddr, (cached ? "" : "not")); + LOG.trace("Address {} is{} local", targetAddr, (cached ? "" : " not")); return cached; } boolean local = NetUtils.isLocalAddress(addr); - LOG.trace("Address {} is {} local", targetAddr, (local ? "" : "not")); + LOG.trace("Address {} is{} local", targetAddr, (local ? "" : " not")); localAddrMap.put(addr.getHostAddress(), local); return local; } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java index c1e25af466d9d..4fa578ab6c03f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java @@ -38,6 +38,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -525,11 +526,13 @@ boolean doWaitForRestart() { // List of congested data nodes. The stream will back off if the DataNodes // are congested private final List congestedNodes = new ArrayList<>(); + private final Map slowNodeMap = new HashMap<>(); private static final int CONGESTION_BACKOFF_MEAN_TIME_IN_MS = 5000; private static final int CONGESTION_BACK_OFF_MAX_TIME_IN_MS = CONGESTION_BACKOFF_MEAN_TIME_IN_MS * 10; private int lastCongestionBackoffTime; private int maxPipelineRecoveryRetries; + private int markSlowNodeAsBadNodeThreshold; protected final LoadingCache excludedNodes; private final String[] favoredNodes; @@ -559,6 +562,7 @@ private DataStreamer(HdfsFileStatus stat, ExtendedBlock block, this.errorState = new ErrorState(conf.getDatanodeRestartTimeout()); this.addBlockFlags = flags; this.maxPipelineRecoveryRetries = conf.getMaxPipelineRecoveryRetries(); + this.markSlowNodeAsBadNodeThreshold = conf.getMarkSlowNodeAsBadNodeThreshold(); } /** @@ -687,11 +691,6 @@ public void run() { continue; } // get packet to be sent. - try { - backOffIfNecessary(); - } catch (InterruptedException e) { - LOG.debug("Thread interrupted", e); - } one = dataQueue.getFirst(); // regular data packet SpanContext[] parents = one.getTraceParents(); if (parents != null && parents.length > 0) { @@ -704,6 +703,14 @@ public void run() { } } + // The DataStreamer has to release the dataQueue before sleeping, + // otherwise it will cause the ResponseProcessor to accept the ACK delay. + try { + backOffIfNecessary(); + } catch (InterruptedException e) { + LOG.debug("Thread interrupted", e); + } + // get new block from namenode. LOG.debug("stage={}, {}", stage, this); @@ -1152,6 +1159,7 @@ public void run() { long seqno = ack.getSeqno(); // processes response status from datanodes. ArrayList congestedNodesFromAck = new ArrayList<>(); + ArrayList slownodesFromAck = new ArrayList<>(); for (int i = ack.getNumOfReplies()-1; i >=0 && dfsClient.clientRunning; i--) { final Status reply = PipelineAck.getStatusFromHeader(ack .getHeaderFlag(i)); @@ -1159,6 +1167,10 @@ public void run() { PipelineAck.ECN.CONGESTED) { congestedNodesFromAck.add(targets[i]); } + if (PipelineAck.getSLOWFromHeader(ack.getHeaderFlag(i)) == + PipelineAck.SLOW.SLOW) { + slownodesFromAck.add(targets[i]); + } // Restart will not be treated differently unless it is // the local node or the only one in the pipeline. if (PipelineAck.isRestartOOBStatus(reply)) { @@ -1188,6 +1200,16 @@ public void run() { } } + if (slownodesFromAck.isEmpty()) { + if (!slowNodeMap.isEmpty()) { + slowNodeMap.clear(); + } + } else { + markSlowNode(slownodesFromAck); + LOG.debug("SlowNodeMap content: {}.", slowNodeMap); + } + + assert seqno != PipelineAck.UNKOWN_SEQNO : "Ack for unknown seqno should be a failed ack: " + ack; if (seqno == DFSPacket.HEART_BEAT_SEQNO) { // a heartbeat ack @@ -1254,10 +1276,51 @@ public void run() { } } + void markSlowNode(List slownodesFromAck) throws IOException { + Set discontinuousNodes = new HashSet<>(slowNodeMap.keySet()); + for (DatanodeInfo slowNode : slownodesFromAck) { + if (!slowNodeMap.containsKey(slowNode)) { + slowNodeMap.put(slowNode, 1); + } else { + int oldCount = slowNodeMap.get(slowNode); + slowNodeMap.put(slowNode, ++oldCount); + } + discontinuousNodes.remove(slowNode); + } + for (DatanodeInfo discontinuousNode : discontinuousNodes) { + slowNodeMap.remove(discontinuousNode); + } + + if (!slowNodeMap.isEmpty()) { + for (Map.Entry entry : slowNodeMap.entrySet()) { + if (entry.getValue() >= markSlowNodeAsBadNodeThreshold) { + DatanodeInfo slowNode = entry.getKey(); + int index = getDatanodeIndex(slowNode); + if (index >= 0) { + errorState.setBadNodeIndex(index); + throw new IOException("Receive reply from slowNode " + slowNode + + " for continuous " + markSlowNodeAsBadNodeThreshold + + " times, treating it as badNode"); + } + slowNodeMap.remove(entry.getKey()); + } + } + } + } + void close() { responderClosed = true; this.interrupt(); } + + int getDatanodeIndex(DatanodeInfo datanodeInfo) { + for (int i = 0; i < targets.length; i++) { + if (targets[i].equals(datanodeInfo)) { + return i; + } + } + return -1; + } } private boolean shouldHandleExternalError(){ diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 499f5a764a014..93db332d738c4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -2400,6 +2400,51 @@ public SnapshotDiffReport next(final FileSystem fs, final Path p) }.resolve(this, absF); } + /** + * Get the difference between two snapshots of a directory iteratively. + * + * @param snapshotDir full path of the directory where snapshots are taken. + * @param fromSnapshotName snapshot name of the from point. Null indicates the current tree. + * @param toSnapshotName snapshot name of the to point. Null indicates the current tree. + * @param snapshotDiffStartPath path relative to the snapshottable root directory from where + * the snapshotdiff computation needs to start. + * @param snapshotDiffIndex index in the created or deleted list of the directory at which the + * snapshotdiff computation stopped during the last rpc call. -1 indicates the diff + * computation needs to start right from the start path. + * @return the difference report represented as a {@link SnapshotDiffReportListing}. + * @throws IOException if an I/O error occurred. + */ + public SnapshotDiffReportListing getSnapshotDiffReportListing(Path snapshotDir, + String fromSnapshotName, String toSnapshotName, String snapshotDiffStartPath, + int snapshotDiffIndex) throws IOException { + statistics.incrementReadOps(1); + storageStatistics.incrementOpCounter(OpType.GET_SNAPSHOT_DIFF); + Path absF = fixRelativePart(snapshotDir); + return new FileSystemLinkResolver() { + + @Override + public SnapshotDiffReportListing doCall(final Path p) throws IOException { + return dfs.getSnapshotDiffReportListing(getPathName(p), fromSnapshotName, toSnapshotName, + DFSUtilClient.string2Bytes(snapshotDiffStartPath), snapshotDiffIndex); + } + + @Override + public SnapshotDiffReportListing next(final FileSystem fs, final Path p) + throws IOException { + if (fs instanceof DistributedFileSystem) { + DistributedFileSystem distributedFileSystem = (DistributedFileSystem)fs; + distributedFileSystem.getSnapshotDiffReportListing(p, fromSnapshotName, toSnapshotName, + snapshotDiffStartPath, snapshotDiffIndex); + } else { + throw new UnsupportedOperationException("Cannot perform snapshot" + + " operations on a symlink to a non-DistributedFileSystem: " + + snapshotDir + " -> " + p); + } + return null; + } + }.resolve(this, absF); + } + /** * Get the close status of a file * @param src The path to the file @@ -3842,4 +3887,15 @@ public MultipartUploaderBuilder createMultipartUploader(final Path basePath) throws IOException { return new FileSystemMultipartUploaderBuilder(this, basePath); } + + /** + * Retrieve stats for slow running datanodes. + * + * @return An array of slow datanode info. + * @throws IOException If an I/O error occurs. + */ + public DatanodeInfo[] getSlowDatanodeStats() throws IOException { + return dfs.slowDatanodeReport(); + } + } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/KeyProviderCache.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/KeyProviderCache.java index c70081c06fa49..d8dd485101bce 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/KeyProviderCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/KeyProviderCache.java @@ -26,6 +26,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.util.KMSUtil; import org.apache.hadoop.classification.VisibleForTesting; @@ -34,6 +35,7 @@ import org.apache.hadoop.thirdparty.com.google.common.cache.RemovalListener; import org.apache.hadoop.thirdparty.com.google.common.cache.RemovalNotification; +import org.apache.hadoop.util.ShutdownHookManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,6 +67,9 @@ public void onRemoval( } }) .build(); + + ShutdownHookManager.get().addShutdownHook(new KeyProviderCacheFinalizer(), + SHUTDOWN_HOOK_PRIORITY); } public KeyProvider get(final Configuration conf, @@ -85,6 +90,26 @@ public KeyProvider call() throws Exception { } } + public static final int SHUTDOWN_HOOK_PRIORITY = FileSystem.SHUTDOWN_HOOK_PRIORITY - 1; + + private class KeyProviderCacheFinalizer implements Runnable { + @Override + public synchronized void run() { + invalidateCache(); + } + } + + /** + * Invalidate cache. KeyProviders in the cache will be closed by cache hook. + */ + @VisibleForTesting + synchronized void invalidateCache() { + LOG.debug("Invalidating all cached KeyProviders."); + if (cache != null) { + cache.invalidateAll(); + } + } + private URI createKeyProviderURI(Configuration conf) { final String providerUriStr = conf.getTrimmed( CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH); diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/LocatedBlocksRefresher.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/LocatedBlocksRefresher.java new file mode 100644 index 0000000000000..454d1f9cd93e4 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/LocatedBlocksRefresher.java @@ -0,0 +1,210 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs; + +import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_CONTEXT_DEFAULT; +import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_REFRESH_READ_BLOCK_LOCATIONS_THREADS_DEFAULT; +import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_REFRESH_READ_BLOCK_LOCATIONS_THREADS_KEY; + +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.WeakHashMap; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Phaser; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.client.impl.DfsClientConf; +import org.apache.hadoop.hdfs.protocol.LocatedBlocks; +import org.apache.hadoop.util.Daemon; +import org.apache.hadoop.util.Time; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Periodically refresh the underlying cached {@link LocatedBlocks} for eligible registered + * {@link DFSInputStream}s. DFSInputStreams are eligible for refreshing if they have any + * deadNodes or any blocks are lacking local replicas. + * Disabled by default, unless an interval is configured. + */ +public class LocatedBlocksRefresher extends Daemon { + private static final Logger LOG = + LoggerFactory.getLogger(LocatedBlocksRefresher.class); + + private static final String THREAD_PREFIX = "located-block-refresher-"; + + private final String name; + private final long interval; + private final long jitter; + private final ExecutorService refreshThreadPool; + + // Use WeakHashMap so that we don't hold onto references that might have not been explicitly + // closed because they were created and thrown away. + private final Set registeredInputStreams = + Collections.newSetFromMap(new WeakHashMap<>()); + + private int runCount; + private int refreshCount; + + LocatedBlocksRefresher(String name, Configuration conf, DfsClientConf dfsClientConf) { + this.name = name; + this.interval = dfsClientConf.getLocatedBlocksRefresherInterval(); + this.jitter = Math.round(this.interval * 0.1); + int rpcThreads = conf.getInt(DFS_CLIENT_REFRESH_READ_BLOCK_LOCATIONS_THREADS_KEY, + DFS_CLIENT_REFRESH_READ_BLOCK_LOCATIONS_THREADS_DEFAULT); + + String threadPrefix; + if (name.equals(DFS_CLIENT_CONTEXT_DEFAULT)) { + threadPrefix = THREAD_PREFIX; + } else { + threadPrefix = THREAD_PREFIX + name + "-"; + } + + this.refreshThreadPool = Executors.newFixedThreadPool(rpcThreads, new Daemon.DaemonFactory() { + private final AtomicInteger threadIndex = new AtomicInteger(0); + + @Override + public Thread newThread(Runnable r) { + Thread t = super.newThread(r); + t.setName(threadPrefix + threadIndex.getAndIncrement()); + return t; + } + }); + + setName(threadPrefix + "main"); + + LOG.info("Start located block refresher for DFSClient {}.", this.name); + } + + @Override + public void run() { + while (!Thread.currentThread().isInterrupted()) { + + if (!waitForInterval()) { + return; + } + + LOG.debug("Running refresh for {} streams", registeredInputStreams.size()); + long start = Time.monotonicNow(); + AtomicInteger neededRefresh = new AtomicInteger(0); + + Phaser phaser = new Phaser(1); + + Map addressCache = new ConcurrentHashMap<>(); + + for (DFSInputStream inputStream : getInputStreams()) { + phaser.register(); + refreshThreadPool.submit(() -> { + try { + if (isInputStreamTracked(inputStream) && + inputStream.refreshBlockLocations(addressCache)) { + neededRefresh.incrementAndGet(); + } + } finally { + phaser.arriveAndDeregister(); + } + }); + } + + phaser.arriveAndAwaitAdvance(); + + synchronized (this) { + runCount++; + refreshCount += neededRefresh.get(); + } + + LOG.debug( + "Finished refreshing {} of {} streams in {}ms", + neededRefresh, + registeredInputStreams.size(), + Time.monotonicNow() - start + ); + } + } + + public synchronized int getRunCount() { + return runCount; + } + + public synchronized int getRefreshCount() { + return refreshCount; + } + + private boolean waitForInterval() { + try { + Thread.sleep(interval + ThreadLocalRandom.current().nextLong(-jitter, jitter)); + return true; + } catch (InterruptedException e) { + LOG.debug("Interrupted during wait interval", e); + Thread.currentThread().interrupt(); + return false; + } + } + + /** + * Shutdown all the threads. + */ + public void shutdown() { + if (isAlive()) { + interrupt(); + try { + join(); + } catch (InterruptedException e) { + } + } + refreshThreadPool.shutdown(); + } + + /** + * Collects the DFSInputStreams to a list within synchronization, so that we can iterate them + * without potentially blocking callers to {@link #addInputStream(DFSInputStream)} or + * {@link #removeInputStream(DFSInputStream)}. We don't care so much about missing additions, + * and we'll guard against removals by doing an additional + * {@link #isInputStreamTracked(DFSInputStream)} track during iteration. + */ + private synchronized Collection getInputStreams() { + return new ArrayList<>(registeredInputStreams); + } + + public synchronized void addInputStream(DFSInputStream dfsInputStream) { + LOG.trace("Registering {} for {}", dfsInputStream, dfsInputStream.getSrc()); + registeredInputStreams.add(dfsInputStream); + } + + public synchronized void removeInputStream(DFSInputStream dfsInputStream) { + if (isInputStreamTracked(dfsInputStream)) { + LOG.trace("De-registering {} for {}", dfsInputStream, dfsInputStream.getSrc()); + registeredInputStreams.remove(dfsInputStream); + } + } + + public synchronized boolean isInputStreamTracked(DFSInputStream dfsInputStream) { + return registeredInputStreams.contains(dfsInputStream); + } + + public long getInterval() { + return interval; + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StatefulStripeReader.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StatefulStripeReader.java index 9069a558b67d7..730307b4434cb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StatefulStripeReader.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StatefulStripeReader.java @@ -52,7 +52,9 @@ void prepareDecodeInputs() { cur = dfsStripedInputStream.getCurStripeBuf().duplicate(); } - this.decodeInputs = new ECChunk[dataBlkNum + parityBlkNum]; + if (this.decodeInputs == null) { + this.decodeInputs = new ECChunk[dataBlkNum + parityBlkNum]; + } int bufLen = (int) alignedStripe.getSpanInBlock(); int bufOff = (int) alignedStripe.getOffsetInBlock(); for (int i = 0; i < dataBlkNum; i++) { @@ -72,11 +74,6 @@ void prepareDecodeInputs() { boolean prepareParityChunk(int index) { Preconditions.checkState(index >= dataBlkNum && alignedStripe.chunks[index] == null); - if (readerInfos[index] != null && readerInfos[index].shouldSkip) { - alignedStripe.chunks[index] = new StripingChunk(StripingChunk.MISSING); - // we have failed the block reader before - return false; - } final int parityIndex = index - dataBlkNum; ByteBuffer buf = dfsStripedInputStream.getParityBuffer().duplicate(); buf.position(cellSize * parityIndex); diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripeReader.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripeReader.java index 932ddb491cc2f..3fc87c7952a19 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripeReader.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripeReader.java @@ -119,6 +119,7 @@ void skip() { protected final int cellSize; protected final RawErasureDecoder decoder; protected final DFSStripedInputStream dfsStripedInputStream; + private long readTo = -1; protected ECChunk[] decodeInputs; @@ -302,7 +303,7 @@ boolean readChunk(final LocatedBlock block, int chunkIndex) if (readerInfos[chunkIndex] == null) { if (!dfsStripedInputStream.createBlockReader(block, alignedStripe.getOffsetInBlock(), targetBlocks, - readerInfos, chunkIndex)) { + readerInfos, chunkIndex, readTo)) { chunk.state = StripingChunk.MISSING; return false; } @@ -478,4 +479,9 @@ void clearFutures() { boolean useDirectBuffer() { return decoder.preferDirectBuffer(); } + + public void setReadTo(long readTo) { + this.readTo = readTo; + } + } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ViewDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ViewDistributedFileSystem.java index 2a4469a5ae585..d34df37ae6133 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ViewDistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ViewDistributedFileSystem.java @@ -2318,4 +2318,14 @@ public long getUsed() throws IOException { } return this.vfs.getUsed(); } + + @Override + public DatanodeInfo[] getSlowDatanodeStats() throws IOException { + if (this.vfs == null) { + return super.getSlowDatanodeStats(); + } + checkDefaultDFS(defaultDFS, "getSlowDatanodeStats"); + return defaultDFS.getSlowDatanodeStats(); + } + } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java index 0664f6414ec64..7750c48ae6bfe 100755 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java @@ -154,6 +154,9 @@ public interface HdfsClientConfigKeys { String DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_KEY = "dfs.client.slow.io.warning.threshold.ms"; long DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_DEFAULT = 30000; + String DFS_CLIENT_MARK_SLOWNODE_AS_BADNODE_THRESHOLD_KEY = + "dfs.client.mark.slownode.as.badnode.threshold"; + int DFS_CLIENT_MARK_SLOWNODE_AS_BADNODE_THRESHOLD_DEFAULT = 10; String DFS_CLIENT_KEY_PROVIDER_CACHE_EXPIRY_MS = "dfs.client.key.provider.cache.expiry"; long DFS_CLIENT_KEY_PROVIDER_CACHE_EXPIRY_DEFAULT = @@ -202,6 +205,19 @@ public interface HdfsClientConfigKeys { "dfs.client.refresh.read-block-locations.ms"; long DFS_CLIENT_REFRESH_READ_BLOCK_LOCATIONS_MS_DEFAULT = 0L; + // Number of threads to use for refreshing LocatedBlocks of registered + // DFSInputStreams. If a DFSClient opens many DFSInputStreams, increasing + // this may help refresh them all in a timely manner. + String DFS_CLIENT_REFRESH_READ_BLOCK_LOCATIONS_THREADS_KEY = + "dfs.client.refresh.read-block-locations.threads"; + int DFS_CLIENT_REFRESH_READ_BLOCK_LOCATIONS_THREADS_DEFAULT = 5; + + // Whether to auto-register all DFSInputStreams for background refreshes. + // If false, user must manually register using DFSClient#addLocatedBlocksRefresh(DFSInputStream) + String DFS_CLIENT_REFRESH_READ_BLOCK_LOCATIONS_AUTOMATICALLY_KEY = + "dfs.client.refresh.read-block-locations.register-automatically"; + boolean DFS_CLIENT_REFRESH_READ_BLOCK_LOCATIONS_AUTOMATICALLY_DEFAULT = true; + String DFS_DATANODE_KERBEROS_PRINCIPAL_KEY = "dfs.datanode.kerberos.principal"; String DFS_DATANODE_READAHEAD_BYTES_KEY = "dfs.datanode.readahead.bytes"; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java index 7a4508d1d3360..6562c81358039 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java @@ -60,6 +60,8 @@ import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_DOMAIN_SOCKET_DATA_TRAFFIC_DEFAULT; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_KEY_PROVIDER_CACHE_EXPIRY_DEFAULT; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_KEY_PROVIDER_CACHE_EXPIRY_MS; +import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_MARK_SLOWNODE_AS_BADNODE_THRESHOLD_DEFAULT; +import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_MARK_SLOWNODE_AS_BADNODE_THRESHOLD_KEY; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_MAX_BLOCK_ACQUIRE_FAILURES_DEFAULT; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_MAX_BLOCK_ACQUIRE_FAILURES_KEY; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_READ_USE_CACHE_PRIORITY; @@ -142,9 +144,11 @@ public class DfsClientConf { private final int retryIntervalForGetLastBlockLength; private final long datanodeRestartTimeout; private final long slowIoWarningThresholdMs; + private final int markSlowNodeAsBadNodeThreshold; /** wait time window before refreshing blocklocation for inputstream. */ private final long refreshReadBlockLocationsMS; + private final boolean refreshReadBlockLocationsAutomatically; private final ShortCircuitConf shortCircuitConf; private final int clientShortCircuitNum; @@ -261,12 +265,19 @@ public DfsClientConf(Configuration conf) { DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_DEFAULT); readUseCachePriority = conf.getBoolean(DFS_CLIENT_READ_USE_CACHE_PRIORITY, DFS_CLIENT_READ_USE_CACHE_PRIORITY_DEFAULT); + markSlowNodeAsBadNodeThreshold = conf.getInt( + DFS_CLIENT_MARK_SLOWNODE_AS_BADNODE_THRESHOLD_KEY, + DFS_CLIENT_MARK_SLOWNODE_AS_BADNODE_THRESHOLD_DEFAULT); refreshReadBlockLocationsMS = conf.getLong( HdfsClientConfigKeys.DFS_CLIENT_REFRESH_READ_BLOCK_LOCATIONS_MS_KEY, HdfsClientConfigKeys. DFS_CLIENT_REFRESH_READ_BLOCK_LOCATIONS_MS_DEFAULT); + refreshReadBlockLocationsAutomatically = conf.getBoolean( + HdfsClientConfigKeys.DFS_CLIENT_REFRESH_READ_BLOCK_LOCATIONS_AUTOMATICALLY_KEY, + HdfsClientConfigKeys.DFS_CLIENT_REFRESH_READ_BLOCK_LOCATIONS_AUTOMATICALLY_DEFAULT); + hedgedReadThresholdMillis = conf.getLong( HedgedRead.THRESHOLD_MILLIS_KEY, HedgedRead.THRESHOLD_MILLIS_DEFAULT); @@ -644,6 +655,13 @@ public long getSlowIoWarningThresholdMs() { return slowIoWarningThresholdMs; } + /** + * @return the continuous slowNode replies received to mark slowNode as badNode + */ + public int getMarkSlowNodeAsBadNodeThreshold() { + return markSlowNodeAsBadNodeThreshold; + } + /* * @return the clientShortCircuitNum */ @@ -701,13 +719,18 @@ public boolean isReadUseCachePriority() { return replicaAccessorBuilderClasses; } - /** - * @return the replicaAccessorBuilderClasses - */ - public long getRefreshReadBlockLocationsMS() { + public boolean isLocatedBlocksRefresherEnabled() { + return refreshReadBlockLocationsMS > 0; + } + + public long getLocatedBlocksRefresherInterval() { return refreshReadBlockLocationsMS; } + public boolean isRefreshReadBlockLocationsAutomatically() { + return refreshReadBlockLocationsAutomatically; + } + /** * @return the shortCircuitConf */ diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java index ea90645ca082b..e9ae803a541b1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java @@ -1868,4 +1868,16 @@ BatchedEntries listOpenFiles(long prevId, */ @AtMostOnce void satisfyStoragePolicy(String path) throws IOException; + + /** + * Get report on all of the slow Datanodes. Slow running datanodes are identified based on + * the Outlier detection algorithm, if slow peer tracking is enabled for the DFS cluster. + * + * @return Datanode report for slow running datanodes. + * @throws IOException If an I/O error occurs. + */ + @Idempotent + @ReadOnly + DatanodeInfo[] getSlowDatanodeReport() throws IOException; + } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java index bba90a05794ba..fbe6bcc4629d8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java @@ -698,9 +698,10 @@ public static class DatanodeInfoBuilder { private long nonDfsUsed = 0L; private long lastBlockReportTime = 0L; private long lastBlockReportMonotonic = 0L; - private int numBlocks; - + private int numBlocks = 0; + // Please use setNumBlocks explicitly to set numBlocks as this method doesn't have + // sufficient info about numBlocks public DatanodeInfoBuilder setFrom(DatanodeInfo from) { this.capacity = from.getCapacity(); this.dfsUsed = from.getDfsUsed(); @@ -717,7 +718,6 @@ public DatanodeInfoBuilder setFrom(DatanodeInfo from) { this.upgradeDomain = from.getUpgradeDomain(); this.lastBlockReportTime = from.getLastBlockReportTime(); this.lastBlockReportMonotonic = from.getLastBlockReportMonotonic(); - this.numBlocks = from.getNumBlocks(); setNodeID(from); return this; } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PipelineAck.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PipelineAck.java index 3d9e48ceebd51..7561a40b0163b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PipelineAck.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PipelineAck.java @@ -27,6 +27,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.PipelineAckProto; import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status; import org.apache.hadoop.thirdparty.protobuf.TextFormat; @@ -42,6 +43,27 @@ public class PipelineAck { final static int OOB_START = Status.OOB_RESTART_VALUE; // the first OOB type final static int OOB_END = Status.OOB_RESERVED3_VALUE; // the last OOB type + public enum SLOW { + DISABLED(0), + NORMAL(1), + SLOW(2), + RESERVED(3); + + private final int value; + private static final SLOW[] VALUES = values(); + static SLOW valueOf(int value) { + return VALUES[value]; + } + + SLOW(int value) { + this.value = value; + } + + public int getValue() { + return value; + } + } + public enum ECN { DISABLED(0), SUPPORTED(1), @@ -66,7 +88,8 @@ public int getValue() { private enum StatusFormat { STATUS(null, 4), RESERVED(STATUS.BITS, 1), - ECN_BITS(RESERVED.BITS, 2); + ECN_BITS(RESERVED.BITS, 2), + SLOW_BITS(ECN_BITS.BITS, 2); private final LongBitFormat BITS; @@ -82,6 +105,10 @@ static ECN getECN(int header) { return ECN.valueOf((int) ECN_BITS.BITS.retrieve(header)); } + static SLOW getSLOW(int header) { + return SLOW.valueOf((int) SLOW_BITS.BITS.retrieve(header)); + } + public static int setStatus(int old, Status status) { return (int) STATUS.BITS.combine(status.getNumber(), old); } @@ -89,6 +116,10 @@ public static int setStatus(int old, Status status) { public static int setECN(int old, ECN ecn) { return (int) ECN_BITS.BITS.combine(ecn.getValue(), old); } + + public static int setSLOW(int old, SLOW slow) { + return (int) SLOW_BITS.BITS.combine(slow.getValue(), old); + } } /** default constructor **/ @@ -149,7 +180,7 @@ public int getHeaderFlag(int i) { if (proto.getFlagCount() > 0) { return proto.getFlag(i); } else { - return combineHeader(ECN.DISABLED, proto.getReply(i)); + return combineHeader(ECN.DISABLED, proto.getReply(i), SLOW.DISABLED); } } @@ -230,14 +261,28 @@ public static ECN getECNFromHeader(int header) { return StatusFormat.getECN(header); } + public static SLOW getSLOWFromHeader(int header) { + return StatusFormat.getSLOW(header); + } + public static int setStatusForHeader(int old, Status status) { return StatusFormat.setStatus(old, status); } + @VisibleForTesting + public static int setSLOWForHeader(int old, SLOW slow) { + return StatusFormat.setSLOW(old, slow); + } + public static int combineHeader(ECN ecn, Status status) { + return combineHeader(ecn, status, SLOW.DISABLED); + } + + public static int combineHeader(ECN ecn, Status status, SLOW slow) { int header = 0; header = StatusFormat.setStatus(header, status); header = StatusFormat.setECN(header, ecn); + header = StatusFormat.setSLOW(header, slow); return header; } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/DataTransferSaslUtil.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/DataTransferSaslUtil.java index 526f3d0c66512..ab5cd0608d9d4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/DataTransferSaslUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/DataTransferSaslUtil.java @@ -34,6 +34,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Function; import javax.security.sasl.Sasl; import org.apache.commons.codec.binary.Base64; @@ -52,6 +53,7 @@ import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.HandshakeSecretProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.CipherOptionProto; import org.apache.hadoop.hdfs.protocolPB.PBHelperClient; +import org.apache.hadoop.hdfs.security.token.block.InvalidBlockTokenException; import org.apache.hadoop.security.SaslPropertiesResolver; import org.apache.hadoop.security.SaslRpcServer.QualityOfProtection; import org.slf4j.Logger; @@ -204,6 +206,26 @@ public static SaslPropertiesResolver getSaslPropertiesResolver( return resolver; } + private static T readSaslMessage(InputStream in, + Function handler) throws IOException { + DataTransferEncryptorMessageProto proto = + DataTransferEncryptorMessageProto.parseFrom(vintPrefixed(in)); + switch (proto.getStatus()) { + case ERROR_UNKNOWN_KEY: + throw new InvalidEncryptionKeyException(proto.getMessage()); + case ERROR: + if (proto.hasAccessTokenError() && proto.getAccessTokenError()) { + throw new InvalidBlockTokenException(proto.getMessage()); + } + throw new IOException(proto.getMessage()); + case SUCCESS: + return handler.apply(proto); + default: + throw new IOException( + "Unknown status: " + proto.getStatus() + ", message: " + proto.getMessage()); + } + } + /** * Reads a SASL negotiation message. * @@ -212,15 +234,7 @@ public static SaslPropertiesResolver getSaslPropertiesResolver( * @throws IOException for any error */ public static byte[] readSaslMessage(InputStream in) throws IOException { - DataTransferEncryptorMessageProto proto = - DataTransferEncryptorMessageProto.parseFrom(vintPrefixed(in)); - if (proto.getStatus() == DataTransferEncryptorStatus.ERROR_UNKNOWN_KEY) { - throw new InvalidEncryptionKeyException(proto.getMessage()); - } else if (proto.getStatus() == DataTransferEncryptorStatus.ERROR) { - throw new IOException(proto.getMessage()); - } else { - return proto.getPayload().toByteArray(); - } + return readSaslMessage(in, proto -> proto.getPayload().toByteArray()); } /** @@ -233,13 +247,7 @@ public static byte[] readSaslMessage(InputStream in) throws IOException { */ public static byte[] readSaslMessageAndNegotiationCipherOptions( InputStream in, List cipherOptions) throws IOException { - DataTransferEncryptorMessageProto proto = - DataTransferEncryptorMessageProto.parseFrom(vintPrefixed(in)); - if (proto.getStatus() == DataTransferEncryptorStatus.ERROR_UNKNOWN_KEY) { - throw new InvalidEncryptionKeyException(proto.getMessage()); - } else if (proto.getStatus() == DataTransferEncryptorStatus.ERROR) { - throw new IOException(proto.getMessage()); - } else { + return readSaslMessage(in, proto -> { List optionProtos = proto.getCipherOptionList(); if (optionProtos != null) { for (CipherOptionProto optionProto : optionProtos) { @@ -247,7 +255,7 @@ public static byte[] readSaslMessageAndNegotiationCipherOptions( } } return proto.getPayload().toByteArray(); - } + }); } static class SaslMessageWithHandshake { @@ -276,13 +284,7 @@ String getBpid() { public static SaslMessageWithHandshake readSaslMessageWithHandshakeSecret( InputStream in) throws IOException { - DataTransferEncryptorMessageProto proto = - DataTransferEncryptorMessageProto.parseFrom(vintPrefixed(in)); - if (proto.getStatus() == DataTransferEncryptorStatus.ERROR_UNKNOWN_KEY) { - throw new InvalidEncryptionKeyException(proto.getMessage()); - } else if (proto.getStatus() == DataTransferEncryptorStatus.ERROR) { - throw new IOException(proto.getMessage()); - } else { + return readSaslMessage(in, proto -> { byte[] payload = proto.getPayload().toByteArray(); byte[] secret = null; String bpid = null; @@ -292,7 +294,7 @@ public static SaslMessageWithHandshake readSaslMessageWithHandshakeSecret( bpid = handshakeSecret.getBpid(); } return new SaslMessageWithHandshake(payload, secret, bpid); - } + }); } /** @@ -467,13 +469,7 @@ public static void sendSaslMessageAndNegotiationCipherOptions( public static SaslResponseWithNegotiatedCipherOption readSaslMessageAndNegotiatedCipherOption(InputStream in) throws IOException { - DataTransferEncryptorMessageProto proto = - DataTransferEncryptorMessageProto.parseFrom(vintPrefixed(in)); - if (proto.getStatus() == DataTransferEncryptorStatus.ERROR_UNKNOWN_KEY) { - throw new InvalidEncryptionKeyException(proto.getMessage()); - } else if (proto.getStatus() == DataTransferEncryptorStatus.ERROR) { - throw new IOException(proto.getMessage()); - } else { + return readSaslMessage(in, proto -> { byte[] response = proto.getPayload().toByteArray(); List options = PBHelperClient.convertCipherOptionProtos( proto.getCipherOptionList()); @@ -482,7 +478,7 @@ public static void sendSaslMessageAndNegotiationCipherOptions( option = options.get(0); } return new SaslResponseWithNegotiatedCipherOption(response, option); - } + }); } /** @@ -558,6 +554,13 @@ public static void sendSaslMessage(OutputStream out, DataTransferEncryptorStatus status, byte[] payload, String message, HandshakeSecretProto handshakeSecret) throws IOException { + sendSaslMessage(out, status, payload, message, handshakeSecret, false); + } + + public static void sendSaslMessage(OutputStream out, + DataTransferEncryptorStatus status, byte[] payload, String message, + HandshakeSecretProto handshakeSecret, boolean accessTokenError) + throws IOException { DataTransferEncryptorMessageProto.Builder builder = DataTransferEncryptorMessageProto.newBuilder(); @@ -571,6 +574,9 @@ public static void sendSaslMessage(OutputStream out, if (handshakeSecret != null) { builder.setHandshakeSecret(handshakeSecret); } + if (accessTokenError) { + builder.setAccessTokenError(true); + } DataTransferEncryptorMessageProto proto = builder.build(); proto.writeDelimitedTo(out); diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferClient.java index 1a61c9664825e..641c7a0ff4790 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferClient.java @@ -588,11 +588,11 @@ private IOStreamPair doSaslHandshake(InetAddress addr, // the client accepts some cipher suites, but the server does not. LOG.debug("Client accepts cipher suites {}, " + "but server {} does not accept any of them", - cipherSuites, addr.toString()); + cipherSuites, addr); } } else { LOG.debug("Client using cipher suite {} with server {}", - cipherOption.getCipherSuite().getName(), addr.toString()); + cipherOption.getCipherSuite().getName(), addr); } } } @@ -603,7 +603,20 @@ private IOStreamPair doSaslHandshake(InetAddress addr, conf, cipherOption, underlyingOut, underlyingIn, false) : sasl.createStreamPair(out, in); } catch (IOException ioe) { - sendGenericSaslErrorMessage(out, ioe.getMessage()); + String message = ioe.getMessage(); + try { + sendGenericSaslErrorMessage(out, message); + } catch (Exception e) { + // If ioe is caused by error response from server, server will close peer connection. + // So sendGenericSaslErrorMessage might cause IOException due to "Broken pipe". + // We suppress IOException from sendGenericSaslErrorMessage + // and always throw `ioe` as top level. + // `ioe` can be InvalidEncryptionKeyException or InvalidBlockTokenException + // that indicates refresh key or token and are important for caller. + LOG.debug("Failed to send generic sasl error to server {} (message: {}), " + + "suppress exception", addr, message, e); + ioe.addSuppressed(e); + } throw ioe; } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java index 2668ec1cefe98..7b8ca42c50f7b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java @@ -143,6 +143,7 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetPreferredBlockSizeRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetQuotaUsageRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetServerDefaultsRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSlowDatanodeReportRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSnapshotDiffReportRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSnapshotDiffReportResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSnapshotDiffReportListingRequestProto; @@ -2065,6 +2066,18 @@ public void satisfyStoragePolicy(String src) throws IOException { } } + @Override + public DatanodeInfo[] getSlowDatanodeReport() throws IOException { + GetSlowDatanodeReportRequestProto req = + GetSlowDatanodeReportRequestProto.newBuilder().build(); + try { + return PBHelperClient.convert( + rpcProxy.getSlowDatanodeReport(null, req).getDatanodeInfoProtoList()); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + } + @Override public HAServiceProtocol.HAServiceState getHAServiceState() throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ObserverReadProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ObserverReadProxyProvider.java index bc89830f1e14f..e1a7c2f8030cb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ObserverReadProxyProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ObserverReadProxyProvider.java @@ -214,7 +214,6 @@ public ObserverReadProxyProvider( OBSERVER_PROBE_RETRY_PERIOD_KEY, OBSERVER_PROBE_RETRY_PERIOD_DEFAULT, TimeUnit.MILLISECONDS); - // TODO : make this configurable or remove this variable if (wrappedProxy instanceof ClientProtocol) { this.observerReadEnabled = true; } else { diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java index a950388a31239..df2a92c75c962 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java @@ -189,6 +189,7 @@ public void run() { final DfsClientShm shm = (DfsClientShm)slot.getShm(); final DomainSocket shmSock = shm.getPeer().getDomainSocket(); final String path = shmSock.getPath(); + DomainSocket domainSocket = pathToDomainSocket.get(path); DataOutputStream out = null; boolean success = false; int retries = 2; @@ -196,9 +197,10 @@ public void run() { while (retries > 0) { try { if (domainSocket == null || !domainSocket.isOpen()) { - // we are running in single thread mode, no protection needed for - // domainSocket domainSocket = DomainSocket.connect(path); + // we are running in single thread mode, no protection needed for + // pathToDomainSocket + pathToDomainSocket.put(path, domainSocket); } out = new DataOutputStream( @@ -221,13 +223,16 @@ public void run() { } catch (SocketException se) { // the domain socket on datanode may be timed out, we retry once retries--; - domainSocket.close(); - domainSocket = null; + if (domainSocket != null) { + domainSocket.close(); + domainSocket = null; + pathToDomainSocket.remove(path); + } if (retries == 0) { throw new SocketException("Create domain socket failed"); } } - } + } // end of while block } catch (IOException e) { LOG.warn(ShortCircuitCache.this + ": failed to release " + "short-circuit shared memory slot " + slot + " by sending " @@ -240,10 +245,10 @@ public void run() { } else { shm.getEndpointShmManager().shutdown(shm); IOUtilsClient.cleanupWithLogger(LOG, domainSocket, out); - domainSocket = null; + pathToDomainSocket.remove(path); } } - } + } // end of run() } public interface ShortCircuitReplicaCreator { @@ -354,7 +359,11 @@ public interface ShortCircuitReplicaCreator { */ private final DfsClientShmManager shmManager; - private DomainSocket domainSocket = null; + /** + * A map contains all DomainSockets used in SlotReleaser. Keys are the domain socket + * paths of short-circuit shared memory segments. + */ + private Map pathToDomainSocket = new HashMap<>(); public static ShortCircuitCache fromConf(ShortCircuitConf conf) { return new ShortCircuitCache( diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index 2cba43b32dd70..ebc12ff04b919 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -478,7 +478,8 @@ private Path makeAbsolute(Path f) { return f.isAbsolute()? f: new Path(workingDir, f); } - static Map jsonParse(final HttpURLConnection c, + @VisibleForTesting + public static Map jsonParse(final HttpURLConnection c, final boolean useErrorStream) throws IOException { if (c.getContentLength() == 0) { return null; @@ -1458,7 +1459,9 @@ SnapshotDiffReport decodeResponse(Map json) { }.run(); } - private SnapshotDiffReportListing getSnapshotDiffReportListing( + // This API should be treated as private to WebHdfsFileSystem. Only tests can use it directly. + @VisibleForTesting + public SnapshotDiffReportListing getSnapshotDiffReportListing( String snapshotDir, final String fromSnapshot, final String toSnapshot, byte[] startPath, int index) throws IOException { return new FsPathResponseRunner( diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientDatanodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientDatanodeProtocol.proto index 84cd771da4912..6c8d1c5fafb80 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientDatanodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientDatanodeProtocol.proto @@ -18,7 +18,7 @@ /** * These .proto interfaces are private and stable. - * Please see http://wiki.apache.org/hadoop/Compatibility + * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html * for what changes are allowed for a *stable* .proto interface. */ syntax="proto2"; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto index 20967cc13ab86..1e8d0b0a2667d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto @@ -18,7 +18,7 @@ /** * These .proto interfaces are private and stable. - * Please see http://wiki.apache.org/hadoop/Compatibility + * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html * for what changes are allowed for a *stable* .proto interface. */ syntax="proto2"; @@ -424,6 +424,13 @@ message GetPreferredBlockSizeResponseProto { required uint64 bsize = 1; } +message GetSlowDatanodeReportRequestProto { +} + +message GetSlowDatanodeReportResponseProto { + repeated DatanodeInfoProto datanodeInfoProto = 1; +} + enum SafeModeActionProto { SAFEMODE_LEAVE = 1; SAFEMODE_ENTER = 2; @@ -1070,4 +1077,6 @@ service ClientNamenodeProtocol { returns(SatisfyStoragePolicyResponseProto); rpc getHAServiceState(HAServiceStateRequestProto) returns(HAServiceStateResponseProto); + rpc getSlowDatanodeReport(GetSlowDatanodeReportRequestProto) + returns(GetSlowDatanodeReportResponseProto); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/datatransfer.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/datatransfer.proto index 28a292e729e2a..5356cd6961699 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/datatransfer.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/datatransfer.proto @@ -18,7 +18,7 @@ /** * These .proto interfaces are private and stable. - * Please see http://wiki.apache.org/hadoop/Compatibility + * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html * for what changes are allowed for a *stable* .proto interface. */ syntax="proto2"; @@ -44,6 +44,7 @@ message DataTransferEncryptorMessageProto { optional string message = 3; repeated CipherOptionProto cipherOption = 4; optional HandshakeSecretProto handshakeSecret = 5; + optional bool accessTokenError = 6; } message HandshakeSecretProto { diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/encryption.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/encryption.proto index bcd82d63e0577..d280947cf5a74 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/encryption.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/encryption.proto @@ -18,7 +18,7 @@ /** * These .proto interfaces are private and stable. - * Please see http://wiki.apache.org/hadoop/Compatibility + * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html * for what changes are allowed for a *stable* .proto interface. */ syntax="proto2"; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto index 08fce71560602..163d3a49d3014 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto @@ -18,7 +18,7 @@ /** * These .proto interfaces are private and stable. - * Please see http://wiki.apache.org/hadoop/Compatibility + * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html * for what changes are allowed for a *stable* .proto interface. */ syntax="proto2"; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/inotify.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/inotify.proto index e1ade19b27ee8..eb7a0c3549b08 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/inotify.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/inotify.proto @@ -18,7 +18,7 @@ /** * These .proto interfaces are private and stable. - * Please see http://wiki.apache.org/hadoop/Compatibility + * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html * for what changes are allowed for a *stable* .proto interface. */ syntax="proto2"; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestReadOnly.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestReadOnly.java index 4e6f4e3f4ba38..f7ea7bcd76dde 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestReadOnly.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestReadOnly.java @@ -76,7 +76,8 @@ public class TestReadOnly { "getQuotaUsage", "msync", "getHAServiceState", - "getECTopologyResultForPolicies" + "getECTopologyResultForPolicies", + "getSlowDatanodeReport" ) ); diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml index 1916ef0e3b7f6..a1b3ab1f92397 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml @@ -247,7 +247,6 @@ org.apache.maven.plugins maven-surefire-plugin - ${ignoreTestFailure} 1 600 @@ -361,7 +360,6 @@ org.apache.maven.plugins maven-surefire-plugin - ${ignoreTestFailure} 1 true 600 diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java index 9f7dfd4647b11..0b4de72844403 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java @@ -53,6 +53,7 @@ import org.apache.hadoop.hdfs.protocol.FsPermissionExtension; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import org.apache.hadoop.hdfs.web.JsonUtilClient; @@ -136,6 +137,8 @@ public class HttpFSFileSystem extends FileSystem public static final String POLICY_NAME_PARAM = "storagepolicy"; public static final String SNAPSHOT_NAME_PARAM = "snapshotname"; public static final String OLD_SNAPSHOT_NAME_PARAM = "oldsnapshotname"; + public static final String SNAPSHOT_DIFF_START_PATH = "snapshotdiffstartpath"; + public static final String SNAPSHOT_DIFF_INDEX = "snapshotdiffindex"; public static final String FSACTION_MODE_PARAM = "fsaction"; public static final String EC_POLICY_NAME_PARAM = "ecpolicy"; @@ -266,8 +269,8 @@ public enum Operation { RENAMESNAPSHOT(HTTP_PUT), GETSNAPSHOTDIFF(HTTP_GET), GETSNAPSHOTTABLEDIRECTORYLIST(HTTP_GET), GETSNAPSHOTLIST(HTTP_GET), GETSERVERDEFAULTS(HTTP_GET), - CHECKACCESS(HTTP_GET), SETECPOLICY(HTTP_PUT), GETECPOLICY( - HTTP_GET), UNSETECPOLICY(HTTP_POST), SATISFYSTORAGEPOLICY(HTTP_PUT); + CHECKACCESS(HTTP_GET), SETECPOLICY(HTTP_PUT), GETECPOLICY(HTTP_GET), UNSETECPOLICY( + HTTP_POST), SATISFYSTORAGEPOLICY(HTTP_PUT), GETSNAPSHOTDIFFLISTING(HTTP_GET); private String httpMethod; @@ -1577,6 +1580,22 @@ public SnapshotDiffReport getSnapshotDiffReport(Path path, return JsonUtilClient.toSnapshotDiffReport(json); } + public SnapshotDiffReportListing getSnapshotDiffReportListing(Path path, String snapshotOldName, + String snapshotNewName, byte[] snapshotDiffStartPath, Integer snapshotDiffIndex) + throws IOException { + Map params = new HashMap<>(); + params.put(OP_PARAM, Operation.GETSNAPSHOTDIFFLISTING.toString()); + params.put(SNAPSHOT_NAME_PARAM, snapshotNewName); + params.put(OLD_SNAPSHOT_NAME_PARAM, snapshotOldName); + params.put(SNAPSHOT_DIFF_START_PATH, DFSUtilClient.bytes2String(snapshotDiffStartPath)); + params.put(SNAPSHOT_DIFF_INDEX, snapshotDiffIndex.toString()); + HttpURLConnection conn = getConnection( + Operation.GETSNAPSHOTDIFFLISTING.getMethod(), params, path, true); + HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); + JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); + return JsonUtilClient.toSnapshotDiffReportListing(json); + } + public SnapshottableDirectoryStatus[] getSnapshottableDirectoryList() throws IOException { Map params = new HashMap(); diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java index e272cdc71b686..33f2abbb319ea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java @@ -45,6 +45,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import org.apache.hadoop.hdfs.web.JsonUtil; @@ -1879,6 +1880,65 @@ public String execute(FileSystem fs) throws IOException { } } + /** + * Executor that performs a getSnapshotDiffListing operation. + */ + @InterfaceAudience.Private + public static class FSGetSnapshotDiffListing implements + FileSystemAccess.FileSystemExecutor { + + private final Path path; + private final String oldSnapshotName; + private final String snapshotName; + private final String snapshotDiffStartPath; + private final int snapshotDiffIndex; + + /** + * Creates a getSnapshotDiffListing executor. + * + * @param path directory path of the snapshots to be examined. + * @param oldSnapshotName Older snapshot name. + * @param snapshotName Newer snapshot name. + * @param snapshotDiffStartPath snapshot diff start path. + * @param snapshotDiffIndex snapshot diff index. + */ + public FSGetSnapshotDiffListing(String path, String oldSnapshotName, + String snapshotName, String snapshotDiffStartPath, int snapshotDiffIndex) { + this.path = new Path(path); + this.oldSnapshotName = oldSnapshotName; + this.snapshotName = snapshotName; + this.snapshotDiffStartPath = snapshotDiffStartPath; + this.snapshotDiffIndex = snapshotDiffIndex; + } + + /** + * Executes the filesystem operation. + * + * @param fs filesystem instance to use. + * @return A serialized JSON string of snapshot diffs. + * @throws IOException thrown if an IO error occurred. + */ + @Override + public String execute(FileSystem fs) throws IOException { + SnapshotDiffReportListing snapshotDiffReportListing = null; + if (fs instanceof DistributedFileSystem) { + DistributedFileSystem dfs = (DistributedFileSystem) fs; + snapshotDiffReportListing = + dfs.getSnapshotDiffReportListing(path, oldSnapshotName, snapshotName, + snapshotDiffStartPath, snapshotDiffIndex); + } else { + throw new UnsupportedOperationException("getSnapshotDiffListing is not " + + "supported for HttpFs on " + fs.getClass() + + ". Please check your fs.defaultFS configuration"); + } + if (snapshotDiffReportListing != null) { + return JsonUtil.toJsonString(snapshotDiffReportListing); + } else { + return ""; + } + } + } + /** * Executor that performs a getSnapshottableDirListing operation. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java index f6c84dcae4e07..41009c7b5169f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java @@ -27,6 +27,7 @@ import org.apache.hadoop.lib.wsrs.BooleanParam; import org.apache.hadoop.lib.wsrs.EnumParam; import org.apache.hadoop.lib.wsrs.EnumSetParam; +import org.apache.hadoop.lib.wsrs.IntegerParam; import org.apache.hadoop.lib.wsrs.LongParam; import org.apache.hadoop.lib.wsrs.Param; import org.apache.hadoop.lib.wsrs.ParametersProvider; @@ -112,6 +113,9 @@ public class HttpFSParametersProvider extends ParametersProvider { PARAMS_DEF.put(Operation.RENAMESNAPSHOT, new Class[] {OldSnapshotNameParam.class, SnapshotNameParam.class}); + PARAMS_DEF.put(Operation.GETSNAPSHOTDIFFLISTING, + new Class[] {OldSnapshotNameParam.class, SnapshotNameParam.class, + SnapshotDiffStartPathParam.class, SnapshotDiffIndexParam.class}); PARAMS_DEF.put(Operation.GETSNAPSHOTDIFF, new Class[] {OldSnapshotNameParam.class, SnapshotNameParam.class}); @@ -669,6 +673,44 @@ public OldSnapshotNameParam() { } } + /** + * Class for SnapshotDiffStartPath parameter. + */ + public static class SnapshotDiffStartPathParam extends StringParam { + + /** + * Parameter name. + */ + public static final String NAME = HttpFSFileSystem.SNAPSHOT_DIFF_START_PATH; + + /** + * Constructor. + */ + public SnapshotDiffStartPathParam() { + super(NAME, ""); + } + + } + + /** + * Class for SnapshotDiffStartPath parameter. + */ + public static class SnapshotDiffIndexParam extends IntegerParam { + + /** + * Parameter name. + */ + public static final String NAME = HttpFSFileSystem.SNAPSHOT_DIFF_INDEX; + + /** + * Constructor. + */ + public SnapshotDiffIndexParam() { + super(NAME, null); + } + + } + /** * Class for FsAction parameter. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java index d0d76d6289912..399ff3bde9c5b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java @@ -450,6 +450,24 @@ public InputStream run() throws Exception { response = Response.ok(js).type(MediaType.APPLICATION_JSON).build(); break; } + case GETSNAPSHOTDIFFLISTING: { + String oldSnapshotName = params.get(OldSnapshotNameParam.NAME, + OldSnapshotNameParam.class); + String snapshotName = params.get(SnapshotNameParam.NAME, + SnapshotNameParam.class); + String snapshotDiffStartPath = params + .get(HttpFSParametersProvider.SnapshotDiffStartPathParam.NAME, + HttpFSParametersProvider.SnapshotDiffStartPathParam.class); + Integer snapshotDiffIndex = params.get(HttpFSParametersProvider.SnapshotDiffIndexParam.NAME, + HttpFSParametersProvider.SnapshotDiffIndexParam.class); + FSOperations.FSGetSnapshotDiffListing command = + new FSOperations.FSGetSnapshotDiffListing(path, oldSnapshotName, + snapshotName, snapshotDiffStartPath, snapshotDiffIndex); + String js = fsExecute(user, command); + AUDIT_LOG.info("[{}]", path); + response = Response.ok(js).type(MediaType.APPLICATION_JSON).build(); + break; + } case GETSNAPSHOTTABLEDIRECTORYLIST: { FSOperations.FSGetSnapshottableDirListing command = new FSOperations.FSGetSnapshottableDirListing(); diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/ServerSetup.md.vm b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/ServerSetup.md.vm index 2d0a5b8cd2e7d..e97de0275ca22 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/ServerSetup.md.vm +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/ServerSetup.md.vm @@ -162,9 +162,10 @@ Name | Description /logs | Display log files /stacks | Display JVM stacks /static/index.html | The static home page +/prof | Async Profiler endpoint To control the access to servlet `/conf`, `/jmx`, `/logLevel`, `/logs`, -and `/stacks`, configure the following properties in `httpfs-site.xml`: +`/stacks` and `/prof`, configure the following properties in `httpfs-site.xml`: ```xml @@ -178,7 +179,7 @@ and `/stacks`, configure the following properties in `httpfs-site.xml`: true Indicates if administrator ACLs are required to access - instrumentation servlets (JMX, METRICS, CONF, STACKS). + instrumentation servlets (JMX, METRICS, CONF, STACKS, PROF). diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java index d8dd7c7c362d7..9ef24ec734ab0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java @@ -42,6 +42,8 @@ import org.apache.hadoop.hdfs.AppendTestUtil; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.DFSUtil; +import org.apache.hadoop.hdfs.DFSUtilClient; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; @@ -50,6 +52,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing; import org.apache.hadoop.hdfs.protocol.SnapshotException; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; import org.apache.hadoop.hdfs.protocol.SnapshotStatus; @@ -1198,7 +1201,7 @@ protected enum Operation { ALLOW_SNAPSHOT, DISALLOW_SNAPSHOT, DISALLOW_SNAPSHOT_EXCEPTION, FILE_STATUS_ATTR, GET_SNAPSHOT_DIFF, GET_SNAPSHOTTABLE_DIRECTORY_LIST, GET_SNAPSHOT_LIST, GET_SERVERDEFAULTS, CHECKACCESS, SETECPOLICY, - SATISFYSTORAGEPOLICY + SATISFYSTORAGEPOLICY, GET_SNAPSHOT_DIFF_LISTING } private void operation(Operation op) throws Exception { @@ -1335,6 +1338,9 @@ private void operation(Operation op) throws Exception { case SATISFYSTORAGEPOLICY: testStoragePolicySatisfier(); break; + case GET_SNAPSHOT_DIFF_LISTING: + testGetSnapshotDiffListing(); + break; } } @@ -1607,29 +1613,30 @@ private void testGetSnapshotDiff() throws Exception { Path file2 = new Path(path, "file2"); testCreate(file2, false); fs.createSnapshot(path, "snap2"); - // Get snapshot diff - SnapshotDiffReport diffReport = null; - if (fs instanceof HttpFSFileSystem) { - HttpFSFileSystem httpFS = (HttpFSFileSystem) fs; - diffReport = httpFS.getSnapshotDiffReport(path, "snap1", "snap2"); - } else if (fs instanceof WebHdfsFileSystem) { - WebHdfsFileSystem webHdfsFileSystem = (WebHdfsFileSystem) fs; - diffReport = webHdfsFileSystem.getSnapshotDiffReport(path, - "snap1", "snap2"); - } else { - Assert.fail(fs.getClass().getSimpleName() + - " doesn't support getSnapshotDiff"); + + try { + // Get snapshot diff + SnapshotDiffReport diffReport = null; + if (fs instanceof HttpFSFileSystem) { + HttpFSFileSystem httpFS = (HttpFSFileSystem) fs; + diffReport = httpFS.getSnapshotDiffReport(path, "snap1", "snap2"); + } else if (fs instanceof WebHdfsFileSystem) { + WebHdfsFileSystem webHdfsFileSystem = (WebHdfsFileSystem) fs; + diffReport = webHdfsFileSystem.getSnapshotDiffReport(path, "snap1", "snap2"); + } else { + Assert.fail(fs.getClass().getSimpleName() + " doesn't support getSnapshotDiff"); + } + // Verify result with DFS + DistributedFileSystem dfs = + (DistributedFileSystem) FileSystem.get(path.toUri(), this.getProxiedFSConf()); + SnapshotDiffReport dfsDiffReport = dfs.getSnapshotDiffReport(path, "snap1", "snap2"); + Assert.assertEquals(diffReport.toString(), dfsDiffReport.toString()); + } finally { + // Cleanup + fs.deleteSnapshot(path, "snap2"); + fs.deleteSnapshot(path, "snap1"); + fs.delete(path, true); } - // Verify result with DFS - DistributedFileSystem dfs = (DistributedFileSystem) - FileSystem.get(path.toUri(), this.getProxiedFSConf()); - SnapshotDiffReport dfsDiffReport = - dfs.getSnapshotDiffReport(path, "snap1", "snap2"); - Assert.assertEquals(diffReport.toString(), dfsDiffReport.toString()); - // Cleanup - fs.deleteSnapshot(path, "snap2"); - fs.deleteSnapshot(path, "snap1"); - fs.delete(path, true); } } @@ -1951,4 +1958,102 @@ public void testStoragePolicySatisfier() throws Exception { dfs.delete(path1, true); } } + + private void testGetSnapshotDiffListing() throws Exception { + if (!this.isLocalFS()) { + // Create a directory with snapshot allowed + Path path = new Path("/tmp/tmp-snap-test"); + createSnapshotTestsPreconditions(path); + // Get the FileSystem instance that's being tested + FileSystem fs = this.getHttpFSFileSystem(); + // Check FileStatus + Assert.assertTrue(fs.getFileStatus(path).isSnapshotEnabled()); + // Create a file and take a snapshot + Path file1 = new Path(path, "file1"); + testCreate(file1, false); + fs.createSnapshot(path, "snap1"); + // Create another file and take a snapshot + Path file2 = new Path(path, "file2"); + testCreate(file2, false); + fs.createSnapshot(path, "snap2"); + // Get snapshot diff listing + try { + SnapshotDiffReportListing diffReportListing = null; + byte[] emptyBytes = new byte[] {}; + if (fs instanceof HttpFSFileSystem) { + HttpFSFileSystem httpFS = (HttpFSFileSystem) fs; + diffReportListing = + httpFS.getSnapshotDiffReportListing(path, "snap1", "snap2", emptyBytes, -1); + } else if (fs instanceof WebHdfsFileSystem) { + WebHdfsFileSystem webHdfsFileSystem = (WebHdfsFileSystem) fs; + diffReportListing = webHdfsFileSystem + .getSnapshotDiffReportListing(path.toUri().getPath(), "snap1", "snap2", emptyBytes, + -1); + } else { + Assert.fail(fs.getClass().getSimpleName() + " doesn't support getSnapshotDiff"); + } + // Verify result with DFS + DistributedFileSystem dfs = + (DistributedFileSystem) FileSystem.get(path.toUri(), this.getProxiedFSConf()); + SnapshotDiffReportListing dfsDiffReportListing = + dfs.getSnapshotDiffReportListing(path, "snap1", "snap2", + DFSUtil.bytes2String(emptyBytes), -1); + assertHttpFsReportListingWithDfsClient(diffReportListing, dfsDiffReportListing); + } finally { + // Cleanup + fs.deleteSnapshot(path, "snap2"); + fs.deleteSnapshot(path, "snap1"); + fs.delete(path, true); + } + } + } + + private void assertHttpFsReportListingWithDfsClient(SnapshotDiffReportListing diffReportListing, + SnapshotDiffReportListing dfsDiffReportListing) { + Assert.assertEquals(diffReportListing.getCreateList().size(), + dfsDiffReportListing.getCreateList().size()); + Assert.assertEquals(diffReportListing.getDeleteList().size(), + dfsDiffReportListing.getDeleteList().size()); + Assert.assertEquals(diffReportListing.getModifyList().size(), + dfsDiffReportListing.getModifyList().size()); + Assert.assertEquals(diffReportListing.getIsFromEarlier(), + dfsDiffReportListing.getIsFromEarlier()); + Assert.assertEquals(diffReportListing.getLastIndex(), dfsDiffReportListing.getLastIndex()); + Assert.assertEquals(DFSUtil.bytes2String(diffReportListing.getLastPath()), + DFSUtil.bytes2String(dfsDiffReportListing.getLastPath())); + int i = 0; + for (SnapshotDiffReportListing.DiffReportListingEntry entry : diffReportListing + .getCreateList()) { + SnapshotDiffReportListing.DiffReportListingEntry dfsDiffEntry = + dfsDiffReportListing.getCreateList().get(i); + Assert.assertEquals(entry.getDirId(), dfsDiffEntry.getDirId()); + Assert.assertEquals(entry.getFileId(), dfsDiffEntry.getFileId()); + Assert.assertArrayEquals(DFSUtilClient.byteArray2bytes(entry.getSourcePath()), + DFSUtilClient.byteArray2bytes(dfsDiffEntry.getSourcePath())); + i++; + } + i = 0; + for (SnapshotDiffReportListing.DiffReportListingEntry entry : diffReportListing + .getDeleteList()) { + SnapshotDiffReportListing.DiffReportListingEntry dfsDiffEntry = + dfsDiffReportListing.getDeleteList().get(i); + Assert.assertEquals(entry.getDirId(), dfsDiffEntry.getDirId()); + Assert.assertEquals(entry.getFileId(), dfsDiffEntry.getFileId()); + Assert.assertArrayEquals(DFSUtilClient.byteArray2bytes(entry.getSourcePath()), + DFSUtilClient.byteArray2bytes(dfsDiffEntry.getSourcePath())); + i++; + } + i = 0; + for (SnapshotDiffReportListing.DiffReportListingEntry entry : diffReportListing + .getModifyList()) { + SnapshotDiffReportListing.DiffReportListingEntry dfsDiffEntry = + dfsDiffReportListing.getModifyList().get(i); + Assert.assertEquals(entry.getDirId(), dfsDiffEntry.getDirId()); + Assert.assertEquals(entry.getFileId(), dfsDiffEntry.getFileId()); + Assert.assertArrayEquals(DFSUtilClient.byteArray2bytes(entry.getSourcePath()), + DFSUtilClient.byteArray2bytes(dfsDiffEntry.getSourcePath())); + i++; + } + } + } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/service/security/DummyGroupMapping.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/service/security/DummyGroupMapping.java deleted file mode 100644 index 4af93182ed341..0000000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/service/security/DummyGroupMapping.java +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.lib.service.security; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Set; - -import org.apache.hadoop.security.GroupMappingServiceProvider; -import org.apache.hadoop.test.HadoopUsersConfTestHelper; -import org.apache.hadoop.util.Sets; - -public class DummyGroupMapping implements GroupMappingServiceProvider { - - @Override - public List getGroups(String user) throws IOException { - if (user.equals("root")) { - return Arrays.asList("admin"); - } - else if (user.equals("nobody")) { - return Arrays.asList("nobody"); - } else { - String[] groups = HadoopUsersConfTestHelper.getHadoopUserGroups(user); - return (groups != null) ? Arrays.asList(groups) : Collections.emptyList(); - } - } - - @Override - public void cacheGroupsRefresh() throws IOException { - } - - @Override - public void cacheGroupsAdd(List groups) throws IOException { - } - - @Override - public Set getGroupsSet(String user) throws IOException { - if (user.equals("root")) { - return Sets.newHashSet("admin"); - } else if (user.equals("nobody")) { - return Sets.newHashSet("nobody"); - } else { - String[] groups = HadoopUsersConfTestHelper.getHadoopUserGroups(user); - return (groups != null) ? Sets.newHashSet(groups) : - Collections.emptySet(); - } - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/doc/README b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/doc/README index e8cc0e509f7ee..0dc17214348d6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/doc/README +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/doc/README @@ -88,6 +88,7 @@ rw -odebug (do not daemonize - aka -d in fuse speak) -obig_writes (use fuse big_writes option so as to allow better performance of writes on kernels >= 2.6.26) -initchecks - have fuse-dfs try to connect to hdfs to ensure all is ok upon startup. recommended to have this on +-omax_background=%d (maximum number of pending "background" requests - see fuse docs) The defaults are: entry,attribute_timeouts = 60 seconds diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_dfs.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_dfs.c index f693032d5c5ed..b9b01009beb1f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_dfs.c +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_dfs.c @@ -81,6 +81,7 @@ int main(int argc, char *argv[]) options.rdbuffer_size = 10*1024*1024; options.attribute_timeout = 60; options.entry_timeout = 60; + options.max_background = 0; if (-1 == fuse_opt_parse(&args, &options, dfs_opts, dfs_options)) { return -1; @@ -114,6 +115,11 @@ int main(int argc, char *argv[]) snprintf(buf, sizeof buf, "-oentry_timeout=%d",options.entry_timeout); fuse_opt_add_arg(&args, buf); + + if (options.max_background > 0) { + snprintf(buf, sizeof buf, "-omax_background=%d",options.max_background); + fuse_opt_add_arg(&args, buf); + } } if (options.nn_uri == NULL) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_init.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_init.c index 205a7f40fded0..65e9e6d27682f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_init.c +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_init.c @@ -91,11 +91,11 @@ static void dfsPrintOptions(FILE *fp, const struct options *o) INFO("Mounting with options: [ protected=%s, nn_uri=%s, nn_port=%d, " "debug=%d, read_only=%d, initchecks=%d, " "no_permissions=%d, usetrash=%d, entry_timeout=%d, " - "attribute_timeout=%d, rdbuffer_size=%zd, direct_io=%d ]", + "attribute_timeout=%d, rdbuffer_size=%zd, direct_io=%d, max_background=%d ]", (o->protected ? o->protected : "(NULL)"), o->nn_uri, o->nn_port, o->debug, o->read_only, o->initchecks, o->no_permissions, o->usetrash, o->entry_timeout, - o->attribute_timeout, o->rdbuffer_size, o->direct_io); + o->attribute_timeout, o->rdbuffer_size, o->direct_io, o->max_background); } void *dfs_init(struct fuse_conn_info *conn) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.c index 8461ce40f9186..b4082c63d783e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.c +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.c @@ -37,11 +37,13 @@ void print_options() { "\tentry_timeout=%d\n" "\tattribute_timeout=%d\n" "\tprivate=%d\n" - "\trdbuffer_size=%d (KBs)\n", - options.protected, options.nn_uri, options.nn_port, options.debug, + "\trdbuffer_size=%d (KBs)\n" + "\tmax_background=%d\n", + options.protected, options.nn_uri, options.nn_port, options.debug, options.read_only, options.usetrash, options.entry_timeout, options.attribute_timeout, options.private, - (int)options.rdbuffer_size / 1024); + (int)options.rdbuffer_size / 1024, + options.max_background); } const char *program; @@ -56,7 +58,7 @@ void print_usage(const char *pname) "[-ousetrash] [-obig_writes] [-oprivate (single user)] [ro] " "[-oserver=] [-oport=] " "[-oentry_timeout=] [-oattribute_timeout=] " - "[-odirect_io] [-onopoermissions] [-o] " + "[-odirect_io] [-onopoermissions] [-omax_background=] [-o] " " [fuse options]\n", pname); printf("NOTE: debugging option for fuse is -debug\n"); } @@ -87,6 +89,7 @@ struct fuse_opt dfs_opts[] = DFSFS_OPT_KEY("protected=%s", protected, 0), DFSFS_OPT_KEY("port=%d", nn_port, 0), DFSFS_OPT_KEY("rdbuffer=%d", rdbuffer_size,0), + DFSFS_OPT_KEY("max_background=%d", max_background, 0), FUSE_OPT_KEY("private", KEY_PRIVATE), FUSE_OPT_KEY("ro", KEY_RO), diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.h index 4bfc2355259b3..2d00f1b30a1dc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.h @@ -34,6 +34,7 @@ struct options { int private; size_t rdbuffer_size; int direct_io; + int max_background; } options; extern struct fuse_opt dfs_opts[]; diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/test/TestFuseDFS.java b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/test/TestFuseDFS.java index 33fe4464e65c6..68cd29a703b3c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/test/TestFuseDFS.java +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/test/TestFuseDFS.java @@ -187,6 +187,7 @@ private static Process establishMount(URI uri) throws IOException { "-ononempty", // Don't complain about junk in mount point "-f", // Don't background the process "-ordbuffer=32768", // Read buffer size in kb + "-omax_background=100", // Set fuse max_background=100 (12 by default) "rw" }; diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/CMakeLists.txt index f16cc9eb1b033..44bc87d17c67d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/CMakeLists.txt @@ -25,6 +25,7 @@ include_directories( ${CMAKE_CURRENT_SOURCE_DIR}/../libhdfs ${JNI_INCLUDE_DIRS} ${OS_DIR} + ../libhdfspp/lib ) add_library(native_mini_dfs diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_ops.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_ops.c index a3058bbe6ec06..359cc2f4e3269 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_ops.c +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_ops.c @@ -454,6 +454,68 @@ int main(int argc, char **argv) { hdfsCloseFile(lfs, localFile); } + + { + // HDFS Open File Builder tests + + exists = hdfsExists(fs, readPath); + + if (exists) { + fprintf(stderr, "Failed to validate existence of %s\n", readPath); + shutdown_and_exit(cl, -1); + } + + hdfsOpenFileBuilder *builder; + builder = hdfsOpenFileBuilderAlloc(fs, readPath); + hdfsOpenFileBuilderOpt(builder, "hello", "world"); + + hdfsOpenFileFuture *future; + future = hdfsOpenFileBuilderBuild(builder); + + readFile = hdfsOpenFileFutureGet(future); + if (!hdfsOpenFileFutureCancel(future, 0)) { + fprintf(stderr, "Cancel on a completed Future should return false"); + shutdown_and_exit(cl, -1); + } + hdfsOpenFileFutureFree(future); + + memset(buffer, 0, sizeof(buffer)); + num_read_bytes = hdfsRead(fs, readFile, (void *) buffer, + sizeof(buffer)); + if (strncmp(fileContents, buffer, strlen(fileContents)) != 0) { + fprintf(stderr, + "Failed to read. Expected %s but got %s (%d bytes)\n", + fileContents, buffer, num_read_bytes); + shutdown_and_exit(cl, -1); + } + hdfsCloseFile(fs, readFile); + + builder = hdfsOpenFileBuilderAlloc(fs, readPath); + hdfsOpenFileBuilderOpt(builder, "hello", "world"); + + future = hdfsOpenFileBuilderBuild(builder); + + readFile = hdfsOpenFileFutureGetWithTimeout(future, 1, jDays); + if (!hdfsOpenFileFutureCancel(future, 0)) { + fprintf(stderr, "Cancel on a completed Future should return " + "false"); + shutdown_and_exit(cl, -1); + } + hdfsOpenFileFutureFree(future); + + memset(buffer, 0, sizeof(buffer)); + num_read_bytes = hdfsRead(fs, readFile, (void*)buffer, + sizeof(buffer)); + if (strncmp(fileContents, buffer, strlen(fileContents)) != 0) { + fprintf(stderr, "Failed to read. Expected %s but got " + "%s (%d bytes)\n", fileContents, buffer, + num_read_bytes); + shutdown_and_exit(cl, -1); + } + memset(buffer, 0, strlen(fileContents + 1)); + hdfsCloseFile(fs, readFile); + } + totalResult = 0; result = 0; { diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/CMakeLists.txt index a7fb311125110..22d18708d6e93 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/CMakeLists.txt @@ -29,6 +29,7 @@ include_directories( main/native main/native/libhdfs ${OS_DIR} + ../libhdfspp/lib ) hadoop_add_dual_library(hdfs diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c index 60f2826c74173..ed150925cdb81 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c @@ -38,6 +38,10 @@ #define KERBEROS_TICKET_CACHE_PATH "hadoop.security.kerberos.ticket.cache.path" +// StreamCapability flags taken from o.a.h.fs.StreamCapabilities +#define IS_READ_BYTE_BUFFER_CAPABILITY "in:readbytebuffer" +#define IS_PREAD_BYTE_BUFFER_CAPABILITY "in:preadbytebuffer" + // Bit fields for hdfsFile_internal flags #define HDFS_FILE_SUPPORTS_DIRECT_READ (1<<0) #define HDFS_FILE_SUPPORTS_DIRECT_PREAD (1<<1) @@ -1075,6 +1079,27 @@ static int hdfsHasStreamCapability(jobject jFile, return 0; } +/** + * Sets the flags of the given hdfsFile based on the capabilities of the + * underlying stream. + * + * @param file file->flags will be updated based on the capabilities of jFile + * @param jFile the underlying stream to check for capabilities + */ +static void setFileFlagCapabilities(hdfsFile file, jobject jFile) { + // Check the StreamCapabilities of jFile to see if we can do direct + // reads + if (hdfsHasStreamCapability(jFile, IS_READ_BYTE_BUFFER_CAPABILITY)) { + file->flags |= HDFS_FILE_SUPPORTS_DIRECT_READ; + } + + // Check the StreamCapabilities of jFile to see if we can do direct + // preads + if (hdfsHasStreamCapability(jFile, IS_PREAD_BYTE_BUFFER_CAPABILITY)) { + file->flags |= HDFS_FILE_SUPPORTS_DIRECT_PREAD; + } +} + static hdfsFile hdfsOpenFileImpl(hdfsFS fs, const char *path, int flags, int32_t bufferSize, int16_t replication, int64_t blockSize) { @@ -1245,17 +1270,7 @@ static hdfsFile hdfsOpenFileImpl(hdfsFS fs, const char *path, int flags, file->flags = 0; if ((flags & O_WRONLY) == 0) { - // Check the StreamCapabilities of jFile to see if we can do direct - // reads - if (hdfsHasStreamCapability(jFile, "in:readbytebuffer")) { - file->flags |= HDFS_FILE_SUPPORTS_DIRECT_READ; - } - - // Check the StreamCapabilities of jFile to see if we can do direct - // preads - if (hdfsHasStreamCapability(jFile, "in:preadbytebuffer")) { - file->flags |= HDFS_FILE_SUPPORTS_DIRECT_PREAD; - } + setFileFlagCapabilities(file, jFile); } ret = 0; @@ -1288,6 +1303,469 @@ hdfsFile hdfsStreamBuilderBuild(struct hdfsStreamBuilder *bld) return file; } +/** + * A wrapper around o.a.h.fs.FutureDataInputStreamBuilder and the file name + * associated with the builder. + */ +struct hdfsOpenFileBuilder { + jobject jBuilder; + const char *path; +}; + +/** + * A wrapper around a java.util.concurrent.Future (created by calling + * FutureDataInputStreamBuilder#build) and the file name associated with the + * builder. + */ +struct hdfsOpenFileFuture { + jobject jFuture; + const char *path; +}; + +hdfsOpenFileBuilder *hdfsOpenFileBuilderAlloc(hdfsFS fs, + const char *path) { + int ret = 0; + jthrowable jthr; + jvalue jVal; + jobject jFS = (jobject) fs; + + jobject jPath = NULL; + jobject jBuilder = NULL; + + JNIEnv *env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return NULL; + } + + hdfsOpenFileBuilder *builder; + builder = calloc(1, sizeof(hdfsOpenFileBuilder)); + if (!builder) { + fprintf(stderr, "hdfsOpenFileBuilderAlloc(%s): OOM when creating " + "hdfsOpenFileBuilder\n", path); + errno = ENOMEM; + goto done; + } + builder->path = path; + + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFileBuilderAlloc(%s): constructNewObjectOfPath", + path); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "openFile", JMETHOD1(JPARAM(HADOOP_PATH), JPARAM(HADOOP_FDISB)), + jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFileBuilderAlloc(%s): %s#openFile(Path) failed", + HADOOP_FS, path); + goto done; + } + jBuilder = jVal.l; + + builder->jBuilder = (*env)->NewGlobalRef(env, jBuilder); + if (!builder->jBuilder) { + printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsOpenFileBuilderAlloc(%s): NewGlobalRef(%s) failed", path, + HADOOP_FDISB); + ret = EINVAL; + goto done; + } + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jBuilder); + if (ret) { + if (builder) { + if (builder->jBuilder) { + (*env)->DeleteGlobalRef(env, builder->jBuilder); + } + free(builder); + } + errno = ret; + return NULL; + } + return builder; +} + +/** + * Used internally by hdfsOpenFileBuilderWithOption to switch between + * FSBuilder#must and #opt. + */ +typedef enum { must, opt } openFileBuilderOptionType; + +/** + * Shared implementation of hdfsOpenFileBuilderMust and hdfsOpenFileBuilderOpt + * that switches between each method depending on the value of + * openFileBuilderOptionType. + */ +static hdfsOpenFileBuilder *hdfsOpenFileBuilderWithOption( + hdfsOpenFileBuilder *builder, const char *key, + const char *value, openFileBuilderOptionType optionType) { + int ret = 0; + jthrowable jthr; + jvalue jVal; + jobject localJBuilder = NULL; + jobject globalJBuilder; + jstring jKeyString = NULL; + jstring jValueString = NULL; + + // If the builder was not previously created by a prior call to + // hdfsOpenFileBuilderAlloc then exit + if (builder == NULL || builder->jBuilder == NULL) { + errno = EINVAL; + return NULL; + } + + JNIEnv *env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return NULL; + } + jthr = newJavaStr(env, key, &jKeyString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFileBuilderWithOption(%s): newJavaStr(%s)", + builder->path, key); + goto done; + } + jthr = newJavaStr(env, value, &jValueString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFileBuilderWithOption(%s): newJavaStr(%s)", + builder->path, value); + goto done; + } + + const char *optionTypeMethodName; + switch (optionType) { + case must: + optionTypeMethodName = "must"; + break; + case opt: + optionTypeMethodName = "opt"; + break; + default: + ret = EINTERNAL; + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, builder->jBuilder, + JC_FUTURE_DATA_IS_BUILDER, optionTypeMethodName, + JMETHOD2(JPARAM(JAVA_STRING), JPARAM(JAVA_STRING), + JPARAM(HADOOP_FS_BLDR)), jKeyString, + jValueString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFileBuilderWithOption(%s): %s#%s(%s, %s) failed", + builder->path, HADOOP_FS_BLDR, optionTypeMethodName, key, + value); + goto done; + } + + localJBuilder = jVal.l; + globalJBuilder = (*env)->NewGlobalRef(env, localJBuilder); + if (!globalJBuilder) { + printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsOpenFileBuilderWithOption(%s): NewGlobalRef(%s) failed", + builder->path, HADOOP_FDISB); + ret = EINVAL; + goto done; + } + (*env)->DeleteGlobalRef(env, builder->jBuilder); + builder->jBuilder = globalJBuilder; + +done: + destroyLocalReference(env, jKeyString); + destroyLocalReference(env, jValueString); + destroyLocalReference(env, localJBuilder); + if (ret) { + errno = ret; + return NULL; + } + return builder; +} + +hdfsOpenFileBuilder *hdfsOpenFileBuilderMust(hdfsOpenFileBuilder *builder, + const char *key, const char *value) { + openFileBuilderOptionType optionType; + optionType = must; + return hdfsOpenFileBuilderWithOption(builder, key, value, optionType); +} + +hdfsOpenFileBuilder *hdfsOpenFileBuilderOpt(hdfsOpenFileBuilder *builder, + const char *key, const char *value) { + openFileBuilderOptionType optionType; + optionType = opt; + return hdfsOpenFileBuilderWithOption(builder, key, value, optionType); +} + +hdfsOpenFileFuture *hdfsOpenFileBuilderBuild(hdfsOpenFileBuilder *builder) { + int ret = 0; + jthrowable jthr; + jvalue jVal; + + jobject jFuture = NULL; + + // If the builder was not previously created by a prior call to + // hdfsOpenFileBuilderAlloc then exit + if (builder == NULL || builder->jBuilder == NULL) { + ret = EINVAL; + return NULL; + } + + JNIEnv *env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return NULL; + } + + hdfsOpenFileFuture *future; + future = calloc(1, sizeof(hdfsOpenFileFuture)); + if (!future) { + fprintf(stderr, "hdfsOpenFileBuilderBuild: OOM when creating " + "hdfsOpenFileFuture\n"); + errno = ENOMEM; + goto done; + } + future->path = builder->path; + + jthr = invokeMethod(env, &jVal, INSTANCE, builder->jBuilder, + JC_FUTURE_DATA_IS_BUILDER, "build", + JMETHOD1("", JPARAM(JAVA_CFUTURE))); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFileBuilderBuild(%s): %s#build() failed", + builder->path, HADOOP_FDISB); + goto done; + } + jFuture = jVal.l; + + future->jFuture = (*env)->NewGlobalRef(env, jFuture); + if (!future->jFuture) { + printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsOpenFileBuilderBuild(%s): NewGlobalRef(%s) failed", + builder->path, JAVA_CFUTURE); + ret = EINVAL; + goto done; + } + +done: + destroyLocalReference(env, jFuture); + if (ret) { + if (future) { + if (future->jFuture) { + (*env)->DeleteGlobalRef(env, future->jFuture); + } + free(future); + } + hdfsOpenFileBuilderFree(builder); + errno = ret; + return NULL; + } + hdfsOpenFileBuilderFree(builder); + return future; +} + +void hdfsOpenFileBuilderFree(hdfsOpenFileBuilder *builder) { + JNIEnv *env; + env = getJNIEnv(); + if (!env) { + return; + } + if (builder->jBuilder) { + (*env)->DeleteGlobalRef(env, builder->jBuilder); + builder->jBuilder = NULL; + } + free(builder); +} + +/** + * Shared implementation of hdfsOpenFileFutureGet and + * hdfsOpenFileFutureGetWithTimeout. If a timeout is specified, calls + * Future#get() otherwise it calls Future#get(long, TimeUnit). + */ +static hdfsFile fileFutureGetWithTimeout(hdfsOpenFileFuture *future, + int64_t timeout, jobject jTimeUnit) { + int ret = 0; + jthrowable jthr; + jvalue jVal; + + hdfsFile file = NULL; + jobject jFile = NULL; + + JNIEnv *env = getJNIEnv(); + if (!env) { + ret = EINTERNAL; + return NULL; + } + + if (!jTimeUnit) { + jthr = invokeMethod(env, &jVal, INSTANCE, future->jFuture, + JC_CFUTURE, "get", JMETHOD1("", JPARAM(JAVA_OBJECT))); + } else { + jthr = invokeMethod(env, &jVal, INSTANCE, future->jFuture, + JC_CFUTURE, "get", JMETHOD2("J", + JPARAM(JAVA_TIMEUNIT), JPARAM(JAVA_OBJECT)), timeout, + jTimeUnit); + } + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFileFutureGet(%s): %s#get failed", future->path, + JAVA_CFUTURE); + goto done; + } + + file = calloc(1, sizeof(struct hdfsFile_internal)); + if (!file) { + fprintf(stderr, "hdfsOpenFileFutureGet(%s): OOM when creating " + "hdfsFile\n", future->path); + ret = ENOMEM; + goto done; + } + jFile = jVal.l; + file->file = (*env)->NewGlobalRef(env, jFile); + if (!file->file) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsOpenFileFutureGet(%s): NewGlobalRef(jFile) failed", + future->path); + goto done; + } + + file->type = HDFS_STREAM_INPUT; + file->flags = 0; + + setFileFlagCapabilities(file, jFile); + +done: + destroyLocalReference(env, jTimeUnit); + destroyLocalReference(env, jFile); + if (ret) { + if (file) { + if (file->file) { + (*env)->DeleteGlobalRef(env, file->file); + } + free(file); + } + errno = ret; + return NULL; + } + return file; +} + +hdfsFile hdfsOpenFileFutureGet(hdfsOpenFileFuture *future) { + return fileFutureGetWithTimeout(future, -1, NULL); +} + +hdfsFile hdfsOpenFileFutureGetWithTimeout(hdfsOpenFileFuture *future, + int64_t timeout, javaConcurrentTimeUnit timeUnit) { + int ret = 0; + jthrowable jthr; + jobject jTimeUnit = NULL; + + JNIEnv *env = getJNIEnv(); + if (!env) { + ret = EINTERNAL; + return NULL; + } + + const char *timeUnitEnumName; + switch (timeUnit) { + case jNanoseconds: + timeUnitEnumName = "NANOSECONDS"; + break; + case jMicroseconds: + timeUnitEnumName = "MICROSECONDS"; + break; + case jMilliseconds: + timeUnitEnumName = "MILLISECONDS"; + break; + case jSeconds: + timeUnitEnumName = "SECONDS"; + break; + case jMinutes: + timeUnitEnumName = "MINUTES"; + break; + case jHours: + timeUnitEnumName = "HOURS"; + break; + case jDays: + timeUnitEnumName = "DAYS"; + break; + default: + ret = EINTERNAL; + goto done; + } + + jthr = fetchEnumInstance(env, JAVA_TIMEUNIT, timeUnitEnumName, &jTimeUnit); + + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFileFutureGet(%s): %s#get failed", future->path, + JAVA_CFUTURE); + goto done; + } + return fileFutureGetWithTimeout(future, timeout, jTimeUnit); + +done: + if (ret) { + errno = ret; + } + return NULL; +} + +int hdfsOpenFileFutureCancel(hdfsOpenFileFuture *future, + int mayInterruptIfRunning) { + int ret = 0; + jthrowable jthr; + jvalue jVal; + + jboolean jMayInterruptIfRunning; + + JNIEnv *env = getJNIEnv(); + if (!env) { + ret = EINTERNAL; + return -1; + } + + jMayInterruptIfRunning = mayInterruptIfRunning ? JNI_TRUE : JNI_FALSE; + jthr = invokeMethod(env, &jVal, INSTANCE, future->jFuture, JC_CFUTURE, + "cancel", JMETHOD1("Z", "Z"), jMayInterruptIfRunning); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFileFutureCancel(%s): %s#cancel failed", future->path, + JAVA_CFUTURE); + goto done; + } + +done: + if (ret) { + errno = ret; + return -1; + } + if (!jVal.z) { + return -1; + } + return 0; +} + +void hdfsOpenFileFutureFree(hdfsOpenFileFuture *future) { + JNIEnv *env; + env = getJNIEnv(); + if (!env) { + return; + } + if (future->jFuture) { + (*env)->DeleteGlobalRef(env, future->jFuture); + future->jFuture = NULL; + } + free(future); +} + int hdfsTruncateFile(hdfsFS fs, const char* path, tOffset newlength) { jobject jFS = (jobject)fs; diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h index e58a6232d205a..eba50ff6eb277 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h @@ -82,6 +82,29 @@ extern "C" { } tObjectKind; struct hdfsStreamBuilder; + /** + * The C reflection of the enum values from java.util.concurrent.TimeUnit . + */ + typedef enum javaConcurrentTimeUnit { + jNanoseconds, + jMicroseconds, + jMilliseconds, + jSeconds, + jMinutes, + jHours, + jDays, + } javaConcurrentTimeUnit; + + /** + * The C reflection of java.util.concurrent.Future specifically used for + * opening HDFS files asynchronously. + */ + typedef struct hdfsOpenFileFuture hdfsOpenFileFuture; + + /** + * The C reflection of o.a.h.fs.FutureDataInputStreamBuilder . + */ + typedef struct hdfsOpenFileBuilder hdfsOpenFileBuilder; /** * The C reflection of org.apache.org.hadoop.FileSystem . @@ -429,6 +452,118 @@ extern "C" { hdfsFile hdfsOpenFile(hdfsFS fs, const char* path, int flags, int bufferSize, short replication, tSize blocksize); + /** + * hdfsOpenFileBuilderAlloc - Allocate a HDFS open file builder. + * + * @param fs The configured filesystem handle. + * @param path The full path to the file. + * @return Returns the hdfsOpenFileBuilder, or NULL on error. + */ + LIBHDFS_EXTERNAL + hdfsOpenFileBuilder *hdfsOpenFileBuilderAlloc(hdfsFS fs, + const char *path); + + /** + * hdfsOpenFileBuilderMust - Specifies a mandatory parameter for the open + * file builder. While the underlying FsBuilder supports various various + * types for the value (boolean, int, float, double), currently only + * strings are supported. + * + * @param builder The open file builder to set the config for. + * @param key The config key + * @param value The config value + * @return Returns the hdfsOpenFileBuilder, or NULL on error. + */ + LIBHDFS_EXTERNAL + hdfsOpenFileBuilder *hdfsOpenFileBuilderMust(hdfsOpenFileBuilder *builder, + const char *key, const char *value); + + /** + * hdfsOpenFileBuilderOpt - Specifies an optional parameter for the open + * file builder. While the underlying FsBuilder supports various various + * types for the value (boolean, int, float, double), currently only + * strings are supported. + * + * @param builder The open file builder to set the config for. + * @param key The config key + * @param value The config value + * @return Returns the hdfsOpenFileBuilder, or NULL on error. + */ + LIBHDFS_EXTERNAL + hdfsOpenFileBuilder *hdfsOpenFileBuilderOpt(hdfsOpenFileBuilder *builder, + const char *key, const char *value); + + /** + * hdfsOpenFileBuilderBuild - Builds the open file builder and returns a + * hdfsOpenFileFuture which tracks the asynchronous call to open the + * specified file. + * + * @param builder The open file builder to build. + * @return Returns the hdfsOpenFileFuture, or NULL on error. + */ + LIBHDFS_EXTERNAL + hdfsOpenFileFuture *hdfsOpenFileBuilderBuild(hdfsOpenFileBuilder *builder); + + /** + * hdfsOpenFileBuilderFree - Free a HDFS open file builder. + * + * It is normally not necessary to call this function since + * hdfsOpenFileBuilderBuild frees the builder. + * + * @param builder The hdfsOpenFileBuilder to free. + */ + LIBHDFS_EXTERNAL + void hdfsOpenFileBuilderFree(hdfsOpenFileBuilder *builder); + + /** + * hdfsOpenFileFutureGet - Call Future#get() on the underlying Java Future + * object. A call to #get() will block until the asynchronous operation has + * completed. In this case, until the open file call has completed. This + * method blocks indefinitely until blocking call completes. + * + * @param future The hdfsOpenFileFuture to call #get on + * @return Returns the opened hdfsFile, or NULL on error. + */ + LIBHDFS_EXTERNAL + hdfsFile hdfsOpenFileFutureGet(hdfsOpenFileFuture *future); + + /** + * hdfsOpenFileFutureGetWithTimeout - Call Future#get(long, TimeUnit) on + * the underlying Java Future object. A call to #get(long, TimeUnit) will + * block until the asynchronous operation has completed (in this case, + * until the open file call has completed) or the specified timeout has + * been reached. + * + * @param future The hdfsOpenFileFuture to call #get on + * @return Returns the opened hdfsFile, or NULL on error or if the timeout + * has been reached. + */ + LIBHDFS_EXTERNAL + hdfsFile hdfsOpenFileFutureGetWithTimeout(hdfsOpenFileFuture *future, + int64_t timeout, javaConcurrentTimeUnit timeUnit); + + /** + * hdfsOpenFileFutureCancel - Call Future#cancel(boolean) on the + * underlying Java Future object. The value of mayInterruptedIfRunning + * controls whether the Java thread running the Future should be + * interrupted or not. + * + * @param future The hdfsOpenFileFuture to call #cancel on + * @param mayInterruptIfRunning if true, interrupts the running thread + * @return Returns 0 if the thread was successfully cancelled, else -1 + */ + LIBHDFS_EXTERNAL + int hdfsOpenFileFutureCancel(hdfsOpenFileFuture *future, + int mayInterruptIfRunning); + + /** + * hdfsOpenFileFutureFree - Free a HDFS open file future. + * + * @param hdfsOpenFileFuture The hdfsOpenFileFuture to free. + */ + LIBHDFS_EXTERNAL + void hdfsOpenFileFutureFree(hdfsOpenFileFuture *future); + /** * hdfsStreamBuilderAlloc - Allocate an HDFS stream builder. * diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jclasses.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jclasses.c index cf880e91b7596..9f589ac257aa1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jclasses.c +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jclasses.c @@ -98,6 +98,8 @@ jthrowable initCachedClasses(JNIEnv* env) { "org/apache/hadoop/hdfs/ReadStatistics"; cachedJavaClasses[JC_HDFS_DATA_INPUT_STREAM].className = "org/apache/hadoop/hdfs/client/HdfsDataInputStream"; + cachedJavaClasses[JC_FUTURE_DATA_IS_BUILDER].className = + "org/apache/hadoop/fs/FutureDataInputStreamBuilder"; cachedJavaClasses[JC_DOMAIN_SOCKET].className = "org/apache/hadoop/net/unix/DomainSocket"; cachedJavaClasses[JC_URI].className = @@ -108,6 +110,8 @@ jthrowable initCachedClasses(JNIEnv* env) { "java/util/EnumSet"; cachedJavaClasses[JC_EXCEPTION_UTILS].className = "org/apache/commons/lang3/exception/ExceptionUtils"; + cachedJavaClasses[JC_CFUTURE].className = + "java/util/concurrent/CompletableFuture"; // Create and set the jclass objects based on the class names set above jthrowable jthr; diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jclasses.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jclasses.h index 92cdd542e2371..0b174e1fecc56 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jclasses.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jclasses.h @@ -54,11 +54,13 @@ typedef enum { JC_FS_PERMISSION, JC_READ_STATISTICS, JC_HDFS_DATA_INPUT_STREAM, + JC_FUTURE_DATA_IS_BUILDER, JC_DOMAIN_SOCKET, JC_URI, JC_BYTE_BUFFER, JC_ENUM_SET, JC_EXCEPTION_UTILS, + JC_CFUTURE, // A special marker enum that counts the number of cached jclasses NUM_CACHED_CLASSES } CachedJavaClass; @@ -95,6 +97,8 @@ const char *getClassName(CachedJavaClass cachedJavaClass); #define HADOOP_FSPERM "org/apache/hadoop/fs/permission/FsPermission" #define HADOOP_RSTAT "org/apache/hadoop/hdfs/ReadStatistics" #define HADOOP_HDISTRM "org/apache/hadoop/hdfs/client/HdfsDataInputStream" +#define HADOOP_FDISB "org/apache/hadoop/fs/FutureDataInputStreamBuilder" +#define HADOOP_FS_BLDR "org/apache/hadoop/fs/FSBuilder" #define HADOOP_RO "org/apache/hadoop/fs/ReadOption" #define HADOOP_DS "org/apache/hadoop/net/unix/DomainSocket" @@ -104,6 +108,9 @@ const char *getClassName(CachedJavaClass cachedJavaClass); #define JAVA_BYTEBUFFER "java/nio/ByteBuffer" #define JAVA_STRING "java/lang/String" #define JAVA_ENUMSET "java/util/EnumSet" +#define JAVA_CFUTURE "java/util/concurrent/CompletableFuture" +#define JAVA_TIMEUNIT "java/util/concurrent/TimeUnit" +#define JAVA_OBJECT "java/lang/Object" /* Some frequently used third-party class names */ diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jni_helper.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jni_helper.c index bbbc8b4602b54..c834c74e4cd09 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jni_helper.c +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jni_helper.c @@ -23,6 +23,7 @@ #include "platform.h" #include "os/mutexes.h" #include "os/thread_local_storage.h" +#include "x-platform/types.h" #include #include diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/CMakeLists.txt index 41a9ee87fde3f..d7dfb0a74438d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/CMakeLists.txt @@ -20,7 +20,7 @@ # it by add -DLIBHDFSPP_DIR=... to your cmake invocation set(LIBHDFSPP_DIR CACHE STRING ${CMAKE_INSTALL_PREFIX}) -include_directories( ${LIBHDFSPP_DIR}/include ) +include_directories( ${LIBHDFSPP_DIR}/include ../../lib ) link_directories( ${LIBHDFSPP_DIR}/lib ) add_executable(cat_c cat.c) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/cat.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/cat.c index bee5382c9ec0d..d0de0dc5fea77 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/cat.c +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/cat.c @@ -28,6 +28,7 @@ #include "hdfspp/hdfs_ext.h" #include "uriparser2/uriparser2.h" #include "common/util_c.h" +#include "x-platform/types.h" #define SCHEME "hdfs" #define BUF_SIZE 1048576 //1 MB diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/cat/cat.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/cat/cat.cc index 9d400e7b00cdc..584af44924e26 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/cat/cat.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/cat/cat.cc @@ -62,7 +62,6 @@ int main(int argc, char *argv[]) { //wrapping file_raw into a unique pointer to guarantee deletion std::unique_ptr file(file_raw); - ssize_t total_bytes_read = 0; size_t last_bytes_read = 0; do{ @@ -71,7 +70,6 @@ int main(int argc, char *argv[]) { if(status.ok()) { //Writing file chunks to stdout fwrite(input_buffer, last_bytes_read, 1, stdout); - total_bytes_read += last_bytes_read; } else { if(status.is_invalid_offset()){ //Reached the end of the file diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc index 7c9e24c0d883a..7cfd6df3b9861 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc @@ -22,6 +22,7 @@ #include "connection/datanodeconnection.h" #include "reader/block_reader.h" #include "hdfspp/events.h" +#include "x-platform/types.h" #include #include diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h index 724b1a14bc21d..0e4eed7af4ec9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h @@ -28,6 +28,7 @@ #include "bad_datanode_tracker.h" #include "ClientNamenodeProtocol.pb.h" +#include "x-platform/types.h" #include #include diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc index 15ec620c9abd7..51907634bbac6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc @@ -24,6 +24,7 @@ #include #include "syscall.h" +#include "types.h" bool XPlatform::Syscall::WriteToStdout(const std::string& message) { return WriteToStdoutImpl(message.c_str()); diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/types.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/types.h new file mode 100644 index 0000000000000..6df5b96f39337 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/types.h @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_TYPES +#define NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_TYPES + +#if _WIN64 +// Windows 64-bit. +typedef long int ssize_t; +#elif _WIN32 +// Windows 32-bit. +typedef int ssize_t; +#else +// ssize_t is correctly defined by taking bit-ness into account on non-Windows +// systems. So we just include the header file where ssize_t is defined. +#include +#endif + +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt index d7d20ec36007a..1a88b5c81e521 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt @@ -136,7 +136,6 @@ target_link_libraries(hdfs_config_connect_bugs_test common gmock_main bindings_c add_memcheck_test(hdfs_config_connect_bugs hdfs_config_connect_bugs_test) - # # # INTEGRATION TESTS - TESTS THE FULL LIBRARY AGAINST ACTUAL SERVERS diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c index bda27b9a43202..2d265b8f03c0c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c @@ -250,6 +250,65 @@ hdfsFile hdfsOpenFile(hdfsFS fs, const char* path, int flags, return ret; } +hdfsOpenFileBuilder *hdfsOpenFileBuilderAlloc(hdfsFS fs, + const char *path) { + return libhdfs_hdfsOpenFileBuilderAlloc(fs->libhdfsRep, path); +} + +hdfsOpenFileBuilder *hdfsOpenFileBuilderMust( + hdfsOpenFileBuilder *builder, const char *key, + const char *value) { + return libhdfs_hdfsOpenFileBuilderMust(builder, key, value); +} + +hdfsOpenFileBuilder *hdfsOpenFileBuilderOpt( + hdfsOpenFileBuilder *builder, const char *key, + const char *value) { + return libhdfs_hdfsOpenFileBuilderOpt(builder, key, value); +} + +hdfsOpenFileFuture *hdfsOpenFileBuilderBuild( + hdfsOpenFileBuilder *builder) { + return libhdfs_hdfsOpenFileBuilderBuild(builder); +} + +void hdfsOpenFileBuilderFree(hdfsOpenFileBuilder *builder) { + libhdfs_hdfsOpenFileBuilderFree(builder); +} + +hdfsFile hdfsOpenFileFutureGet(hdfsOpenFileFuture *future) { + hdfsFile ret = calloc(1, sizeof(struct hdfsFile_internal)); + ret->libhdfsppRep = 0; + ret->libhdfsRep = libhdfs_hdfsOpenFileFutureGet(future); + if (!ret->libhdfsRep) { + free(ret); + ret = NULL; + } + return ret; +} + +hdfsFile hdfsOpenFileFutureGetWithTimeout(hdfsOpenFileFuture *future, + int64_t timeout, javaConcurrentTimeUnit timeUnit) { + hdfsFile ret = calloc(1, sizeof(struct hdfsFile_internal)); + ret->libhdfsppRep = 0; + ret->libhdfsRep = libhdfs_hdfsOpenFileFutureGetWithTimeout(future, timeout, + timeUnit); + if (!ret->libhdfsRep) { + free(ret); + ret = NULL; + } + return ret; +} + +int hdfsOpenFileFutureCancel(hdfsOpenFileFuture *future, + int mayInterruptIfRunning) { + return libhdfs_hdfsOpenFileFutureCancel(future, mayInterruptIfRunning); +} + +void hdfsOpenFileFutureFree(hdfsOpenFileFuture *future) { + libhdfs_hdfsOpenFileFutureFree(future); +} + int hdfsTruncateFile(hdfsFS fs, const char* path, tOffset newlength) { return libhdfs_hdfsTruncateFile(fs->libhdfsRep, path, newlength); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_defines.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_defines.h index 0d014341b4c57..165744142558a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_defines.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_defines.h @@ -39,6 +39,23 @@ #define hdfsConfStrFree libhdfs_hdfsConfStrFree #define hdfsDisconnect libhdfs_hdfsDisconnect #define hdfsOpenFile libhdfs_hdfsOpenFile +#define hdfsOpenFileBuilderAlloc libhdfs_hdfsOpenFileBuilderAlloc +#define hdfsOpenFileBuilderMust libhdfs_hdfsOpenFileBuilderMust +#define hdfsOpenFileBuilderOpt libhdfs_hdfsOpenFileBuilderOpt +#define hdfsOpenFileBuilderBuild libhdfs_hdfsOpenFileBuilderBuild +#define hdfsOpenFileBuilderFree libhdfs_hdfsOpenFileBuilderFree +#define hdfsOpenFileFutureGet libhdfs_hdfsOpenFileFutureGet +#define javaConcurrentTimeUnit libhdfs_javaConcurrentTimeUnit +#define jNanoseconds libhdfs_jNanoseconds +#define jMicroseconds libhdfs_jMicroseconds +#define jMilliseconds libhdfs_jMilliseconds +#define jSeconds libhdfsj_jSeconds +#define jMinutes libhdfs_jMinutes +#define jHours libhdfs_jHours +#define jDays libhdfs_jDays +#define hdfsOpenFileFutureGetWithTimeout libhdfs_hdfsOpenFileFutureGetWithTimeout +#define hdfsOpenFileFutureCancel libhdfs_hdfsOpenFileFutureCancel +#define hdfsOpenFileFutureFree libhdfs_hdfsOpenFileFutureFree #define hdfsTruncateFile libhdfs_hdfsTruncateFile #define hdfsUnbufferFile libhdfs_hdfsUnbufferFile #define hdfsCloseFile libhdfs_hdfsCloseFile diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_undefs.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_undefs.h index d46768c02ad39..d84b8ba287525 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_undefs.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_undefs.h @@ -39,6 +39,23 @@ #undef hdfsConfStrFree #undef hdfsDisconnect #undef hdfsOpenFile +#undef hdfsOpenFileBuilderAlloc +#undef hdfsOpenFileBuilderMust +#undef hdfsOpenFileBuilderOpt +#undef hdfsOpenFileBuilderBuild +#undef hdfsOpenFileBuilderFree +#undef hdfsOpenFileFutureGet +#undef javaConcurrentTimeUnit +#undef jNanoseconds +#undef jMicroseconds +#undef jMilliseconds +#undef jSeconds +#undef jMinutes +#undef jHours +#undef jDays +#undef hdfsOpenFileFutureGetWithTimeout +#undef hdfsOpenFileFutureCancel +#undef hdfsOpenFileFutureFree #undef hdfsTruncateFile #undef hdfsUnbufferFile #undef hdfsCloseFile diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfspp_wrapper_defines.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfspp_wrapper_defines.h index 4b08d0556c3aa..0a6d987409fec 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfspp_wrapper_defines.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfspp_wrapper_defines.h @@ -39,6 +39,23 @@ #define hdfsConfStrFree libhdfspp_hdfsConfStrFree #define hdfsDisconnect libhdfspp_hdfsDisconnect #define hdfsOpenFile libhdfspp_hdfsOpenFile +#define hdfsOpenFileBuilderAlloc libhdfspp_hdfsOpenFileBuilderAlloc +#define hdfsOpenFileBuilderMust libhdfspp_hdfsOpenFileBuilderMust +#define hdfsOpenFileBuilderOpt libhdfspp_hdfsOpenFileBuilderOpt +#define hdfsOpenFileBuilderBuild libhdfspp_hdfsOpenFileBuilderBuild +#define hdfsOpenFileBuilderFree libhdfspp_hdfsOpenFileBuilderFree +#define hdfsOpenFileFutureGet libhdfspp_hdfsOpenFileFutureGet +#define javaConcurrentTimeUnit libhdfspp_javaConcurrentTimeUnit +#define jNanoseconds libhdfspp_jNanoseconds +#define jMicroseconds libhdfspp_jMicroseconds +#define jMilliseconds libhdfspp_jMilliseconds +#define jSeconds libhdfspp_jSeconds +#define jMinutes libhdfspp_jMinutes +#define jHours libhdfspp_jHours +#define jDays libhdfspp_jDays +#define hdfsOpenFileFutureGetWithTimeout libhdfspp_hdfsOpenFileFutureGetWithTimeout +#define hdfsOpenFileFutureCancel libhdfspp_hdfsOpenFileFutureCancel +#define hdfsOpenFileFutureFree libhdfspp_hdfsOpenFileFutureFree #define hdfsTruncateFile libhdfspp_hdfsTruncateFile #define hdfsUnbufferFile libhdfspp_hdfsUnbufferFile #define hdfsCloseFile libhdfspp_hdfsCloseFile diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/CMakeLists.txt index b5c21c7915baf..7bbe63153b77c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/CMakeLists.txt @@ -23,27 +23,72 @@ add_executable(hdfs_tool_tests hdfs-delete-snapshot-mock.cc hdfs-create-snapshot-mock.cc hdfs-cat-mock.cc + hdfs-chown-mock.cc + hdfs-chmod-mock.cc + hdfs-chgrp-mock.cc hdfs-tool-test-fixtures.cc hdfs-tool-tests.cc hdfs-df-mock.cc + hdfs-du-mock.cc + hdfs-copy-to-local-mock.cc + hdfs-move-to-local-mock.cc + hdfs-count-mock.cc + hdfs-mkdir-mock.cc + hdfs-rm-mock.cc + hdfs-get-mock.cc + hdfs-find-mock.cc + hdfs-ls-mock.cc + hdfs-setrep-mock.cc + hdfs-stat-mock.cc + hdfs-tail-mock.cc main.cc) target_include_directories(hdfs_tool_tests PRIVATE ../tools ../../tools ../../tools/hdfs-df + ../../tools/hdfs-du ../../tools/hdfs-allow-snapshot ../../tools/hdfs-disallow-snapshot ../../tools/hdfs-delete-snapshot ../../tools/hdfs-create-snapshot ../../tools/hdfs-rename-snapshot + ../../tools/hdfs-chown + ../../tools/hdfs-chgrp + ../../tools/hdfs-chmod + ../../tools/hdfs-copy-to-local + ../../tools/hdfs-move-to-local + ../../tools/hdfs-count + ../../tools/hdfs-mkdir + ../../tools/hdfs-rm + ../../tools/hdfs-get + ../../tools/hdfs-find + ../../tools/hdfs-ls + ../../tools/hdfs-setrep + ../../tools/hdfs-stat + ../../tools/hdfs-tail ../../tools/hdfs-cat) target_link_libraries(hdfs_tool_tests PRIVATE gmock_main hdfs_df_lib + hdfs_du_lib hdfs_allowSnapshot_lib hdfs_disallowSnapshot_lib hdfs_deleteSnapshot_lib hdfs_createSnapshot_lib hdfs_renameSnapshot_lib + hdfs_chown_lib + hdfs_chgrp_lib + hdfs_chmod_lib + hdfs_copyToLocal_lib + hdfs_moveToLocal_lib + hdfs_count_lib + hdfs_mkdir_lib + hdfs_rm_lib + hdfs_get_lib + hdfs_find_lib + hdfs_ls_lib + hdfs_setrep_lib + hdfs_stat_lib + hdfs_tail_lib hdfs_cat_lib) add_test(hdfs_tool_tests hdfs_tool_tests) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chgrp-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chgrp-mock.cc new file mode 100644 index 0000000000000..5948a7cc942bd --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chgrp-mock.cc @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include +#include + +#include "hdfs-chgrp-mock.h" +#include "hdfs-chgrp.h" +#include "hdfs-tool-tests.h" + +namespace hdfs::tools::test { +ChgrpMock::~ChgrpMock() = default; + +void ChgrpMock::SetExpectations( + std::function()> test_case, + const std::vector &args) const { + // Get the pointer to the function that defines the test case + const auto test_case_func = + test_case.target (*)()>(); + ASSERT_NE(test_case_func, nullptr); + + // Set the expected method calls and their corresponding arguments for each + // test case + if (*test_case_func == &CallHelp) { + EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true)); + return; + } + + if (*test_case_func == &PassOwnerAndAPath) { + const auto arg1 = args[0]; + const auto arg2 = args[1]; + + EXPECT_CALL(*this, HandlePath(arg1, false, arg2)) + .Times(1) + .WillOnce(testing::Return(true)); + } + + if (*test_case_func == &PassRecursiveOwnerAndAPath) { + const auto arg1 = args[1]; + const auto arg2 = args[2]; + + EXPECT_CALL(*this, HandlePath(arg1, true, arg2)) + .Times(1) + .WillOnce(testing::Return(true)); + } +} +} // namespace hdfs::tools::test diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chgrp-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chgrp-mock.h new file mode 100644 index 0000000000000..a7fb95cc81723 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chgrp-mock.h @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_CHGRP_MOCK +#define LIBHDFSPP_TOOLS_HDFS_CHGRP_MOCK + +#include +#include +#include +#include + +#include + +#include "hdfs-chgrp.h" + +namespace hdfs::tools::test { +/** + * {@class ChgrpMock} is an {@class Chgrp} whereby it mocks the + * HandleHelp and HandlePath methods for testing their functionality. + */ +class ChgrpMock : public hdfs::tools::Chgrp { +public: + /** + * {@inheritdoc} + */ + ChgrpMock(const int argc, char **argv) : Chgrp(argc, argv) {} + + // Abiding to the Rule of 5 + ChgrpMock(const ChgrpMock &) = delete; + ChgrpMock(ChgrpMock &&) = delete; + ChgrpMock &operator=(const ChgrpMock &) = delete; + ChgrpMock &operator=(ChgrpMock &&) = delete; + ~ChgrpMock() override; + + /** + * Defines the methods and the corresponding arguments that are expected + * to be called on this instance of {@link HdfsTool} for the given test case. + * + * @param test_case An {@link std::function} object that points to the + * function defining the test case + * @param args The arguments that are passed to this test case + */ + void SetExpectations(std::function()> test_case, + const std::vector &args = {}) const; + + MOCK_METHOD(bool, HandleHelp, (), (const, override)); + + MOCK_METHOD(bool, HandlePath, + (const std::string &, bool, const std::string &), + (const, override)); +}; +} // namespace hdfs::tools::test + +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chmod-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chmod-mock.cc new file mode 100644 index 0000000000000..e0df32cc2a16a --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chmod-mock.cc @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include +#include + +#include "hdfs-chmod-mock.h" +#include "hdfs-chmod.h" +#include "hdfs-tool-tests.h" + +namespace hdfs::tools::test { +ChmodMock::~ChmodMock() = default; + +void ChmodMock::SetExpectations( + std::function()> test_case, + const std::vector &args) const { + // Get the pointer to the function that defines the test case + const auto test_case_func = + test_case.target (*)()>(); + ASSERT_NE(test_case_func, nullptr); + + // Set the expected method calls and their corresponding arguments for each + // test case + if (*test_case_func == &CallHelp) { + EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true)); + return; + } + + if (*test_case_func == &PassPermissionsAndAPath) { + const auto arg1 = args[0]; + const auto arg2 = args[1]; + + EXPECT_CALL(*this, HandlePath(arg1, false, arg2)) + .Times(1) + .WillOnce(testing::Return(true)); + } + + if (*test_case_func == &PassRecursivePermissionsAndAPath) { + const auto arg1 = args[1]; + const auto arg2 = args[2]; + + EXPECT_CALL(*this, HandlePath(arg1, true, arg2)) + .Times(1) + .WillOnce(testing::Return(true)); + } +} +} // namespace hdfs::tools::test diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chmod-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chmod-mock.h new file mode 100644 index 0000000000000..cd193b9048914 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chmod-mock.h @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_CHMOD_MOCK +#define LIBHDFSPP_TOOLS_HDFS_CHMOD_MOCK + +#include +#include +#include +#include + +#include + +#include "hdfs-chmod.h" + +namespace hdfs::tools::test { +/** + * {@class ChmodMock} is an {@class Chmod} whereby it mocks the + * HandleHelp and HandlePath methods for testing their functionality. + */ +class ChmodMock : public hdfs::tools::Chmod { +public: + /** + * {@inheritdoc} + */ + ChmodMock(const int argc, char **argv) : Chmod(argc, argv) {} + + // Abiding to the Rule of 5 + ChmodMock(const ChmodMock &) = delete; + ChmodMock(ChmodMock &&) = delete; + ChmodMock &operator=(const ChmodMock &) = delete; + ChmodMock &operator=(ChmodMock &&) = delete; + ~ChmodMock() override; + + /** + * Defines the methods and the corresponding arguments that are expected + * to be called on this instance of {@link HdfsTool} for the given test case. + * + * @param test_case An {@link std::function} object that points to the + * function defining the test case + * @param args The arguments that are passed to this test case + */ + void SetExpectations(std::function()> test_case, + const std::vector &args = {}) const; + + MOCK_METHOD(bool, HandleHelp, (), (const, override)); + + MOCK_METHOD(bool, HandlePath, + (const std::string &, bool, const std::string &), + (const, override)); +}; +} // namespace hdfs::tools::test + +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chown-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chown-mock.cc new file mode 100644 index 0000000000000..a126e9c6fe29a --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chown-mock.cc @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include +#include + +#include "hdfs-chown-mock.h" +#include "hdfs-chown.h" +#include "hdfs-tool-tests.h" + +namespace hdfs::tools::test { +ChownMock::~ChownMock() = default; + +void ChownMock::SetExpectations( + std::function()> test_case, + const std::vector &args) const { + // Get the pointer to the function that defines the test case + const auto test_case_func = + test_case.target (*)()>(); + ASSERT_NE(test_case_func, nullptr); + + // Set the expected method calls and their corresponding arguments for each + // test case + if (*test_case_func == &CallHelp) { + EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true)); + return; + } + + if (*test_case_func == &PassOwnerAndAPath) { + const auto arg1 = args[0]; + const auto arg2 = args[1]; + const Ownership ownership(arg1); + + EXPECT_CALL(*this, HandlePath(ownership, false, arg2)) + .Times(1) + .WillOnce(testing::Return(true)); + } + + if (*test_case_func == &PassRecursiveOwnerAndAPath) { + const auto arg1 = args[1]; + const auto arg2 = args[2]; + const Ownership ownership(arg1); + + EXPECT_CALL(*this, HandlePath(ownership, true, arg2)) + .Times(1) + .WillOnce(testing::Return(true)); + } +} +} // namespace hdfs::tools::test diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chown-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chown-mock.h new file mode 100644 index 0000000000000..5cde9416bb6d4 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chown-mock.h @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_CHOWN_MOCK +#define LIBHDFSPP_TOOLS_HDFS_CHOWN_MOCK + +#include +#include +#include +#include + +#include + +#include "hdfs-chown.h" + +namespace hdfs::tools::test { +/** + * {@class ChownMock} is an {@class Chown} whereby it mocks the + * HandleHelp and HandlePath methods for testing their functionality. + */ +class ChownMock : public hdfs::tools::Chown { +public: + /** + * {@inheritdoc} + */ + ChownMock(const int argc, char **argv) : Chown(argc, argv) {} + + // Abiding to the Rule of 5 + ChownMock(const ChownMock &) = delete; + ChownMock(ChownMock &&) = delete; + ChownMock &operator=(const ChownMock &) = delete; + ChownMock &operator=(ChownMock &&) = delete; + ~ChownMock() override; + + /** + * Defines the methods and the corresponding arguments that are expected + * to be called on this instance of {@link HdfsTool} for the given test case. + * + * @param test_case An {@link std::function} object that points to the + * function defining the test case + * @param args The arguments that are passed to this test case + */ + void SetExpectations(std::function()> test_case, + const std::vector &args = {}) const; + + MOCK_METHOD(bool, HandleHelp, (), (const, override)); + + MOCK_METHOD(bool, HandlePath, (const Ownership &, bool, const std::string &), + (const, override)); +}; +} // namespace hdfs::tools::test + +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-copy-to-local-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-copy-to-local-mock.cc new file mode 100644 index 0000000000000..f76a2c917795f --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-copy-to-local-mock.cc @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include +#include + +#include "hdfs-copy-to-local-mock.h" +#include "hdfs-tool-tests.h" + +namespace hdfs::tools::test { +CopyToLocalMock::~CopyToLocalMock() = default; + +void CopyToLocalMock::SetExpectations( + std::function()> test_case, + const std::vector &args) const { + // Get the pointer to the function that defines the test case + const auto test_case_func = + test_case.target (*)()>(); + ASSERT_NE(test_case_func, nullptr); + + // Set the expected method calls and their corresponding arguments for each + // test case + if (*test_case_func == &CallHelp) { + EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true)); + return; + } + + if (*test_case_func == &Pass2Paths) { + const auto arg1 = args[0]; + const auto arg2 = args[1]; + EXPECT_CALL(*this, HandlePath(arg1, arg2)) + .Times(1) + .WillOnce(testing::Return(true)); + } +} +} // namespace hdfs::tools::test diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-copy-to-local-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-copy-to-local-mock.h new file mode 100644 index 0000000000000..32973dd2f2e6f --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-copy-to-local-mock.h @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_COPY_TO_LOCAL_MOCK +#define LIBHDFSPP_TOOLS_HDFS_COPY_TO_LOCAL_MOCK + +#include +#include +#include +#include + +#include + +#include "hdfs-copy-to-local.h" + +namespace hdfs::tools::test { +/** + * {@class CopyToLocalMock} is an {@class CopyToLocal} whereby it mocks the + * HandleHelp and HandlePath methods for testing their functionality. + */ +class CopyToLocalMock : public hdfs::tools::CopyToLocal { +public: + /** + * {@inheritdoc} + */ + CopyToLocalMock(const int argc, char **argv) : CopyToLocal(argc, argv) {} + + // Abiding to the Rule of 5 + CopyToLocalMock(const CopyToLocalMock &) = delete; + CopyToLocalMock(CopyToLocalMock &&) = delete; + CopyToLocalMock &operator=(const CopyToLocalMock &) = delete; + CopyToLocalMock &operator=(CopyToLocalMock &&) = delete; + ~CopyToLocalMock() override; + + /** + * Defines the methods and the corresponding arguments that are expected + * to be called on this instance of {@link HdfsTool} for the given test case. + * + * @param test_case An {@link std::function} object that points to the + * function defining the test case + * @param args The arguments that are passed to this test case + */ + void + SetExpectations(std::function()> test_case, + const std::vector &args = {}) const; + + MOCK_METHOD(bool, HandleHelp, (), (const, override)); + + MOCK_METHOD(bool, HandlePath, (const std::string &, const std::string &), + (const, override)); +}; +} // namespace hdfs::tools::test + +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-count-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-count-mock.cc new file mode 100644 index 0000000000000..649a71036079a --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-count-mock.cc @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include +#include + +#include "hdfs-count-mock.h" +#include "hdfs-tool-tests.h" + +namespace hdfs::tools::test { +CountMock::~CountMock() = default; + +void CountMock::SetExpectations( + std::function()> test_case, + const std::vector &args) const { + // Get the pointer to the function that defines the test case + const auto test_case_func = + test_case.target (*)()>(); + ASSERT_NE(test_case_func, nullptr); + + // Set the expected method calls and their corresponding arguments for each + // test case + if (*test_case_func == &CallHelp) { + EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true)); + return; + } + + if (*test_case_func == &PassAPath) { + const auto arg1 = args[0]; + EXPECT_CALL(*this, HandlePath(false, arg1)) + .Times(1) + .WillOnce(testing::Return(true)); + } + + if (*test_case_func == &PassQOptAndPath) { + const auto arg1 = args[0]; + const auto arg2 = args[1]; + EXPECT_CALL(*this, HandlePath(true, arg2)) + .Times(1) + .WillOnce(testing::Return(true)); + } +} +} // namespace hdfs::tools::test diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-count-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-count-mock.h new file mode 100644 index 0000000000000..6f0e5c00cea65 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-count-mock.h @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_COUNT_MOCK +#define LIBHDFSPP_TOOLS_HDFS_COUNT_MOCK + +#include +#include +#include +#include + +#include + +#include "hdfs-count.h" + +namespace hdfs::tools::test { +/** + * {@class CountMock} is an {@class Count} whereby it mocks the + * HandleHelp and HandlePath methods for testing their functionality. + */ +class CountMock : public hdfs::tools::Count { +public: + /** + * {@inheritdoc} + */ + CountMock(const int argc, char **argv) : Count(argc, argv) {} + + // Abiding to the Rule of 5 + CountMock(const CountMock &) = delete; + CountMock(CountMock &&) = delete; + CountMock &operator=(const CountMock &) = delete; + CountMock &operator=(CountMock &&) = delete; + ~CountMock() override; + + /** + * Defines the methods and the corresponding arguments that are expected + * to be called on this instance of {@link HdfsTool} for the given test case. + * + * @param test_case An {@link std::function} object that points to the + * function defining the test case + * @param args The arguments that are passed to this test case + */ + void SetExpectations(std::function()> test_case, + const std::vector &args = {}) const; + + MOCK_METHOD(bool, HandleHelp, (), (const, override)); + + MOCK_METHOD(bool, HandlePath, (const bool, const std::string &), + (const, override)); +}; +} // namespace hdfs::tools::test + +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-create-snapshot-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-create-snapshot-mock.cc index 323963181fc37..6a4f96a83fd61 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-create-snapshot-mock.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-create-snapshot-mock.cc @@ -48,9 +48,11 @@ void CreateSnapshotMock::SetExpectations( } if (*test_case_func == &PassNOptAndAPath) { - const auto arg1 = args[1]; - const auto arg2 = std::optional{args[0]}; - EXPECT_CALL(*this, HandleSnapshot(arg1, arg2)) + const auto opt_n = args[0]; + const auto path = args[2]; + const auto opt_n_value = std::optional{args[1]}; + ASSERT_EQ(opt_n, "-n"); + EXPECT_CALL(*this, HandleSnapshot(path, opt_n_value)) .Times(1) .WillOnce(testing::Return(true)); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-du-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-du-mock.cc new file mode 100644 index 0000000000000..e0c2ebf74546e --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-du-mock.cc @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include +#include + +#include "hdfs-du-mock.h" +#include "hdfs-tool-tests.h" + +namespace hdfs::tools::test { +DuMock::~DuMock() = default; + +void DuMock::SetExpectations(std::function()> test_case, + const std::vector &args) const { + // Get the pointer to the function that defines the test case + const auto test_case_func = test_case.target (*)()>(); + ASSERT_NE(test_case_func, nullptr); + + // Set the expected method calls and their corresponding arguments for each + // test case + if (*test_case_func == &CallHelp) { + EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true)); + return; + } + + if (*test_case_func == &PassAPath) { + const auto arg1 = args[0]; + EXPECT_CALL(*this, HandlePath(arg1, false)) + .Times(1) + .WillOnce(testing::Return(true)); + } + + if (*test_case_func == &PassRecursivePath) { + const auto arg1 = args[0]; + const auto arg2 = args[1]; + ASSERT_EQ(arg1, "-R"); + EXPECT_CALL(*this, HandlePath(arg2, true)) + .Times(1) + .WillOnce(testing::Return(true)); + } + + if (*test_case_func == &PassRecursive) { + const auto arg1 = args[0]; + ASSERT_EQ(arg1, "-R"); + } +} +} // namespace hdfs::tools::test diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-du-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-du-mock.h new file mode 100644 index 0000000000000..de5caeb76b318 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-du-mock.h @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_DU_MOCK +#define LIBHDFSPP_TOOLS_HDFS_DU_MOCK + +#include +#include +#include +#include + +#include + +#include "hdfs-du.h" + +namespace hdfs::tools::test { +/** + * {@class DuMock} is an {@class Du} whereby it mocks the + * HandleHelp and HandlePath methods for testing their functionality. + */ +class DuMock : public hdfs::tools::Du { +public: + /** + * {@inheritdoc} + */ + DuMock(const int argc, char **argv) : Du(argc, argv) {} + + // Abiding to the Rule of 5 + DuMock(const DuMock &) = delete; + DuMock(DuMock &&) = delete; + DuMock &operator=(const DuMock &) = delete; + DuMock &operator=(DuMock &&) = delete; + ~DuMock() override; + + /** + * Defines the methods and the corresponding arguments that are expected + * to be called on this instance of {@link HdfsTool} for the given test case. + * + * @param test_case An {@link std::function} object that points to the + * function defining the test case + * @param args The arguments that are passed to this test case + */ + void SetExpectations(std::function()> test_case, + const std::vector &args = {}) const; + + MOCK_METHOD(bool, HandleHelp, (), (const, override)); + + MOCK_METHOD(bool, HandlePath, (const std::string &, const bool), + (const, override)); +}; +} // namespace hdfs::tools::test + +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-find-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-find-mock.cc new file mode 100644 index 0000000000000..9fd57ec270c07 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-find-mock.cc @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include +#include + +#include "hdfs-find-mock.h" +#include "hdfs-tool-tests.h" +#include "hdfspp/hdfspp.h" + +namespace hdfs::tools::test { +FindMock::~FindMock() = default; + +void FindMock::SetExpectations( + std::function()> test_case, + const std::vector &args) const { + // Get the pointer to the function that defines the test case + const auto test_case_func = + test_case.target (*)()>(); + ASSERT_NE(test_case_func, nullptr); + + // Set the expected method calls and their corresponding arguments for each + // test case + if (*test_case_func == &CallHelp) { + EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true)); + return; + } + + if (*test_case_func == &PassAPath) { + const auto arg1 = args[0]; + EXPECT_CALL(*this, HandlePath(arg1, "*", + hdfs::FileSystem::GetDefaultFindMaxDepth())) + .Times(1) + .WillOnce(testing::Return(true)); + } + + if (*test_case_func == &PassNOptAndAPath) { + const auto arg1 = args[0]; + const auto arg2 = args[1]; + const auto arg3 = args[2]; + ASSERT_EQ(arg1, "-n"); + EXPECT_CALL(*this, HandlePath(arg3, arg2, + hdfs::FileSystem::GetDefaultFindMaxDepth())) + .Times(1) + .WillOnce(testing::Return(true)); + } + + if (*test_case_func == &PassMOptPermissionsAndAPath) { + const auto arg1 = args[0]; + const auto arg2 = args[1]; + const auto arg3 = args[2]; + ASSERT_EQ(arg1, "-m"); + EXPECT_CALL(*this, + HandlePath(arg3, "*", static_cast(std::stoi(arg2)))) + .Times(1) + .WillOnce(testing::Return(true)); + } + + if (*test_case_func == &PassNStrMNumAndAPath) { + const auto arg1 = args[0]; + const auto arg2 = args[1]; + const auto arg3 = args[2]; + const auto arg4 = args[3]; + const auto arg5 = args[4]; + ASSERT_EQ(arg1, "-n"); + ASSERT_EQ(arg3, "-m"); + EXPECT_CALL(*this, + HandlePath(arg5, arg2, static_cast(std::stoi(arg4)))) + .Times(1) + .WillOnce(testing::Return(true)); + } +} +} // namespace hdfs::tools::test diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-find-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-find-mock.h new file mode 100644 index 0000000000000..7520ac7c0da03 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-find-mock.h @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_FIND_MOCK +#define LIBHDFSPP_TOOLS_HDFS_FIND_MOCK + +#include +#include +#include +#include + +#include + +#include "hdfs-find.h" + +namespace hdfs::tools::test { +/** + * {@class FindMock} is an {@class Find} whereby it mocks the + * HandleHelp and HandlePath methods for testing their functionality. + */ +class FindMock : public hdfs::tools::Find { +public: + /** + * {@inheritdoc} + */ + FindMock(const int argc, char **argv) : Find(argc, argv) {} + + // Abiding to the Rule of 5 + FindMock(const FindMock &) = delete; + FindMock(FindMock &&) = delete; + FindMock &operator=(const FindMock &) = delete; + FindMock &operator=(FindMock &&) = delete; + ~FindMock() override; + + /** + * Defines the methods and the corresponding arguments that are expected + * to be called on this instance of {@link HdfsTool} for the given test case. + * + * @param test_case An {@link std::function} object that points to the + * function defining the test case + * @param args The arguments that are passed to this test case + */ + void SetExpectations(std::function()> test_case, + const std::vector &args = {}) const; + + MOCK_METHOD(bool, HandleHelp, (), (const, override)); + + MOCK_METHOD(bool, HandlePath, + (const std::string &, const std::string &, uint32_t), + (const, override)); +}; +} // namespace hdfs::tools::test + +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-get-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-get-mock.cc new file mode 100644 index 0000000000000..713564e45b160 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-get-mock.cc @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include +#include + +#include "hdfs-get-mock.h" +#include "hdfs-tool-tests.h" + +namespace hdfs::tools::test { +GetMock::~GetMock() = default; + +void GetMock::SetExpectations( + std::function()> test_case, + const std::vector &args) const { + // Get the pointer to the function that defines the test case + const auto test_case_func = + test_case.target (*)()>(); + ASSERT_NE(test_case_func, nullptr); + + // Set the expected method calls and their corresponding arguments for each + // test case + if (*test_case_func == &CallHelp) { + EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true)); + return; + } + + if (*test_case_func == &Pass2Paths) { + const auto arg1 = args[0]; + const auto arg2 = args[1]; + EXPECT_CALL(*this, HandlePath(arg1, arg2)) + .Times(1) + .WillOnce(testing::Return(true)); + } +} +} // namespace hdfs::tools::test diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-get-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-get-mock.h new file mode 100644 index 0000000000000..535f7153f1f98 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-get-mock.h @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_GET_MOCK +#define LIBHDFSPP_TOOLS_HDFS_GET_MOCK + +#include +#include +#include +#include + +#include + +#include "hdfs-get.h" + +namespace hdfs::tools::test { +/** + * {@class GetMock} is an {@class Get} whereby it mocks the + * HandleHelp and HandlePath methods for testing their functionality. + */ +class GetMock : public hdfs::tools::Get { +public: + /** + * {@inheritdoc} + */ + GetMock(const int argc, char **argv) : Get(argc, argv) {} + + // Abiding to the Rule of 5 + GetMock(const GetMock &) = delete; + GetMock(GetMock &&) = delete; + GetMock &operator=(const GetMock &) = delete; + GetMock &operator=(GetMock &&) = delete; + ~GetMock() override; + + /** + * Defines the methods and the corresponding arguments that are expected + * to be called on this instance of {@link HdfsTool} for the given test case. + * + * @param test_case An {@link std::function} object that points to the + * function defining the test case + * @param args The arguments that are passed to this test case + */ + void SetExpectations(std::function()> test_case, + const std::vector &args = {}) const; + + MOCK_METHOD(bool, HandleHelp, (), (const, override)); + + MOCK_METHOD(bool, HandlePath, (const std::string &, const std::string &), + (const, override)); +}; +} // namespace hdfs::tools::test + +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-ls-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-ls-mock.cc new file mode 100644 index 0000000000000..6f1cbcf1d074d --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-ls-mock.cc @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include +#include + +#include "hdfs-ls-mock.h" +#include "hdfs-tool-tests.h" + +namespace hdfs::tools::test { +LsMock::~LsMock() = default; + +void LsMock::SetExpectations(std::function()> test_case, + const std::vector &args) const { + // Get the pointer to the function that defines the test case + const auto test_case_func = test_case.target (*)()>(); + ASSERT_NE(test_case_func, nullptr); + + // Set the expected method calls and their corresponding arguments for each + // test case + if (*test_case_func == &CallHelp) { + EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true)); + return; + } + + if (*test_case_func == &PassAPath) { + const auto arg1 = args[0]; + EXPECT_CALL(*this, HandlePath(arg1, false)) + .Times(1) + .WillOnce(testing::Return(true)); + } + + if (*test_case_func == &PassRecursivePath) { + const auto arg1 = args[0]; + const auto arg2 = args[1]; + ASSERT_EQ(arg1, "-R"); + EXPECT_CALL(*this, HandlePath(arg2, true)) + .Times(1) + .WillOnce(testing::Return(true)); + } + + if (*test_case_func == &PassRecursive) { + const auto arg1 = args[0]; + ASSERT_EQ(arg1, "-R"); + } +} +} // namespace hdfs::tools::test diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-ls-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-ls-mock.h new file mode 100644 index 0000000000000..2218549e95cb7 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-ls-mock.h @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_LS_MOCK +#define LIBHDFSPP_TOOLS_HDFS_LS_MOCK + +#include +#include +#include +#include + +#include + +#include "hdfs-ls.h" + +namespace hdfs::tools::test { +/** + * {@class LsMock} is an {@class Ls} whereby it mocks the + * HandleHelp and HandlePath methods for testing their functionality. + */ +class LsMock : public hdfs::tools::Ls { +public: + /** + * {@inheritdoc} + */ + LsMock(const int argc, char **argv) : Ls(argc, argv) {} + + // Abiding to the Rule of 5 + LsMock(const LsMock &) = delete; + LsMock(LsMock &&) = delete; + LsMock &operator=(const LsMock &) = delete; + LsMock &operator=(LsMock &&) = delete; + ~LsMock() override; + + /** + * Defines the methods and the corresponding arguments that are expected + * to be called on this instance of {@link HdfsTool} for the given test case. + * + * @param test_case An {@link std::function} object that points to the + * function defining the test case + * @param args The arguments that are passed to this test case + */ + void SetExpectations(std::function()> test_case, + const std::vector &args = {}) const; + + MOCK_METHOD(bool, HandleHelp, (), (const, override)); + + MOCK_METHOD(bool, HandlePath, (const std::string &, const bool), + (const, override)); +}; +} // namespace hdfs::tools::test + +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-mkdir-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-mkdir-mock.cc new file mode 100644 index 0000000000000..54ed0b0990ee1 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-mkdir-mock.cc @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include + +#include +#include + +#include "hdfs-mkdir-mock.h" +#include "hdfs-tool-tests.h" + +namespace hdfs::tools::test { +MkdirMock::~MkdirMock() = default; + +void MkdirMock::SetExpectations( + std::function()> test_case, + const std::vector &args) const { + // Get the pointer to the function that defines the test case + const auto test_case_func = + test_case.target (*)()>(); + ASSERT_NE(test_case_func, nullptr); + + // Set the expected method calls and their corresponding arguments for each + // test case + if (*test_case_func == &CallHelp) { + EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true)); + return; + } + + if (*test_case_func == &PassAPath) { + const auto arg1 = args[0]; + const std::optional permissions = std::nullopt; + EXPECT_CALL(*this, HandlePath(false, permissions, arg1)) + .Times(1) + .WillOnce(testing::Return(true)); + } + + if (*test_case_func == &PassPOptAndPath) { + const auto arg1 = args[1]; + const std::optional permissions = std::nullopt; + EXPECT_CALL(*this, HandlePath(true, permissions, arg1)) + .Times(1) + .WillOnce(testing::Return(true)); + } + + if (*test_case_func == &PassMOptPermissionsAndAPath) { + const auto arg1 = args[1]; + const auto arg2 = args[2]; + const auto permissions = std::optional(arg1); + EXPECT_CALL(*this, HandlePath(false, permissions, arg2)) + .Times(1) + .WillOnce(testing::Return(true)); + } + + if (*test_case_func == &PassMPOptsPermissionsAndAPath) { + const auto arg1 = args[1]; + const auto arg2 = args[3]; + const auto permissions = std::optional(arg1); + EXPECT_CALL(*this, HandlePath(true, permissions, arg2)) + .Times(1) + .WillOnce(testing::Return(true)); + } +} +} // namespace hdfs::tools::test diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-mkdir-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-mkdir-mock.h new file mode 100644 index 0000000000000..e112a14cd2049 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-mkdir-mock.h @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_MKDIR_MOCK +#define LIBHDFSPP_TOOLS_HDFS_MKDIR_MOCK + +#include +#include +#include +#include +#include + +#include + +#include "hdfs-mkdir.h" + +namespace hdfs::tools::test { +/** + * {@class MkdirMock} is an {@class Mkdir} whereby it mocks the + * HandleHelp and HandlePath methods for testing their functionality. + */ +class MkdirMock : public hdfs::tools::Mkdir { +public: + /** + * {@inheritdoc} + */ + MkdirMock(const int argc, char **argv) : Mkdir(argc, argv) {} + + // Abiding to the Rule of 5 + MkdirMock(const MkdirMock &) = delete; + MkdirMock(MkdirMock &&) = delete; + MkdirMock &operator=(const MkdirMock &) = delete; + MkdirMock &operator=(MkdirMock &&) = delete; + ~MkdirMock() override; + + /** + * Defines the methods and the corresponding arguments that are expected + * to be called on this instance of {@link HdfsTool} for the given test case. + * + * @param test_case An {@link std::function} object that points to the + * function defining the test case + * @param args The arguments that are passed to this test case + */ + void SetExpectations(std::function()> test_case, + const std::vector &args = {}) const; + + MOCK_METHOD(bool, HandleHelp, (), (const, override)); + + MOCK_METHOD(bool, HandlePath, + (bool, const std::optional &, const std::string &), + (const, override)); +}; +} // namespace hdfs::tools::test + +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-move-to-local-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-move-to-local-mock.cc new file mode 100644 index 0000000000000..33d1a7c99486f --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-move-to-local-mock.cc @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include +#include + +#include "hdfs-move-to-local-mock.h" +#include "hdfs-tool-tests.h" + +namespace hdfs::tools::test { +MoveToLocalMock::~MoveToLocalMock() = default; + +void MoveToLocalMock::SetExpectations( + std::function()> test_case, + const std::vector &args) const { + // Get the pointer to the function that defines the test case + const auto test_case_func = + test_case.target (*)()>(); + ASSERT_NE(test_case_func, nullptr); + + // Set the expected method calls and their corresponding arguments for each + // test case + if (*test_case_func == &CallHelp) { + EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true)); + return; + } + + if (*test_case_func == &Pass2Paths) { + const auto arg1 = args[0]; + const auto arg2 = args[1]; + EXPECT_CALL(*this, HandlePath(arg1, arg2)) + .Times(1) + .WillOnce(testing::Return(true)); + } +} +} // namespace hdfs::tools::test diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-move-to-local-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-move-to-local-mock.h new file mode 100644 index 0000000000000..a068b5a9e2302 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-move-to-local-mock.h @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_MOVE_TO_LOCAL_MOCK +#define LIBHDFSPP_TOOLS_HDFS_MOVE_TO_LOCAL_MOCK + +#include +#include +#include +#include + +#include + +#include "hdfs-move-to-local.h" + +namespace hdfs::tools::test { +/** + * {@class MoveToLocalMock} is an {@class MoveToLocal} whereby it mocks the + * HandleHelp and HandlePath methods for testing their functionality. + */ +class MoveToLocalMock : public hdfs::tools::MoveToLocal { +public: + /** + * {@inheritdoc} + */ + MoveToLocalMock(const int argc, char **argv) : MoveToLocal(argc, argv) {} + + // Abiding to the Rule of 5 + MoveToLocalMock(const MoveToLocalMock &) = delete; + MoveToLocalMock(MoveToLocalMock &&) = delete; + MoveToLocalMock &operator=(const MoveToLocalMock &) = delete; + MoveToLocalMock &operator=(MoveToLocalMock &&) = delete; + ~MoveToLocalMock() override; + + /** + * Defines the methods and the corresponding arguments that are expected + * to be called on this instance of {@link HdfsTool} for the given test case. + * + * @param test_case An {@link std::function} object that points to the + * function defining the test case + * @param args The arguments that are passed to this test case + */ + void + SetExpectations(std::function()> test_case, + const std::vector &args = {}) const; + + MOCK_METHOD(bool, HandleHelp, (), (const, override)); + + MOCK_METHOD(bool, HandlePath, (const std::string &, const std::string &), + (const, override)); +}; +} // namespace hdfs::tools::test + +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-rm-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-rm-mock.cc new file mode 100644 index 0000000000000..29cf95fcced6d --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-rm-mock.cc @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include +#include + +#include "hdfs-rm-mock.h" +#include "hdfs-tool-tests.h" + +namespace hdfs::tools::test { +RmMock::~RmMock() = default; + +void RmMock::SetExpectations(std::function()> test_case, + const std::vector &args) const { + // Get the pointer to the function that defines the test case + const auto test_case_func = test_case.target (*)()>(); + ASSERT_NE(test_case_func, nullptr); + + // Set the expected method calls and their corresponding arguments for each + // test case + if (*test_case_func == &CallHelp) { + EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true)); + return; + } + + if (*test_case_func == &PassAPath) { + const auto arg1 = args[0]; + EXPECT_CALL(*this, HandlePath(false, arg1)) + .Times(1) + .WillOnce(testing::Return(true)); + } + + if (*test_case_func == &PassRecursivePath) { + const auto arg1 = args[1]; + EXPECT_CALL(*this, HandlePath(true, arg1)) + .Times(1) + .WillOnce(testing::Return(true)); + } +} +} // namespace hdfs::tools::test diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-rm-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-rm-mock.h new file mode 100644 index 0000000000000..632716bf0a64b --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-rm-mock.h @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_RM_MOCK +#define LIBHDFSPP_TOOLS_HDFS_RM_MOCK + +#include +#include +#include +#include + +#include + +#include "hdfs-rm.h" + +namespace hdfs::tools::test { +/** + * {@class RmMock} is an {@class Rm} whereby it mocks the + * HandleHelp and HandlePath methods for testing their functionality. + */ +class RmMock : public hdfs::tools::Rm { +public: + /** + * {@inheritdoc} + */ + RmMock(const int argc, char **argv) : Rm(argc, argv) {} + + // Abiding to the Rule of 5 + RmMock(const RmMock &) = delete; + RmMock(RmMock &&) = delete; + RmMock &operator=(const RmMock &) = delete; + RmMock &operator=(RmMock &&) = delete; + ~RmMock() override; + + /** + * Defines the methods and the corresponding arguments that are expected + * to be called on this instance of {@link HdfsTool} for the given test case. + * + * @param test_case An {@link std::function} object that points to the + * function defining the test case + * @param args The arguments that are passed to this test case + */ + void SetExpectations(std::function()> test_case, + const std::vector &args = {}) const; + + MOCK_METHOD(bool, HandleHelp, (), (const, override)); + + MOCK_METHOD(bool, HandlePath, (const bool, const std::string &), + (const, override)); +}; +} // namespace hdfs::tools::test + +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-setrep-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-setrep-mock.cc new file mode 100644 index 0000000000000..d33f49b6aec67 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-setrep-mock.cc @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include +#include + +#include "hdfs-setrep-mock.h" +#include "hdfs-tool-tests.h" + +namespace hdfs::tools::test { +SetrepMock::~SetrepMock() = default; + +void SetrepMock::SetExpectations( + std::function()> test_case, + const std::vector &args) const { + // Get the pointer to the function that defines the test case + const auto test_case_func = + test_case.target (*)()>(); + ASSERT_NE(test_case_func, nullptr); + + // Set the expected method calls and their corresponding arguments for each + // test case + if (*test_case_func == &CallHelp) { + EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true)); + return; + } + + if (*test_case_func == &PassPermissionsAndAPath) { + const auto number = args[0]; + const auto path = args[1]; + EXPECT_CALL(*this, HandlePath(path, number)) + .Times(1) + .WillOnce(testing::Return(true)); + } +} +} // namespace hdfs::tools::test diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-setrep-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-setrep-mock.h new file mode 100644 index 0000000000000..db1e0960ae0a7 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-setrep-mock.h @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_SETREP_MOCK +#define LIBHDFSPP_TOOLS_HDFS_SETREP_MOCK + +#include +#include +#include +#include + +#include + +#include "hdfs-setrep.h" + +namespace hdfs::tools::test { +/** + * {@class SetrepMock} is an {@class Setrep} whereby it mocks the + * HandleHelp and HandlePath methods for testing their functionality. + */ +class SetrepMock : public hdfs::tools::Setrep { +public: + /** + * {@inheritdoc} + */ + SetrepMock(const int argc, char **argv) : Setrep(argc, argv) {} + + // Abiding to the Rule of 5 + SetrepMock(const SetrepMock &) = delete; + SetrepMock(SetrepMock &&) = delete; + SetrepMock &operator=(const SetrepMock &) = delete; + SetrepMock &operator=(SetrepMock &&) = delete; + ~SetrepMock() override; + + /** + * Defines the methods and the corresponding arguments that are expected + * to be called on this instance of {@link HdfsTool} for the given test case. + * + * @param test_case An {@link std::function} object that points to the + * function defining the test case + * @param args The arguments that are passed to this test case + */ + void SetExpectations(std::function()> test_case, + const std::vector &args = {}) const; + + MOCK_METHOD(bool, HandleHelp, (), (const, override)); + + MOCK_METHOD(bool, HandlePath, (const std::string &, const std::string &), + (const, override)); +}; +} // namespace hdfs::tools::test + +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-stat-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-stat-mock.cc new file mode 100644 index 0000000000000..efa773ce70fdc --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-stat-mock.cc @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include +#include + +#include "hdfs-stat-mock.h" +#include "hdfs-tool-tests.h" + +namespace hdfs::tools::test { +StatMock::~StatMock() = default; + +void StatMock::SetExpectations( + std::function()> test_case, + const std::vector &args) const { + // Get the pointer to the function that defines the test case + const auto test_case_func = + test_case.target (*)()>(); + ASSERT_NE(test_case_func, nullptr); + + // Set the expected method calls and their corresponding arguments for each + // test case + if (*test_case_func == &CallHelp) { + EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true)); + return; + } + + if (*test_case_func == &PassAPath) { + const auto path = args[0]; + EXPECT_CALL(*this, HandlePath(path)) + .Times(1) + .WillOnce(testing::Return(true)); + } +} +} // namespace hdfs::tools::test diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-stat-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-stat-mock.h new file mode 100644 index 0000000000000..01781f5f1751c --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-stat-mock.h @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_STAT_MOCK +#define LIBHDFSPP_TOOLS_HDFS_STAT_MOCK + +#include +#include +#include +#include + +#include + +#include "hdfs-stat.h" + +namespace hdfs::tools::test { +/** + * {@class StatMock} is an {@class Stat} whereby it mocks the + * HandleHelp and HandlePath methods for testing their functionality. + */ +class StatMock : public hdfs::tools::Stat { +public: + /** + * {@inheritdoc} + */ + StatMock(const int argc, char **argv) : Stat(argc, argv) {} + + // Abiding to the Rule of 5 + StatMock(const StatMock &) = delete; + StatMock(StatMock &&) = delete; + StatMock &operator=(const StatMock &) = delete; + StatMock &operator=(StatMock &&) = delete; + ~StatMock() override; + + /** + * Defines the methods and the corresponding arguments that are expected + * to be called on this instance of {@link HdfsTool} for the given test case. + * + * @param test_case An {@link std::function} object that points to the + * function defining the test case + * @param args The arguments that are passed to this test case + */ + void SetExpectations(std::function()> test_case, + const std::vector &args = {}) const; + + MOCK_METHOD(bool, HandleHelp, (), (const, override)); + + MOCK_METHOD(bool, HandlePath, (const std::string &), (const, override)); +}; +} // namespace hdfs::tools::test + +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tail-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tail-mock.cc new file mode 100644 index 0000000000000..59c9f3a23b371 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tail-mock.cc @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include +#include + +#include "hdfs-tail-mock.h" +#include "hdfs-tool-tests.h" + +namespace hdfs::tools::test { +TailMock::~TailMock() = default; + +void TailMock::SetExpectations( + std::function()> test_case, + const std::vector &args) const { + // Get the pointer to the function that defines the test case + const auto test_case_func = + test_case.target (*)()>(); + ASSERT_NE(test_case_func, nullptr); + + // Set the expected method calls and their corresponding arguments for each + // test case + if (*test_case_func == &CallHelp) { + EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true)); + return; + } + + if (*test_case_func == &PassAPath) { + const auto path = args[0]; + EXPECT_CALL(*this, HandlePath(path, false)) + .Times(1) + .WillOnce(testing::Return(true)); + } + + if (*test_case_func == &PassFOptAndAPath) { + const auto f_opt = args[0]; + const auto path = args[1]; + ASSERT_EQ(f_opt, "-f"); + EXPECT_CALL(*this, HandlePath(path, true)) + .Times(1) + .WillOnce(testing::Return(true)); + } +} +} // namespace hdfs::tools::test diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tail-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tail-mock.h new file mode 100644 index 0000000000000..e9fb9b7521bfa --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tail-mock.h @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_TAIL_MOCK +#define LIBHDFSPP_TOOLS_HDFS_TAIL_MOCK + +#include +#include +#include +#include + +#include + +#include "hdfs-tail.h" + +namespace hdfs::tools::test { +/** + * {@class TailMock} is an {@class Tail} whereby it mocks the + * HandleHelp and HandlePath methods for testing their functionality. + */ +class TailMock : public hdfs::tools::Tail { +public: + /** + * {@inheritdoc} + */ + TailMock(const int argc, char **argv) : Tail(argc, argv) {} + + // Abiding to the Rule of 5 + TailMock(const TailMock &) = delete; + TailMock(TailMock &&) = delete; + TailMock &operator=(const TailMock &) = delete; + TailMock &operator=(TailMock &&) = delete; + ~TailMock() override; + + /** + * Defines the methods and the corresponding arguments that are expected + * to be called on this instance of {@link HdfsTool} for the given test case. + * + * @param test_case An {@link std::function} object that points to the + * function defining the test case + * @param args The arguments that are passed to this test case + */ + void SetExpectations(std::function()> test_case, + const std::vector &args = {}) const; + + MOCK_METHOD(bool, HandleHelp, (), (const, override)); + + MOCK_METHOD(bool, HandlePath, (const std::string &, const bool), + (const, override)); +}; +} // namespace hdfs::tools::test + +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.cc index 33480a933df0b..2a16c3c0d97f0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.cc @@ -21,11 +21,26 @@ #include "hdfs-allow-snapshot-mock.h" #include "hdfs-cat-mock.h" +#include "hdfs-chgrp-mock.h" +#include "hdfs-chmod-mock.h" +#include "hdfs-chown-mock.h" +#include "hdfs-copy-to-local-mock.h" +#include "hdfs-count-mock.h" #include "hdfs-create-snapshot-mock.h" #include "hdfs-delete-snapshot-mock.h" #include "hdfs-df-mock.h" #include "hdfs-disallow-snapshot-mock.h" +#include "hdfs-du-mock.h" +#include "hdfs-find-mock.h" +#include "hdfs-get-mock.h" +#include "hdfs-ls-mock.h" +#include "hdfs-mkdir-mock.h" +#include "hdfs-move-to-local-mock.h" #include "hdfs-rename-snapshot-mock.h" +#include "hdfs-rm-mock.h" +#include "hdfs-setrep-mock.h" +#include "hdfs-stat-mock.h" +#include "hdfs-tail-mock.h" #include "hdfs-tool-test-fixtures.h" #include "hdfs-tool-tests.h" @@ -64,11 +79,102 @@ INSTANTIATE_TEST_SUITE_P(HdfsDf, HdfsToolBasicTest, testing::Values(PassAPath, CallHelp)); +INSTANTIATE_TEST_SUITE_P( + HdfsDu, HdfsToolBasicTest, + testing::Values(PassAPath, + CallHelp, + PassRecursivePath)); + +INSTANTIATE_TEST_SUITE_P( + HdfsLs, HdfsToolBasicTest, + testing::Values(PassAPath, + CallHelp, + PassRecursivePath)); + INSTANTIATE_TEST_SUITE_P( HdfsDeleteSnapshot, HdfsToolBasicTest, testing::Values(CallHelp, Pass2Paths)); +INSTANTIATE_TEST_SUITE_P( + HdfsChown, HdfsToolBasicTest, + testing::Values(CallHelp, + PassOwnerAndAPath, + PassRecursiveOwnerAndAPath)); + +INSTANTIATE_TEST_SUITE_P( + HdfsChmod, HdfsToolBasicTest, + testing::Values( + CallHelp, + PassPermissionsAndAPath, + PassRecursivePermissionsAndAPath)); + +INSTANTIATE_TEST_SUITE_P( + HdfsChgrp, HdfsToolBasicTest, + testing::Values(CallHelp, + PassOwnerAndAPath, + PassRecursiveOwnerAndAPath)); + +INSTANTIATE_TEST_SUITE_P( + HdfsCopyToLocal, HdfsToolBasicTest, + testing::Values(CallHelp, + Pass2Paths)); + +INSTANTIATE_TEST_SUITE_P( + HdfsGet, HdfsToolBasicTest, + testing::Values(CallHelp, + Pass2Paths)); + +INSTANTIATE_TEST_SUITE_P( + HdfsMoveToLocal, HdfsToolBasicTest, + testing::Values(CallHelp, + Pass2Paths)); + +INSTANTIATE_TEST_SUITE_P( + HdfsCount, HdfsToolBasicTest, + testing::Values(CallHelp, + PassAPath, + PassQOptAndPath)); + +INSTANTIATE_TEST_SUITE_P( + HdfsMkdir, HdfsToolBasicTest, + testing::Values( + CallHelp, + PassAPath, + PassPOptAndPath, + PassMOptPermissionsAndAPath, + PassMPOptsPermissionsAndAPath)); + +INSTANTIATE_TEST_SUITE_P( + HdfsRm, HdfsToolBasicTest, + testing::Values(CallHelp, + PassAPath, + PassRecursivePath)); + +INSTANTIATE_TEST_SUITE_P( + HdfsFind, HdfsToolBasicTest, + testing::Values(CallHelp, + PassAPath, + PassNStrMNumAndAPath, + PassMOptPermissionsAndAPath, + PassNOptAndAPath)); + +INSTANTIATE_TEST_SUITE_P( + HdfsSetrep, HdfsToolBasicTest, + testing::Values(CallHelp, + PassPermissionsAndAPath)); + +INSTANTIATE_TEST_SUITE_P( + HdfsStat, HdfsToolBasicTest, + testing::Values(CallHelp, + PassAPath)); + +INSTANTIATE_TEST_SUITE_P( + HdfsTail, HdfsToolBasicTest, + testing::Values(PassAPath, + CallHelp, + PassFOptAndAPath)); + // Negative tests INSTANTIATE_TEST_SUITE_P( HdfsAllowSnapshot, HdfsToolNegativeTestThrows, @@ -92,10 +198,171 @@ INSTANTIATE_TEST_SUITE_P( HdfsDf, HdfsToolNegativeTestThrows, testing::Values(Pass2Paths)); +INSTANTIATE_TEST_SUITE_P( + HdfsDu, HdfsToolNegativeTestThrows, + testing::Values(Pass2Paths, + Pass3Paths, + PassNOptAndAPath, + PassOwnerAndAPath, + PassPermissionsAndAPath)); + +INSTANTIATE_TEST_SUITE_P( + HdfsLs, HdfsToolNegativeTestThrows, + testing::Values(Pass2Paths, + Pass3Paths, + PassNOptAndAPath, + PassOwnerAndAPath, + PassPermissionsAndAPath)); + INSTANTIATE_TEST_SUITE_P( HdfsCat, HdfsToolNegativeTestThrows, testing::Values(Pass2Paths)); +INSTANTIATE_TEST_SUITE_P( + HdfsCopyToLocal, HdfsToolNegativeTestThrows, + testing::Values(Pass3Paths)); + +INSTANTIATE_TEST_SUITE_P( + HdfsGet, HdfsToolNegativeTestThrows, + testing::Values(Pass3Paths)); + +INSTANTIATE_TEST_SUITE_P( + HdfsMoveToLocal, HdfsToolNegativeTestThrows, + testing::Values(Pass3Paths)); + +INSTANTIATE_TEST_SUITE_P( + HdfsCount, HdfsToolNegativeTestThrows, + testing::Values(Pass2Paths, + Pass3Paths, + PassNOptAndAPath, + PassRecursive)); + +INSTANTIATE_TEST_SUITE_P( + HdfsMkdir, HdfsToolNegativeTestThrows, + testing::Values(Pass2Paths, + Pass3Paths, + PassNOptAndAPath, + PassRecursive, + PassMOpt)); + +INSTANTIATE_TEST_SUITE_P( + HdfsRm, HdfsToolNegativeTestThrows, + testing::Values(Pass2Paths, + Pass3Paths, + PassNOptAndAPath, + PassRecursiveOwnerAndAPath, + PassMOpt)); + +INSTANTIATE_TEST_SUITE_P( + HdfsFind, HdfsToolNegativeTestThrows, + testing::Values(Pass2Paths, + Pass3Paths, + PassRecursiveOwnerAndAPath, + PassRecursive, + PassRecursivePath, + PassMPOptsPermissionsAndAPath, + PassMOpt, + PassNOpt)); + +INSTANTIATE_TEST_SUITE_P( + HdfsChgrp, HdfsToolNegativeTestThrows, + testing::Values(PassNOptAndAPath)); + +INSTANTIATE_TEST_SUITE_P( + HdfsSetrep, HdfsToolNegativeTestThrows, + testing::Values( + Pass3Paths, + PassRecursiveOwnerAndAPath, + PassRecursive, + PassMPOptsPermissionsAndAPath, + PassMOpt, + PassNOpt)); + +INSTANTIATE_TEST_SUITE_P( + HdfsStat, HdfsToolNegativeTestThrows, + testing::Values(Pass2Paths, + Pass3Paths, + PassRecursiveOwnerAndAPath, + PassRecursive, + PassRecursivePath, + PassMPOptsPermissionsAndAPath, + PassMOpt, + PassNOpt)); + +INSTANTIATE_TEST_SUITE_P( + HdfsTail, HdfsToolNegativeTestThrows, + testing::Values(Pass2Paths, + Pass3Paths, + PassNOptAndAPath, + PassRecursiveOwnerAndAPath, + PassMOpt, + PassRecursive, + PassRecursivePath, + PassNOpt, + PassOwnerAndAPath, + PassMPOptsPermissionsAndAPath, + PassPermissionsAndAPath)); + +INSTANTIATE_TEST_SUITE_P( + HdfsRm, HdfsToolNegativeTestNoThrow, + testing::Values(PassRecursive)); + +INSTANTIATE_TEST_SUITE_P( + HdfsMkdir, HdfsToolNegativeTestNoThrow, + testing::Values(PassPOpt)); + +INSTANTIATE_TEST_SUITE_P( + HdfsCount, HdfsToolNegativeTestNoThrow, + testing::Values(PassQOpt)); + +INSTANTIATE_TEST_SUITE_P( + HdfsMoveToLocal, HdfsToolNegativeTestNoThrow, + testing::Values(PassAPath)); + +INSTANTIATE_TEST_SUITE_P( + HdfsCopyToLocal, HdfsToolNegativeTestNoThrow, + testing::Values(PassAPath)); + +INSTANTIATE_TEST_SUITE_P( + HdfsGet, HdfsToolNegativeTestNoThrow, + testing::Values(PassAPath)); + INSTANTIATE_TEST_SUITE_P( HdfsDeleteSnapshot, HdfsToolNegativeTestNoThrow, testing::Values(PassAPath)); + +INSTANTIATE_TEST_SUITE_P( + HdfsDu, HdfsToolNegativeTestNoThrow, + testing::Values(PassRecursive)); + +INSTANTIATE_TEST_SUITE_P( + HdfsLs, HdfsToolNegativeTestNoThrow, + testing::Values(PassRecursive)); + +INSTANTIATE_TEST_SUITE_P( + HdfsChown, HdfsToolNegativeTestNoThrow, + testing::Values(PassAPath)); + +INSTANTIATE_TEST_SUITE_P( + HdfsChown, HdfsToolNegativeTestThrows, + testing::Values(PassNOptAndAPath)); + +INSTANTIATE_TEST_SUITE_P( + HdfsChmod, HdfsToolNegativeTestNoThrow, + testing::Values(PassAPath)); + +INSTANTIATE_TEST_SUITE_P( + HdfsChmod, HdfsToolNegativeTestThrows, + testing::Values(PassNOptAndAPath)); + +INSTANTIATE_TEST_SUITE_P( + HdfsChgrp, HdfsToolNegativeTestNoThrow, + testing::Values(PassAPath)); + +INSTANTIATE_TEST_SUITE_P( + HdfsSetrep, HdfsToolNegativeTestNoThrow, + testing::Values(PassAPath)); + +INSTANTIATE_TEST_SUITE_P( + HdfsTail, HdfsToolNegativeTestNoThrow, + testing::Values(PassFOpt)); diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.h index 188a76c439478..4fef261d0e72f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.h @@ -44,6 +44,44 @@ template std::unique_ptr PassAPath() { return hdfs_tool; } +template std::unique_ptr PassRecursive() { + constexpr auto argc = 2; + static std::string exe("hdfs_tool_name"); + static std::string arg1("-R"); + + static char *argv[] = {exe.data(), arg1.data()}; + + auto hdfs_tool = std::make_unique(argc, argv); + hdfs_tool->SetExpectations(PassRecursive, {arg1}); + return hdfs_tool; +} + +template std::unique_ptr PassRecursivePath() { + constexpr auto argc = 3; + static std::string exe("hdfs_tool_name"); + static std::string arg1("-R"); + static std::string arg2("a/b/c"); + + static char *argv[] = {exe.data(), arg1.data(), arg2.data()}; + + auto hdfs_tool = std::make_unique(argc, argv); + hdfs_tool->SetExpectations(PassRecursivePath, {arg1, arg2}); + return hdfs_tool; +} + +template std::unique_ptr PassFOptAndAPath() { + constexpr auto argc = 3; + static std::string exe("hdfs_tool_name"); + static std::string arg1("-f"); + static std::string arg2("a/b/c"); + + static char *argv[] = {exe.data(), arg1.data(), arg2.data()}; + + auto hdfs_tool = std::make_unique(argc, argv); + hdfs_tool->SetExpectations(PassFOptAndAPath, {arg1, arg2}); + return hdfs_tool; +} + template std::unique_ptr CallHelp() { constexpr auto argc = 2; static std::string exe("hdfs_tool_name"); @@ -93,7 +131,198 @@ template std::unique_ptr PassNOptAndAPath() { static char *argv[] = {exe.data(), arg1.data(), arg2.data(), arg3.data()}; auto hdfs_tool = std::make_unique(argc, argv); - hdfs_tool->SetExpectations(PassNOptAndAPath, {arg2, arg3}); + hdfs_tool->SetExpectations(PassNOptAndAPath, {arg1, arg2, arg3}); + return hdfs_tool; +} + +template std::unique_ptr PassOwnerAndAPath() { + constexpr auto argc = 3; + static std::string exe("hdfs_tool_name"); + static std::string arg1("new_owner:new_group"); + static std::string arg2("g/h/i"); + + static char *argv[] = {exe.data(), arg1.data(), arg2.data()}; + + auto hdfs_tool = std::make_unique(argc, argv); + hdfs_tool->SetExpectations(PassOwnerAndAPath, {arg1, arg2}); + return hdfs_tool; +} + +template std::unique_ptr PassRecursiveOwnerAndAPath() { + constexpr auto argc = 4; + static std::string exe("hdfs_tool_name"); + static std::string arg1("-R"); + static std::string arg2("new_owner:new_group"); + static std::string arg3("g/h/i"); + + static char *argv[] = {exe.data(), arg1.data(), arg2.data(), arg3.data()}; + + auto hdfs_tool = std::make_unique(argc, argv); + hdfs_tool->SetExpectations(PassRecursiveOwnerAndAPath, {arg1, arg2, arg3}); + return hdfs_tool; +} + +template std::unique_ptr PassPermissionsAndAPath() { + constexpr auto argc = 3; + static std::string exe("hdfs_tool_name"); + static std::string arg1("757"); + static std::string arg2("g/h/i"); + + static char *argv[] = {exe.data(), arg1.data(), arg2.data()}; + + auto hdfs_tool = std::make_unique(argc, argv); + hdfs_tool->SetExpectations(PassPermissionsAndAPath, {arg1, arg2}); + return hdfs_tool; +} + +template std::unique_ptr PassRecursivePermissionsAndAPath() { + constexpr auto argc = 4; + static std::string exe("hdfs_tool_name"); + static std::string arg1("-R"); + static std::string arg2("757"); + static std::string arg3("g/h/i"); + + static char *argv[] = {exe.data(), arg1.data(), arg2.data(), arg3.data()}; + + auto hdfs_tool = std::make_unique(argc, argv); + hdfs_tool->SetExpectations(PassRecursivePermissionsAndAPath, + {arg1, arg2, arg3}); + return hdfs_tool; +} + +template std::unique_ptr PassQOpt() { + constexpr auto argc = 2; + static std::string exe("hdfs_tool_name"); + static std::string arg1("-q"); + + static char *argv[] = {exe.data(), arg1.data()}; + + auto hdfs_tool = std::make_unique(argc, argv); + hdfs_tool->SetExpectations(PassQOpt, {arg1}); + return hdfs_tool; +} + +template std::unique_ptr PassQOptAndPath() { + constexpr auto argc = 3; + static std::string exe("hdfs_tool_name"); + static std::string arg1("-q"); + static std::string arg2("a/b/c"); + + static char *argv[] = {exe.data(), arg1.data(), arg2.data()}; + + auto hdfs_tool = std::make_unique(argc, argv); + hdfs_tool->SetExpectations(PassQOptAndPath, {arg1, arg2}); + return hdfs_tool; +} + +template std::unique_ptr PassPOpt() { + constexpr auto argc = 2; + static std::string exe("hdfs_tool_name"); + static std::string arg1("-p"); + + static char *argv[] = {exe.data(), arg1.data()}; + + auto hdfs_tool = std::make_unique(argc, argv); + hdfs_tool->SetExpectations(PassPOpt, {arg1}); + return hdfs_tool; +} + +template std::unique_ptr PassMOpt() { + constexpr auto argc = 2; + static std::string exe("hdfs_tool_name"); + static std::string arg1("-m"); + + static char *argv[] = {exe.data(), arg1.data()}; + + auto hdfs_tool = std::make_unique(argc, argv); + hdfs_tool->SetExpectations(PassMOpt, {arg1}); + return hdfs_tool; +} + +template std::unique_ptr PassFOpt() { + constexpr auto argc = 2; + static std::string exe("hdfs_tool_name"); + static std::string arg1("-f"); + + static char *argv[] = {exe.data(), arg1.data()}; + + auto hdfs_tool = std::make_unique(argc, argv); + hdfs_tool->SetExpectations(PassFOpt, {arg1}); + return hdfs_tool; +} + +template std::unique_ptr PassPOptAndPath() { + constexpr auto argc = 3; + static std::string exe("hdfs_tool_name"); + static std::string arg1("-p"); + static std::string arg2("a/b/c"); + + static char *argv[] = {exe.data(), arg1.data(), arg2.data()}; + + auto hdfs_tool = std::make_unique(argc, argv); + hdfs_tool->SetExpectations(PassPOptAndPath, {arg1, arg2}); + return hdfs_tool; +} + +template std::unique_ptr PassMOptPermissionsAndAPath() { + constexpr auto argc = 4; + static std::string exe("hdfs_tool_name"); + static std::string arg1("-m"); + static std::string arg2("757"); + static std::string arg3("g/h/i"); + + static char *argv[] = {exe.data(), arg1.data(), arg2.data(), arg3.data()}; + + auto hdfs_tool = std::make_unique(argc, argv); + hdfs_tool->SetExpectations(PassMOptPermissionsAndAPath, + {arg1, arg2, arg3}); + return hdfs_tool; +} + +template std::unique_ptr PassMPOptsPermissionsAndAPath() { + constexpr auto argc = 5; + static std::string exe("hdfs_tool_name"); + static std::string arg1("-m"); + static std::string arg2("757"); + static std::string arg3("-p"); + static std::string arg4("g/h/i"); + + static char *argv[] = {exe.data(), arg1.data(), arg2.data(), arg3.data(), + arg4.data()}; + + auto hdfs_tool = std::make_unique(argc, argv); + hdfs_tool->SetExpectations(PassMPOptsPermissionsAndAPath, + {arg1, arg2, arg3, arg4}); + return hdfs_tool; +} + +template std::unique_ptr PassNStrMNumAndAPath() { + constexpr auto argc = 6; + static std::string exe("hdfs_tool_name"); + static std::string arg1("-n"); + static std::string arg2("some_str"); + static std::string arg3("-m"); + static std::string arg4("757"); + static std::string arg5("some/path"); + + static char *argv[] = {exe.data(), arg1.data(), arg2.data(), + arg3.data(), arg4.data(), arg5.data()}; + + auto hdfs_tool = std::make_unique(argc, argv); + hdfs_tool->SetExpectations(PassNStrMNumAndAPath, + {arg1, arg2, arg3, arg4, arg5}); + return hdfs_tool; +} + +template std::unique_ptr PassNOpt() { + constexpr auto argc = 2; + static std::string exe("hdfs_tool_name"); + static std::string arg1("-n"); + + static char *argv[] = {exe.data(), arg1.data()}; + + auto hdfs_tool = std::make_unique(argc, argv); + hdfs_tool->SetExpectations(PassNOpt, {arg1}); return hdfs_tool; } diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/CMakeLists.txt index 6a7d0bec37ed4..e481ebc31e862 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/CMakeLists.txt @@ -31,3 +31,8 @@ add_test(x_platform_utils_test x_platform_utils_test) target_include_directories(x_platform_syscall_test PRIVATE ${LIBHDFSPP_LIB_DIR}) target_link_libraries(x_platform_syscall_test gmock_main) add_test(x_platform_syscall_test x_platform_syscall_test) + +add_executable(x_platform_types_test types_test.cc) +target_include_directories(x_platform_types_test PRIVATE ${LIBHDFSPP_LIB_DIR}) +target_link_libraries(x_platform_types_test gtest_main) +add_test(x_platform_types_test x_platform_types_test) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/types_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/types_test.cc new file mode 100644 index 0000000000000..b234fa2f107eb --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/types_test.cc @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "types_test.h" +#include "x-platform/types.h" + +INSTANTIATE_TYPED_TEST_SUITE_P(SSizeTTest, XPlatformTypesTest, ssize_t); diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/types_test.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/types_test.h new file mode 100644 index 0000000000000..b24f657880615 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/types_test.h @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_CROSS_PLATFORM_TYPES_TEST +#define LIBHDFSPP_CROSS_PLATFORM_TYPES_TEST + +#include + +#include + +/** + * {@class XPlatformTypesTest} tests the types defined in the XPlatform library. + */ +template class XPlatformTypesTest : public testing::Test { +public: + XPlatformTypesTest() = default; + XPlatformTypesTest(const XPlatformTypesTest &) = delete; + XPlatformTypesTest(XPlatformTypesTest &&) = delete; + XPlatformTypesTest &operator=(const XPlatformTypesTest &) = delete; + XPlatformTypesTest &operator=(XPlatformTypesTest &&) = delete; + ~XPlatformTypesTest() override; +}; + +template XPlatformTypesTest::~XPlatformTypesTest() = default; + +TYPED_TEST_SUITE_P(XPlatformTypesTest); + +/** + * Tests whether ssize_t can hold -1. + */ +TYPED_TEST_P(XPlatformTypesTest, SSizeTMinusOne) { + constexpr TypeParam value = -1; + ASSERT_EQ(value, -1); +} + +/** + * Tests whether ssize_t can hold at least an int. + */ +TYPED_TEST_P(XPlatformTypesTest, SSizeTCanHoldInts) { + constexpr auto actual = std::numeric_limits::max(); + constexpr auto expected = std::numeric_limits::max(); + ASSERT_GE(actual, expected); +} + +// For 64-bit systems. +#if _WIN64 || __x86_64__ || __ppc64__ +/** + * Tests whether ssize_t can hold at least a long int. + */ +TYPED_TEST_P(XPlatformTypesTest, SSizeTCanHoldLongInts) { + constexpr auto actual = std::numeric_limits::max(); + constexpr auto expected = std::numeric_limits::max(); + ASSERT_GE(actual, expected); +} + +REGISTER_TYPED_TEST_SUITE_P(XPlatformTypesTest, SSizeTMinusOne, + SSizeTCanHoldInts, SSizeTCanHoldLongInts); +#else +REGISTER_TYPED_TEST_SUITE_P(XPlatformTypesTest, SSizeTMinusOne, + SSizeTCanHoldInts); +#endif + +#endif \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt index c4aed54eeabb7..52caf01915653 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt @@ -28,64 +28,30 @@ link_directories( ${LIBHDFSPP_DIR}/lib ) add_library(tools_common_obj OBJECT tools_common.cc) add_library(tools_common $) +add_subdirectory(internal) + add_library(hdfs_tool_obj OBJECT hdfs-tool.cc) target_include_directories(hdfs_tool_obj PRIVATE ../tools) add_subdirectory(hdfs-cat) - -add_executable(hdfs_chgrp hdfs_chgrp.cc) -target_link_libraries(hdfs_chgrp tools_common hdfspp_static) - -add_executable(hdfs_chown hdfs_chown.cc) -target_link_libraries(hdfs_chown tools_common hdfspp_static) - -add_executable(hdfs_chmod hdfs_chmod.cc) -target_link_libraries(hdfs_chmod tools_common hdfspp_static) - -add_executable(hdfs_find hdfs_find.cc) -target_link_libraries(hdfs_find tools_common hdfspp_static) - -add_executable(hdfs_mkdir hdfs_mkdir.cc) -target_link_libraries(hdfs_mkdir tools_common hdfspp_static) - -add_executable(hdfs_rm hdfs_rm.cc) -target_link_libraries(hdfs_rm tools_common hdfspp_static) - -add_executable(hdfs_ls hdfs_ls.cc) -target_link_libraries(hdfs_ls tools_common hdfspp_static) - -add_executable(hdfs_stat hdfs_stat.cc) -target_link_libraries(hdfs_stat tools_common hdfspp_static) - -add_executable(hdfs_count hdfs_count.cc) -target_link_libraries(hdfs_count tools_common hdfspp_static) - +add_subdirectory(hdfs-chgrp) +add_subdirectory(hdfs-chown) +add_subdirectory(hdfs-chmod) +add_subdirectory(hdfs-find) +add_subdirectory(hdfs-mkdir) +add_subdirectory(hdfs-rm) +add_subdirectory(hdfs-ls) +add_subdirectory(hdfs-stat) +add_subdirectory(hdfs-count) add_subdirectory(hdfs-df) - -add_executable(hdfs_du hdfs_du.cc) -target_link_libraries(hdfs_du tools_common hdfspp_static) - -add_executable(hdfs_get hdfs_get.cc) -target_link_libraries(hdfs_get tools_common hdfspp_static) - -add_executable(hdfs_copyToLocal hdfs_copyToLocal.cc) -target_link_libraries(hdfs_copyToLocal tools_common hdfspp_static) - -add_executable(hdfs_moveToLocal hdfs_moveToLocal.cc) -target_link_libraries(hdfs_moveToLocal tools_common hdfspp_static) - -add_executable(hdfs_setrep hdfs_setrep.cc) -target_link_libraries(hdfs_setrep tools_common hdfspp_static) - +add_subdirectory(hdfs-du) +add_subdirectory(hdfs-get) +add_subdirectory(hdfs-copy-to-local) +add_subdirectory(hdfs-move-to-local) +add_subdirectory(hdfs-setrep) add_subdirectory(hdfs-allow-snapshot) - add_subdirectory(hdfs-disallow-snapshot) - add_subdirectory(hdfs-create-snapshot) - add_subdirectory(hdfs-rename-snapshot) - add_subdirectory(hdfs-delete-snapshot) - -add_executable(hdfs_tail hdfs_tail.cc) -target_link_libraries(hdfs_tail tools_common hdfspp_static) +add_subdirectory(hdfs-tail) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chgrp/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chgrp/CMakeLists.txt new file mode 100644 index 0000000000000..101365900a404 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chgrp/CMakeLists.txt @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +add_library(hdfs_chgrp_lib STATIC $ $ hdfs-chgrp.cc) +target_include_directories(hdfs_chgrp_lib PRIVATE ../../tools hdfs-chgrp ${Boost_INCLUDE_DIRS}) +target_link_libraries(hdfs_chgrp_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static) + +add_executable(hdfs_chgrp main.cc) +target_include_directories(hdfs_chgrp PRIVATE ../../tools) +target_link_libraries(hdfs_chgrp PRIVATE hdfs_chgrp_lib) + +install(TARGETS hdfs_chgrp RUNTIME DESTINATION bin) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chgrp/hdfs-chgrp.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chgrp/hdfs-chgrp.cc new file mode 100644 index 0000000000000..cbb3b4514b13d --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chgrp/hdfs-chgrp.cc @@ -0,0 +1,220 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hdfs-chgrp.h" +#include "internal/hdfs-ownership.h" +#include "tools_common.h" + +namespace hdfs::tools { +Chgrp::Chgrp(const int argc, char **argv) : HdfsTool(argc, argv) {} + +bool Chgrp::Initialize() { + auto add_options = opt_desc_.add_options(); + add_options("help,h", "Change the group association of each FILE to GROUP."); + add_options("file", po::value(), + "The path to the file whose group needs to be modified"); + add_options("recursive,R", "Operate on files and directories recursively"); + add_options( + "group", po::value(), + "The group to which the file's group association needs to be changed to"); + + // An exception is thrown if these arguments are missing or if the arguments' + // count doesn't tally. + pos_opt_desc_.add("group", 1); + pos_opt_desc_.add("file", 1); + + po::store(po::command_line_parser(argc_, argv_) + .options(opt_desc_) + .positional(pos_opt_desc_) + .run(), + opt_val_); + po::notify(opt_val_); + return true; +} + +bool Chgrp::ValidateConstraints() const { + // Only "help" is allowed as single argument + if (argc_ == 2) { + return opt_val_.count("help"); + } + + // Rest of the cases must contain more than 2 arguments on the command line + return argc_ > 2; +} + +std::string Chgrp ::GetDescription() const { + std::stringstream desc; + desc << "Usage: hdfs_chgrp [OPTION] GROUP FILE" << std::endl + << std::endl + << "Change the group association of each FILE to GROUP." << std::endl + << "The user must be the owner of files. Additional information is in " + "the Permissions Guide:" + << std::endl + << "https://hadoop.apache.org/docs/r2.7.1/hadoop-project-dist/" + "hadoop-hdfs/HdfsPermissionsGuide.html" + << std::endl + << std::endl + << " -R operate on files and directories recursively" << std::endl + << " -h display this help and exit" << std::endl + << std::endl + << "Examples:" << std::endl + << "hdfs_chgrp -R new_group hdfs://localhost.localdomain:8020/dir/file" + << std::endl + << "hdfs_chgrp new_group /dir/file" << std::endl; + return desc.str(); +} + +bool Chgrp::Do() { + if (!Initialize()) { + std::cerr << "Unable to initialize HDFS chgrp tool" << std::endl; + return false; + } + + if (!ValidateConstraints()) { + std::cout << GetDescription(); + return false; + } + + if (opt_val_.count("help") > 0) { + return HandleHelp(); + } + + if (opt_val_.count("file") > 0 && opt_val_.count("group") > 0) { + const auto file = opt_val_["file"].as(); + const auto recursive = opt_val_.count("recursive") > 0; + const auto group = opt_val_["group"].as(); + return HandlePath(group, recursive, file); + } + + return true; +} + +bool Chgrp::HandleHelp() const { + std::cout << GetDescription(); + return true; +} + +bool Chgrp::HandlePath(const std::string &group, const bool recursive, + const std::string &file) const { + // Building a URI object from the given file + auto uri = hdfs::parse_path_or_exit(file); + + const auto fs = hdfs::doConnect(uri, true); + if (!fs) { + std::cerr << "Could not connect the file system. " << std::endl; + return false; + } + + // Wrap async FileSystem::SetOwner with promise to make it a blocking call + const auto promise = std::make_shared>(); + auto future(promise->get_future()); + auto handler = [promise](const hdfs::Status &s) { promise->set_value(s); }; + + if (!recursive) { + fs->SetOwner(uri.get_path(), "", group, handler); + } else { + /* + * Allocating shared state, which includes: username and groupname to be + * set, handler to be called, request counter, and a boolean to keep track + * if find is done + */ + const auto state = + std::make_shared("", group, handler, 0, false); + + /* + * Keep requesting more from Find until we process the entire listing. Call + * handler when Find is done and request counter is 0. Find guarantees that + * the handler will only be called once at a time so we do not need locking + * in handler_find. + */ + auto handler_find = [fs, + state](const hdfs::Status &status_find, + const std::vector &stat_infos, + const bool has_more_results) -> bool { + /* + * For each result returned by Find we call async SetOwner with the + * handler below. SetOwner DOES NOT guarantee that the handler will only + * be called once at a time, so we DO need locking in handler_set_owner. + */ + auto handler_set_owner = [state](const hdfs::Status &status_set_owner) { + std::lock_guard guard(state->lock); + + // Decrement the counter once since we are done with this async call + if (!status_set_owner.ok() && state->status.ok()) { + // We make sure we set state->status only on the first error. + state->status = status_set_owner; + } + + state->request_counter--; + if (state->request_counter == 0 && state->find_is_done) { + state->handler(state->status); // exit + } + }; + + if (!stat_infos.empty() && state->status.ok()) { + for (const auto &s : stat_infos) { + // Launch an asynchronous call to SetOwner for every returned result + state->request_counter++; + fs->SetOwner(s.full_path, state->user, state->group, + handler_set_owner); + } + } + + /* + * Lock this section because handler_set_owner might be accessing the same + * shared variables simultaneously. + */ + std::lock_guard guard(state->lock); + if (!status_find.ok() && state->status.ok()) { + // We make sure we set state->status only on the first error. + state->status = status_find; + } + + if (!has_more_results) { + state->find_is_done = true; + if (state->request_counter == 0) { + state->handler(state->status); // exit + } + return false; + } + return true; + }; + + // Asynchronous call to Find + fs->Find(uri.get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), + handler_find); + } + + // Block until promise is set + const auto status = future.get(); + if (!status.ok()) { + std::cerr << "Error: " << status.ToString() << std::endl; + return false; + } + return true; +} +} // namespace hdfs::tools diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chgrp/hdfs-chgrp.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chgrp/hdfs-chgrp.h new file mode 100644 index 0000000000000..bd1920ff07492 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chgrp/hdfs-chgrp.h @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_CHGRP +#define LIBHDFSPP_TOOLS_HDFS_CHGRP + +#include + +#include + +#include "hdfs-tool.h" + +namespace hdfs::tools { +/** + * {@class Chgrp} is an {@class HdfsTool} that changes the owner and/or group of + * each file to owner and/or group. + */ +class Chgrp : public HdfsTool { +public: + /** + * {@inheritdoc} + */ + Chgrp(int argc, char **argv); + + // Abiding to the Rule of 5 + Chgrp(const Chgrp &) = default; + Chgrp(Chgrp &&) = default; + Chgrp &operator=(const Chgrp &) = delete; + Chgrp &operator=(Chgrp &&) = delete; + ~Chgrp() override = default; + + /** + * {@inheritdoc} + */ + [[nodiscard]] std::string GetDescription() const override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Do() override; + +protected: + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Initialize() override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool ValidateConstraints() const override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool HandleHelp() const override; + + /** + * Handle the path to the file argument that's passed to this tool. + * + * @param group The name of the group to which to change to. + * @param recursive Whether this operation needs to be performed recursively + * on all the files in the given path's sub-directory. + * @param file The path to the file whose group needs to be changed. + * + * @return A boolean indicating the result of this operation. + */ + [[nodiscard]] virtual bool HandlePath(const std::string &group, + bool recursive, + const std::string &file) const; + +private: + /** + * A boost data-structure containing the description of positional arguments + * passed to the command-line. + */ + po::positional_options_description pos_opt_desc_; +}; +} // namespace hdfs::tools + +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chgrp/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chgrp/main.cc new file mode 100644 index 0000000000000..52f36748b5e46 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chgrp/main.cc @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include + +#include "hdfs-chgrp.h" + +int main(int argc, char *argv[]) { + const auto result = std::atexit([]() -> void { + // Clean up static data on exit and prevent valgrind memory leaks + google::protobuf::ShutdownProtobufLibrary(); + }); + if (result != 0) { + std::cerr << "Error: Unable to schedule clean-up tasks for HDFS chgrp " + "tool, exiting" + << std::endl; + std::exit(EXIT_FAILURE); + } + + hdfs::tools::Chgrp chgrp(argc, argv); + auto success = false; + + try { + success = chgrp.Do(); + } catch (const std::exception &e) { + std::cerr << "Error: " << e.what() << std::endl; + } + + if (!success) { + std::exit(EXIT_FAILURE); + } + return 0; +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chmod/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chmod/CMakeLists.txt new file mode 100644 index 0000000000000..a1e17f87ebac5 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chmod/CMakeLists.txt @@ -0,0 +1,28 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +add_library(hdfs_chmod_lib STATIC $ $ hdfs-chmod.cc) +target_include_directories(hdfs_chmod_lib PRIVATE ../../tools hdfs-chmod ${Boost_INCLUDE_DIRS}) +target_link_libraries(hdfs_chmod_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static) + +add_executable(hdfs_chmod main.cc) +target_include_directories(hdfs_chmod PRIVATE ../../tools) +target_link_libraries(hdfs_chmod PRIVATE hdfs_chmod_lib) + +install(TARGETS hdfs_chmod RUNTIME DESTINATION bin) + diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chmod/hdfs-chmod.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chmod/hdfs-chmod.cc new file mode 100644 index 0000000000000..4775860fb4483 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chmod/hdfs-chmod.cc @@ -0,0 +1,232 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hdfs-chmod.h" +#include "tools_common.h" + +namespace hdfs::tools { +Chmod::Chmod(const int argc, char **argv) : HdfsTool(argc, argv) {} + +bool Chmod::Initialize() { + auto add_options = opt_desc_.add_options(); + add_options("help,h", "Change the permissions of each FILE to MODE."); + add_options("file", po::value(), + "The path to the file whose permissions needs to be modified"); + add_options("recursive,R", "Operate on files and directories recursively"); + add_options("permissions", po::value(), + "Octal representation of the permission bits"); + + // An exception is thrown if these arguments are missing or if the arguments' + // count doesn't tally. + pos_opt_desc_.add("permissions", 1); + pos_opt_desc_.add("file", 1); + + po::store(po::command_line_parser(argc_, argv_) + .options(opt_desc_) + .positional(pos_opt_desc_) + .run(), + opt_val_); + po::notify(opt_val_); + return true; +} + +bool Chmod::ValidateConstraints() const { + // Only "help" is allowed as single argument + if (argc_ == 2) { + return opt_val_.count("help"); + } + + // Rest of the cases must contain more than 2 arguments on the command line + return argc_ > 2; +} + +std::string Chmod ::GetDescription() const { + std::stringstream desc; + desc << "Usage: hdfs_chmod [OPTION] FILE" + << std::endl + << std::endl + << "Change the permissions of each FILE to MODE." << std::endl + << "The user must be the owner of the file, or else a super-user." + << std::endl + << "Additional information is in the Permissions Guide:" << std::endl + << "https://hadoop.apache.org/docs/r2.7.1/hadoop-project-dist/" + "hadoop-hdfs/HdfsPermissionsGuide.html" + << std::endl + << std::endl + << " -R operate on files and directories recursively" << std::endl + << " -h display this help and exit" << std::endl + << std::endl + << "Examples:" << std::endl + << "hdfs_chmod -R 755 hdfs://localhost.localdomain:8020/dir/file" + << std::endl + << "hdfs_chmod 777 /dir/file" << std::endl; + return desc.str(); +} + +bool Chmod::Do() { + if (!Initialize()) { + std::cerr << "Unable to initialize HDFS chmod tool" << std::endl; + return false; + } + + if (!ValidateConstraints()) { + std::cout << GetDescription(); + return false; + } + + if (opt_val_.count("help") > 0) { + return HandleHelp(); + } + + if (opt_val_.count("file") > 0 && opt_val_.count("permissions") > 0) { + const auto file = opt_val_["file"].as(); + const auto recursive = opt_val_.count("recursive") > 0; + const auto permissions = opt_val_["permissions"].as(); + return HandlePath(permissions, recursive, file); + } + + return true; +} + +bool Chmod::HandleHelp() const { + std::cout << GetDescription(); + return true; +} + +bool Chmod::HandlePath(const std::string &permissions, const bool recursive, + const std::string &file) const { + // Building a URI object from the given uri_path + auto uri = hdfs::parse_path_or_exit(file); + + const auto fs = hdfs::doConnect(uri, true); + if (!fs) { + std::cerr << "Could not connect the file system. " << std::endl; + return false; + } + + /* + * Wrap async FileSystem::SetPermission with promise to make it a blocking + * call. + */ + const auto promise = std::make_shared>(); + auto future(promise->get_future()); + auto handler = [promise](const hdfs::Status &s) { promise->set_value(s); }; + + /* + * strtol is reading the value with base 8, NULL because we are reading in + * just one value. + */ + auto perm = static_cast(strtol(permissions.c_str(), nullptr, 8)); + if (!recursive) { + fs->SetPermission(uri.get_path(), perm, handler); + } else { + /* + * Allocating shared state, which includes - + * 1. Permissions to be set + * 2. Handler to be called + * 3. Request counter + * 4. A boolean to keep track if find is done + */ + auto state = std::make_shared(perm, handler, 0, false); + + /* + * Keep requesting more from Find until we process the entire listing. Call + * handler when Find is done and request counter is 0. Find guarantees that + * the handler will only be called once at a time so we do not need locking + * in handler_find. + */ + auto handler_find = [fs, + state](const hdfs::Status &status_find, + const std::vector &stat_infos, + const bool has_more_results) -> bool { + /* + * For each result returned by Find we call async SetPermission with the + * handler below. SetPermission DOES NOT guarantee that the handler will + * only be called once at a time, so we DO need locking in + * handler_set_permission. + */ + auto handler_set_permission = + [state](const hdfs::Status &status_set_permission) { + std::lock_guard guard(state->lock); + + // Decrement the counter once since we are done with this async call + if (!status_set_permission.ok() && state->status.ok()) { + // We make sure we set state->status only on the first error. + state->status = status_set_permission; + } + state->request_counter--; + if (state->request_counter == 0 && state->find_is_done) { + state->handler(state->status); // exit + } + }; + + if (!stat_infos.empty() && state->status.ok()) { + for (const auto &s : stat_infos) { + /* + * Launch an asynchronous call to SetPermission for every returned + * result + */ + state->request_counter++; + fs->SetPermission(s.full_path, state->permissions, + handler_set_permission); + } + } + + /* + * Lock this section because handler_set_permission might be accessing the + * same shared variables simultaneously + */ + std::lock_guard guard(state->lock); + if (!status_find.ok() && state->status.ok()) { + // We make sure we set state->status only on the first error. + state->status = status_find; + } + if (!has_more_results) { + state->find_is_done = true; + if (state->request_counter == 0) { + state->handler(state->status); // exit + } + return false; + } + return true; + }; + + // Asynchronous call to Find + fs->Find(uri.get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), + handler_find); + } + + // Block until promise is set + const auto status = future.get(); + if (!status.ok()) { + std::cerr << "Error: " << status.ToString() << std::endl; + return false; + } + return true; +} +} // namespace hdfs::tools diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chmod/hdfs-chmod.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chmod/hdfs-chmod.h new file mode 100644 index 0000000000000..4400625a3b947 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chmod/hdfs-chmod.h @@ -0,0 +1,133 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_CHMOD +#define LIBHDFSPP_TOOLS_HDFS_CHMOD + +#include +#include +#include +#include + +#include + +#include "hdfs-tool.h" +#include "hdfspp/status.h" + +namespace hdfs::tools { +struct PermissionState { + PermissionState(const uint16_t permissions, + std::function handler, + const uint64_t request_counter, const bool find_is_done) + : permissions(permissions), handler(std::move(handler)), + request_counter(request_counter), find_is_done(find_is_done) {} + + const uint16_t permissions; + const std::function handler; + + /** + * The request counter is incremented once every time SetOwner async call is + * made + */ + uint64_t request_counter; + + /** + * This boolean will be set when find returns the last result + */ + bool find_is_done{false}; + + /** + * Final status to be returned + */ + hdfs::Status status; + + /** + * Shared variables will need protection with a lock + */ + std::mutex lock; +}; + +/** + * {@class Chmod} is an {@class HdfsTool} that changes the permissions to a + * file or folder. + */ +class Chmod : public HdfsTool { +public: + /** + * {@inheritdoc} + */ + Chmod(int argc, char **argv); + + // Abiding to the Rule of 5 + Chmod(const Chmod &) = default; + Chmod(Chmod &&) = default; + Chmod &operator=(const Chmod &) = delete; + Chmod &operator=(Chmod &&) = delete; + ~Chmod() override = default; + + /** + * {@inheritdoc} + */ + [[nodiscard]] std::string GetDescription() const override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Do() override; + +protected: + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Initialize() override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool ValidateConstraints() const override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool HandleHelp() const override; + + /** + * Handle the file to the file argument that's passed to this tool. + * + * @param permissions An octal representation of the new permissions to be + * assigned. + * @param recursive Whether this operation needs to be performed recursively + * on all the files in the given path's sub-directory. + * @param file The path to the file whose ownership needs to be changed. + * + * @return A boolean indicating the result of this operation. + */ + [[nodiscard]] virtual bool HandlePath(const std::string &permissions, + bool recursive, + const std::string &file) const; + +private: + /** + * A boost data-structure containing the description of positional arguments + * passed to the command-line. + */ + po::positional_options_description pos_opt_desc_; +}; +} // namespace hdfs::tools + +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chmod/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chmod/main.cc new file mode 100644 index 0000000000000..f79be66ccc6aa --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chmod/main.cc @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include + +#include "hdfs-chmod.h" + +int main(int argc, char *argv[]) { + const auto result = std::atexit([]() -> void { + // Clean up static data on exit and prevent valgrind memory leaks + google::protobuf::ShutdownProtobufLibrary(); + }); + if (result != 0) { + std::cerr << "Error: Unable to schedule clean-up tasks for HDFS chmod " + "tool, exiting" + << std::endl; + std::exit(EXIT_FAILURE); + } + + hdfs::tools::Chmod chmod(argc, argv); + auto success = false; + + try { + success = chmod.Do(); + } catch (const std::exception &e) { + std::cerr << "Error: " << e.what() << std::endl; + } + + if (!success) { + std::exit(EXIT_FAILURE); + } + return 0; +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chown/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chown/CMakeLists.txt new file mode 100644 index 0000000000000..6773d8f88683a --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chown/CMakeLists.txt @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +add_library(hdfs_chown_lib STATIC $ $ hdfs-chown.cc) +target_include_directories(hdfs_chown_lib PRIVATE ../../tools ${Boost_INCLUDE_DIRS}) +target_link_libraries(hdfs_chown_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static) + +add_executable(hdfs_chown main.cc) +target_include_directories(hdfs_chown PRIVATE ../../tools) +target_link_libraries(hdfs_chown PRIVATE hdfs_chown_lib) + +install(TARGETS hdfs_chown RUNTIME DESTINATION bin) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chown/hdfs-chown.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chown/hdfs-chown.cc new file mode 100644 index 0000000000000..41e4ca91bf92e --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chown/hdfs-chown.cc @@ -0,0 +1,229 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hdfs-chown.h" +#include "tools_common.h" + +namespace hdfs::tools { +Chown::Chown(const int argc, char **argv) : HdfsTool(argc, argv) {} + +bool Chown::Initialize() { + auto add_options = opt_desc_.add_options(); + add_options( + "help,h", + "Change the owner and/or group of each FILE to OWNER and/or GROUP."); + add_options("file", po::value(), + "The path to the file whose ownership needs to be modified"); + add_options("recursive,R", "Operate on files and directories recursively"); + add_options( + "user-group", po::value(), + "The user:group to which the file's ownership needs to be changed to"); + + // An exception is thrown if these arguments are missing or if the arguments' + // count doesn't tally. + pos_opt_desc_.add("user-group", 1); + pos_opt_desc_.add("file", 1); + + po::store(po::command_line_parser(argc_, argv_) + .options(opt_desc_) + .positional(pos_opt_desc_) + .run(), + opt_val_); + po::notify(opt_val_); + return true; +} + +bool Chown::ValidateConstraints() const { + // Only "help" is allowed as single argument + if (argc_ == 2) { + return opt_val_.count("help"); + } + + // Rest of the cases must contain more than 2 arguments on the command line + return argc_ > 2; +} + +std::string Chown ::GetDescription() const { + std::stringstream desc; + desc << "Usage: hdfs_chown [OPTION] [OWNER][:[GROUP]] FILE" << std::endl + << std::endl + << "Change the owner and/or group of each FILE to OWNER and/or GROUP." + << std::endl + << "The user must be a super-user. Additional information is in the " + "Permissions Guide:" + << std::endl + << "https://hadoop.apache.org/docs/r2.7.1/hadoop-project-dist/" + "hadoop-hdfs/HdfsPermissionsGuide.html" + << std::endl + << std::endl + << " -R operate on files and directories recursively" << std::endl + << " -h display this help and exit" << std::endl + << std::endl + << "Owner is unchanged if missing. Group is unchanged if missing." + << std::endl + << "OWNER and GROUP may be numeric as well as symbolic." << std::endl + << std::endl + << "Examples:" << std::endl + << "hdfs_chown -R new_owner:new_group " + "hdfs://localhost.localdomain:8020/dir/file" + << std::endl + << "hdfs_chown new_owner /dir/file" << std::endl; + return desc.str(); +} + +bool Chown::Do() { + if (!Initialize()) { + std::cerr << "Unable to initialize HDFS chown tool" << std::endl; + return false; + } + + if (!ValidateConstraints()) { + std::cout << GetDescription(); + return false; + } + + if (opt_val_.count("help") > 0) { + return HandleHelp(); + } + + if (opt_val_.count("file") > 0 && opt_val_.count("user-group") > 0) { + const auto file = opt_val_["file"].as(); + const auto recursive = opt_val_.count("recursive") > 0; + const Ownership ownership(opt_val_["user-group"].as()); + return HandlePath(ownership, recursive, file); + } + + return true; +} + +bool Chown::HandleHelp() const { + std::cout << GetDescription(); + return true; +} + +bool Chown::HandlePath(const Ownership &ownership, const bool recursive, + const std::string &file) const { + // Building a URI object from the given uri_path + auto uri = hdfs::parse_path_or_exit(file); + + const auto fs = hdfs::doConnect(uri, true); + if (!fs) { + std::cerr << "Could not connect the file system. " << std::endl; + return false; + } + + // Wrap async FileSystem::SetOwner with promise to make it a blocking call + auto promise = std::make_shared>(); + auto future(promise->get_future()); + auto handler = [promise](const hdfs::Status &s) { promise->set_value(s); }; + + if (!recursive) { + fs->SetOwner(uri.get_path(), ownership.GetUser(), + ownership.GetGroup().value_or(""), handler); + } else { + /* + * Allocating shared state, which includes: username and groupname to be + * set, handler to be called, request counter, and a boolean to keep track + * if find is done + */ + auto state = std::make_shared(ownership.GetUser(), + ownership.GetGroup().value_or(""), + handler, 0, false); + + /* + * Keep requesting more from Find until we process the entire listing. Call + * handler when Find is done and request counter is 0. Find guarantees that + * the handler will only be called once at a time so we do not need locking + * in handler_find. + */ + auto handler_find = [fs, + state](const hdfs::Status &status_find, + const std::vector &stat_infos, + const bool has_more_results) -> bool { + /* + * For each result returned by Find we call async SetOwner with the + * handler below. SetOwner DOES NOT guarantee that the handler will only + * be called once at a time, so we DO need locking in handler_set_owner. + */ + auto handler_set_owner = [state](const hdfs::Status &status_set_owner) { + std::lock_guard guard(state->lock); + + // Decrement the counter once since we are done with this async call + if (!status_set_owner.ok() && state->status.ok()) { + // We make sure we set state->status only on the first error. + state->status = status_set_owner; + } + + state->request_counter--; + if (state->request_counter == 0 && state->find_is_done) { + state->handler(state->status); // exit + } + }; + + if (!stat_infos.empty() && state->status.ok()) { + for (const auto &s : stat_infos) { + // Launch an asynchronous call to SetOwner for every returned result + state->request_counter++; + fs->SetOwner(s.full_path, state->user, state->group, + handler_set_owner); + } + } + + /* + * Lock this section because handler_set_owner might be accessing the same + * shared variables simultaneously. + */ + std::lock_guard guard(state->lock); + if (!status_find.ok() && state->status.ok()) { + // We make sure we set state->status only on the first error. + state->status = status_find; + } + + if (!has_more_results) { + state->find_is_done = true; + if (state->request_counter == 0) { + state->handler(state->status); // exit + } + return false; + } + return true; + }; + + // Asynchronous call to Find + fs->Find(uri.get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), + handler_find); + } + + // Block until promise is set + const auto status = future.get(); + if (!status.ok()) { + std::cerr << "Error: " << status.ToString() << std::endl; + return false; + } + return true; +} +} // namespace hdfs::tools diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chown/hdfs-chown.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chown/hdfs-chown.h new file mode 100644 index 0000000000000..25774be25ec6e --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chown/hdfs-chown.h @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_CHOWN +#define LIBHDFSPP_TOOLS_HDFS_CHOWN + +#include + +#include + +#include "hdfs-tool.h" +#include "internal/hdfs-ownership.h" + +namespace hdfs::tools { +/** + * {@class Chown} is an {@class HdfsTool} that changes the owner and/or group of + * each file to owner and/or group. + */ +class Chown : public HdfsTool { +public: + /** + * {@inheritdoc} + */ + Chown(int argc, char **argv); + + // Abiding to the Rule of 5 + Chown(const Chown &) = default; + Chown(Chown &&) = default; + Chown &operator=(const Chown &) = delete; + Chown &operator=(Chown &&) = delete; + ~Chown() override = default; + + /** + * {@inheritdoc} + */ + [[nodiscard]] std::string GetDescription() const override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Do() override; + +protected: + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Initialize() override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool ValidateConstraints() const override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool HandleHelp() const override; + + /** + * Handle the file to the file argument that's passed to this tool. + * + * @param ownership The owner's user and group names. + * @param recursive Whether this operation needs to be performed recursively + * on all the files in the given path's sub-directory. + * @param file The path to the file whose ownership needs to be changed. + * + * @return A boolean indicating the result of this operation. + */ + [[nodiscard]] virtual bool HandlePath(const Ownership &ownership, + bool recursive, + const std::string &file) const; + +private: + /** + * A boost data-structure containing the description of positional arguments + * passed to the command-line. + */ + po::positional_options_description pos_opt_desc_; +}; +} // namespace hdfs::tools + +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chown/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chown/main.cc new file mode 100644 index 0000000000000..26492c7679ef2 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chown/main.cc @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include + +#include "hdfs-chown.h" + +int main(int argc, char *argv[]) { + const auto result = std::atexit([]() -> void { + // Clean up static data on exit and prevent valgrind memory leaks + google::protobuf::ShutdownProtobufLibrary(); + }); + if (result != 0) { + std::cerr << "Error: Unable to schedule clean-up tasks for HDFS chown " + "tool, exiting" + << std::endl; + std::exit(EXIT_FAILURE); + } + + hdfs::tools::Chown chown(argc, argv); + auto success = false; + + try { + success = chown.Do(); + } catch (const std::exception &e) { + std::cerr << "Error: " << e.what() << std::endl; + } + + if (!success) { + std::exit(EXIT_FAILURE); + } + return 0; +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/CMakeLists.txt new file mode 100644 index 0000000000000..3546f9b67a74c --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/CMakeLists.txt @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +add_library(hdfs_copyToLocal_lib STATIC $ hdfs-copy-to-local.cc) +target_include_directories(hdfs_copyToLocal_lib PRIVATE ../../tools hdfs-copyToLocal ${Boost_INCLUDE_DIRS}) +target_link_libraries(hdfs_copyToLocal_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static) + +add_executable(hdfs_copyToLocal main.cc) +target_include_directories(hdfs_copyToLocal PRIVATE ../../tools) +target_link_libraries(hdfs_copyToLocal PRIVATE hdfs_copyToLocal_lib) + +install(TARGETS hdfs_copyToLocal RUNTIME DESTINATION bin) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/hdfs-copy-to-local.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/hdfs-copy-to-local.cc new file mode 100644 index 0000000000000..7affa1fbdc92a --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/hdfs-copy-to-local.cc @@ -0,0 +1,138 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include "hdfs-copy-to-local.h" +#include "tools_common.h" + +namespace hdfs::tools { +CopyToLocal::CopyToLocal(const int argc, char **argv) : HdfsTool(argc, argv) {} + +bool CopyToLocal::Initialize() { + auto add_options = opt_desc_.add_options(); + add_options("help,h", "Copies the file from the given HDFS source path to " + "the destination path on the local machine"); + + add_options("source", po::value(), "The HDFS source file path"); + add_options("target", po::value(), + "The target local system file path"); + + // We allow only two arguments to be passed to this tool. An exception is + // thrown if more arguments are passed. + pos_opt_desc_.add("source", 1); + pos_opt_desc_.add("target", 1); + + po::store(po::command_line_parser(argc_, argv_) + .options(opt_desc_) + .positional(pos_opt_desc_) + .run(), + opt_val_); + po::notify(opt_val_); + return true; +} + +bool CopyToLocal::ValidateConstraints() const { + // Only "help" is allowed as single argument + if (argc_ == 2) { + return opt_val_.count("help") > 0; + } + + // Rest of the cases must contain exactly 2 arguments + return argc_ == 3; +} + +std::string CopyToLocal::GetDescription() const { + std::stringstream desc; + desc << "Usage: hdfs_" << GetToolName() << " [OPTION] SRC_FILE DST_FILE" + << std::endl + << std::endl + << "Copy SRC_FILE from hdfs to DST_FILE on the local file system." + << std::endl + << std::endl + << " -h display this help and exit" << std::endl + << std::endl + << "Examples:" << std::endl + << "hdfs_" << GetToolName() + << " hdfs://localhost.localdomain:8020/dir/file " + "/home/usr/myfile" + << std::endl + << "hdfs_" << GetToolName() << " /dir/file /home/usr/dir/file" + << std::endl; + return desc.str(); +} + +bool CopyToLocal::Do() { + if (!Initialize()) { + std::cerr << "Unable to initialize HDFS " << GetToolName() << " tool" + << std::endl; + return false; + } + + if (!ValidateConstraints()) { + std::cout << GetDescription(); + return false; + } + + if (opt_val_.count("help") > 0) { + return HandleHelp(); + } + + if (opt_val_.count("source") > 0 && opt_val_.count("target") > 0) { + const auto source = opt_val_["source"].as(); + const auto target = opt_val_["target"].as(); + return HandlePath(source, target); + } + + return false; +} + +bool CopyToLocal::HandleHelp() const { + std::cout << GetDescription(); + return true; +} + +bool CopyToLocal::HandlePath(const std::string &source, + const std::string &target) const { + // Building a URI object from the given path + auto uri = hdfs::parse_path_or_exit(source); + + const auto fs = hdfs::doConnect(uri, true); + if (!fs) { + std::cerr << "Could not connect the file system. " << std::endl; + return false; + } + + auto dst_file = std::fopen(target.c_str(), "wb"); + if (!dst_file) { + std::cerr << "Unable to open the destination file: " << target << std::endl; + return false; + } + + readFile(fs, uri.get_path(), 0, dst_file, false); + std::fclose(dst_file); + return true; +} + +std::string CopyToLocal::GetToolName() const { return "copyToLocal"; } +} // namespace hdfs::tools diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/hdfs-copy-to-local.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/hdfs-copy-to-local.h new file mode 100644 index 0000000000000..0137f1e614d1c --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/hdfs-copy-to-local.h @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_COPY_TO_LOCAL +#define LIBHDFSPP_TOOLS_HDFS_COPY_TO_LOCAL + +#include + +#include + +#include "hdfs-tool.h" + +namespace hdfs::tools { +/** + * {@class CopyToLocal} is an {@class HdfsTool} that copies the file from the + * given HDFS source path to the destination path on the local machine. + */ +class CopyToLocal : public HdfsTool { +public: + /** + * {@inheritdoc} + */ + CopyToLocal(int argc, char **argv); + + // Abiding to the Rule of 5 + CopyToLocal(const CopyToLocal &) = default; + CopyToLocal(CopyToLocal &&) = default; + CopyToLocal &operator=(const CopyToLocal &) = delete; + CopyToLocal &operator=(CopyToLocal &&) = delete; + ~CopyToLocal() override = default; + + /** + * {@inheritdoc} + */ + [[nodiscard]] std::string GetDescription() const override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Do() override; + +protected: + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Initialize() override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool ValidateConstraints() const override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool HandleHelp() const override; + + /** + * Handle the path argument that's passed to this tool. + * + * @param source The source file path in HDFS. + * @param target The target file path on the local machine. + * + * @return A boolean indicating the result of this operation. + */ + [[nodiscard]] virtual bool HandlePath(const std::string &source, + const std::string &target) const; + + /** + * @return The name of the tool. + */ + [[nodiscard]] virtual std::string GetToolName() const; + +private: + /** + * A boost data-structure containing the description of positional arguments + * passed to the command-line. + */ + po::positional_options_description pos_opt_desc_; +}; +} // namespace hdfs::tools +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/main.cc new file mode 100644 index 0000000000000..4dfc4d1c59165 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/main.cc @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include + +#include "hdfs-copy-to-local.h" + +int main(int argc, char *argv[]) { + const auto result = std::atexit([]() -> void { + // Clean up static data on exit and prevent valgrind memory leaks + google::protobuf::ShutdownProtobufLibrary(); + }); + if (result != 0) { + std::cerr + << "Error: Unable to schedule clean-up tasks for HDFS copy_to_local tool, exiting" + << std::endl; + std::exit(EXIT_FAILURE); + } + + hdfs::tools::CopyToLocal copy_to_local(argc, argv); + auto success = false; + + try { + success = copy_to_local.Do(); + } catch (const std::exception &e) { + std::cerr << "Error: " << e.what() << std::endl; + } + + if (!success) { + std::exit(EXIT_FAILURE); + } + return 0; +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/CMakeLists.txt new file mode 100644 index 0000000000000..fbf21fecbe26a --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/CMakeLists.txt @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +add_library(hdfs_count_lib STATIC $ hdfs-count.cc) +target_include_directories(hdfs_count_lib PRIVATE ../../tools hdfs-count ${Boost_INCLUDE_DIRS}) +target_link_libraries(hdfs_count_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static) + +add_executable(hdfs_count main.cc) +target_include_directories(hdfs_count PRIVATE ../../tools) +target_link_libraries(hdfs_count PRIVATE hdfs_count_lib) + +install(TARGETS hdfs_count RUNTIME DESTINATION bin) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/hdfs-count.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/hdfs-count.cc new file mode 100644 index 0000000000000..fca5c9713b6d3 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/hdfs-count.cc @@ -0,0 +1,125 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include + +#include "hdfs-count.h" +#include "tools_common.h" + +namespace hdfs::tools { +Count::Count(const int argc, char **argv) : HdfsTool(argc, argv) {} + +bool Count::Initialize() { + auto add_options = opt_desc_.add_options(); + add_options( + "help,h", + "Count the number of directories, files and bytes under the given path"); + add_options("show-quota,q", "Output additional columns before the rest: " + "QUOTA, SPACE_QUOTA, SPACE_CONSUMED"); + add_options("path", po::value(), + "The path to the file that needs to be count-ed"); + + // We allow only one argument to be passed to this tool. An exception is + // thrown if multiple arguments are passed. + pos_opt_desc_.add("path", 1); + + po::store(po::command_line_parser(argc_, argv_) + .options(opt_desc_) + .positional(pos_opt_desc_) + .run(), + opt_val_); + po::notify(opt_val_); + return true; +} + +std::string Count::GetDescription() const { + std::stringstream desc; + desc << "Usage: hdfs_count [OPTION] FILE" << std::endl + << std::endl + << "Count the number of directories, files and bytes under the path " + "that match the specified FILE pattern." + << std::endl + << "The output columns with -count are: DIR_COUNT, FILE_COUNT, " + "CONTENT_SIZE, PATHNAME" + << std::endl + << std::endl + << " -q output additional columns before the rest: QUOTA, " + "SPACE_QUOTA, SPACE_CONSUMED" + << std::endl + << " -h display this help and exit" << std::endl + << std::endl + << "Examples:" << std::endl + << "hdfs_count hdfs://localhost.localdomain:8020/dir" << std::endl + << "hdfs_count -q /dir1/dir2" << std::endl; + return desc.str(); +} + +bool Count::Do() { + if (!Initialize()) { + std::cerr << "Unable to initialize HDFS count tool" << std::endl; + return false; + } + + if (!ValidateConstraints()) { + std::cout << GetDescription(); + return false; + } + + if (opt_val_.count("help") > 0) { + return HandleHelp(); + } + + if (opt_val_.count("path") > 0) { + const auto path = opt_val_["path"].as(); + const auto show_quota = opt_val_.count("show-quota") > 0; + return HandlePath(show_quota, path); + } + + return false; +} + +bool Count::HandleHelp() const { + std::cout << GetDescription(); + return true; +} + +bool Count::HandlePath(const bool show_quota, const std::string &path) const { + // Building a URI object from the given uri_path + auto uri = hdfs::parse_path_or_exit(path); + + const auto fs = hdfs::doConnect(uri, false); + if (fs == nullptr) { + std::cerr << "Could not connect the file system." << std::endl; + return false; + } + + hdfs::ContentSummary content_summary; + const auto status = fs->GetContentSummary(uri.get_path(), content_summary); + if (!status.ok()) { + std::cerr << "Error: " << status.ToString() << std::endl; + return false; + } + + std::cout << content_summary.str(show_quota) << std::endl; + return true; +} +} // namespace hdfs::tools diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/hdfs-count.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/hdfs-count.h new file mode 100644 index 0000000000000..473c60967fead --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/hdfs-count.h @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_COUNT +#define LIBHDFSPP_TOOLS_HDFS_COUNT + +#include + +#include + +#include "hdfs-tool.h" + +namespace hdfs::tools { +/** + * {@class Count} is an {@class HdfsTool} that counts the number of directories, + * files and bytes under the given path. + */ +class Count : public HdfsTool { +public: + /** + * {@inheritdoc} + */ + Count(int argc, char **argv); + + // Abiding to the Rule of 5 + Count(const Count &) = default; + Count(Count &&) = default; + Count &operator=(const Count &) = delete; + Count &operator=(Count &&) = delete; + ~Count() override = default; + + /** + * {@inheritdoc} + */ + [[nodiscard]] std::string GetDescription() const override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Do() override; + +protected: + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Initialize() override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool ValidateConstraints() const override { return argc_ > 1; } + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool HandleHelp() const override; + + /** + * Handle the path argument that's passed to this tool. + * + * @param show_quota Output additional columns before the rest: QUOTA, + * SPACE_QUOTA, SPACE_CONSUMED. + * @param path The path to the directory for which the files, directories and + * bytes need to be counted. + * + * @return A boolean indicating the result of this operation. + */ + [[nodiscard]] virtual bool HandlePath(bool show_quota, + const std::string &path) const; + +private: + /** + * A boost data-structure containing the description of positional arguments + * passed to the command-line. + */ + po::positional_options_description pos_opt_desc_; +}; +} // namespace hdfs::tools +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/main.cc new file mode 100644 index 0000000000000..807ad3b2c39d0 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/main.cc @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include + +#include "hdfs-count.h" + +int main(int argc, char *argv[]) { + const auto result = std::atexit([]() -> void { + // Clean up static data on exit and prevent valgrind memory leaks + google::protobuf::ShutdownProtobufLibrary(); + }); + if (result != 0) { + std::cerr << "Error: Unable to schedule clean-up tasks for HDFS count " + "tool, exiting" + << std::endl; + std::exit(EXIT_FAILURE); + } + + hdfs::tools::Count count(argc, argv); + auto success = false; + + try { + success = count.Do(); + } catch (const std::exception &e) { + std::cerr << "Error: " << e.what() << std::endl; + } + + if (!success) { + std::exit(EXIT_FAILURE); + } + return 0; +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-du/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-du/CMakeLists.txt new file mode 100644 index 0000000000000..7164136fa10d3 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-du/CMakeLists.txt @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +add_library(hdfs_du_lib STATIC $ hdfs-du.cc) +target_include_directories(hdfs_du_lib PRIVATE ../../tools ${Boost_INCLUDE_DIRS}) +target_link_libraries(hdfs_du_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static) + +add_executable(hdfs_du main.cc) +target_include_directories(hdfs_du PRIVATE ../../tools) +target_link_libraries(hdfs_du PRIVATE hdfs_du_lib) + +install(TARGETS hdfs_du RUNTIME DESTINATION bin) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-du/hdfs-du.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-du/hdfs-du.cc new file mode 100644 index 0000000000000..5b5cab67d060b --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-du/hdfs-du.cc @@ -0,0 +1,205 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include "hdfs-du.h" +#include "internal/get-content-summary-state.h" +#include "tools_common.h" + +namespace hdfs::tools { +Du::Du(const int argc, char **argv) : HdfsTool(argc, argv) {} + +bool Du::Initialize() { + auto add_options = opt_desc_.add_options(); + add_options("help,h", + "Displays sizes of files and directories contained in the given " + "PATH or the length of a file in case PATH is just a file"); + add_options("recursive,R", "Operate on files and directories recursively"); + add_options("path", po::value(), + "The path indicating the filesystem that needs to be du-ed"); + + // We allow only one positional argument to be passed to this tool. An + // exception is thrown if multiple arguments are passed. + pos_opt_desc_.add("path", 1); + + po::store(po::command_line_parser(argc_, argv_) + .options(opt_desc_) + .positional(pos_opt_desc_) + .run(), + opt_val_); + po::notify(opt_val_); + return true; +} + +std::string Du::GetDescription() const { + std::stringstream desc; + desc << "Usage: hdfs_du [OPTION] PATH" << std::endl + << std::endl + << "Displays sizes of files and directories contained in the given PATH" + << std::endl + << "or the length of a file in case PATH is just a file" << std::endl + << std::endl + << " -R operate on files and directories recursively" + << std::endl + << " -h display this help and exit" << std::endl + << std::endl + << "Examples:" << std::endl + << "hdfs_du hdfs://localhost.localdomain:8020/dir/file" << std::endl + << "hdfs_du -R /dir1/dir2" << std::endl; + return desc.str(); +} + +bool Du::Do() { + if (!Initialize()) { + std::cerr << "Unable to initialize HDFS du tool" << std::endl; + return false; + } + + if (!ValidateConstraints()) { + std::cout << GetDescription(); + return false; + } + + if (opt_val_.count("help") > 0) { + return HandleHelp(); + } + + if (opt_val_.count("path") > 0) { + const auto path = opt_val_["path"].as(); + const auto recursive = opt_val_.count("recursive") > 0; + return HandlePath(path, recursive); + } + + return false; +} + +bool Du::HandleHelp() const { + std::cout << GetDescription(); + return true; +} + +bool Du::HandlePath(const std::string &path, const bool recursive) const { + // Building a URI object from the given path. + auto uri = hdfs::parse_path_or_exit(path); + + const auto fs = hdfs::doConnect(uri, true); + if (!fs) { + std::cerr << "Could not connect to the file system." << std::endl; + return false; + } + + /* + * Wrap async FileSystem::GetContentSummary with promise to make it a blocking + * call. + */ + const auto promise = std::make_shared>(); + std::future future(promise->get_future()); + auto handler = [promise](const hdfs::Status &s) { promise->set_value(s); }; + + /* + * Allocating shared state, which includes: handler to be called, request + * counter, and a boolean to keep track if find is done. + */ + const auto state = + std::make_shared(handler, 0, false); + + /* + * Keep requesting more from Find until we process the entire listing. Call + * handler when Find is done and request counter is 0. Find guarantees that + * the handler will only be called once at a time so we do not need locking in + * handler_find. + */ + auto handler_find = [fs, state](const hdfs::Status &status_find, + const std::vector &stat_infos, + const bool has_more_results) -> bool { + /* + * For each result returned by Find we call async GetContentSummary with the + * handler below. GetContentSummary DOES NOT guarantee that the handler will + * only be called once at a time, so we DO need locking in + * handler_get_content_summary. + */ + auto handler_get_content_summary = + [state](const hdfs::Status &status_get_summary, + const hdfs::ContentSummary &si) { + std::lock_guard guard(state->lock); + std::cout << si.str_du() << std::endl; + // Decrement the counter once since we are done with this async call. + if (!status_get_summary.ok() && state->status.ok()) { + // We make sure we set state->status only on the first error. + state->status = status_get_summary; + } + state->request_counter--; + if (state->request_counter == 0 && state->find_is_done) { + state->handler(state->status); // exit + } + }; + + if (!stat_infos.empty() && state->status.ok()) { + for (hdfs::StatInfo const &s : stat_infos) { + /* + * Launch an asynchronous call to GetContentSummary for every returned + * result. + */ + state->request_counter++; + fs->GetContentSummary(s.full_path, handler_get_content_summary); + } + } + + /* + * Lock this section because handler_get_content_summary might be accessing + * the same shared variables simultaneously. + */ + std::lock_guard guard(state->lock); + if (!status_find.ok() && state->status.ok()) { + // We make sure we set state->status only on the first error. + state->status = status_find; + } + + if (!has_more_results) { + state->find_is_done = true; + if (state->request_counter == 0) { + state->handler(state->status); // exit + } + return false; + } + return true; + }; + + // Asynchronous call to Find. + if (!recursive) { + fs->GetListing(uri.get_path(), handler_find); + } else { + fs->Find(uri.get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), + handler_find); + } + + // Block until promise is set. + const auto status = future.get(); + if (!status.ok()) { + std::cerr << "Error: " << status.ToString() << std::endl; + return false; + } + return true; +} +} // namespace hdfs::tools diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-du/hdfs-du.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-du/hdfs-du.h new file mode 100644 index 0000000000000..494270868eec6 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-du/hdfs-du.h @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_DU +#define LIBHDFSPP_TOOLS_HDFS_DU + +#include + +#include + +#include "hdfs-tool.h" + +namespace hdfs::tools { +/** + * {@class Du} is an {@class HdfsTool} that displays the size of the directories + * and files. + */ +class Du : public HdfsTool { +public: + /** + * {@inheritdoc} + */ + Du(int argc, char **argv); + + // Abiding to the Rule of 5 + Du(const Du &) = default; + Du(Du &&) = default; + Du &operator=(const Du &) = delete; + Du &operator=(Du &&) = delete; + ~Du() override = default; + + /** + * {@inheritdoc} + */ + [[nodiscard]] std::string GetDescription() const override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Do() override; + +protected: + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Initialize() override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool ValidateConstraints() const override { return argc_ > 1; } + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool HandleHelp() const override; + + /** + * Handle the path argument that's passed to this tool. + * + * @param path The path to the directory for which we need du info. + * @param recursive A boolean indicating whether du needs to be + * performed recursively for the given path. + * + * @return A boolean indicating the result of this operation. + */ + [[nodiscard]] virtual bool HandlePath(const std::string &path, + bool recursive) const; + +private: + /** + * A boost data-structure containing the description of positional arguments + * passed to the command-line. + */ + po::positional_options_description pos_opt_desc_; +}; +} // namespace hdfs::tools +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-du/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-du/main.cc new file mode 100644 index 0000000000000..0d8738fe3d1f7 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-du/main.cc @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include + +#include "hdfs-du.h" + +int main(int argc, char *argv[]) { + const auto result = std::atexit([]() -> void { + // Clean up static data on exit and prevent valgrind memory leaks + google::protobuf::ShutdownProtobufLibrary(); + }); + if (result != 0) { + std::cerr + << "Error: Unable to schedule clean-up tasks for HDFS df tool, exiting" + << std::endl; + std::exit(EXIT_FAILURE); + } + + hdfs::tools::Du du(argc, argv); + auto success = false; + + try { + success = du.Do(); + } catch (const std::exception &e) { + std::cerr << "Error: " << e.what() << std::endl; + } + + if (!success) { + std::exit(EXIT_FAILURE); + } + return 0; +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-find/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-find/CMakeLists.txt new file mode 100644 index 0000000000000..c6ce02132875a --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-find/CMakeLists.txt @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +add_library(hdfs_find_lib STATIC $ hdfs-find.cc) +target_include_directories(hdfs_find_lib PRIVATE ../../tools ${Boost_INCLUDE_DIRS}) +target_link_libraries(hdfs_find_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static) + +add_executable(hdfs_find main.cc) +target_include_directories(hdfs_find PRIVATE ../../tools) +target_link_libraries(hdfs_find PRIVATE hdfs_find_lib) + +install(TARGETS hdfs_find RUNTIME DESTINATION bin) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-find/hdfs-find.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-find/hdfs-find.cc new file mode 100644 index 0000000000000..ee1a04019f295 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-find/hdfs-find.cc @@ -0,0 +1,194 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "hdfs-find.h" +#include "tools_common.h" + +namespace hdfs::tools { +Find::Find(const int argc, char **argv) : HdfsTool(argc, argv) {} + +bool Find::Initialize() { + auto add_options = opt_desc_.add_options(); + add_options( + "help,h", + "Finds all files recursively starting from the specified PATH and prints " + "their file paths. This hdfs_find tool mimics the POSIX find."); + add_options( + "name,n", po::value(), + "If provided, all results will be matching the NAME pattern otherwise, " + "the implicit '*' will be used NAME allows wild-cards"); + add_options( + "max-depth,m", po::value(), + "If provided, the maximum depth to recurse after the end of the path is " + "reached will be limited by MAX_DEPTH otherwise, the maximum depth to " + "recurse is unbound MAX_DEPTH can be set to 0 for pure globbing and " + "ignoring the NAME option (no recursion after the end of the path)"); + add_options("path", po::value(), + "The path where we want to start the find operation"); + + // We allow only one positional argument to be passed to this tool. An + // exception is thrown if multiple arguments are passed. + pos_opt_desc_.add("path", 1); + + po::store(po::command_line_parser(argc_, argv_) + .options(opt_desc_) + .positional(pos_opt_desc_) + .run(), + opt_val_); + po::notify(opt_val_); + return true; +} + +std::string Find::GetDescription() const { + std::stringstream desc; + desc << "Usage: hdfs_find [OPTION] PATH" << std::endl + << std::endl + << "Finds all files recursively starting from the" << std::endl + << "specified PATH and prints their file paths." << std::endl + << "This hdfs_find tool mimics the POSIX find." << std::endl + << std::endl + << "Both PATH and NAME can have wild-cards." << std::endl + << std::endl + << " -n NAME if provided all results will be matching the NAME " + "pattern" + << std::endl + << " otherwise, the implicit '*' will be used" + << std::endl + << " NAME allows wild-cards" << std::endl + << std::endl + << " -m MAX_DEPTH if provided the maximum depth to recurse after the " + "end of" + << std::endl + << " the path is reached will be limited by MAX_DEPTH" + << std::endl + << " otherwise, the maximum depth to recurse is unbound" + << std::endl + << " MAX_DEPTH can be set to 0 for pure globbing and " + "ignoring" + << std::endl + << " the NAME option (no recursion after the end of the " + "path)" + << std::endl + << std::endl + << " -h display this help and exit" << std::endl + << std::endl + << "Examples:" << std::endl + << "hdfs_find hdfs://localhost.localdomain:8020/dir?/tree* -n " + "some?file*name" + << std::endl + << "hdfs_find / -n file_name -m 3" << std::endl; + return desc.str(); +} + +bool Find::Do() { + if (!Initialize()) { + std::cerr << "Unable to initialize HDFS find tool" << std::endl; + return false; + } + + if (!ValidateConstraints()) { + std::cout << GetDescription(); + return false; + } + + if (opt_val_.count("help") > 0) { + return HandleHelp(); + } + + if (opt_val_.count("path") > 0) { + const auto path = opt_val_["path"].as(); + const auto name = + opt_val_.count("name") > 0 ? opt_val_["name"].as() : "*"; + const auto max_depth = opt_val_.count("max-depth") <= 0 + ? hdfs::FileSystem::GetDefaultFindMaxDepth() + : opt_val_["max-depth"].as(); + return HandlePath(path, name, max_depth); + } + + return false; +} + +bool Find::HandleHelp() const { + std::cout << GetDescription(); + return true; +} + +bool Find::HandlePath(const std::string &path, const std::string &name, + const uint32_t max_depth) const { + // Building a URI object from the given path + auto uri = hdfs::parse_path_or_exit(path); + + const auto fs = hdfs::doConnect(uri, true); + if (!fs) { + std::cerr << "Could not connect the file system." << std::endl; + return false; + } + + const auto promise = std::make_shared>(); + std::future future(promise->get_future()); + auto final_status = hdfs::Status::OK(); + + /** + * Keep requesting more until we get the entire listing. Set the promise + * when we have the entire listing to stop. + * + * Find guarantees that the handler will only be called once at a time, + * so we do not need any locking here. It also guarantees that the handler + * will be only called once with has_more_results set to false. + */ + auto handler = [promise, + &final_status](const hdfs::Status &status, + const std::vector &stat_info, + const bool has_more_results) -> bool { + // Print result chunks as they arrive + if (!stat_info.empty()) { + for (hdfs::StatInfo const &info : stat_info) { + std::cout << info.str() << std::endl; + } + } + if (!status.ok() && final_status.ok()) { + // We make sure we set 'status' only on the first error + final_status = status; + } + if (!has_more_results) { + promise->set_value(); // Set promise + return false; // Request stop sending results + } + return true; // request more results + }; + + // Asynchronous call to Find + fs->Find(uri.get_path(), name, max_depth, handler); + + // Block until promise is set + future.get(); + if (!final_status.ok()) { + std::cerr << "Error: " << final_status.ToString() << std::endl; + return false; + } + return true; +} +} // namespace hdfs::tools diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-find/hdfs-find.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-find/hdfs-find.h new file mode 100644 index 0000000000000..9adde3c622bf1 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-find/hdfs-find.h @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_FIND +#define LIBHDFSPP_TOOLS_HDFS_FIND + +#include + +#include + +#include "hdfs-tool.h" + +namespace hdfs::tools { +/** + * {@class Find} is an {@class HdfsTool} finds all files recursively starting + * from the specified PATH and prints their file paths. This tool mimics the + * POSIX find. + */ +class Find : public HdfsTool { +public: + /** + * {@inheritdoc} + */ + Find(int argc, char **argv); + + // Abiding to the Rule of 5 + Find(const Find &) = default; + Find(Find &&) = default; + Find &operator=(const Find &) = delete; + Find &operator=(Find &&) = delete; + ~Find() override = default; + + /** + * {@inheritdoc} + */ + [[nodiscard]] std::string GetDescription() const override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Do() override; + +protected: + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Initialize() override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool ValidateConstraints() const override { return argc_ > 1; } + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool HandleHelp() const override; + + /** + * Handle the path argument that's passed to this tool. + * + * @param path The path to the directory to begin the find. + * @param name The pattern name of the search term. + * @param max_depth The maximum depth of the traversal while searching through + * the folders. + * + * @return A boolean indicating the result of this operation. + */ + [[nodiscard]] virtual bool HandlePath(const std::string &path, + const std::string &name, + uint32_t max_depth) const; + +private: + /** + * A boost data-structure containing the description of positional arguments + * passed to the command-line. + */ + po::positional_options_description pos_opt_desc_; +}; +} // namespace hdfs::tools +#endif diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLogAppender.java b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-find/main.cc similarity index 52% rename from hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLogAppender.java rename to hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-find/main.cc index eda1d1fee402e..1f63aa7a2d8e3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLogAppender.java +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-find/main.cc @@ -15,48 +15,38 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.http; -import org.apache.log4j.spi.LoggingEvent; -import org.apache.log4j.AppenderSkeleton; +#include +#include +#include -/** - * Log4j Appender adapter for HttpRequestLog - */ -public class HttpRequestLogAppender extends AppenderSkeleton { - - private String filename; - private int retainDays; +#include - public HttpRequestLogAppender() { - } - - public void setRetainDays(int retainDays) { - this.retainDays = retainDays; - } +#include "hdfs-find.h" - public int getRetainDays() { - return retainDays; +int main(int argc, char *argv[]) { + const auto result = std::atexit([]() -> void { + // Clean up static data on exit and prevent valgrind memory leaks + google::protobuf::ShutdownProtobufLibrary(); + }); + if (result != 0) { + std::cerr + << "Error: Unable to schedule clean-up tasks for HDFS find tool, exiting" + << std::endl; + std::exit(EXIT_FAILURE); } - public void setFilename(String filename) { - this.filename = filename; - } - - public String getFilename() { - return filename; - } - - @Override - public void append(LoggingEvent event) { - } + hdfs::tools::Find find(argc, argv); + auto success = false; - @Override - public void close() { + try { + success = find.Do(); + } catch (const std::exception &e) { + std::cerr << "Error: " << e.what() << std::endl; } - @Override - public boolean requiresLayout() { - return false; + if (!success) { + std::exit(EXIT_FAILURE); } + return 0; } diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/CMakeLists.txt new file mode 100644 index 0000000000000..367bca6a0279c --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/CMakeLists.txt @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +add_library(hdfs_get_lib STATIC $ hdfs-get.cc) +target_include_directories(hdfs_get_lib PRIVATE ../../tools hdfs-get ${Boost_INCLUDE_DIRS}) +target_link_libraries(hdfs_get_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static hdfs_copyToLocal_lib) + +add_executable(hdfs_get main.cc) +target_include_directories(hdfs_get PRIVATE ../../tools) +target_link_libraries(hdfs_get PRIVATE hdfs_get_lib) + +install(TARGETS hdfs_get RUNTIME DESTINATION bin) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/hdfs-get.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/hdfs-get.cc new file mode 100644 index 0000000000000..e45c1d0215b26 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/hdfs-get.cc @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "hdfs-get.h" + +namespace hdfs::tools { +Get::Get(const int argc, char **argv) : CopyToLocal(argc, argv) {} + +std::string Get::GetToolName() const { return "get"; } +} // namespace hdfs::tools diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/hdfs-get.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/hdfs-get.h new file mode 100644 index 0000000000000..8153264a72a3e --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/hdfs-get.h @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_GET +#define LIBHDFSPP_TOOLS_HDFS_GET + +#include "hdfs-copy-to-local/hdfs-copy-to-local.h" + +namespace hdfs::tools { +class Get : public CopyToLocal { +public: + /** + * {@inheritdoc} + */ + Get(int argc, char **argv); + + // Abiding to the Rule of 5 + Get(const Get &) = default; + Get(Get &&) = default; + Get &operator=(const Get &) = delete; + Get &operator=(Get &&) = delete; + ~Get() override = default; + +protected: + /** + * {@inheritdoc} + */ + [[nodiscard]] std::string GetToolName() const override; +}; +} // namespace hdfs::tools + +#endif \ No newline at end of file diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetastoreInstrumentationImpl.java b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/main.cc similarity index 51% rename from hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetastoreInstrumentationImpl.java rename to hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/main.cc index 7884d8e830f59..916e60a4631c6 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetastoreInstrumentationImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/main.cc @@ -1,4 +1,4 @@ -/* +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -16,57 +16,37 @@ * limitations under the License. */ -package org.apache.hadoop.fs.s3a.s3guard; - -/** - * A no-op implementation of {@link MetastoreInstrumentation} - * which allows metastores to always return an instance - * when requested. - */ -public class MetastoreInstrumentationImpl implements MetastoreInstrumentation { - - @Override - public void initialized() { - - } - - @Override - public void storeClosed() { - - } - - @Override - public void throttled() { +#include +#include +#include - } - - @Override - public void retrying() { - - } +#include - @Override - public void recordsDeleted(final int count) { +#include "hdfs-get.h" +int main(int argc, char *argv[]) { + const auto result = std::atexit([]() -> void { + // Clean up static data on exit and prevent valgrind memory leaks + google::protobuf::ShutdownProtobufLibrary(); + }); + if (result != 0) { + std::cerr << "Error: Unable to schedule clean-up tasks for HDFS " + "get tool, exiting" + << std::endl; + std::exit(EXIT_FAILURE); } - @Override - public void recordsRead(final int count) { + hdfs::tools::Get get(argc, argv); + auto success = false; + try { + success = get.Do(); + } catch (const std::exception &e) { + std::cerr << "Error: " << e.what() << std::endl; } - @Override - public void recordsWritten(final int count) { - - } - - @Override - public void directoryMarkedAuthoritative() { - - } - - @Override - public void entryAdded(final long durationNanos) { - + if (!success) { + std::exit(EXIT_FAILURE); } + return 0; } diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-ls/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-ls/CMakeLists.txt new file mode 100644 index 0000000000000..196d6356e2ee2 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-ls/CMakeLists.txt @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +add_library(hdfs_ls_lib STATIC $ hdfs-ls.cc) +target_include_directories(hdfs_ls_lib PRIVATE ../../tools ${Boost_INCLUDE_DIRS}) +target_link_libraries(hdfs_ls_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static) + +add_executable(hdfs_ls main.cc) +target_include_directories(hdfs_ls PRIVATE ../../tools) +target_link_libraries(hdfs_ls PRIVATE hdfs_ls_lib) + +install(TARGETS hdfs_ls RUNTIME DESTINATION bin) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-ls/hdfs-ls.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-ls/hdfs-ls.cc new file mode 100644 index 0000000000000..8367bbc71feee --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-ls/hdfs-ls.cc @@ -0,0 +1,156 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include "hdfs-ls.h" +#include "tools_common.h" + +namespace hdfs::tools { +Ls::Ls(const int argc, char **argv) : HdfsTool(argc, argv) {} + +bool Ls::Initialize() { + auto add_options = opt_desc_.add_options(); + add_options("help,h", "List information about the files"); + add_options("recursive,R", "Operate on files and directories recursively"); + add_options("path", po::value(), + "The path for which we need to do ls"); + + // We allow only one positional argument to be passed to this tool. An + // exception is thrown if multiple arguments are passed. + pos_opt_desc_.add("path", 1); + + po::store(po::command_line_parser(argc_, argv_) + .options(opt_desc_) + .positional(pos_opt_desc_) + .run(), + opt_val_); + po::notify(opt_val_); + return true; +} + +std::string Ls::GetDescription() const { + std::stringstream desc; + desc << "Usage: hdfs_ls [OPTION] FILE" << std::endl + << std::endl + << "List information about the FILEs." << std::endl + << std::endl + << " -R list subdirectories recursively" << std::endl + << " -h display this help and exit" << std::endl + << std::endl + << "Examples:" << std::endl + << "hdfs_ls hdfs://localhost.localdomain:8020/dir" << std::endl + << "hdfs_ls -R /dir1/dir2" << std::endl; + return desc.str(); +} + +bool Ls::Do() { + if (!Initialize()) { + std::cerr << "Unable to initialize HDFS ls tool" << std::endl; + return false; + } + + if (!ValidateConstraints()) { + std::cout << GetDescription(); + return false; + } + + if (opt_val_.count("help") > 0) { + return HandleHelp(); + } + + if (opt_val_.count("path") > 0) { + const auto path = opt_val_["path"].as(); + const auto recursive = opt_val_.count("recursive") > 0; + return HandlePath(path, recursive); + } + + return false; +} + +bool Ls::HandleHelp() const { + std::cout << GetDescription(); + return true; +} + +bool Ls::HandlePath(const std::string &path, const bool recursive) const { + // Building a URI object from the given path + auto uri = hdfs::parse_path_or_exit(path); + + const auto fs = hdfs::doConnect(uri, true); + if (!fs) { + std::cerr << "Could not connect the file system. " << std::endl; + return false; + } + + const auto promise = std::make_shared>(); + auto future(promise->get_future()); + auto result = hdfs::Status::OK(); + + /* + * Keep requesting more until we get the entire listing. Set the promise + * when we have the entire listing to stop. + * + * Find and GetListing guarantee that the handler will only be called once at + * a time, so we do not need any locking here. They also guarantee that the + * handler will be only called once with has_more_results set to false. + */ + auto handler = [promise, + &result](const hdfs::Status &status, + const std::vector &stat_info, + const bool has_more_results) -> bool { + // Print result chunks as they arrive + if (!stat_info.empty()) { + for (const auto &info : stat_info) { + std::cout << info.str() << std::endl; + } + } + if (!status.ok() && result.ok()) { + // We make sure we set the result only on the first error + result = status; + } + if (!has_more_results) { + promise->set_value(); // Set promise + return false; // Request to stop sending results + } + return true; // Request more results + }; + + if (!recursive) { + // Asynchronous call to GetListing + fs->GetListing(uri.get_path(), handler); + } else { + // Asynchronous call to Find + fs->Find(uri.get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), + handler); + } + + // Block until promise is set + future.get(); + if (!result.ok()) { + std::cerr << "Error: " << result.ToString() << std::endl; + return false; + } + return true; +} +} // namespace hdfs::tools diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-ls/hdfs-ls.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-ls/hdfs-ls.h new file mode 100644 index 0000000000000..c38731b67a5fe --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-ls/hdfs-ls.h @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_LS +#define LIBHDFSPP_TOOLS_HDFS_LS + +#include + +#include + +#include "hdfs-tool.h" + +namespace hdfs::tools { +/** + * {@class Ls} is an {@class HdfsTool} that lists information about the files. + */ +class Ls : public HdfsTool { +public: + /** + * {@inheritdoc} + */ + Ls(int argc, char **argv); + + // Abiding to the Rule of 5 + Ls(const Ls &) = default; + Ls(Ls &&) = default; + Ls &operator=(const Ls &) = delete; + Ls &operator=(Ls &&) = delete; + ~Ls() override = default; + + /** + * {@inheritdoc} + */ + [[nodiscard]] std::string GetDescription() const override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Do() override; + +protected: + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Initialize() override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool ValidateConstraints() const override { return argc_ > 1; } + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool HandleHelp() const override; + + /** + * Handle the path argument that's passed to this tool. + * + * @param path The path to the directory for which we need to ls. + * @param recursive A boolean indicating whether ls needs to be + * performed recursively for the given path. + * + * @return A boolean indicating the result of this operation. + */ + [[nodiscard]] virtual bool HandlePath(const std::string &path, + bool recursive) const; + +private: + /** + * A boost data-structure containing the description of positional arguments + * passed to the command-line. + */ + po::positional_options_description pos_opt_desc_; +}; +} // namespace hdfs::tools +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-ls/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-ls/main.cc new file mode 100644 index 0000000000000..5d5312f492d45 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-ls/main.cc @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include + +#include "hdfs-ls.h" + +int main(int argc, char *argv[]) { + const auto result = std::atexit([]() -> void { + // Clean up static data on exit and prevent valgrind memory leaks + google::protobuf::ShutdownProtobufLibrary(); + }); + if (result != 0) { + std::cerr << "Error: Unable to schedule clean-up tasks for HDFS ls tool, " + "exiting" + << std::endl; + std::exit(EXIT_FAILURE); + } + + hdfs::tools::Ls ls(argc, argv); + auto success = false; + + try { + success = ls.Do(); + } catch (const std::exception &e) { + std::cerr << "Error: " << e.what() << std::endl; + } + + if (!success) { + std::exit(EXIT_FAILURE); + } + return 0; +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/CMakeLists.txt new file mode 100644 index 0000000000000..5efce8de86be0 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/CMakeLists.txt @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +add_library(hdfs_mkdir_lib STATIC $ hdfs-mkdir.cc) +target_include_directories(hdfs_mkdir_lib PRIVATE ../../tools hdfs-mkdir ${Boost_INCLUDE_DIRS}) +target_link_libraries(hdfs_mkdir_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static) + +add_executable(hdfs_mkdir main.cc) +target_include_directories(hdfs_mkdir PRIVATE ../../tools) +target_link_libraries(hdfs_mkdir PRIVATE hdfs_mkdir_lib) + +install(TARGETS hdfs_mkdir RUNTIME DESTINATION bin) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/hdfs-mkdir.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/hdfs-mkdir.cc new file mode 100644 index 0000000000000..9c406fb959fc0 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/hdfs-mkdir.cc @@ -0,0 +1,140 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include "hdfs-mkdir.h" +#include "tools_common.h" + +namespace hdfs::tools { +Mkdir::Mkdir(const int argc, char **argv) : HdfsTool(argc, argv) {} + +bool Mkdir::Initialize() { + auto add_options = opt_desc_.add_options(); + add_options("help,h", "Create directory if it does not exist"); + add_options("create-parents,p", "Create parent directories as needed"); + add_options( + "mode,m", po::value(), + "Set the permissions for the new directory (and newly created parents if " + "any). The permissions are specified in octal representation"); + add_options("path", po::value(), + "The path to the directory that needs to be created"); + + // We allow only one argument to be passed to this tool. An exception is + // thrown if multiple arguments are passed. + pos_opt_desc_.add("path", 1); + + po::store(po::command_line_parser(argc_, argv_) + .options(opt_desc_) + .positional(pos_opt_desc_) + .run(), + opt_val_); + po::notify(opt_val_); + return true; +} + +std::string Mkdir::GetDescription() const { + std::stringstream desc; + desc << "Usage: hdfs_mkdir [OPTION] DIRECTORY" << std::endl + << std::endl + << "Create the DIRECTORY(ies), if they do not already exist." + << std::endl + << std::endl + << " -p make parent directories as needed" << std::endl + << " -m MODE set file mode (octal permissions) for the new " + "DIRECTORY(ies)" + << std::endl + << " -h display this help and exit" << std::endl + << std::endl + << "Examples:" << std::endl + << "hdfs_mkdir hdfs://localhost.localdomain:8020/dir1/dir2" << std::endl + << "hdfs_mkdir -p /extant_dir/non_extant_dir/non_extant_dir/new_dir" + << std::endl; + return desc.str(); +} + +bool Mkdir::Do() { + if (!Initialize()) { + std::cerr << "Unable to initialize HDFS mkdir tool" << std::endl; + return false; + } + + if (!ValidateConstraints()) { + std::cout << GetDescription(); + return false; + } + + if (opt_val_.count("help") > 0) { + return HandleHelp(); + } + + if (opt_val_.count("path") > 0) { + const auto path = opt_val_["path"].as(); + const auto create_parents = opt_val_.count("create-parents") > 0; + const auto permissions = + opt_val_.count("mode") > 0 + ? std::optional(opt_val_["mode"].as()) + : std::nullopt; + return HandlePath(create_parents, permissions, path); + } + + return false; +} + +bool Mkdir::HandleHelp() const { + std::cout << GetDescription(); + return true; +} + +bool Mkdir::HandlePath(const bool create_parents, + const std::optional &permissions, + const std::string &path) const { + // Building a URI object from the given uri_path + auto uri = hdfs::parse_path_or_exit(path); + + const auto fs = hdfs::doConnect(uri, false); + if (fs == nullptr) { + std::cerr << "Could not connect the file system." << std::endl; + return false; + } + + const auto status = + fs->Mkdirs(uri.get_path(), GetPermissions(permissions), create_parents); + if (!status.ok()) { + std::cerr << "Error: " << status.ToString() << std::endl; + return false; + } + + return true; +} + +uint16_t Mkdir::GetPermissions(const std::optional &permissions) { + if (permissions) { + // TODO : Handle the error returned by std::strtol. + return static_cast( + std::strtol(permissions.value().c_str(), nullptr, 8)); + } + + return hdfs::FileSystem::GetDefaultPermissionMask(); +} +} // namespace hdfs::tools diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/hdfs-mkdir.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/hdfs-mkdir.h new file mode 100644 index 0000000000000..60875f3c7af7c --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/hdfs-mkdir.h @@ -0,0 +1,105 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_MKDIR +#define LIBHDFSPP_TOOLS_HDFS_MKDIR + +#include +#include + +#include + +#include "hdfs-tool.h" + +namespace hdfs::tools { +/** + * {@class Mkdir} is an {@class HdfsTool} that creates directory if it does not + * exist. + */ +class Mkdir : public HdfsTool { +public: + /** + * {@inheritdoc} + */ + Mkdir(int argc, char **argv); + + // Abiding to the Rule of 5 + Mkdir(const Mkdir &) = default; + Mkdir(Mkdir &&) = default; + Mkdir &operator=(const Mkdir &) = delete; + Mkdir &operator=(Mkdir &&) = delete; + ~Mkdir() override = default; + + /** + * {@inheritdoc} + */ + [[nodiscard]] std::string GetDescription() const override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Do() override; + +protected: + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Initialize() override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool ValidateConstraints() const override { return argc_ > 1; } + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool HandleHelp() const override; + + /** + * Handle the path argument that's passed to this tool. + * + * @param create_parents Creates parent directories as needed if this boolean + * is set to true. + * @param permissions An octal representation of the permissions to be stamped + * to each directory that gets created. + * @param path The path in the filesystem where the directory must be created. + * + * @return A boolean indicating the result of this operation. + */ + [[nodiscard]] virtual bool + HandlePath(bool create_parents, const std::optional &permissions, + const std::string &path) const; + + /** + * @param permissions The permissions string to convert to octal value. + * @return The octal representation of the permissions supplied as parameter + * to this tool. + */ + [[nodiscard]] static uint16_t + GetPermissions(const std::optional &permissions); + +private: + /** + * A boost data-structure containing the description of positional arguments + * passed to the command-line. + */ + po::positional_options_description pos_opt_desc_; +}; +} // namespace hdfs::tools +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/main.cc new file mode 100644 index 0000000000000..3f70ef6dd5772 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/main.cc @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include + +#include "hdfs-mkdir.h" + +int main(int argc, char *argv[]) { + const auto result = std::atexit([]() -> void { + // Clean up static data on exit and prevent valgrind memory leaks + google::protobuf::ShutdownProtobufLibrary(); + }); + if (result != 0) { + std::cerr << "Error: Unable to schedule clean-up tasks for HDFS mkdir " + "tool, exiting" + << std::endl; + std::exit(EXIT_FAILURE); + } + + hdfs::tools::Mkdir mkdir(argc, argv); + auto success = false; + + try { + success = mkdir.Do(); + } catch (const std::exception &e) { + std::cerr << "Error: " << e.what() << std::endl; + } + + if (!success) { + std::exit(EXIT_FAILURE); + } + return 0; +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-move-to-local/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-move-to-local/CMakeLists.txt new file mode 100644 index 0000000000000..bd3486d472524 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-move-to-local/CMakeLists.txt @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +add_library(hdfs_moveToLocal_lib STATIC $ hdfs-move-to-local.cc) +target_include_directories(hdfs_moveToLocal_lib PRIVATE ../../tools hdfs-moveToLocal ${Boost_INCLUDE_DIRS}) +target_link_libraries(hdfs_moveToLocal_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static) + +add_executable(hdfs_moveToLocal main.cc) +target_include_directories(hdfs_moveToLocal PRIVATE ../../tools) +target_link_libraries(hdfs_moveToLocal PRIVATE hdfs_moveToLocal_lib) + +install(TARGETS hdfs_moveToLocal RUNTIME DESTINATION bin) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-move-to-local/hdfs-move-to-local.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-move-to-local/hdfs-move-to-local.cc new file mode 100644 index 0000000000000..9aeecea7b6e87 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-move-to-local/hdfs-move-to-local.cc @@ -0,0 +1,135 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include "hdfs-move-to-local.h" +#include "tools_common.h" + +namespace hdfs::tools { +MoveToLocal::MoveToLocal(const int argc, char **argv) : HdfsTool(argc, argv) {} + +bool MoveToLocal::Initialize() { + auto add_options = opt_desc_.add_options(); + add_options("help,h", "Moves the file from the given HDFS source path to " + "the destination path on the local machine"); + + add_options("source", po::value(), "The HDFS source file path"); + add_options("target", po::value(), + "The target local system file path"); + + // We allow only two arguments to be passed to this tool. An exception is + // thrown if more arguments are passed. + pos_opt_desc_.add("source", 1); + pos_opt_desc_.add("target", 1); + + po::store(po::command_line_parser(argc_, argv_) + .options(opt_desc_) + .positional(pos_opt_desc_) + .run(), + opt_val_); + po::notify(opt_val_); + return true; +} + +bool MoveToLocal::ValidateConstraints() const { + // Only "help" is allowed as single argument + if (argc_ == 2) { + return opt_val_.count("help") > 0; + } + + // Rest of the cases must contain exactly 2 arguments + return argc_ == 3; +} + +std::string MoveToLocal::GetDescription() const { + std::stringstream desc; + desc << "Usage: hdfs_moveToLocal [OPTION] SRC_FILE DST_FILE" << std::endl + << std::endl + << "Move SRC_FILE from hdfs to DST_FILE on the local file system." + << std::endl + << "Moving is done by copying SRC_FILE to DST_FILE, and then" + << std::endl + << "deleting DST_FILE if copy succeeded." << std::endl + << std::endl + << " -h display this help and exit" << std::endl + << std::endl + << "Examples:" << std::endl + << "hdfs_moveToLocal hdfs://localhost.localdomain:8020/dir/file " + "/home/usr/myfile" + << std::endl + << "hdfs_moveToLocal /dir/file /home/usr/dir/file" << std::endl; + return desc.str(); +} + +bool MoveToLocal::Do() { + if (!Initialize()) { + std::cerr << "Unable to initialize HDFS moveToLocal tool" << std::endl; + return false; + } + + if (!ValidateConstraints()) { + std::cout << GetDescription(); + return false; + } + + if (opt_val_.count("help") > 0) { + return HandleHelp(); + } + + if (opt_val_.count("source") > 0 && opt_val_.count("target") > 0) { + const auto source = opt_val_["source"].as(); + const auto target = opt_val_["target"].as(); + return HandlePath(source, target); + } + + return false; +} + +bool MoveToLocal::HandleHelp() const { + std::cout << GetDescription(); + return true; +} + +bool MoveToLocal::HandlePath(const std::string &source, + const std::string &target) const { + // Building a URI object from the given path + auto uri = hdfs::parse_path_or_exit(source); + + const auto fs = hdfs::doConnect(uri, true); + if (!fs) { + std::cerr << "Could not connect the file system. " << std::endl; + return false; + } + + auto dst_file = std::fopen(target.c_str(), "wb"); + if (!dst_file) { + std::cerr << "Unable to open the destination file: " << target << std::endl; + return false; + } + + readFile(fs, uri.get_path(), 0, dst_file, true); + std::fclose(dst_file); + return true; +} +} // namespace hdfs::tools diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-move-to-local/hdfs-move-to-local.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-move-to-local/hdfs-move-to-local.h new file mode 100644 index 0000000000000..982a67914fa8b --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-move-to-local/hdfs-move-to-local.h @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_MOVE_TO_LOCAL +#define LIBHDFSPP_TOOLS_HDFS_MOVE_TO_LOCAL + +#include + +#include + +#include "hdfs-tool.h" + +namespace hdfs::tools { +/** + * {@class MoveToLocal} is an {@class HdfsTool} that copies the file from the + * given HDFS source path to the destination path on the local machine. + */ +class MoveToLocal : public HdfsTool { +public: + /** + * {@inheritdoc} + */ + MoveToLocal(int argc, char **argv); + + // Abiding to the Rule of 5 + MoveToLocal(const MoveToLocal &) = default; + MoveToLocal(MoveToLocal &&) = default; + MoveToLocal &operator=(const MoveToLocal &) = delete; + MoveToLocal &operator=(MoveToLocal &&) = delete; + ~MoveToLocal() override = default; + + /** + * {@inheritdoc} + */ + [[nodiscard]] std::string GetDescription() const override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Do() override; + +protected: + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Initialize() override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool ValidateConstraints() const override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool HandleHelp() const override; + + /** + * Handle the path argument that's passed to this tool. + * + * @param source The source file path in HDFS. + * @param target The target file path on the local machine. + * + * @return A boolean indicating the result of this operation. + */ + [[nodiscard]] virtual bool HandlePath(const std::string &source, + const std::string &target) const; + +private: + /** + * A boost data-structure containing the description of positional arguments + * passed to the command-line. + */ + po::positional_options_description pos_opt_desc_; +}; +} // namespace hdfs::tools +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-move-to-local/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-move-to-local/main.cc new file mode 100644 index 0000000000000..2d13464185934 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-move-to-local/main.cc @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include + +#include "hdfs-move-to-local.h" + +int main(int argc, char *argv[]) { + const auto result = std::atexit([]() -> void { + // Clean up static data on exit and prevent valgrind memory leaks + google::protobuf::ShutdownProtobufLibrary(); + }); + if (result != 0) { + std::cerr << "Error: Unable to schedule clean-up tasks for HDFS " + "move_to_local tool, exiting" + << std::endl; + std::exit(EXIT_FAILURE); + } + + hdfs::tools::MoveToLocal move_to_local(argc, argv); + auto success = false; + + try { + success = move_to_local.Do(); + } catch (const std::exception &e) { + std::cerr << "Error: " << e.what() << std::endl; + } + + if (!success) { + std::exit(EXIT_FAILURE); + } + return 0; +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/CMakeLists.txt new file mode 100644 index 0000000000000..70a7db5167463 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/CMakeLists.txt @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +add_library(hdfs_rm_lib STATIC $ hdfs-rm.cc) +target_include_directories(hdfs_rm_lib PRIVATE ../../tools hdfs-rm ${Boost_INCLUDE_DIRS}) +target_link_libraries(hdfs_rm_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static) + +add_executable(hdfs_rm main.cc) +target_include_directories(hdfs_rm PRIVATE ../../tools) +target_link_libraries(hdfs_rm PRIVATE hdfs_rm_lib) + +install(TARGETS hdfs_rm RUNTIME DESTINATION bin) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/hdfs-rm.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/hdfs-rm.cc new file mode 100644 index 0000000000000..ef1a85649d257 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/hdfs-rm.cc @@ -0,0 +1,114 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include + +#include "hdfs-rm.h" +#include "tools_common.h" + +namespace hdfs::tools { +Rm::Rm(const int argc, char **argv) : HdfsTool(argc, argv) {} + +bool Rm::Initialize() { + auto add_options = opt_desc_.add_options(); + add_options("help,h", "Remove/unlink the files or directories."); + add_options("recursive,R", + "Remove the directories and their contents recursively."); + add_options("path", po::value(), + "The path to the file that needs to be removed."); + + // We allow only one argument to be passed to this tool. An exception is + // thrown if multiple arguments are passed. + pos_opt_desc_.add("path", 1); + + po::store(po::command_line_parser(argc_, argv_) + .options(opt_desc_) + .positional(pos_opt_desc_) + .run(), + opt_val_); + po::notify(opt_val_); + return true; +} + +std::string Rm::GetDescription() const { + std::stringstream desc; + desc << "Usage: hdfs_rm [OPTION] FILE" << std::endl + << std::endl + << "Remove (unlink) the FILE(s) or directory(ies)." << std::endl + << std::endl + << " -R remove directories and their contents recursively" + << std::endl + << " -h display this help and exit" << std::endl + << std::endl + << "Examples:" << std::endl + << "hdfs_rm hdfs://localhost.localdomain:8020/dir/file" << std::endl + << "hdfs_rm -R /dir1/dir2" << std::endl; + return desc.str(); +} + +bool Rm::Do() { + if (!Initialize()) { + std::cerr << "Unable to initialize HDFS rm tool" << std::endl; + return false; + } + + if (!ValidateConstraints()) { + std::cout << GetDescription(); + return false; + } + + if (opt_val_.count("help") > 0) { + return HandleHelp(); + } + + if (opt_val_.count("path") > 0) { + const auto path = opt_val_["path"].as(); + const auto recursive = opt_val_.count("recursive") > 0; + return HandlePath(recursive, path); + } + + return false; +} + +bool Rm::HandleHelp() const { + std::cout << GetDescription(); + return true; +} + +bool Rm::HandlePath(const bool recursive, const std::string &path) const { + // Building a URI object from the given uri_path + auto uri = hdfs::parse_path_or_exit(path); + + const auto fs = hdfs::doConnect(uri, false); + if (fs == nullptr) { + std::cerr << "Could not connect the file system." << std::endl; + return false; + } + + const auto status = fs->Delete(uri.get_path(), recursive); + if (!status.ok()) { + std::cerr << "Error: " << status.ToString() << std::endl; + return false; + } + return true; +} +} // namespace hdfs::tools diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/hdfs-rm.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/hdfs-rm.h new file mode 100644 index 0000000000000..af5faf65be489 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/hdfs-rm.h @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_RM +#define LIBHDFSPP_TOOLS_HDFS_RM + +#include + +#include + +#include "hdfs-tool.h" + +namespace hdfs::tools { +/** + * {@class Rm} is an {@class HdfsTool} that removes/unlinks the files or + * directories. + */ +class Rm : public HdfsTool { +public: + /** + * {@inheritdoc} + */ + Rm(int argc, char **argv); + + // Abiding to the Rule of 5 + Rm(const Rm &) = default; + Rm(Rm &&) = default; + Rm &operator=(const Rm &) = delete; + Rm &operator=(Rm &&) = delete; + ~Rm() override = default; + + /** + * {@inheritdoc} + */ + [[nodiscard]] std::string GetDescription() const override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Do() override; + +protected: + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Initialize() override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool ValidateConstraints() const override { return argc_ > 1; } + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool HandleHelp() const override; + + /** + * Handle the path argument that's passed to this tool. + * + * @param recursive Perform this operation recursively on the sub-directories. + * @param path The path to the file/directory that needs to be removed. + * + * @return A boolean indicating the result of this operation. + */ + [[nodiscard]] virtual bool HandlePath(bool recursive, + const std::string &path) const; + +private: + /** + * A boost data-structure containing the description of positional arguments + * passed to the command-line. + */ + po::positional_options_description pos_opt_desc_; +}; +} // namespace hdfs::tools +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/main.cc new file mode 100644 index 0000000000000..16577f52a849a --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/main.cc @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include + +#include "hdfs-rm.h" + +int main(int argc, char *argv[]) { + const auto result = std::atexit([]() -> void { + // Clean up static data on exit and prevent valgrind memory leaks + google::protobuf::ShutdownProtobufLibrary(); + }); + if (result != 0) { + std::cerr << "Error: Unable to schedule clean-up tasks for HDFS rm " + "tool, exiting" + << std::endl; + std::exit(EXIT_FAILURE); + } + + hdfs::tools::Rm rm(argc, argv); + auto success = false; + + try { + success = rm.Do(); + } catch (const std::exception &e) { + std::cerr << "Error: " << e.what() << std::endl; + } + + if (!success) { + std::exit(EXIT_FAILURE); + } + return 0; +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-setrep/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-setrep/CMakeLists.txt new file mode 100644 index 0000000000000..a0d8bafa63080 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-setrep/CMakeLists.txt @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +add_library(hdfs_setrep_lib STATIC $ hdfs-setrep.cc) +target_include_directories(hdfs_setrep_lib PRIVATE ../../tools ${Boost_INCLUDE_DIRS}) +target_link_libraries(hdfs_setrep_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static) + +add_executable(hdfs_setrep main.cc) +target_include_directories(hdfs_setrep PRIVATE ../../tools) +target_link_libraries(hdfs_setrep PRIVATE hdfs_setrep_lib) + +install(TARGETS hdfs_setrep RUNTIME DESTINATION bin) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-setrep/hdfs-setrep.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-setrep/hdfs-setrep.cc new file mode 100644 index 0000000000000..542659b29f141 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-setrep/hdfs-setrep.cc @@ -0,0 +1,220 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include "hdfs-setrep.h" +#include "internal/set-replication-state.h" +#include "tools_common.h" + +namespace hdfs::tools { +Setrep::Setrep(const int argc, char **argv) : HdfsTool(argc, argv) {} + +bool Setrep::Initialize() { + auto add_options = opt_desc_.add_options(); + add_options("help,h", + "Changes the replication factor of a file at PATH. If PATH is a " + "directory then the command recursively changes the replication " + "factor of all files under the directory tree rooted at PATH."); + add_options( + "replication-factor", po::value(), + "The replication factor to set for the given path and its children."); + add_options("path", po::value(), + "The path for which the replication factor needs to be set."); + + // We allow only one positional argument to be passed to this tool. An + // exception is thrown if multiple arguments are passed. + pos_opt_desc_.add("replication-factor", 1); + pos_opt_desc_.add("path", 1); + + po::store(po::command_line_parser(argc_, argv_) + .options(opt_desc_) + .positional(pos_opt_desc_) + .run(), + opt_val_); + po::notify(opt_val_); + return true; +} + +bool Setrep::ValidateConstraints() const { + // Only "help" is allowed as single argument. + if (argc_ == 2) { + return opt_val_.count("help"); + } + + // Rest of the cases must contain more than 2 arguments on the command line. + return argc_ > 2; +} + +std::string Setrep::GetDescription() const { + std::stringstream desc; + desc << "Usage: hdfs_setrep [OPTION] NUM_REPLICAS PATH" << std::endl + << std::endl + << "Changes the replication factor of a file at PATH. If PATH is a " + "directory then the command" + << std::endl + << "recursively changes the replication factor of all files under the " + "directory tree rooted at PATH." + << std::endl + << std::endl + << " -h display this help and exit" << std::endl + << std::endl + << "Examples:" << std::endl + << "hdfs_setrep 5 hdfs://localhost.localdomain:8020/dir/file" + << std::endl + << "hdfs_setrep 3 /dir1/dir2" << std::endl; + return desc.str(); +} + +bool Setrep::Do() { + if (!Initialize()) { + std::cerr << "Unable to initialize HDFS setrep tool" << std::endl; + return false; + } + + if (!ValidateConstraints()) { + std::cout << GetDescription(); + return false; + } + + if (opt_val_.count("help") > 0) { + return HandleHelp(); + } + + if (opt_val_.count("path") > 0 && opt_val_.count("replication-factor") > 0) { + const auto replication_factor = + opt_val_["replication-factor"].as(); + const auto path = opt_val_["path"].as(); + return HandlePath(path, replication_factor); + } + + return false; +} + +bool Setrep::HandleHelp() const { + std::cout << GetDescription(); + return true; +} + +bool Setrep::HandlePath(const std::string &path, + const std::string &replication_factor) const { + // Building a URI object from the given path. + auto uri = hdfs::parse_path_or_exit(path); + + const auto fs = hdfs::doConnect(uri, true); + if (!fs) { + std::cerr << "Could not connect to the file system." << std::endl; + return false; + } + + /* + * Wrap async FileSystem::SetReplication with promise to make it a blocking + * call. + */ + auto promise = std::make_shared>(); + std::future future(promise->get_future()); + auto handler = [promise](const hdfs::Status &s) { promise->set_value(s); }; + + const auto replication = static_cast( + std::strtol(replication_factor.c_str(), nullptr, 8)); + /* + * Allocating shared state, which includes: + * replication to be set, handler to be called, request counter, and a boolean + * to keep track if find is done + */ + auto state = + std::make_shared(replication, handler, 0, false); + + /* + * Keep requesting more from Find until we process the entire listing. Call + * handler when Find is done and request counter is 0. Find guarantees that + * the handler will only be called once at a time so we do not need locking in + * handler_find. + */ + auto handler_find = [fs, state](const hdfs::Status &status_find, + const std::vector &stat_infos, + const bool has_more_results) -> bool { + /* + * For each result returned by Find we call async SetReplication with the + * handler below. SetReplication DOES NOT guarantee that the handler will + * only be called once at a time, so we DO need locking in + * handler_set_replication. + */ + auto handler_set_replication = + [state](const hdfs::Status &status_set_replication) { + std::lock_guard guard(state->lock); + + // Decrement the counter once since we are done with this async call. + if (!status_set_replication.ok() && state->status.ok()) { + // We make sure we set state->status only on the first error. + state->status = status_set_replication; + } + state->request_counter--; + if (state->request_counter == 0 && state->find_is_done) { + state->handler(state->status); // Exit. + } + }; + if (!stat_infos.empty() && state->status.ok()) { + for (hdfs::StatInfo const &stat_info : stat_infos) { + // Launch an asynchronous call to SetReplication for every returned + // file. + if (stat_info.file_type == hdfs::StatInfo::IS_FILE) { + state->request_counter++; + fs->SetReplication(stat_info.full_path, state->replication, + handler_set_replication); + } + } + } + + /* + * Lock this section because handlerSetReplication might be accessing the + * same shared variables simultaneously. + */ + std::lock_guard guard(state->lock); + if (!status_find.ok() && state->status.ok()) { + // We make sure we set state->status only on the first error. + state->status = status_find; + } + if (!has_more_results) { + state->find_is_done = true; + if (state->request_counter == 0) { + state->handler(state->status); // Exit. + } + return false; + } + return true; + }; + + // Asynchronous call to Find. + fs->Find(uri.get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), + handler_find); + + // Block until promise is set. + const auto status = future.get(); + if (!status.ok()) { + std::cerr << "Error: " << status.ToString() << std::endl; + return false; + } + return true; +} +} // namespace hdfs::tools diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-setrep/hdfs-setrep.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-setrep/hdfs-setrep.h new file mode 100644 index 0000000000000..20ee7405b6de9 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-setrep/hdfs-setrep.h @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_SETREP +#define LIBHDFSPP_TOOLS_HDFS_SETREP + +#include + +#include + +#include "hdfs-tool.h" + +namespace hdfs::tools { +/** + * {@class Setrep} is an {@class HdfsTool} that changes the replication factor + * of a file at a given path. If the path is a directory, then it recursively + * changes the replication factor of all files under the directory tree rooted + * at the given path. + */ +class Setrep : public HdfsTool { +public: + /** + * {@inheritdoc} + */ + Setrep(int argc, char **argv); + + // Abiding to the Rule of 5 + Setrep(const Setrep &) = default; + Setrep(Setrep &&) = default; + Setrep &operator=(const Setrep &) = delete; + Setrep &operator=(Setrep &&) = delete; + ~Setrep() override = default; + + /** + * {@inheritdoc} + */ + [[nodiscard]] std::string GetDescription() const override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Do() override; + +protected: + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Initialize() override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool ValidateConstraints() const override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool HandleHelp() const override; + + /** + * Handle the path argument that's passed to this tool. + * + * @param path The path to the directory for which we need setrep info. + * @param replication_factor The replication factor to set to given path and + * its children. + * + * @return A boolean indicating the result of this operation. + */ + [[nodiscard]] virtual bool + HandlePath(const std::string &path, + const std::string &replication_factor) const; + +private: + /** + * A boost data-structure containing the description of positional arguments + * passed to the command-line. + */ + po::positional_options_description pos_opt_desc_; +}; +} // namespace hdfs::tools +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-setrep/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-setrep/main.cc new file mode 100644 index 0000000000000..a3d8399c575ab --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-setrep/main.cc @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include + +#include "hdfs-setrep.h" + +int main(int argc, char *argv[]) { + const auto result = std::atexit([]() -> void { + // Clean up static data on exit and prevent valgrind memory leaks + google::protobuf::ShutdownProtobufLibrary(); + }); + if (result != 0) { + std::cerr << "Error: Unable to schedule clean-up tasks for HDFS setrep " + "tool, exiting" + << std::endl; + std::exit(EXIT_FAILURE); + } + + hdfs::tools::Setrep setrep(argc, argv); + auto success = false; + + try { + success = setrep.Do(); + } catch (const std::exception &e) { + std::cerr << "Error: " << e.what() << std::endl; + } + + if (!success) { + std::exit(EXIT_FAILURE); + } + return 0; +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-stat/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-stat/CMakeLists.txt new file mode 100644 index 0000000000000..3a390cc2ed20a --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-stat/CMakeLists.txt @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +add_library(hdfs_stat_lib STATIC $ hdfs-stat.cc) +target_include_directories(hdfs_stat_lib PRIVATE ../../tools ${Boost_INCLUDE_DIRS}) +target_link_libraries(hdfs_stat_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static) + +add_executable(hdfs_stat main.cc) +target_include_directories(hdfs_stat PRIVATE ../../tools) +target_link_libraries(hdfs_stat PRIVATE hdfs_stat_lib) + +install(TARGETS hdfs_stat RUNTIME DESTINATION bin) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-stat/hdfs-stat.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-stat/hdfs-stat.cc new file mode 100644 index 0000000000000..e20fc4576d31c --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-stat/hdfs-stat.cc @@ -0,0 +1,111 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include "hdfs-stat.h" +#include "tools_common.h" + +namespace hdfs::tools { +Stat::Stat(const int argc, char **argv) : HdfsTool(argc, argv) {} + +bool Stat::Initialize() { + auto add_options = opt_desc_.add_options(); + add_options("help,h", "Displays the stat information for the given path. The " + "path can be a file or a directory."); + add_options("path", po::value(), + "The path in the filesystem for which to display the " + "stat information."); + + // We allow only one positional argument to be passed to this tool. An + // exception is thrown if multiple arguments are passed. + pos_opt_desc_.add("path", 1); + + po::store(po::command_line_parser(argc_, argv_) + .options(opt_desc_) + .positional(pos_opt_desc_) + .run(), + opt_val_); + po::notify(opt_val_); + return true; +} + +std::string Stat::GetDescription() const { + std::stringstream desc; + desc << "Usage: hdfs_stat PATH" << std::endl + << std::endl + << "Displays the stat information for the given path." << std::endl + << "The path can be a file or a directory." << std::endl + << "Examples:" << std::endl + << "hdfs_stat hdfs://localhost.localdomain:8020/dir/file" << std::endl; + return desc.str(); +} + +bool Stat::Do() { + if (!Initialize()) { + std::cerr << "Unable to initialize HDFS stat tool" << std::endl; + return false; + } + + if (!ValidateConstraints()) { + std::cout << GetDescription(); + return false; + } + + if (opt_val_.count("help") > 0) { + return HandleHelp(); + } + + if (opt_val_.count("path") > 0) { + const auto path = opt_val_["path"].as(); + return HandlePath(path); + } + + return false; +} + +bool Stat::HandleHelp() const { + std::cout << GetDescription(); + return true; +} + +bool Stat::HandlePath(const std::string &path) const { + // Building a URI object from the given uri_path + auto uri = hdfs::parse_path_or_exit(path); + + const auto fs = hdfs::doConnect(uri, false); + if (!fs) { + std::cerr << "Could not connect the file system. " << std::endl; + return false; + } + + hdfs::StatInfo stat_info; + const auto status = fs->GetFileInfo(uri.get_path(), stat_info); + if (!status.ok()) { + std::cerr << "Error: " << status.ToString() << std::endl; + return false; + } + std::cout << stat_info.str() << std::endl; + return true; +} +} // namespace hdfs::tools diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-stat/hdfs-stat.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-stat/hdfs-stat.h new file mode 100644 index 0000000000000..a23affc1c461c --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-stat/hdfs-stat.h @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_STAT +#define LIBHDFSPP_TOOLS_HDFS_STAT + +#include + +#include + +#include "hdfs-tool.h" + +namespace hdfs::tools { +/** + * {@class Stat} is an {@class HdfsTool} that displays the stat information for + * the given path. The path can be a file or a directory. + */ +class Stat : public HdfsTool { +public: + /** + * {@inheritdoc} + */ + Stat(int argc, char **argv); + + // Abiding to the Rule of 5 + Stat(const Stat &) = default; + Stat(Stat &&) = default; + Stat &operator=(const Stat &) = delete; + Stat &operator=(Stat &&) = delete; + ~Stat() override = default; + + /** + * {@inheritdoc} + */ + [[nodiscard]] std::string GetDescription() const override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Do() override; + +protected: + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Initialize() override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool ValidateConstraints() const override { return argc_ > 1; } + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool HandleHelp() const override; + + /** + * Handle the path argument that's passed to this tool. + * + * @param path The path to the directory for which we need the stat info. + * + * @return A boolean indicating the result of this operation. + */ + [[nodiscard]] virtual bool HandlePath(const std::string &path) const; + +private: + /** + * A boost data-structure containing the description of positional arguments + * passed to the command-line. + */ + po::positional_options_description pos_opt_desc_; +}; +} // namespace hdfs::tools +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-stat/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-stat/main.cc new file mode 100644 index 0000000000000..4437419259ca0 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-stat/main.cc @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include + +#include "hdfs-stat.h" + +int main(int argc, char *argv[]) { + const auto result = std::atexit([]() -> void { + // Clean up static data on exit and prevent valgrind memory leaks + google::protobuf::ShutdownProtobufLibrary(); + }); + if (result != 0) { + std::cerr << "Error: Unable to schedule the clean-up tasks for HDFS stat" + "tool, exiting" + << std::endl; + std::exit(EXIT_FAILURE); + } + + hdfs::tools::Stat stat(argc, argv); + auto success = false; + + try { + success = stat.Do(); + } catch (const std::exception &e) { + std::cerr << "Error: " << e.what() << std::endl; + } + + if (!success) { + std::exit(EXIT_FAILURE); + } + return 0; +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-tail/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-tail/CMakeLists.txt new file mode 100644 index 0000000000000..5169e00f1996f --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-tail/CMakeLists.txt @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +add_library(hdfs_tail_lib STATIC $ hdfs-tail.cc) +target_include_directories(hdfs_tail_lib PRIVATE ../../tools ${Boost_INCLUDE_DIRS}) +target_link_libraries(hdfs_tail_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static) + +add_executable(hdfs_tail main.cc) +target_include_directories(hdfs_tail PRIVATE ../../tools) +target_link_libraries(hdfs_tail PRIVATE hdfs_tail_lib) + +install(TARGETS hdfs_tail RUNTIME DESTINATION bin) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-tail/hdfs-tail.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-tail/hdfs-tail.cc new file mode 100644 index 0000000000000..7eaa4602c894f --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-tail/hdfs-tail.cc @@ -0,0 +1,150 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hdfs-tail.h" +#include "tools_common.h" + +namespace hdfs::tools { +Tail::Tail(const int argc, char **argv) : HdfsTool(argc, argv) {} + +bool Tail::Initialize() { + auto add_options = opt_desc_.add_options(); + add_options("help,h", "Displays last kilobyte of the file to stdout."); + add_options("follow,f", + "Append data to the output as the file grows, as in Unix."); + add_options("path", po::value(), + "The path indicating the filesystem that needs to be tailed."); + + // We allow only one positional argument to be passed to this tool. An + // exception is thrown if multiple arguments are passed. + pos_opt_desc_.add("path", 1); + + po::store(po::command_line_parser(argc_, argv_) + .options(opt_desc_) + .positional(pos_opt_desc_) + .run(), + opt_val_); + po::notify(opt_val_); + return true; +} + +std::string Tail::GetDescription() const { + std::stringstream desc; + desc << "Usage: hdfs_tail [OPTION] FILE" << std::endl + << std::endl + << "Displays last kilobyte of the file to stdout." << std::endl + << std::endl + << " -f append data to the output as the file grows, as in Unix" + << std::endl + << " -h display this help and exit" << std::endl + << std::endl + << "Examples:" << std::endl + << "hdfs_tail hdfs://localhost.localdomain:8020/dir/file" << std::endl + << "hdfs_tail /dir/file" << std::endl; + return desc.str(); +} + +bool Tail::Do() { + if (!Initialize()) { + std::cerr << "Unable to initialize HDFS tail tool" << std::endl; + return false; + } + + if (!ValidateConstraints()) { + std::cout << GetDescription(); + return false; + } + + if (opt_val_.count("help") > 0) { + return HandleHelp(); + } + + if (opt_val_.count("path") > 0) { + const auto path = opt_val_["path"].as(); + const auto follow = opt_val_.count("follow") > 0; + return HandlePath(path, follow); + } + + return false; +} + +bool Tail::HandleHelp() const { + std::cout << GetDescription(); + return true; +} + +bool Tail::HandlePath(const std::string &path, const bool follow) const { + // Building a URI object from the given path. + auto uri = hdfs::parse_path_or_exit(path); + + const auto fs = hdfs::doConnect(uri, true); + if (!fs) { + std::cerr << "Could not connect to the file system." << std::endl; + return false; + } + + // We need to get the size of the file using stat. + hdfs::StatInfo stat_info; + auto status = fs->GetFileInfo(uri.get_path(), stat_info); + if (!status.ok()) { + std::cerr << "Error: " << status.ToString() << std::endl; + return false; + } + + // Determine where to start reading. + off_t offset{0}; + if (stat_info.length > tail_size_in_bytes) { + offset = static_cast(stat_info.length - tail_size_in_bytes); + } + + do { + const auto current_length = static_cast(stat_info.length); + readFile(fs, uri.get_path(), offset, stdout, false); + + // Exit if -f flag was not set. + if (!follow) { + break; + } + + do { + // Sleep for the refresh rate. + std::this_thread::sleep_for(std::chrono::seconds(refresh_rate_in_sec)); + + // Use stat to check the new file size. + status = fs->GetFileInfo(uri.get_path(), stat_info); + if (!status.ok()) { + std::cerr << "Error: " << status.ToString() << std::endl; + return false; + } + + // If file became longer, loop back and print the difference. + } while (static_cast(stat_info.length) <= current_length); + } while (true); + + return true; +} +} // namespace hdfs::tools diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-tail/hdfs-tail.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-tail/hdfs-tail.h new file mode 100644 index 0000000000000..f103d0d5b9470 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-tail/hdfs-tail.h @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_TAIL +#define LIBHDFSPP_TOOLS_HDFS_TAIL + +#include + +#include + +#include "hdfs-tool.h" + +namespace hdfs::tools { +/** + * {@class Tail} is an {@class HdfsTool} displays last kilobyte of the file to + * stdout. + */ +class Tail : public HdfsTool { +public: + /** + * {@inheritdoc} + */ + Tail(int argc, char **argv); + + // Abiding to the Rule of 5 + Tail(const Tail &) = default; + Tail(Tail &&) = default; + Tail &operator=(const Tail &) = delete; + Tail &operator=(Tail &&) = delete; + ~Tail() override = default; + + /** + * {@inheritdoc} + */ + [[nodiscard]] std::string GetDescription() const override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Do() override; + +protected: + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Initialize() override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool ValidateConstraints() const override { return argc_ > 1; } + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool HandleHelp() const override; + + /** + * Handle the path argument that's passed to this tool. + * + * @param path The path to the file which needs to be tailed. + * @param follow Append data to the output as the file grows, as in Unix. + * + * @return A boolean indicating the result of this operation. + */ + [[nodiscard]] virtual bool HandlePath(const std::string &path, + bool follow) const; + + /** + * The tail size in bytes. + */ + static constexpr uint64_t tail_size_in_bytes{1024}; + + /** + * The refresh rate for {@link hdfs::tools::Tail} in seconds. + */ + static constexpr int refresh_rate_in_sec{1}; + +private: + /** + * A boost data-structure containing the description of positional arguments + * passed to the command-line. + */ + po::positional_options_description pos_opt_desc_; +}; +} // namespace hdfs::tools +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-tail/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-tail/main.cc new file mode 100644 index 0000000000000..fb85915e1d3f2 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-tail/main.cc @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include + +#include "hdfs-tail.h" + +int main(int argc, char *argv[]) { + const auto result = std::atexit([]() -> void { + // Clean up static data on exit and prevent valgrind memory leaks + google::protobuf::ShutdownProtobufLibrary(); + }); + if (result != 0) { + std::cerr << "Error: Unable to schedule clean-up tasks for HDFS tail tool, " + "exiting" + << std::endl; + std::exit(EXIT_FAILURE); + } + + hdfs::tools::Tail tail(argc, argv); + auto success = false; + + try { + success = tail.Do(); + } catch (const std::exception &e) { + std::cerr << "Error: " << e.what() << std::endl; + } + + if (!success) { + std::exit(EXIT_FAILURE); + } + return 0; +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chgrp.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chgrp.cc deleted file mode 100644 index d64affeb14467..0000000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chgrp.cc +++ /dev/null @@ -1,185 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -*/ - -#include -#include -#include -#include "tools_common.h" - -void usage(){ - std::cout << "Usage: hdfs_chgrp [OPTION] GROUP FILE" - << std::endl - << std::endl << "Change the group association of each FILE to GROUP." - << std::endl << "The user must be the owner of files. Additional information is in the Permissions Guide:" - << std::endl << "https://hadoop.apache.org/docs/r2.7.1/hadoop-project-dist/hadoop-hdfs/HdfsPermissionsGuide.html" - << std::endl - << std::endl << " -R operate on files and directories recursively" - << std::endl << " -h display this help and exit" - << std::endl - << std::endl << "Examples:" - << std::endl << "hdfs_chgrp -R new_group hdfs://localhost.localdomain:8020/dir/file" - << std::endl << "hdfs_chgrp new_group /dir/file" - << std::endl; -} - -struct SetOwnerState { - const std::string username; - const std::string groupname; - const std::function handler; - //The request counter is incremented once every time SetOwner async call is made - uint64_t request_counter; - //This boolean will be set when find returns the last result - bool find_is_done; - //Final status to be returned - hdfs::Status status; - //Shared variables will need protection with a lock - std::mutex lock; - SetOwnerState(const std::string & username_, const std::string & groupname_, - const std::function & handler_, - uint64_t request_counter_, bool find_is_done_) - : username(username_), - groupname(groupname_), - handler(handler_), - request_counter(request_counter_), - find_is_done(find_is_done_), - status(), - lock() { - } -}; - -int main(int argc, char *argv[]) { - //We should have 3 or 4 parameters - if (argc != 3 && argc != 4) { - usage(); - exit(EXIT_FAILURE); - } - - bool recursive = false; - int input; - - //Using GetOpt to read in the values - opterr = 0; - while ((input = getopt(argc, argv, "Rh")) != -1) { - switch (input) - { - case 'R': - recursive = 1; - break; - case 'h': - usage(); - exit(EXIT_SUCCESS); - case '?': - if (isprint(optopt)) - std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl; - else - std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl; - usage(); - exit(EXIT_FAILURE); - default: - exit(EXIT_FAILURE); - } - } - std::string group = argv[optind]; - //Owner stays the same, just group association changes. - std::string owner = ""; - std::string uri_path = argv[optind + 1]; - - //Building a URI object from the given uri_path - hdfs::URI uri = hdfs::parse_path_or_exit(uri_path); - - std::shared_ptr fs = hdfs::doConnect(uri, true); - if (!fs) { - std::cerr << "Could not connect the file system. " << std::endl; - exit(EXIT_FAILURE); - } - - /* wrap async FileSystem::SetOwner with promise to make it a blocking call */ - std::shared_ptr> promise = std::make_shared>(); - std::future future(promise->get_future()); - auto handler = [promise](const hdfs::Status &s) { - promise->set_value(s); - }; - - if(!recursive){ - fs->SetOwner(uri.get_path(), owner, group, handler); - } - else { - //Allocating shared state, which includes: - //username and groupname to be set, handler to be called, request counter, and a boolean to keep track if find is done - std::shared_ptr state = std::make_shared(owner, group, handler, 0, false); - - // Keep requesting more from Find until we process the entire listing. Call handler when Find is done and reques counter is 0. - // Find guarantees that the handler will only be called once at a time so we do not need locking in handlerFind. - auto handlerFind = [fs, state](const hdfs::Status &status_find, const std::vector & stat_infos, bool has_more_results) -> bool { - - //For each result returned by Find we call async SetOwner with the handler below. - //SetOwner DOES NOT guarantee that the handler will only be called once at a time, so we DO need locking in handlerSetOwner. - auto handlerSetOwner = [state](const hdfs::Status &status_set_owner) { - std::lock_guard guard(state->lock); - - //Decrement the counter once since we are done with this async call - if (!status_set_owner.ok() && state->status.ok()){ - //We make sure we set state->status only on the first error. - state->status = status_set_owner; - } - state->request_counter--; - if(state->request_counter == 0 && state->find_is_done){ - state->handler(state->status); //exit - } - }; - if(!stat_infos.empty() && state->status.ok()) { - for (hdfs::StatInfo const& s : stat_infos) { - //Launch an asynchronous call to SetOwner for every returned result - state->request_counter++; - fs->SetOwner(s.full_path, state->username, state->groupname, handlerSetOwner); - } - } - - //Lock this section because handlerSetOwner might be accessing the same - //shared variables simultaneously - std::lock_guard guard(state->lock); - if (!status_find.ok() && state->status.ok()){ - //We make sure we set state->status only on the first error. - state->status = status_find; - } - if(!has_more_results){ - state->find_is_done = true; - if(state->request_counter == 0){ - state->handler(state->status); //exit - } - return false; - } - return true; - }; - - //Asynchronous call to Find - fs->Find(uri.get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), handlerFind); - } - - /* block until promise is set */ - hdfs::Status status = future.get(); - if (!status.ok()) { - std::cerr << "Error: " << status.ToString() << std::endl; - exit(EXIT_FAILURE); - } - - // Clean up static data and prevent valgrind memory leaks - google::protobuf::ShutdownProtobufLibrary(); - return 0; -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chmod.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chmod.cc deleted file mode 100644 index 091f18f904ecc..0000000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chmod.cc +++ /dev/null @@ -1,183 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -*/ - -#include -#include -#include -#include "tools_common.h" - -void usage(){ - std::cout << "Usage: hdfs_chmod [OPTION] FILE" - << std::endl - << std::endl << "Change the permissions of each FILE to MODE." - << std::endl << "The user must be the owner of the file, or else a super-user." - << std::endl << "Additional information is in the Permissions Guide:" - << std::endl << "https://hadoop.apache.org/docs/r2.7.1/hadoop-project-dist/hadoop-hdfs/HdfsPermissionsGuide.html" - << std::endl - << std::endl << " -R operate on files and directories recursively" - << std::endl << " -h display this help and exit" - << std::endl - << std::endl << "Examples:" - << std::endl << "hdfs_chmod -R 755 hdfs://localhost.localdomain:8020/dir/file" - << std::endl << "hdfs_chmod 777 /dir/file" - << std::endl; -} - -struct SetPermissionState { - const uint16_t permissions; - const std::function handler; - //The request counter is incremented once every time SetOwner async call is made - uint64_t request_counter; - //This boolean will be set when find returns the last result - bool find_is_done; - //Final status to be returned - hdfs::Status status; - //Shared variables will need protection with a lock - std::mutex lock; - SetPermissionState(const uint16_t permissions_, const std::function & handler_, - uint64_t request_counter_, bool find_is_done_) - : permissions(permissions_), - handler(handler_), - request_counter(request_counter_), - find_is_done(find_is_done_), - status(), - lock() { - } -}; - -int main(int argc, char *argv[]) { - //We should have 3 or 4 parameters - if (argc != 3 && argc != 4) { - usage(); - exit(EXIT_FAILURE); - } - - bool recursive = false; - int input; - - //Using GetOpt to read in the values - opterr = 0; - while ((input = getopt(argc, argv, "Rh")) != -1) { - switch (input) - { - case 'R': - recursive = 1; - break; - case 'h': - usage(); - exit(EXIT_SUCCESS); - case '?': - if (isprint(optopt)) - std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl; - else - std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl; - usage(); - exit(EXIT_FAILURE); - default: - exit(EXIT_FAILURE); - } - } - std::string permissions = argv[optind]; - std::string uri_path = argv[optind + 1]; - - //Building a URI object from the given uri_path - hdfs::URI uri = hdfs::parse_path_or_exit(uri_path); - - std::shared_ptr fs = hdfs::doConnect(uri, true); - if (!fs) { - std::cerr << "Could not connect the file system. " << std::endl; - exit(EXIT_FAILURE); - } - - /* wrap async FileSystem::SetPermission with promise to make it a blocking call */ - std::shared_ptr> promise = std::make_shared>(); - std::future future(promise->get_future()); - auto handler = [promise](const hdfs::Status &s) { - promise->set_value(s); - }; - - //strtol() is reading the value with base 8, NULL because we are reading in just one value. - uint16_t perm = strtol(permissions.c_str(), NULL, 8); - if(!recursive){ - fs->SetPermission(uri.get_path(), perm, handler); - } - else { - //Allocating shared state, which includes: - //permissions to be set, handler to be called, request counter, and a boolean to keep track if find is done - std::shared_ptr state = std::make_shared(perm, handler, 0, false); - - // Keep requesting more from Find until we process the entire listing. Call handler when Find is done and reques counter is 0. - // Find guarantees that the handler will only be called once at a time so we do not need locking in handlerFind. - auto handlerFind = [fs, state](const hdfs::Status &status_find, const std::vector & stat_infos, bool has_more_results) -> bool { - - //For each result returned by Find we call async SetPermission with the handler below. - //SetPermission DOES NOT guarantee that the handler will only be called once at a time, so we DO need locking in handlerSetPermission. - auto handlerSetPermission = [state](const hdfs::Status &status_set_permission) { - std::lock_guard guard(state->lock); - - //Decrement the counter once since we are done with this async call - if (!status_set_permission.ok() && state->status.ok()){ - //We make sure we set state->status only on the first error. - state->status = status_set_permission; - } - state->request_counter--; - if(state->request_counter == 0 && state->find_is_done){ - state->handler(state->status); //exit - } - }; - if(!stat_infos.empty() && state->status.ok()) { - for (hdfs::StatInfo const& s : stat_infos) { - //Launch an asynchronous call to SetPermission for every returned result - state->request_counter++; - fs->SetPermission(s.full_path, state->permissions, handlerSetPermission); - } - } - - //Lock this section because handlerSetPermission might be accessing the same - //shared variables simultaneously - std::lock_guard guard(state->lock); - if (!status_find.ok() && state->status.ok()){ - //We make sure we set state->status only on the first error. - state->status = status_find; - } - if(!has_more_results){ - state->find_is_done = true; - if(state->request_counter == 0){ - state->handler(state->status); //exit - } - return false; - } - return true; - }; - - //Asynchronous call to Find - fs->Find(uri.get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), handlerFind); - } - - /* block until promise is set */ - hdfs::Status status = future.get(); - if (!status.ok()) { - std::cerr << "Error: " << status.ToString() << std::endl; - exit(EXIT_FAILURE); - } - - // Clean up static data and prevent valgrind memory leaks - google::protobuf::ShutdownProtobufLibrary(); - return 0; -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chown.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chown.cc deleted file mode 100644 index 24f7470b4163e..0000000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chown.cc +++ /dev/null @@ -1,195 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -*/ - -#include -#include -#include -#include "tools_common.h" - -void usage(){ - std::cout << "Usage: hdfs_chown [OPTION] [OWNER][:[GROUP]] FILE" - << std::endl - << std::endl << "Change the owner and/or group of each FILE to OWNER and/or GROUP." - << std::endl << "The user must be a super-user. Additional information is in the Permissions Guide:" - << std::endl << "https://hadoop.apache.org/docs/r2.7.1/hadoop-project-dist/hadoop-hdfs/HdfsPermissionsGuide.html" - << std::endl - << std::endl << " -R operate on files and directories recursively" - << std::endl << " -h display this help and exit" - << std::endl - << std::endl << "Owner is unchanged if missing. Group is unchanged if missing." - << std::endl << "OWNER and GROUP may be numeric as well as symbolic." - << std::endl - << std::endl << "Examples:" - << std::endl << "hdfs_chown -R new_owner:new_group hdfs://localhost.localdomain:8020/dir/file" - << std::endl << "hdfs_chown new_owner /dir/file" - << std::endl; -} - -struct SetOwnerState { - const std::string username; - const std::string groupname; - const std::function handler; - //The request counter is incremented once every time SetOwner async call is made - uint64_t request_counter; - //This boolean will be set when find returns the last result - bool find_is_done; - //Final status to be returned - hdfs::Status status; - //Shared variables will need protection with a lock - std::mutex lock; - SetOwnerState(const std::string & username_, const std::string & groupname_, - const std::function & handler_, - uint64_t request_counter_, bool find_is_done_) - : username(username_), - groupname(groupname_), - handler(handler_), - request_counter(request_counter_), - find_is_done(find_is_done_), - status(), - lock() { - } -}; - -int main(int argc, char *argv[]) { - //We should have 3 or 4 parameters - if (argc != 3 && argc != 4) { - usage(); - exit(EXIT_FAILURE); - } - - bool recursive = false; - int input; - - //Using GetOpt to read in the values - opterr = 0; - while ((input = getopt(argc, argv, "Rh")) != -1) { - switch (input) - { - case 'R': - recursive = 1; - break; - case 'h': - usage(); - exit(EXIT_SUCCESS); - case '?': - if (isprint(optopt)) - std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl; - else - std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl; - usage(); - exit(EXIT_FAILURE); - default: - exit(EXIT_FAILURE); - } - } - std::string owner_and_group = argv[optind]; - std::string uri_path = argv[optind + 1]; - - std::string owner, group; - size_t owner_end = owner_and_group.find(":"); - if(owner_end == std::string::npos) { - owner = owner_and_group; - } else { - owner = owner_and_group.substr(0, owner_end); - group = owner_and_group.substr(owner_end + 1); - } - - //Building a URI object from the given uri_path - hdfs::URI uri = hdfs::parse_path_or_exit(uri_path); - - std::shared_ptr fs = hdfs::doConnect(uri, true); - if (!fs) { - std::cerr << "Could not connect the file system. " << std::endl; - exit(EXIT_FAILURE); - } - - /* wrap async FileSystem::SetOwner with promise to make it a blocking call */ - std::shared_ptr> promise = std::make_shared>(); - std::future future(promise->get_future()); - auto handler = [promise](const hdfs::Status &s) { - promise->set_value(s); - }; - - if(!recursive){ - fs->SetOwner(uri.get_path(), owner, group, handler); - } - else { - //Allocating shared state, which includes: - //username and groupname to be set, handler to be called, request counter, and a boolean to keep track if find is done - std::shared_ptr state = std::make_shared(owner, group, handler, 0, false); - - // Keep requesting more from Find until we process the entire listing. Call handler when Find is done and reques counter is 0. - // Find guarantees that the handler will only be called once at a time so we do not need locking in handlerFind. - auto handlerFind = [fs, state](const hdfs::Status &status_find, const std::vector & stat_infos, bool has_more_results) -> bool { - - //For each result returned by Find we call async SetOwner with the handler below. - //SetOwner DOES NOT guarantee that the handler will only be called once at a time, so we DO need locking in handlerSetOwner. - auto handlerSetOwner = [state](const hdfs::Status &status_set_owner) { - std::lock_guard guard(state->lock); - - //Decrement the counter once since we are done with this async call - if (!status_set_owner.ok() && state->status.ok()){ - //We make sure we set state->status only on the first error. - state->status = status_set_owner; - } - state->request_counter--; - if(state->request_counter == 0 && state->find_is_done){ - state->handler(state->status); //exit - } - }; - if(!stat_infos.empty() && state->status.ok()) { - for (hdfs::StatInfo const& s : stat_infos) { - //Launch an asynchronous call to SetOwner for every returned result - state->request_counter++; - fs->SetOwner(s.full_path, state->username, state->groupname, handlerSetOwner); - } - } - - //Lock this section because handlerSetOwner might be accessing the same - //shared variables simultaneously - std::lock_guard guard(state->lock); - if (!status_find.ok() && state->status.ok()){ - //We make sure we set state->status only on the first error. - state->status = status_find; - } - if(!has_more_results){ - state->find_is_done = true; - if(state->request_counter == 0){ - state->handler(state->status); //exit - } - return false; - } - return true; - }; - - //Asynchronous call to Find - fs->Find(uri.get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), handlerFind); - } - - /* block until promise is set */ - hdfs::Status status = future.get(); - if (!status.ok()) { - std::cerr << "Error: " << status.ToString() << std::endl; - exit(EXIT_FAILURE); - } - - // Clean up static data and prevent valgrind memory leaks - google::protobuf::ShutdownProtobufLibrary(); - return 0; -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_copyToLocal.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_copyToLocal.cc deleted file mode 100644 index c9eb193e1baa0..0000000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_copyToLocal.cc +++ /dev/null @@ -1,88 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -*/ - -#include -#include -#include "tools_common.h" - -void usage(){ - std::cout << "Usage: hdfs_copyToLocal [OPTION] SRC_FILE DST_FILE" - << std::endl - << std::endl << "Copy SRC_FILE from hdfs to DST_FILE on the local file system." - << std::endl - << std::endl << " -h display this help and exit" - << std::endl - << std::endl << "Examples:" - << std::endl << "hdfs_copyToLocal hdfs://localhost.localdomain:8020/dir/file /home/usr/myfile" - << std::endl << "hdfs_copyToLocal /dir/file /home/usr/dir/file" - << std::endl; -} - -int main(int argc, char *argv[]) { - if (argc > 4) { - usage(); - exit(EXIT_FAILURE); - } - - int input; - - //Using GetOpt to read in the values - opterr = 0; - while ((input = getopt(argc, argv, "h")) != -1) { - switch (input) - { - case 'h': - usage(); - exit(EXIT_SUCCESS); - case '?': - if (isprint(optopt)) - std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl; - else - std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl; - usage(); - exit(EXIT_FAILURE); - default: - exit(EXIT_FAILURE); - } - } - - std::string uri_path = argv[optind]; - std::string dest = argv[optind+1]; - - //Building a URI object from the given uri_path - hdfs::URI uri = hdfs::parse_path_or_exit(uri_path); - - std::shared_ptr fs = hdfs::doConnect(uri, false); - if (!fs) { - std::cerr << "Could not connect the file system. " << std::endl; - exit(EXIT_FAILURE); - } - - std::FILE* dst_file = std::fopen(dest.c_str(), "wb"); - if(!dst_file){ - std::cerr << "Unable to open the destination file: " << dest << std::endl; - exit(EXIT_FAILURE); - } - readFile(fs, uri.get_path(), 0, dst_file, false); - std::fclose(dst_file); - - // Clean up static data and prevent valgrind memory leaks - google::protobuf::ShutdownProtobufLibrary(); - return 0; -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_count.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_count.cc deleted file mode 100644 index 345ccc6e27959..0000000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_count.cc +++ /dev/null @@ -1,93 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -*/ - -#include -#include -#include "tools_common.h" - -void usage(){ - std::cout << "Usage: hdfs_count [OPTION] FILE" - << std::endl - << std::endl << "Count the number of directories, files and bytes under the path that match the specified FILE pattern." - << std::endl << "The output columns with -count are: DIR_COUNT, FILE_COUNT, CONTENT_SIZE, PATHNAME" - << std::endl - << std::endl << " -q output additional columns before the rest: QUOTA, SPACE_QUOTA, SPACE_CONSUMED" - << std::endl << " -h display this help and exit" - << std::endl - << std::endl << "Examples:" - << std::endl << "hdfs_count hdfs://localhost.localdomain:8020/dir" - << std::endl << "hdfs_count -q /dir1/dir2" - << std::endl; -} - -int main(int argc, char *argv[]) { - //We should have at least 2 arguments - if (argc < 2) { - usage(); - exit(EXIT_FAILURE); - } - - bool quota = false; - int input; - - //Using GetOpt to read in the values - opterr = 0; - while ((input = getopt(argc, argv, "qh")) != -1) { - switch (input) - { - case 'q': - quota = true; - break; - case 'h': - usage(); - exit(EXIT_SUCCESS); - case '?': - if (isprint(optopt)) - std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl; - else - std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl; - usage(); - exit(EXIT_FAILURE); - default: - exit(EXIT_FAILURE); - } - } - std::string uri_path = argv[optind]; - - //Building a URI object from the given uri_path - hdfs::URI uri = hdfs::parse_path_or_exit(uri_path); - - std::shared_ptr fs = hdfs::doConnect(uri, false); - if (!fs) { - std::cerr << "Could not connect the file system. " << std::endl; - exit(EXIT_FAILURE); - } - - hdfs::ContentSummary content_summary; - hdfs::Status status = fs->GetContentSummary(uri.get_path(), content_summary); - if (!status.ok()) { - std::cerr << "Error: " << status.ToString() << std::endl; - exit(EXIT_FAILURE); - } - std::cout << content_summary.str(quota) << std::endl; - - // Clean up static data and prevent valgrind memory leaks - google::protobuf::ShutdownProtobufLibrary(); - return 0; -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_du.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_du.cc deleted file mode 100644 index f6b6e73f09a18..0000000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_du.cc +++ /dev/null @@ -1,176 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -*/ - -#include -#include -#include -#include "tools_common.h" - -void usage(){ - std::cout << "Usage: hdfs_du [OPTION] PATH" - << std::endl - << std::endl << "Displays sizes of files and directories contained in the given PATH" - << std::endl << "or the length of a file in case PATH is just a file" - << std::endl - << std::endl << " -R operate on files and directories recursively" - << std::endl << " -h display this help and exit" - << std::endl - << std::endl << "Examples:" - << std::endl << "hdfs_du hdfs://localhost.localdomain:8020/dir/file" - << std::endl << "hdfs_du -R /dir1/dir2" - << std::endl; -} - -struct GetContentSummaryState { - const std::function handler; - //The request counter is incremented once every time GetContentSummary async call is made - uint64_t request_counter; - //This boolean will be set when find returns the last result - bool find_is_done; - //Final status to be returned - hdfs::Status status; - //Shared variables will need protection with a lock - std::mutex lock; - GetContentSummaryState(const std::function & handler_, - uint64_t request_counter_, bool find_is_done_) - : handler(handler_), - request_counter(request_counter_), - find_is_done(find_is_done_), - status(), - lock() { - } -}; - -int main(int argc, char *argv[]) { - //We should have at least 2 arguments - if (argc < 2) { - usage(); - exit(EXIT_FAILURE); - } - - bool recursive = false; - int input; - - //Using GetOpt to read in the values - opterr = 0; - while ((input = getopt(argc, argv, "Rh")) != -1) { - switch (input) - { - case 'R': - recursive = true; - break; - case 'h': - usage(); - exit(EXIT_SUCCESS); - case '?': - if (isprint(optopt)) - std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl; - else - std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl; - usage(); - exit(EXIT_FAILURE); - default: - exit(EXIT_FAILURE); - } - } - std::string uri_path = argv[optind]; - - //Building a URI object from the given uri_path - hdfs::URI uri = hdfs::parse_path_or_exit(uri_path); - - std::shared_ptr fs = hdfs::doConnect(uri, true); - if (!fs) { - std::cerr << "Could not connect the file system. " << std::endl; - exit(EXIT_FAILURE); - } - - /* wrap async FileSystem::GetContentSummary with promise to make it a blocking call */ - std::shared_ptr> promise = std::make_shared>(); - std::future future(promise->get_future()); - auto handler = [promise](const hdfs::Status &s) { - promise->set_value(s); - }; - - //Allocating shared state, which includes: - //handler to be called, request counter, and a boolean to keep track if find is done - std::shared_ptr state = std::make_shared(handler, 0, false); - - // Keep requesting more from Find until we process the entire listing. Call handler when Find is done and reques counter is 0. - // Find guarantees that the handler will only be called once at a time so we do not need locking in handlerFind. - auto handlerFind = [fs, state](const hdfs::Status &status_find, const std::vector & stat_infos, bool has_more_results) -> bool { - - //For each result returned by Find we call async GetContentSummary with the handler below. - //GetContentSummary DOES NOT guarantee that the handler will only be called once at a time, so we DO need locking in handlerGetContentSummary. - auto handlerGetContentSummary = [state](const hdfs::Status &status_get_summary, const hdfs::ContentSummary &si) { - std::lock_guard guard(state->lock); - std::cout << si.str_du() << std::endl; - //Decrement the counter once since we are done with this async call - if (!status_get_summary.ok() && state->status.ok()){ - //We make sure we set state->status only on the first error. - state->status = status_get_summary; - } - state->request_counter--; - if(state->request_counter == 0 && state->find_is_done){ - state->handler(state->status); //exit - } - }; - if(!stat_infos.empty() && state->status.ok()) { - for (hdfs::StatInfo const& s : stat_infos) { - //Launch an asynchronous call to GetContentSummary for every returned result - state->request_counter++; - fs->GetContentSummary(s.full_path, handlerGetContentSummary); - } - } - - //Lock this section because handlerGetContentSummary might be accessing the same - //shared variables simultaneously - std::lock_guard guard(state->lock); - if (!status_find.ok() && state->status.ok()){ - //We make sure we set state->status only on the first error. - state->status = status_find; - } - if(!has_more_results){ - state->find_is_done = true; - if(state->request_counter == 0){ - state->handler(state->status); //exit - } - return false; - } - return true; - }; - - if(!recursive){ - //Asynchronous call to Find - fs->GetListing(uri.get_path(), handlerFind); - } else { - //Asynchronous call to Find - fs->Find(uri.get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), handlerFind); - } - - /* block until promise is set */ - hdfs::Status status = future.get(); - if (!status.ok()) { - std::cerr << "Error: " << status.ToString() << std::endl; - exit(EXIT_FAILURE); - } - - // Clean up static data and prevent valgrind memory leaks - google::protobuf::ShutdownProtobufLibrary(); - return 0; -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_find.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_find.cc deleted file mode 100644 index 348f851ad3835..0000000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_find.cc +++ /dev/null @@ -1,146 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -*/ - -#include -#include -#include -#include "tools_common.h" - -void usage(){ - std::cout << "Usage: hdfs_find [OPTION] PATH" - << std::endl - << std::endl << "Finds all files recursively starting from the" - << std::endl << "specified PATH and prints their file paths." - << std::endl << "This hdfs_find tool mimics the POSIX find." - << std::endl - << std::endl << "Both PATH and NAME can have wild-cards." - << std::endl - << std::endl << " -n NAME if provided all results will be matching the NAME pattern" - << std::endl << " otherwise, the implicit '*' will be used" - << std::endl << " NAME allows wild-cards" - << std::endl - << std::endl << " -m MAX_DEPTH if provided the maximum depth to recurse after the end of" - << std::endl << " the path is reached will be limited by MAX_DEPTH" - << std::endl << " otherwise, the maximum depth to recurse is unbound" - << std::endl << " MAX_DEPTH can be set to 0 for pure globbing and ignoring" - << std::endl << " the NAME option (no recursion after the end of the path)" - << std::endl - << std::endl << " -h display this help and exit" - << std::endl - << std::endl << "Examples:" - << std::endl << "hdfs_find hdfs://localhost.localdomain:8020/dir?/tree* -n some?file*name" - << std::endl << "hdfs_find / -n file_name -m 3" - << std::endl; -} - -int main(int argc, char *argv[]) { - //We should have at least 2 arguments - if (argc < 2) { - usage(); - exit(EXIT_FAILURE); - } - - int input; - //If NAME is not specified we use implicit "*" - std::string name = "*"; - //If MAX_DEPTH is not specified we use the max value of uint_32_t - uint32_t max_depth = hdfs::FileSystem::GetDefaultFindMaxDepth(); - - //Using GetOpt to read in the values - opterr = 0; - while ((input = getopt(argc, argv, "hn:m:")) != -1) { - switch (input) - { - case 'h': - usage(); - exit(EXIT_SUCCESS); - case 'n': - name = optarg; - break; - case 'm': - max_depth = std::stoi(optarg); - break; - case '?': - if (optopt == 'n' || optopt == 'm') - std::cerr << "Option -" << (char) optopt << " requires an argument." << std::endl; - else if (isprint(optopt)) - std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl; - else - std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl; - usage(); - exit(EXIT_FAILURE); - default: - exit(EXIT_FAILURE); - } - } - std::string uri_path = argv[optind]; - - //Building a URI object from the given uri_path - hdfs::URI uri = hdfs::parse_path_or_exit(uri_path); - - std::shared_ptr fs = hdfs::doConnect(uri, true); - if (!fs) { - std::cerr << "Could not connect the file system. " << std::endl; - exit(EXIT_FAILURE); - } - - std::shared_ptr> promise = std::make_shared>(); - std::future future(promise->get_future()); - hdfs::Status status = hdfs::Status::OK(); - - /** - * Keep requesting more until we get the entire listing. Set the promise - * when we have the entire listing to stop. - * - * Find guarantees that the handler will only be called once at a time, - * so we do not need any locking here. It also guarantees that the handler will be - * only called once with has_more_results set to false. - */ - auto handler = [promise, &status] - (const hdfs::Status &s, const std::vector & si, bool has_more_results) -> bool { - //Print result chunks as they arrive - if(!si.empty()) { - for (hdfs::StatInfo const& s : si) { - std::cout << s.str() << std::endl; - } - } - if(!s.ok() && status.ok()){ - //We make sure we set 'status' only on the first error. - status = s; - } - if (!has_more_results) { - promise->set_value(); //set promise - return false; //request stop sending results - } - return true; //request more results - }; - - //Asynchronous call to Find - fs->Find(uri.get_path(), name, max_depth, handler); - - //block until promise is set - future.get(); - if(!status.ok()) { - std::cerr << "Error: " << status.ToString() << std::endl; - } - - // Clean up static data and prevent valgrind memory leaks - google::protobuf::ShutdownProtobufLibrary(); - return 0; -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_get.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_get.cc deleted file mode 100644 index 16dd72d24bf72..0000000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_get.cc +++ /dev/null @@ -1,88 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -*/ - -#include -#include -#include "tools_common.h" - -void usage(){ - std::cout << "Usage: hdfs_get [OPTION] SRC_FILE DST_FILE" - << std::endl - << std::endl << "Copy SRC_FILE from hdfs to DST_FILE on the local file system." - << std::endl - << std::endl << " -h display this help and exit" - << std::endl - << std::endl << "Examples:" - << std::endl << "hdfs_get hdfs://localhost.localdomain:8020/dir/file /home/usr/myfile" - << std::endl << "hdfs_get /dir/file /home/usr/dir/file" - << std::endl; -} - -int main(int argc, char *argv[]) { - if (argc > 4) { - usage(); - exit(EXIT_FAILURE); - } - - int input; - - //Using GetOpt to read in the values - opterr = 0; - while ((input = getopt(argc, argv, "h")) != -1) { - switch (input) - { - case 'h': - usage(); - exit(EXIT_SUCCESS); - case '?': - if (isprint(optopt)) - std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl; - else - std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl; - usage(); - exit(EXIT_FAILURE); - default: - exit(EXIT_FAILURE); - } - } - - std::string uri_path = argv[optind]; - std::string dest = argv[optind+1]; - - //Building a URI object from the given uri_path - hdfs::URI uri = hdfs::parse_path_or_exit(uri_path); - - std::shared_ptr fs = hdfs::doConnect(uri, false); - if (!fs) { - std::cerr << "Could not connect the file system. " << std::endl; - exit(EXIT_FAILURE); - } - - std::FILE* dst_file = std::fopen(dest.c_str(), "wb"); - if(!dst_file){ - std::cerr << "Unable to open the destination file: " << dest << std::endl; - exit(EXIT_FAILURE); - } - readFile(fs, uri.get_path(), 0, dst_file, false); - std::fclose(dst_file); - - // Clean up static data and prevent valgrind memory leaks - google::protobuf::ShutdownProtobufLibrary(); - return 0; -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_ls.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_ls.cc deleted file mode 100644 index e35e51da1d32c..0000000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_ls.cc +++ /dev/null @@ -1,130 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -*/ - -#include -#include -#include -#include "tools_common.h" - -void usage(){ - std::cout << "Usage: hdfs_ls [OPTION] FILE" - << std::endl - << std::endl << "List information about the FILEs." - << std::endl - << std::endl << " -R list subdirectories recursively" - << std::endl << " -h display this help and exit" - << std::endl - << std::endl << "Examples:" - << std::endl << "hdfs_ls hdfs://localhost.localdomain:8020/dir" - << std::endl << "hdfs_ls -R /dir1/dir2" - << std::endl; -} - -int main(int argc, char *argv[]) { - //We should have at least 2 arguments - if (argc < 2) { - usage(); - exit(EXIT_FAILURE); - } - - bool recursive = false; - int input; - - //Using GetOpt to read in the values - opterr = 0; - while ((input = getopt(argc, argv, "Rh")) != -1) { - switch (input) - { - case 'R': - recursive = true; - break; - case 'h': - usage(); - exit(EXIT_SUCCESS); - case '?': - if (isprint(optopt)) - std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl; - else - std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl; - usage(); - exit(EXIT_FAILURE); - default: - exit(EXIT_FAILURE); - } - } - std::string uri_path = argv[optind]; - - //Building a URI object from the given uri_path - hdfs::URI uri = hdfs::parse_path_or_exit(uri_path); - - std::shared_ptr fs = hdfs::doConnect(uri, true); - if (!fs) { - std::cerr << "Could not connect the file system. " << std::endl; - exit(EXIT_FAILURE); - } - - std::shared_ptr> promise = std::make_shared>(); - std::future future(promise->get_future()); - hdfs::Status status = hdfs::Status::OK(); - - /** - * Keep requesting more until we get the entire listing. Set the promise - * when we have the entire listing to stop. - * - * Find and GetListing guarantee that the handler will only be called once at a time, - * so we do not need any locking here. They also guarantee that the handler will be - * only called once with has_more_results set to false. - */ - auto handler = [promise, &status] - (const hdfs::Status &s, const std::vector & si, bool has_more_results) -> bool { - //Print result chunks as they arrive - if(!si.empty()) { - for (hdfs::StatInfo const& s : si) { - std::cout << s.str() << std::endl; - } - } - if(!s.ok() && status.ok()){ - //We make sure we set 'status' only on the first error. - status = s; - } - if (!has_more_results) { - promise->set_value(); //set promise - return false; //request stop sending results - } - return true; //request more results - }; - - if(!recursive){ - //Asynchronous call to GetListing - fs->GetListing(uri.get_path(), handler); - } else { - //Asynchronous call to Find - fs->Find(uri.get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), handler); - } - - //block until promise is set - future.get(); - if(!status.ok()) { - std::cerr << "Error: " << status.ToString() << std::endl; - } - - // Clean up static data and prevent valgrind memory leaks - google::protobuf::ShutdownProtobufLibrary(); - return 0; -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_mkdir.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_mkdir.cc deleted file mode 100644 index 3ccc6017b58e5..0000000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_mkdir.cc +++ /dev/null @@ -1,98 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -*/ - -#include -#include -#include "tools_common.h" - -void usage(){ - std::cout << "Usage: hdfs_mkdir [OPTION] DIRECTORY" - << std::endl - << std::endl << "Create the DIRECTORY(ies), if they do not already exist." - << std::endl - << std::endl << " -p make parent directories as needed" - << std::endl << " -m MODE set file mode (octal permissions) for the new DIRECTORY(ies)" - << std::endl << " -h display this help and exit" - << std::endl - << std::endl << "Examples:" - << std::endl << "hdfs_mkdir hdfs://localhost.localdomain:8020/dir1/dir2" - << std::endl << "hdfs_mkdir -p /extant_dir/non_extant_dir/non_extant_dir/new_dir" - << std::endl; -} - -int main(int argc, char *argv[]) { - //We should have at least 2 arguments - if (argc < 2) { - usage(); - exit(EXIT_FAILURE); - } - - bool create_parents = false; - uint16_t permissions = hdfs::FileSystem::GetDefaultPermissionMask(); - int input; - - //Using GetOpt to read in the values - opterr = 0; - while ((input = getopt(argc, argv, "pm:h")) != -1) { - switch (input) - { - case 'p': - create_parents = true; - break; - case 'h': - usage(); - exit(EXIT_SUCCESS); - case 'm': - //Get octal permissions for the new DIRECTORY(ies) - permissions = strtol(optarg, NULL, 8); - break; - case '?': - if (optopt == 'm') - std::cerr << "Option -" << (char) optopt << " requires an argument." << std::endl; - else if (isprint(optopt)) - std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl; - else - std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl; - usage(); - exit(EXIT_FAILURE); - default: - exit(EXIT_FAILURE); - } - } - std::string uri_path = argv[optind]; - - //Building a URI object from the given uri_path - hdfs::URI uri = hdfs::parse_path_or_exit(uri_path); - - std::shared_ptr fs = hdfs::doConnect(uri, false); - if (!fs) { - std::cerr << "Could not connect the file system. " << std::endl; - exit(EXIT_FAILURE); - } - - hdfs::Status status = fs->Mkdirs(uri.get_path(), permissions, create_parents); - if (!status.ok()) { - std::cerr << "Error: " << status.ToString() << std::endl; - exit(EXIT_FAILURE); - } - - // Clean up static data and prevent valgrind memory leaks - google::protobuf::ShutdownProtobufLibrary(); - return 0; -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_moveToLocal.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_moveToLocal.cc deleted file mode 100644 index 0b8393747ab11..0000000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_moveToLocal.cc +++ /dev/null @@ -1,90 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -*/ - -#include -#include -#include "tools_common.h" - -void usage(){ - std::cout << "Usage: hdfs_moveToLocal [OPTION] SRC_FILE DST_FILE" - << std::endl - << std::endl << "Move SRC_FILE from hdfs to DST_FILE on the local file system." - << std::endl << "Moving is done by copying SRC_FILE to DST_FILE, and then" - << std::endl << "deleting DST_FILE if copy succeeded." - << std::endl - << std::endl << " -h display this help and exit" - << std::endl - << std::endl << "Examples:" - << std::endl << "hdfs_moveToLocal hdfs://localhost.localdomain:8020/dir/file /home/usr/myfile" - << std::endl << "hdfs_moveToLocal /dir/file /home/usr/dir/file" - << std::endl; -} - -int main(int argc, char *argv[]) { - if (argc > 4) { - usage(); - exit(EXIT_FAILURE); - } - - int input; - - //Using GetOpt to read in the values - opterr = 0; - while ((input = getopt(argc, argv, "h")) != -1) { - switch (input) - { - case 'h': - usage(); - exit(EXIT_SUCCESS); - case '?': - if (isprint(optopt)) - std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl; - else - std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl; - usage(); - exit(EXIT_FAILURE); - default: - exit(EXIT_FAILURE); - } - } - - std::string uri_path = argv[optind]; - std::string dest = argv[optind+1]; - - //Building a URI object from the given uri_path - hdfs::URI uri = hdfs::parse_path_or_exit(uri_path); - - std::shared_ptr fs = hdfs::doConnect(uri, false); - if (!fs) { - std::cerr << "Could not connect the file system. " << std::endl; - exit(EXIT_FAILURE); - } - - std::FILE* dst_file = std::fopen(dest.c_str(), "wb"); - if(!dst_file){ - std::cerr << "Unable to open the destination file: " << dest << std::endl; - exit(EXIT_FAILURE); - } - readFile(fs, uri.get_path(), 0, dst_file, true); - std::fclose(dst_file); - - // Clean up static data and prevent valgrind memory leaks - google::protobuf::ShutdownProtobufLibrary(); - return 0; -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_rm.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_rm.cc deleted file mode 100644 index 7056cf9a674d5..0000000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_rm.cc +++ /dev/null @@ -1,90 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -*/ - -#include -#include -#include "tools_common.h" - -void usage(){ - std::cout << "Usage: hdfs_rm [OPTION] FILE" - << std::endl - << std::endl << "Remove (unlink) the FILE(s) or directory(ies)." - << std::endl - << std::endl << " -R remove directories and their contents recursively" - << std::endl << " -h display this help and exit" - << std::endl - << std::endl << "Examples:" - << std::endl << "hdfs_rm hdfs://localhost.localdomain:8020/dir/file" - << std::endl << "hdfs_rm -R /dir1/dir2" - << std::endl; -} - -int main(int argc, char *argv[]) { - //We should have at least 2 arguments - if (argc < 2) { - usage(); - exit(EXIT_FAILURE); - } - - bool recursive = false; - int input; - - //Using GetOpt to read in the values - opterr = 0; - while ((input = getopt(argc, argv, "Rh")) != -1) { - switch (input) - { - case 'R': - recursive = true; - break; - case 'h': - usage(); - exit(EXIT_SUCCESS); - case '?': - if (isprint(optopt)) - std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl; - else - std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl; - usage(); - exit(EXIT_FAILURE); - default: - exit(EXIT_FAILURE); - } - } - std::string uri_path = argv[optind]; - - //Building a URI object from the given uri_path - hdfs::URI uri = hdfs::parse_path_or_exit(uri_path); - - std::shared_ptr fs = hdfs::doConnect(uri, true); - if (!fs) { - std::cerr << "Could not connect the file system. " << std::endl; - exit(EXIT_FAILURE); - } - - hdfs::Status status = fs->Delete(uri.get_path(), recursive); - if (!status.ok()) { - std::cerr << "Error: " << status.ToString() << std::endl; - exit(EXIT_FAILURE); - } - - // Clean up static data and prevent valgrind memory leaks - google::protobuf::ShutdownProtobufLibrary(); - return 0; -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_setrep.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_setrep.cc deleted file mode 100644 index 019e24d63fea0..0000000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_setrep.cc +++ /dev/null @@ -1,172 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -*/ - -#include -#include -#include -#include "tools_common.h" - -void usage(){ - std::cout << "Usage: hdfs_setrep [OPTION] NUM_REPLICAS PATH" - << std::endl - << std::endl << "Changes the replication factor of a file at PATH. If PATH is a directory then the command" - << std::endl << "recursively changes the replication factor of all files under the directory tree rooted at PATH." - << std::endl - << std::endl << " -h display this help and exit" - << std::endl - << std::endl << "Examples:" - << std::endl << "hdfs_setrep 5 hdfs://localhost.localdomain:8020/dir/file" - << std::endl << "hdfs_setrep 3 /dir1/dir2" - << std::endl; -} - -struct SetReplicationState { - const uint16_t replication; - const std::function handler; - //The request counter is incremented once every time SetReplication async call is made - uint64_t request_counter; - //This boolean will be set when find returns the last result - bool find_is_done; - //Final status to be returned - hdfs::Status status; - //Shared variables will need protection with a lock - std::mutex lock; - SetReplicationState(const uint16_t replication_, const std::function & handler_, - uint64_t request_counter_, bool find_is_done_) - : replication(replication_), - handler(handler_), - request_counter(request_counter_), - find_is_done(find_is_done_), - status(), - lock() { - } -}; - -int main(int argc, char *argv[]) { - //We should have 3 or 4 parameters - if (argc < 3) { - usage(); - exit(EXIT_FAILURE); - } - - int input; - - //Using GetOpt to read in the values - opterr = 0; - while ((input = getopt(argc, argv, "h")) != -1) { - switch (input) - { - case 'h': - usage(); - exit(EXIT_SUCCESS); - case '?': - if (isprint(optopt)) - std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl; - else - std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl; - usage(); - exit(EXIT_FAILURE); - default: - exit(EXIT_FAILURE); - } - } - std::string repl = argv[optind]; - std::string uri_path = argv[optind + 1]; - - //Building a URI object from the given uri_path - hdfs::URI uri = hdfs::parse_path_or_exit(uri_path); - - std::shared_ptr fs = hdfs::doConnect(uri, true); - if (!fs) { - std::cerr << "Could not connect the file system. " << std::endl; - exit(EXIT_FAILURE); - } - - /* wrap async FileSystem::SetReplication with promise to make it a blocking call */ - std::shared_ptr> promise = std::make_shared>(); - std::future future(promise->get_future()); - auto handler = [promise](const hdfs::Status &s) { - promise->set_value(s); - }; - - uint16_t replication = std::stoi(repl.c_str(), NULL, 8); - //Allocating shared state, which includes: - //replication to be set, handler to be called, request counter, and a boolean to keep track if find is done - std::shared_ptr state = std::make_shared(replication, handler, 0, false); - - // Keep requesting more from Find until we process the entire listing. Call handler when Find is done and reques counter is 0. - // Find guarantees that the handler will only be called once at a time so we do not need locking in handlerFind. - auto handlerFind = [fs, state](const hdfs::Status &status_find, const std::vector & stat_infos, bool has_more_results) -> bool { - - //For each result returned by Find we call async SetReplication with the handler below. - //SetReplication DOES NOT guarantee that the handler will only be called once at a time, so we DO need locking in handlerSetReplication. - auto handlerSetReplication = [state](const hdfs::Status &status_set_replication) { - std::lock_guard guard(state->lock); - - //Decrement the counter once since we are done with this async call - if (!status_set_replication.ok() && state->status.ok()){ - //We make sure we set state->status only on the first error. - state->status = status_set_replication; - } - state->request_counter--; - if(state->request_counter == 0 && state->find_is_done){ - state->handler(state->status); //exit - } - }; - if(!stat_infos.empty() && state->status.ok()) { - for (hdfs::StatInfo const& s : stat_infos) { - //Launch an asynchronous call to SetReplication for every returned file - if(s.file_type == hdfs::StatInfo::IS_FILE){ - state->request_counter++; - fs->SetReplication(s.full_path, state->replication, handlerSetReplication); - } - } - } - - //Lock this section because handlerSetReplication might be accessing the same - //shared variables simultaneously - std::lock_guard guard(state->lock); - if (!status_find.ok() && state->status.ok()){ - //We make sure we set state->status only on the first error. - state->status = status_find; - } - if(!has_more_results){ - state->find_is_done = true; - if(state->request_counter == 0){ - state->handler(state->status); //exit - } - return false; - } - return true; - }; - - //Asynchronous call to Find - fs->Find(uri.get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), handlerFind); - - /* block until promise is set */ - hdfs::Status status = future.get(); - if (!status.ok()) { - std::cerr << "Error: " << status.ToString() << std::endl; - exit(EXIT_FAILURE); - } - - // Clean up static data and prevent valgrind memory leaks - google::protobuf::ShutdownProtobufLibrary(); - return 0; -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_stat.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_stat.cc deleted file mode 100644 index 59d513b21afd3..0000000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_stat.cc +++ /dev/null @@ -1,87 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -*/ - -#include -#include -#include "tools_common.h" - -void usage(){ - std::cout << "Usage: hdfs_rm [OPTION] FILE" - << std::endl - << std::endl << "Display FILE status." - << std::endl - << std::endl << " -h display this help and exit" - << std::endl - << std::endl << "Examples:" - << std::endl << "hdfs_rm hdfs://localhost.localdomain:8020/dir/file" - << std::endl << "hdfs_rm -R /dir1/dir2" - << std::endl; -} - -int main(int argc, char *argv[]) { - //We should have at least 2 arguments - if (argc < 2) { - usage(); - exit(EXIT_FAILURE); - } - - int input; - - //Using GetOpt to read in the values - opterr = 0; - while ((input = getopt(argc, argv, "h")) != -1) { - switch (input) - { - case 'h': - usage(); - exit(EXIT_SUCCESS); - case '?': - if (isprint(optopt)) - std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl; - else - std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl; - usage(); - exit(EXIT_FAILURE); - default: - exit(EXIT_FAILURE); - } - } - std::string uri_path = argv[optind]; - - //Building a URI object from the given uri_path - hdfs::URI uri = hdfs::parse_path_or_exit(uri_path); - - std::shared_ptr fs = hdfs::doConnect(uri, false); - if (!fs) { - std::cerr << "Could not connect the file system. " << std::endl; - exit(EXIT_FAILURE); - } - - hdfs::StatInfo stat_info; - hdfs::Status status = fs->GetFileInfo(uri.get_path(), stat_info); - if (!status.ok()) { - std::cerr << "Error: " << status.ToString() << std::endl; - exit(EXIT_FAILURE); - } - std::cout << stat_info.str() << std::endl; - - // Clean up static data and prevent valgrind memory leaks - google::protobuf::ShutdownProtobufLibrary(); - return 0; -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_tail.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_tail.cc deleted file mode 100644 index 8125bdcfecfeb..0000000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_tail.cc +++ /dev/null @@ -1,124 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -*/ - -#include -#include -#include "tools_common.h" - -void usage(){ - std::cout << "Usage: hdfs_tail [OPTION] FILE" - << std::endl - << std::endl << "Displays last kilobyte of the file to stdout." - << std::endl - << std::endl << " -f output appended data as the file grows, as in Unix" - << std::endl << " -h display this help and exit" - << std::endl - << std::endl << "Examples:" - << std::endl << "hdfs_tail hdfs://localhost.localdomain:8020/dir/file" - << std::endl << "hdfs_tail /dir/file" - << std::endl; -} - -#define TAIL_SIZE 1024 -#define REFRESH_RATE 1 //seconds - -int main(int argc, char *argv[]) { - if (argc < 2) { - usage(); - exit(EXIT_FAILURE); - } - - bool follow = false; - int input; - - //Using GetOpt to read in the values - opterr = 0; - while ((input = getopt(argc, argv, "hf")) != -1) { - switch (input) - { - case 'h': - usage(); - exit(EXIT_SUCCESS); - case 'f': - follow = true; - break; - case '?': - if (isprint(optopt)) - std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl; - else - std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl; - usage(); - exit(EXIT_FAILURE); - default: - exit(EXIT_FAILURE); - } - } - - std::string uri_path = argv[optind]; - - //Building a URI object from the given uri_path - hdfs::URI uri = hdfs::parse_path_or_exit(uri_path); - - std::shared_ptr fs = hdfs::doConnect(uri, false); - if (!fs) { - std::cerr << "Could not connect the file system. " << std::endl; - exit(EXIT_FAILURE); - } - - //We need to get the size of the file using stat - hdfs::StatInfo stat_info; - hdfs::Status status = fs->GetFileInfo(uri.get_path(), stat_info); - if (!status.ok()) { - std::cerr << "Error: " << status.ToString() << std::endl; - exit(EXIT_FAILURE); - } - - //Determine where to start reading - off_t offset = 0; - if(stat_info.length > TAIL_SIZE){ - offset = stat_info.length - TAIL_SIZE; - } - - do { - off_t current_length = (off_t) stat_info.length; - readFile(fs, uri.get_path(), offset, stdout, false); - - //Exit if -f flag was not set - if(!follow){ - break; - } - - do{ - //Sleep for the REFRESH_RATE - sleep(REFRESH_RATE); - //Use stat to check the new filesize. - status = fs->GetFileInfo(uri.get_path(), stat_info); - if (!status.ok()) { - std::cerr << "Error: " << status.ToString() << std::endl; - exit(EXIT_FAILURE); - } - //If file became longer, loop back and print the difference - } - while((off_t) stat_info.length <= current_length); - } while (true); - - // Clean up static data and prevent valgrind memory leaks - google::protobuf::ShutdownProtobufLibrary(); - return 0; -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/internal/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/internal/CMakeLists.txt new file mode 100644 index 0000000000000..b223905c9cc99 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/internal/CMakeLists.txt @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +add_library(hdfs_ownership_obj OBJECT hdfs-ownership.cc) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/internal/get-content-summary-state.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/internal/get-content-summary-state.h new file mode 100644 index 0000000000000..f35b55b3cfa6f --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/internal/get-content-summary-state.h @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_DU_GET_CONTENT_SUMMARY_STATE +#define LIBHDFSPP_TOOLS_HDFS_DU_GET_CONTENT_SUMMARY_STATE + +#include +#include +#include + +#include "hdfspp/hdfspp.h" + +namespace hdfs::tools { +/** + * The {@class GetContentSummaryState} is used to hold intermediate information + * during the execution of {@link hdfs::FileSystem#GetContentSummary}. + */ +struct GetContentSummaryState { + GetContentSummaryState(std::function handler, + const uint64_t request_counter, + const bool find_is_done) + : handler{std::move(handler)}, request_counter{request_counter}, + find_is_done{find_is_done} {} + + /** + * The handler that is used to update the status asynchronously. + */ + const std::function handler; + + /** + * The request counter is incremented once every time GetContentSummary async + * call is made. + */ + uint64_t request_counter; + + /** + * This boolean will be set when find returns the last result. + */ + bool find_is_done; + + /** + * Final status to be returned. + */ + hdfs::Status status; + + /** + * Shared variables will need protection with a lock. + */ + std::mutex lock; +}; +} // namespace hdfs::tools + +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/internal/hdfs-ownership.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/internal/hdfs-ownership.cc new file mode 100644 index 0000000000000..e9dc74e541424 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/internal/hdfs-ownership.cc @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "hdfs-ownership.h" + +namespace hdfs::tools { +Ownership::Ownership(const std::string &user_and_group) { + const auto owner_end = user_and_group.find(':'); + if (owner_end == std::string::npos) { + user_ = user_and_group; + return; + } + + user_ = user_and_group.substr(0, owner_end); + group_ = user_and_group.substr(owner_end + 1); +} + +bool Ownership::operator==(const Ownership &other) const { + const auto same_user = user_ == other.user_; + if (group_.has_value() && other.group_.has_value()) { + return same_user && group_.value() == other.group_.value(); + } + + if (!group_.has_value() && !other.group_.has_value()) { + return same_user; + } + return false; +} +} // namespace hdfs::tools diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/internal/hdfs-ownership.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/internal/hdfs-ownership.h new file mode 100644 index 0000000000000..037ab61e58cda --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/internal/hdfs-ownership.h @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_OWNERSHIP +#define LIBHDFSPP_TOOLS_HDFS_OWNERSHIP + +#include +#include +#include +#include + +#include "hdfspp/status.h" + +namespace hdfs::tools { +/** + * {@class Ownership} contains the user and group ownership information. + */ +struct Ownership { + explicit Ownership(const std::string &user_and_group); + + [[nodiscard]] const std::string &GetUser() const { return user_; } + + [[nodiscard]] const std::optional &GetGroup() const { + return group_; + } + + bool operator==(const Ownership &other) const; + +private: + std::string user_; + std::optional group_; +}; + +/** + * {@class OwnerState} holds information needed for recursive traversal of some + * of the HDFS APIs. + */ +struct OwnerState { + OwnerState(std::string username, std::string group, + std::function handler, + const uint64_t request_counter, const bool find_is_done) + : user{std::move(username)}, group{std::move(group)}, handler{std::move( + handler)}, + request_counter{request_counter}, find_is_done{find_is_done} {} + + const std::string user; + const std::string group; + const std::function handler; + + /** + * The request counter is incremented once every time SetOwner async call is + * made. + */ + uint64_t request_counter; + + /** + * This boolean will be set when find returns the last result. + */ + bool find_is_done{false}; + + /** + * Final status to be returned. + */ + hdfs::Status status{}; + + /** + * Shared variables will need protection with a lock. + */ + std::mutex lock; +}; +} // namespace hdfs::tools + +#endif \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/internal/set-replication-state.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/internal/set-replication-state.h new file mode 100644 index 0000000000000..5d432eddbf711 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/internal/set-replication-state.h @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_SET_REPLICATION_STATE +#define LIBHDFSPP_TOOLS_HDFS_SET_REPLICATION_STATE + +#include +#include + +#include "hdfspp/hdfspp.h" + +namespace hdfs::tools { +/** + * {@class SetReplicationState} helps in handling the intermediate results while + * running {@link Setrep}. + */ +struct SetReplicationState { + SetReplicationState(const uint16_t replication, + std::function handler, + const uint64_t request_counter, const bool find_is_done) + : replication{replication}, handler{std::move(handler)}, + request_counter{request_counter}, find_is_done{find_is_done} {} + + /** + * The replication factor. + */ + const uint16_t replication; + + /** + * Handle the given {@link hdfs::Status}. + */ + const std::function handler; + + /** + * The request counter is incremented once every time SetReplication async + * call is made. + */ + uint64_t request_counter; + + /** + * This boolean will be set when find returns the last result. + */ + bool find_is_done; + + /** + * Final status to be returned. + */ + hdfs::Status status; + + /** + * Shared variables will need protection with a lock. + */ + std::mutex lock; +}; +} // namespace hdfs::tools + +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.cc index 6cc5a5b692290..ec59dfbcd24c9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.cc @@ -85,7 +85,6 @@ namespace hdfs { static char input_buffer[BUF_SIZE]; void readFile(std::shared_ptr fs, std::string path, off_t offset, std::FILE* dst_file, bool to_delete) { - ssize_t total_bytes_read = 0; size_t last_bytes_read = 0; hdfs::FileHandle *file_raw = nullptr; @@ -103,7 +102,6 @@ namespace hdfs { if(status.ok()) { //Writing file chunks to stdout fwrite(input_buffer, last_bytes_read, 1, dst_file); - total_bytes_read += last_bytes_read; offset += last_bytes_read; } else { if(status.is_invalid_offset()){ diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestViewfsWithNfs3.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestViewfsWithNfs3.java index a5997b46a9154..4899d9bd4606c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestViewfsWithNfs3.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestViewfsWithNfs3.java @@ -154,8 +154,6 @@ public static void setup() throws Exception { DFSTestUtil.createFile(viewFs, new Path("/hdfs2/write2"), 0, (short) 1, 0); DFSTestUtil.createFile(viewFs, new Path("/hdfs1/renameMultiNN"), 0, (short) 1, 0); - DFSTestUtil.createFile(viewFs, new Path("/hdfs1/renameSingleNN"), - 0, (short) 1, 0); } @AfterClass @@ -307,6 +305,8 @@ public void testNfsRenameMultiNN() throws Exception { @Test (timeout = 60000) public void testNfsRenameSingleNN() throws Exception { + DFSTestUtil.createFile(viewFs, new Path("/hdfs1/renameSingleNN"), + 0, (short) 1, 0); HdfsFileStatus fromFileStatus = nn1.getRpcServer().getFileInfo("/user1"); int fromNNId = Nfs3Utils.getNamenodeId(config, hdfs1.getUri()); FileHandle fromHandle = @@ -316,6 +316,8 @@ public void testNfsRenameSingleNN() throws Exception { nn1.getRpcServer().getFileInfo("/user1/renameSingleNN"); Assert.assertEquals(statusBeforeRename.isDirectory(), false); + Path successFilePath = new Path("/user1/renameSingleNNSucess"); + hdfs1.delete(successFilePath, false); testNfsRename(fromHandle, "renameSingleNN", fromHandle, "renameSingleNNSucess", Nfs3Status.NFS3_OK); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml index e17602d1f6466..d4d5c1eb33983 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml @@ -63,6 +63,12 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> org.apache.hadoop hadoop-hdfs provided + + + org.ow2.asm + asm-commons + + org.apache.hadoop diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/AbstractRouterRpcFairnessPolicyController.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/AbstractRouterRpcFairnessPolicyController.java index 548f1a82f6d83..fe498c66b7ee8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/AbstractRouterRpcFairnessPolicyController.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/AbstractRouterRpcFairnessPolicyController.java @@ -36,7 +36,7 @@ public class AbstractRouterRpcFairnessPolicyController implements RouterRpcFairnessPolicyController { - private static final Logger LOG = + public static final Logger LOG = LoggerFactory.getLogger(AbstractRouterRpcFairnessPolicyController.class); /** Hash table to hold semaphore for each configured name service. */ @@ -64,6 +64,7 @@ public void releasePermit(String nsId) { @Override public void shutdown() { + LOG.debug("Shutting down router fairness policy controller"); // drain all semaphores for (Semaphore sema: this.permits.values()) { sema.drainPermits(); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/RefreshFairnessPolicyControllerHandler.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/RefreshFairnessPolicyControllerHandler.java new file mode 100644 index 0000000000000..f7bc0e8f5a6e0 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/RefreshFairnessPolicyControllerHandler.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.server.federation.fairness; + +import org.apache.hadoop.hdfs.server.federation.router.Router; +import org.apache.hadoop.ipc.RefreshHandler; +import org.apache.hadoop.ipc.RefreshResponse; + +public class RefreshFairnessPolicyControllerHandler implements RefreshHandler { + + final static public String HANDLER_IDENTIFIER = "RefreshFairnessPolicyController"; + private final Router router; + + public RefreshFairnessPolicyControllerHandler(Router router) { + this.router = router; + } + + @Override + public RefreshResponse handleRefresh(String identifier, String[] args) { + if (HANDLER_IDENTIFIER.equals(identifier)) { + return new RefreshResponse(0, router.getRpcServer().refreshFairnessPolicyController()); + } + return new RefreshResponse(-1, "Failed"); + } +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/StaticRouterRpcFairnessPolicyController.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/StaticRouterRpcFairnessPolicyController.java index b4e3dc36b0e37..aa0777fc03d69 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/StaticRouterRpcFairnessPolicyController.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/StaticRouterRpcFairnessPolicyController.java @@ -96,7 +96,8 @@ public void init(Configuration conf) } // Assign remaining handlers if any to fan out calls. - int leftOverHandlers = handlerCount % unassignedNS.size(); + int leftOverHandlers = unassignedNS.isEmpty() ? handlerCount : + handlerCount % unassignedNS.size(); int existingPermits = getAvailablePermits(CONCURRENT_NS); if (leftOverHandlers > 0) { LOG.info("Assigned extra {} handlers to commons pool", leftOverHandlers); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMBean.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMBean.java index b9ea8709e90f9..e8b00d0b5dcfb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMBean.java @@ -343,4 +343,11 @@ public interface FederationMBean { * with the highest risk of loss. */ long getHighestPriorityLowRedundancyECBlocks(); + + /** + * Returns the number of paths to be processed by storage policy satisfier. + * + * @return The number of paths to be processed by sps. + */ + int getPendingSPSPaths(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMetrics.java index 018bf0c60a9e4..823bc7b8af21c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMetrics.java @@ -189,41 +189,49 @@ public long getRouterFailureLockedOps() { } @Override + @Metric({"RpcServerCallQueue", "Length of the rpc server call queue"}) public int getRpcServerCallQueue() { return rpcServer.getServer().getCallQueueLen(); } @Override + @Metric({"RpcServerNumOpenConnections", "Number of the rpc server open connections"}) public int getRpcServerNumOpenConnections() { return rpcServer.getServer().getNumOpenConnections(); } @Override + @Metric({"RpcClientNumConnections", "Number of the rpc client open connections"}) public int getRpcClientNumConnections() { return rpcServer.getRPCClient().getNumConnections(); } @Override + @Metric({"RpcClientNumActiveConnections", "Number of the rpc client active connections"}) public int getRpcClientNumActiveConnections() { return rpcServer.getRPCClient().getNumActiveConnections(); } @Override + @Metric({"RpcClientNumIdleConnections", "Number of the rpc client idle connections"}) public int getRpcClientNumIdleConnections() { return rpcServer.getRPCClient().getNumIdleConnections(); } @Override + @Metric({"RpcClientNumActiveConnectionsRecently", "Number of the rpc client active connections recently"}) public int getRpcClientNumActiveConnectionsRecently() { return rpcServer.getRPCClient().getNumActiveConnectionsRecently(); } @Override + @Metric({"RpcClientNumCreatingConnections", "Number of the rpc client creating connections"}) public int getRpcClientNumCreatingConnections() { return rpcServer.getRPCClient().getNumCreatingConnections(); } @Override + @Metric({"RpcClientNumConnectionPools", "Number of the rpc client connection pools"}) public int getRpcClientNumConnectionPools() { return rpcServer.getRPCClient().getNumConnectionPools(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java index c48728a923c0d..0c62922146311 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java @@ -874,6 +874,16 @@ public long getCurrentTokensCount() { return 0; } + @Override + public int getPendingSPSPaths() { + try { + return getRBFMetrics().getPendingSPSPaths(); + } catch (IOException e) { + LOG.debug("Failed to get number of paths to be processed by sps", e); + } + return 0; + } + private Router getRouter() throws IOException { if (this.router == null) { throw new IOException("Router is not initialized"); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/RBFMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/RBFMetrics.java index 5bba819fd0756..d5eabd1a3da82 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/RBFMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/RBFMetrics.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.server.federation.metrics; +import static org.apache.hadoop.metrics2.impl.MsInfo.ProcessName; import static org.apache.hadoop.util.Time.now; import java.io.IOException; @@ -79,7 +80,11 @@ import org.apache.hadoop.hdfs.server.federation.store.records.MountTable; import org.apache.hadoop.hdfs.server.federation.store.records.RouterState; import org.apache.hadoop.hdfs.server.federation.store.records.StateStoreVersion; +import org.apache.hadoop.metrics2.MetricsSystem; +import org.apache.hadoop.metrics2.annotation.Metric; import org.apache.hadoop.metrics2.annotation.Metrics; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.metrics2.lib.MetricsRegistry; import org.apache.hadoop.metrics2.util.MBeans; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.StringUtils; @@ -100,6 +105,8 @@ public class RBFMetrics implements RouterMBean, FederationMBean { private static final Logger LOG = LoggerFactory.getLogger(RBFMetrics.class); + private final MetricsRegistry registry = new MetricsRegistry("RBFMetrics"); + /** Format for a date. */ private static final String DATE_FORMAT = "yyyy/MM/dd HH:mm:ss"; @@ -171,6 +178,10 @@ public RBFMetrics(Router router) throws IOException { this.topTokenRealOwners = conf.getInt( RBFConfigKeys.DFS_ROUTER_METRICS_TOP_NUM_TOKEN_OWNERS_KEY, RBFConfigKeys.DFS_ROUTER_METRICS_TOP_NUM_TOKEN_OWNERS_KEY_DEFAULT); + + registry.tag(ProcessName, "Router"); + MetricsSystem ms = DefaultMetricsSystem.instance(); + ms.register(RBFMetrics.class.getName(), "RBFActivity Metrics", this); } /** @@ -183,6 +194,8 @@ public void close() { if (this.federationBeanName != null) { MBeans.unregister(federationBeanName); } + MetricsSystem ms = DefaultMetricsSystem.instance(); + ms.unregisterSource(RBFMetrics.class.getName()); } @Override @@ -458,53 +471,65 @@ public int getNumExpiredNamenodes() { } @Override + @Metric({"NumLiveNodes", "Number of live data nodes"}) public int getNumLiveNodes() { return getNameserviceAggregatedInt( MembershipStats::getNumOfActiveDatanodes); } @Override + @Metric({"NumDeadNodes", "Number of dead data nodes"}) public int getNumDeadNodes() { return getNameserviceAggregatedInt(MembershipStats::getNumOfDeadDatanodes); } @Override + @Metric({"NumStaleNodes", "Number of stale data nodes"}) public int getNumStaleNodes() { return getNameserviceAggregatedInt( MembershipStats::getNumOfStaleDatanodes); } @Override + @Metric({"NumDecommissioningNodes", "Number of Decommissioning data nodes"}) public int getNumDecommissioningNodes() { return getNameserviceAggregatedInt( MembershipStats::getNumOfDecommissioningDatanodes); } @Override + @Metric({"NumDecomLiveNodes", "Number of decommissioned Live data nodes"}) public int getNumDecomLiveNodes() { return getNameserviceAggregatedInt( MembershipStats::getNumOfDecomActiveDatanodes); } @Override + @Metric({"NumDecomDeadNodes", "Number of decommissioned dead data nodes"}) public int getNumDecomDeadNodes() { return getNameserviceAggregatedInt( MembershipStats::getNumOfDecomDeadDatanodes); } @Override + @Metric({"NumInMaintenanceLiveDataNodes", + "Number of IN_MAINTENANCE live data nodes"}) public int getNumInMaintenanceLiveDataNodes() { return getNameserviceAggregatedInt( MembershipStats::getNumOfInMaintenanceLiveDataNodes); } @Override + @Metric({"NumInMaintenanceDeadDataNodes", + "Number of IN_MAINTENANCE dead data nodes"}) public int getNumInMaintenanceDeadDataNodes() { return getNameserviceAggregatedInt( MembershipStats::getNumOfInMaintenanceDeadDataNodes); } @Override + @Metric({"NumEnteringMaintenanceDataNodes", + "Number of ENTERING_MAINTENANCE data nodes"}) public int getNumEnteringMaintenanceDataNodes() { return getNameserviceAggregatedInt( MembershipStats::getNumOfEnteringMaintenanceDataNodes); @@ -557,34 +582,41 @@ public String getNodeUsage() { } @Override + @Metric({"NumBlocks", "Total number of blocks"}) public long getNumBlocks() { return getNameserviceAggregatedLong(MembershipStats::getNumOfBlocks); } @Override + @Metric({"NumOfMissingBlocks", "Number of missing blocks"}) public long getNumOfMissingBlocks() { return getNameserviceAggregatedLong(MembershipStats::getNumOfBlocksMissing); } @Override + @Metric({"NumOfBlocksPendingReplication", + "Number of blocks pending replication"}) public long getNumOfBlocksPendingReplication() { return getNameserviceAggregatedLong( MembershipStats::getNumOfBlocksPendingReplication); } @Override + @Metric({"NumOfBlocksUnderReplicated", "Number of blocks under replication"}) public long getNumOfBlocksUnderReplicated() { return getNameserviceAggregatedLong( MembershipStats::getNumOfBlocksUnderReplicated); } @Override + @Metric({"NumOfBlocksPendingDeletion", "Number of blocks pending deletion"}) public long getNumOfBlocksPendingDeletion() { return getNameserviceAggregatedLong( MembershipStats::getNumOfBlocksPendingDeletion); } @Override + @Metric({"NumFiles", "Number of files"}) public long getNumFiles() { return getNameserviceAggregatedLong(MembershipStats::getNumOfFiles); } @@ -659,6 +691,7 @@ public String getRouterStatus() { } @Override + @Metric({"CurrentTokensCount", "Number of router's current tokens"}) public long getCurrentTokensCount() { RouterSecurityManager mgr = this.router.getRpcServer().getRouterSecurityManager(); @@ -714,11 +747,19 @@ public long getHighestPriorityLowRedundancyECBlocks() { } @Override + public int getPendingSPSPaths() { + return getNameserviceAggregatedInt( + MembershipStats::getPendingSPSPaths); + } + + @Override + @Metric({"RouterFederationRenameCount", "Number of federation rename"}) public int getRouterFederationRenameCount() { return this.router.getRpcServer().getRouterFederationRenameCount(); } @Override + @Metric({"SchedulerJobCount", "Number of scheduler job"}) public int getSchedulerJobCount() { return this.router.getRpcServer().getSchedulerJobCount(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java index 13593e694a80e..9f0f78067aedd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java @@ -306,6 +306,7 @@ public boolean registerNamenode(NamenodeStatusReport report) report.getHighestPriorityLowRedundancyReplicatedBlocks()); stats.setHighestPriorityLowRedundancyECBlocks( report.getHighestPriorityLowRedundancyECBlocks()); + stats.setPendingSPSPaths(report.getPendingSPSPaths()); record.setStats(stats); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodeStatusReport.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodeStatusReport.java index feb5a86dba83b..d7da11e6420ef 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodeStatusReport.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodeStatusReport.java @@ -75,6 +75,7 @@ public class NamenodeStatusReport { private long numberOfMissingBlocksWithReplicationFactorOne = -1; private long highestPriorityLowRedundancyReplicatedBlocks = -1; private long highestPriorityLowRedundancyECBlocks = -1; + private int pendingSPSPaths = -1; /** If the fields are valid. */ private boolean registrationValid = false; @@ -367,12 +368,13 @@ public int getNumEnteringMaintenanceDataNodes() { * @param numBlocksPendingReplication Number of blocks pending replication. * @param numBlocksUnderReplicated Number of blocks under replication. * @param numBlocksPendingDeletion Number of blocks pending deletion. - * @param providedSpace Space in provided storage. + * @param providedStorageSpace Space in provided storage. + * @param numPendingSPSPaths The number of paths to be processed by storage policy satisfier. */ public void setNamesystemInfo(long available, long total, long numFiles, long numBlocks, long numBlocksMissing, long numBlocksPendingReplication, long numBlocksUnderReplicated, - long numBlocksPendingDeletion, long providedSpace) { + long numBlocksPendingDeletion, long providedStorageSpace, int numPendingSPSPaths) { this.totalSpace = total; this.availableSpace = available; this.numOfBlocks = numBlocks; @@ -382,7 +384,8 @@ public void setNamesystemInfo(long available, long total, this.numOfBlocksPendingDeletion = numBlocksPendingDeletion; this.numOfFiles = numFiles; this.statsValid = true; - this.providedSpace = providedSpace; + this.providedSpace = providedStorageSpace; + this.pendingSPSPaths = numPendingSPSPaths; } /** @@ -460,6 +463,15 @@ public long getHighestPriorityLowRedundancyECBlocks() { return this.highestPriorityLowRedundancyECBlocks; } + /** + * Returns the number of paths to be processed by storage policy satisfier. + * + * @return The number of paths to be processed by sps. + */ + public int getPendingSPSPaths() { + return this.pendingSPSPaths; + } + /** * Get the number of blocks. * diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java index 1afce83a7e55d..ad9d5e2c2a72c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java @@ -48,6 +48,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.classification.VisibleForTesting; + /** * The {@link Router} periodically checks the state of a Namenode (usually on * the same server) and reports their high availability (HA) state and @@ -267,12 +269,16 @@ private void updateState() { LOG.error("Namenode is not operational: {}", getNamenodeDesc()); } else if (report.haStateValid()) { // block and HA status available - LOG.debug("Received service state: {} from HA namenode: {}", - report.getState(), getNamenodeDesc()); + if (LOG.isDebugEnabled()) { + LOG.debug("Received service state: {} from HA namenode: {}", + report.getState(), getNamenodeDesc()); + } } else if (localTarget == null) { // block info available, HA status not expected - LOG.debug( - "Reporting non-HA namenode as operational: " + getNamenodeDesc()); + if (LOG.isDebugEnabled()) { + LOG.debug( + "Reporting non-HA namenode as operational: {}", getNamenodeDesc()); + } } else { // block info available, HA status should be available, but was not // fetched do nothing and let the current state stand @@ -342,7 +348,8 @@ protected NamenodeStatusReport getNamenodeStatusReport() { // Determine if NN is active // TODO: dynamic timeout if (localTargetHAProtocol == null) { - localTargetHAProtocol = localTarget.getProxy(conf, 30*1000); + localTargetHAProtocol = localTarget.getHealthMonitorProxy(conf, 30*1000); + LOG.debug("Get HA status with address {}", lifelineAddress); } HAServiceStatus status = localTargetHAProtocol.getServiceStatus(); report.setHAServiceState(status.getState()); @@ -369,6 +376,11 @@ protected NamenodeStatusReport getNamenodeStatusReport() { return report; } + @VisibleForTesting + NNHAServiceTarget getLocalTarget(){ + return this.localTarget; + } + /** * Get the description of the Namenode to monitor. * @return Description of the Namenode to monitor. @@ -466,7 +478,8 @@ private void getFsNamesystemMetrics(String address, jsonObject.getLong("PendingReplicationBlocks"), jsonObject.getLong("UnderReplicatedBlocks"), jsonObject.getLong("PendingDeletionBlocks"), - jsonObject.optLong("ProvidedCapacityTotal")); + jsonObject.optLong("ProvidedCapacityTotal"), + jsonObject.getInt("PendingSPSPaths")); } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java index dbfed250156a7..6d52bc26cdf63 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java @@ -196,6 +196,8 @@ protected void serviceInit(Configuration configuration) throws Exception { this.setRpcServerAddress(rpcServer.getRpcAddress()); } + checkRouterId(); + if (conf.getBoolean( RBFConfigKeys.DFS_ROUTER_ADMIN_ENABLE, RBFConfigKeys.DFS_ROUTER_ADMIN_ENABLE_DEFAULT)) { @@ -308,6 +310,21 @@ protected void serviceInit(Configuration configuration) throws Exception { } } + /** + * Set the router id if not set to prevent RouterHeartbeatService + * update state store with a null router id. + */ + private void checkRouterId() { + if (this.routerId == null) { + InetSocketAddress confRpcAddress = conf.getSocketAddr( + RBFConfigKeys.DFS_ROUTER_RPC_BIND_HOST_KEY, + RBFConfigKeys.DFS_ROUTER_RPC_ADDRESS_KEY, + RBFConfigKeys.DFS_ROUTER_RPC_ADDRESS_DEFAULT, + RBFConfigKeys.DFS_ROUTER_RPC_PORT_DEFAULT); + setRpcServerAddress(confRpcAddress); + } + } + private String getDisabledDependentServices() { if (this.stateStore == null && this.adminServer == null) { return StateStoreService.class.getSimpleName() + "," diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java index deb933ad5adb8..127470a1264ed 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java @@ -20,6 +20,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PERMISSIONS_ENABLED_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY; +import static org.apache.hadoop.hdfs.server.federation.fairness.RefreshFairnessPolicyControllerHandler.HANDLER_IDENTIFIER; import java.io.IOException; import java.net.InetSocketAddress; @@ -45,6 +46,7 @@ import org.apache.hadoop.hdfs.protocolPB.RouterAdminProtocolPB; import org.apache.hadoop.hdfs.protocolPB.RouterAdminProtocolServerSideTranslatorPB; import org.apache.hadoop.hdfs.protocolPB.RouterPolicyProvider; +import org.apache.hadoop.hdfs.server.federation.fairness.RefreshFairnessPolicyControllerHandler; import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver; import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamespaceInfo; import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver; @@ -211,6 +213,8 @@ public RouterAdminServer(Configuration conf, Router router) genericRefreshService, adminServer); DFSUtil.addPBProtocol(conf, RefreshCallQueueProtocolPB.class, refreshCallQueueService, adminServer); + + registerRefreshFairnessPolicyControllerHandler(); } /** @@ -784,4 +788,9 @@ public void refreshCallQueue() throws IOException { Configuration configuration = new Configuration(); router.getRpcServer().getServer().refreshCallQueue(configuration); } + + private void registerRefreshFairnessPolicyControllerHandler() { + RefreshRegistry.defaultRegistry() + .register(HANDLER_IDENTIFIER, new RefreshFairnessPolicyControllerHandler(router)); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java index d5bbdf046c2f5..1bd7d65836dbe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java @@ -1815,6 +1815,12 @@ public void satisfyStoragePolicy(String path) throws IOException { storagePolicy.satisfyStoragePolicy(path); } + @Override + public DatanodeInfo[] getSlowDatanodeReport() throws IOException { + rpcServer.checkOperation(NameNode.OperationCategory.UNCHECKED); + return rpcServer.getSlowDatanodeReport(true, 0); + } + @Override public HAServiceProtocol.HAServiceState getHAServiceState() { if (rpcServer.isSafeMode()) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java index 5683f416b3809..34a2c47c3ef29 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java @@ -131,11 +131,8 @@ public class RouterRpcClient { private static final Pattern STACK_TRACE_PATTERN = Pattern.compile("\\tat (.*)\\.(.*)\\((.*):(\\d*)\\)"); - private static final String CLIENT_IP_STR = "clientIp"; - private static final String CLIENT_PORT_STR = "clientPort"; - /** Fairness manager to control handlers assigned per NS. */ - private RouterRpcFairnessPolicyController routerRpcFairnessPolicyController; + private volatile RouterRpcFairnessPolicyController routerRpcFairnessPolicyController; private Map rejectedPermitsPerNs = new ConcurrentHashMap<>(); private Map acceptedPermitsPerNs = new ConcurrentHashMap<>(); @@ -454,7 +451,8 @@ private RetryDecision shouldRetry(final IOException ioe, final int retryCount, * @throws StandbyException If all Namenodes are in Standby. * @throws IOException If it cannot invoke the method. */ - private Object invokeMethod( + @VisibleForTesting + public Object invokeMethod( final UserGroupInformation ugi, final List namenodes, final Class protocol, final Method method, final Object... params) @@ -466,7 +464,7 @@ private Object invokeMethod( + router.getRouterId()); } - appendClientIpPortToCallerContextIfAbsent(); + addClientIpToCallerContext(); Object ret = null; if (rpcMonitor != null) { @@ -588,19 +586,32 @@ private Object invokeMethod( /** * For tracking which is the actual client address. * It adds trace info "clientIp:ip" and "clientPort:port" - * to caller context if they are absent. + * in the caller context, removing the old values if they were + * already present. */ - private void appendClientIpPortToCallerContextIfAbsent() { + private void addClientIpToCallerContext() { CallerContext ctx = CallerContext.getCurrent(); String origContext = ctx == null ? null : ctx.getContext(); byte[] origSignature = ctx == null ? null : ctx.getSignature(); - CallerContext.setCurrent( - new CallerContext.Builder(origContext, contextFieldSeparator) - .appendIfAbsent(CLIENT_IP_STR, Server.getRemoteAddress()) - .appendIfAbsent(CLIENT_PORT_STR, + CallerContext.Builder builder = + new CallerContext.Builder("", contextFieldSeparator) + .append(CallerContext.CLIENT_IP_STR, Server.getRemoteAddress()) + .append(CallerContext.CLIENT_PORT_STR, Integer.toString(Server.getRemotePort())) - .setSignature(origSignature) - .build()); + .setSignature(origSignature); + // Append the original caller context + if (origContext != null) { + for (String part : origContext.split(contextFieldSeparator)) { + String[] keyValue = + part.split(CallerContext.Builder.KEY_VALUE_SEPARATOR, 2); + if (keyValue.length == 2) { + builder.appendIfAbsent(keyValue[0], keyValue[1]); + } else if (keyValue.length == 1) { + builder.append(keyValue[0]); + } + } + } + CallerContext.setCurrent(builder.build()); } /** @@ -818,7 +829,8 @@ public Object invokeSingleBlockPool(final String bpId, RemoteMethod method) public Object invokeSingle(final String nsId, RemoteMethod method) throws IOException { UserGroupInformation ugi = RouterRpcServer.getRemoteUser(); - acquirePermit(nsId, ugi, method); + RouterRpcFairnessPolicyController controller = getRouterRpcFairnessPolicyController(); + acquirePermit(nsId, ugi, method, controller); try { List nns = getNamenodesForNameservice(nsId); @@ -828,7 +840,7 @@ public Object invokeSingle(final String nsId, RemoteMethod method) Object[] params = method.getParams(loc); return invokeMethod(ugi, nns, proto, m, params); } finally { - releasePermit(nsId, ugi, method); + releasePermit(nsId, ugi, method, controller); } } @@ -979,6 +991,7 @@ public RemoteResult invokeSequential( Class expectedResultClass, Object expectedResultValue) throws IOException { + RouterRpcFairnessPolicyController controller = getRouterRpcFairnessPolicyController(); final UserGroupInformation ugi = RouterRpcServer.getRemoteUser(); final Method m = remoteMethod.getMethod(); List thrownExceptions = new ArrayList<>(); @@ -986,7 +999,7 @@ public RemoteResult invokeSequential( // Invoke in priority order for (final RemoteLocationContext loc : locations) { String ns = loc.getNameserviceId(); - acquirePermit(ns, ugi, remoteMethod); + acquirePermit(ns, ugi, remoteMethod, controller); List namenodes = getNamenodesForNameservice(ns); try { @@ -1021,7 +1034,7 @@ public RemoteResult invokeSequential( "Unexpected exception proxying API " + e.getMessage(), e); thrownExceptions.add(ioe); } finally { - releasePermit(ns, ugi, remoteMethod); + releasePermit(ns, ugi, remoteMethod, controller); } } @@ -1346,7 +1359,8 @@ public Map invokeConcurrent( // Shortcut, just one call T location = locations.iterator().next(); String ns = location.getNameserviceId(); - acquirePermit(ns, ugi, method); + RouterRpcFairnessPolicyController controller = getRouterRpcFairnessPolicyController(); + acquirePermit(ns, ugi, method, controller); final List namenodes = getNamenodesForNameservice(ns); try { @@ -1359,7 +1373,7 @@ public Map invokeConcurrent( // Localize the exception throw processException(ioe, location); } finally { - releasePermit(ns, ugi, method); + releasePermit(ns, ugi, method, controller); } } @@ -1409,7 +1423,8 @@ public Map invokeConcurrent( this.router.getRouterClientMetrics().incInvokedConcurrent(m); } - acquirePermit(CONCURRENT_NS, ugi, method); + RouterRpcFairnessPolicyController controller = getRouterRpcFairnessPolicyController(); + acquirePermit(CONCURRENT_NS, ugi, method, controller); try { List> futures = null; if (timeOutMs > 0) { @@ -1467,7 +1482,7 @@ public Map invokeConcurrent( throw new IOException( "Unexpected error while invoking API " + ex.getMessage(), ex); } finally { - releasePermit(CONCURRENT_NS, ugi, method); + releasePermit(CONCURRENT_NS, ugi, method, controller); } } @@ -1548,13 +1563,14 @@ private String getNameserviceForBlockPoolId(final String bpId) * @param nsId Identifier of the block pool. * @param ugi UserGroupIdentifier associated with the user. * @param m Remote method that needs to be invoked. + * @param controller fairness policy controller to acquire permit from * @throws IOException If permit could not be acquired for the nsId. */ - private void acquirePermit( - final String nsId, final UserGroupInformation ugi, final RemoteMethod m) + private void acquirePermit(final String nsId, final UserGroupInformation ugi, + final RemoteMethod m, RouterRpcFairnessPolicyController controller) throws IOException { - if (routerRpcFairnessPolicyController != null) { - if (!routerRpcFairnessPolicyController.acquirePermit(nsId)) { + if (controller != null) { + if (!controller.acquirePermit(nsId)) { // Throw StandByException, // Clients could fail over and try another router. if (rpcMonitor != null) { @@ -1575,21 +1591,20 @@ private void acquirePermit( /** * Release permit for specific nsId after processing against downstream * nsId is completed. - * - * @param nsId Identifier of the block pool. + * @param nsId Identifier of the block pool. * @param ugi UserGroupIdentifier associated with the user. * @param m Remote method that needs to be invoked. + * @param controller fairness policy controller to release permit from */ - private void releasePermit( - final String nsId, final UserGroupInformation ugi, final RemoteMethod m) { - if (routerRpcFairnessPolicyController != null) { - routerRpcFairnessPolicyController.releasePermit(nsId); + private void releasePermit(final String nsId, final UserGroupInformation ugi, + final RemoteMethod m, RouterRpcFairnessPolicyController controller) { + if (controller != null) { + controller.releasePermit(nsId); LOG.trace("Permit released for ugi: {} for method: {}", ugi, m.getMethodName()); } } - @VisibleForTesting public RouterRpcFairnessPolicyController getRouterRpcFairnessPolicyController() { return routerRpcFairnessPolicyController; @@ -1612,4 +1627,35 @@ public Long getAcceptedPermitForNs(String ns) { return acceptedPermitsPerNs.containsKey(ns) ? acceptedPermitsPerNs.get(ns).longValue() : 0L; } + + /** + * Refreshes/changes the fairness policy controller implementation if possible + * and returns the controller class name + * @param conf Configuration + * @return New controller class name if successfully refreshed, else old controller class name + */ + public synchronized String refreshFairnessPolicyController(Configuration conf) { + RouterRpcFairnessPolicyController newController; + try { + newController = FederationUtil.newFairnessPolicyController(conf); + } catch (RuntimeException e) { + LOG.error("Failed to create router fairness policy controller", e); + return getCurrentFairnessPolicyControllerClassName(); + } + + if (newController != null) { + if (routerRpcFairnessPolicyController != null) { + routerRpcFairnessPolicyController.shutdown(); + } + routerRpcFairnessPolicyController = newController; + } + return getCurrentFairnessPolicyControllerClassName(); + } + + private String getCurrentFairnessPolicyControllerClassName() { + if (routerRpcFairnessPolicyController != null) { + return routerRpcFairnessPolicyController.getClass().getCanonicalName(); + } + return null; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java index b8405daca5406..2bec2c726a116 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java @@ -43,7 +43,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.EnumSet; -import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -538,7 +537,7 @@ public FileSubclusterResolver getSubclusterResolver() { } /** - * Get the active namenode resolver + * Get the active namenode resolver. * * @return Active namenode resolver. */ @@ -671,8 +670,8 @@ static String getMethodName() { /** * Invokes the method at default namespace, if default namespace is not - * available then at the first available namespace. - * If the namespace is unavailable, retry once with other namespace. + * available then at the other available namespaces. + * If the namespace is unavailable, retry with other namespaces. * @param expected return type. * @param method the remote method. * @return the response received after invoking method. @@ -681,28 +680,29 @@ static String getMethodName() { T invokeAtAvailableNs(RemoteMethod method, Class clazz) throws IOException { String nsId = subclusterResolver.getDefaultNamespace(); - // If default Ns is not present return result from first namespace. Set nss = namenodeResolver.getNamespaces(); - try { - if (!nsId.isEmpty()) { + // If no namespace is available, then throw this IOException. + IOException io = new IOException("No namespace available."); + // If default Ns is present return result from that namespace. + if (!nsId.isEmpty()) { + try { return rpcClient.invokeSingle(nsId, method, clazz); + } catch (IOException ioe) { + if (!clientProto.isUnavailableSubclusterException(ioe)) { + LOG.debug("{} exception cannot be retried", + ioe.getClass().getSimpleName()); + throw ioe; + } + // Remove the already tried namespace. + nss.removeIf(n -> n.getNameserviceId().equals(nsId)); + return invokeOnNs(method, clazz, io, nss); } - // If no namespace is available, throw IOException. - IOException io = new IOException("No namespace available."); - return invokeOnNs(method, clazz, io, nss); - } catch (IOException ioe) { - if (!clientProto.isUnavailableSubclusterException(ioe)) { - LOG.debug("{} exception cannot be retried", - ioe.getClass().getSimpleName()); - throw ioe; - } - Set nssWithoutFailed = getNameSpaceInfo(nsId); - return invokeOnNs(method, clazz, ioe, nssWithoutFailed); } + return invokeOnNs(method, clazz, io, nss); } /** - * Invoke the method on first available namespace, + * Invoke the method sequentially on available namespaces, * throw no namespace available exception, if no namespaces are available. * @param method the remote method. * @param clazz Class for the return type. @@ -716,24 +716,22 @@ T invokeOnNs(RemoteMethod method, Class clazz, IOException ioe, if (nss.isEmpty()) { throw ioe; } - String nsId = nss.iterator().next().getNameserviceId(); - return rpcClient.invokeSingle(nsId, method, clazz); - } - - /** - * Get set of namespace info's removing the already invoked namespaceinfo. - * @param nsId already invoked namespace id - * @return List of name spaces in the federation on - * removing the already invoked namespaceinfo. - */ - private Set getNameSpaceInfo(String nsId) { - Set namespaceInfos = new HashSet<>(); - for (FederationNamespaceInfo ns : namespaceInfos) { - if (!nsId.equals(ns.getNameserviceId())) { - namespaceInfos.add(ns); + for (FederationNamespaceInfo fnInfo : nss) { + String nsId = fnInfo.getNameserviceId(); + LOG.debug("Invoking {} on namespace {}", method, nsId); + try { + return rpcClient.invokeSingle(nsId, method, clazz); + } catch (IOException e) { + LOG.debug("Failed to invoke {} on namespace {}", method, nsId, e); + // Ignore the exception and try on other namespace, if the tried + // namespace is unavailable, else throw the received exception. + if (!clientProto.isUnavailableSubclusterException(e)) { + throw e; + } } } - return namespaceInfos; + // Couldn't get a response from any of the namespace, throw ioe. + throw ioe; } @Override // ClientProtocol @@ -1097,24 +1095,7 @@ public DatanodeInfo[] getDatanodeReport( Map results = rpcClient.invokeConcurrent(nss, method, requireResponse, false, timeOutMs, DatanodeInfo[].class); - for (Entry entry : - results.entrySet()) { - FederationNamespaceInfo ns = entry.getKey(); - DatanodeInfo[] result = entry.getValue(); - for (DatanodeInfo node : result) { - String nodeId = node.getXferAddr(); - DatanodeInfo dn = datanodesMap.get(nodeId); - if (dn == null || node.getLastUpdate() > dn.getLastUpdate()) { - // Add the subcluster as a suffix to the network location - node.setNetworkLocation( - NodeBase.PATH_SEPARATOR_STR + ns.getNameserviceId() + - node.getNetworkLocation()); - datanodesMap.put(nodeId, node); - } else { - LOG.debug("{} is in multiple subclusters", nodeId); - } - } - } + updateDnMap(results, datanodesMap); // Map -> Array Collection datanodes = datanodesMap.values(); return toArray(datanodes, DatanodeInfo.class); @@ -1580,6 +1561,11 @@ public void satisfyStoragePolicy(String path) throws IOException { clientProto.satisfyStoragePolicy(path); } + @Override // ClientProtocol + public DatanodeInfo[] getSlowDatanodeReport() throws IOException { + return clientProto.getSlowDatanodeReport(); + } + @Override // NamenodeProtocol public BlocksWithLocations getBlocks(DatanodeInfo datanode, long size, long minBlockSize, long hotBlockTimeInterval) throws IOException { @@ -1992,6 +1978,57 @@ public int getSchedulerJobCount() { return fedRenameScheduler.getAllJobs().size(); } + public String refreshFairnessPolicyController() { + return rpcClient.refreshFairnessPolicyController(new Configuration()); + } + + /** + * Get the slow running datanodes report with a timeout. + * + * @param requireResponse If we require all the namespaces to report. + * @param timeOutMs Time out for the reply in milliseconds. + * @return List of datanodes. + * @throws IOException If it cannot get the report. + */ + public DatanodeInfo[] getSlowDatanodeReport(boolean requireResponse, long timeOutMs) + throws IOException { + checkOperation(OperationCategory.UNCHECKED); + + Map datanodesMap = new LinkedHashMap<>(); + RemoteMethod method = new RemoteMethod("getSlowDatanodeReport"); + + Set nss = namenodeResolver.getNamespaces(); + Map results = + rpcClient.invokeConcurrent(nss, method, requireResponse, false, + timeOutMs, DatanodeInfo[].class); + updateDnMap(results, datanodesMap); + // Map -> Array + Collection datanodes = datanodesMap.values(); + return toArray(datanodes, DatanodeInfo.class); + } + + private void updateDnMap(Map results, + Map datanodesMap) { + for (Entry entry : + results.entrySet()) { + FederationNamespaceInfo ns = entry.getKey(); + DatanodeInfo[] result = entry.getValue(); + for (DatanodeInfo node : result) { + String nodeId = node.getXferAddr(); + DatanodeInfo dn = datanodesMap.get(nodeId); + if (dn == null || node.getLastUpdate() > dn.getLastUpdate()) { + // Add the subcluster as a suffix to the network location + node.setNetworkLocation( + NodeBase.PATH_SEPARATOR_STR + ns.getNameserviceId() + + node.getNetworkLocation()); + datanodesMap.put(nodeId, node); + } else { + LOG.debug("{} is in multiple subclusters", nodeId); + } + } + } + } + /** * Deals with loading datanode report into the cache and refresh. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java index a812f198f5e57..accec4627eda8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java @@ -19,6 +19,8 @@ import static org.apache.hadoop.util.StringUtils.getTrimmedStringCollection; +import org.apache.hadoop.fs.InvalidPathException; +import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; @@ -419,6 +421,9 @@ private URI redirectURI(final Router router, final UserGroupInformation ugi, final DoAsParam doAsUser, final String path, final HttpOpParam.Op op, final long openOffset, final String excludeDatanodes, final Param... parameters) throws URISyntaxException, IOException { + if (!DFSUtil.isValidName(path)) { + throw new InvalidPathException(path); + } final DatanodeInfo dn = chooseDatanode(router, path, op, openOffset, excludeDatanodes); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipStats.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipStats.java index 21c8c2f79fce4..3e05a12cd9b9a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipStats.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipStats.java @@ -133,6 +133,10 @@ public abstract void setHighestPriorityLowRedundancyECBlocks( public abstract long getHighestPriorityLowRedundancyECBlocks(); + public abstract void setPendingSPSPaths(int pendingSPSPaths); + + public abstract int getPendingSPSPaths(); + @Override public SortedMap getPrimaryKeys() { // This record is not stored directly, no key needed diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MembershipStatsPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MembershipStatsPBImpl.java index 2caa59dfca7e5..9dff84befa4ee 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MembershipStatsPBImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MembershipStatsPBImpl.java @@ -297,4 +297,14 @@ public long getHighestPriorityLowRedundancyECBlocks() { return this.translator.getProtoOrBuilder() .getHighestPriorityLowRedundancyECBlocks(); } + + @Override + public void setPendingSPSPaths(int pendingSPSPaths) { + this.translator.getBuilder().setPendingSPSPaths(pendingSPSPaths); + } + + @Override + public int getPendingSPSPaths() { + return this.translator.getProtoOrBuilder().getPendingSPSPaths(); + } } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/FederationProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/FederationProtocol.proto index 4a83ebc6ca267..336130e419a3a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/FederationProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/FederationProtocol.proto @@ -54,6 +54,7 @@ message NamenodeMembershipStatsRecordProto { optional uint64 numberOfMissingBlocksWithReplicationFactorOne = 31; optional uint64 highestPriorityLowRedundancyReplicatedBlocks = 32; optional uint64 HighestPriorityLowRedundancyECBlocks = 33; + optional uint32 pendingSPSPaths = 34; } message NamenodeMembershipRecordProto { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/explorer.html b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/explorer.html index 49c3e6606accc..15f7d7feb3848 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/explorer.html +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/explorer.html @@ -311,7 +311,7 @@

- + diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.js b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.js index dc24a322d770b..6220ca14a8cbb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.js +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.js @@ -475,6 +475,22 @@ var base = dust.makeBase(HELPERS); dust.render('mounttable', base.push(data), function(err, out) { $('#tab-mounttable').html(out); + $('#table-mounttable').dataTable( { + 'lengthMenu': [ [25, 50, 100, -1], [25, 50, 100, "All"] ], + 'columns': [ + { 'orderDataType': 'ng-value', 'searchable': true }, + { 'orderDataType': 'ng-value', 'searchable': true }, + { 'orderDataType': 'ng-value', 'searchable': true }, + { 'type': 'string' , "defaultContent": "" }, + { 'type': 'string' , "defaultContent": "" }, + { 'type': 'string' , "defaultContent": "" }, + { 'type': 'string' , "defaultContent": "" }, + { 'type': 'string' , "defaultContent": "" }, + { 'type': 'string' , "defaultContent": "" }, + { 'type': 'string' , "defaultContent": "" }, + { 'type': 'string' , "defaultContent": "" }, + { 'type': 'string' , "defaultContent": "" } + ]}); $('#ui-tabs a[href="#tab-mounttable"]').tab('show'); }); })).fail(ajax_error_handler); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md index de92f86d63c52..ff2cea5761273 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md @@ -375,6 +375,20 @@ With this setting a user can interact with `ns-fed` as a regular namespace: This federated namespace can also be set as the default one at **core-site.xml** using `fs.defaultFS`. +NameNode configuration +-------------------- + +In order for the system to support data-locality, you must configure your NameNodes so that they will trust the routers to supply the user's client IP address. `dfs.namenode.ip-proxy-users` defines a comma separated list of users that are allowed to provide the client ip address via the caller context. + +```xml + + + dfs.namenode.ip-proxy-users + hdfs + + +``` + Router configuration -------------------- @@ -513,7 +527,7 @@ More metrics info can see [RBF Metrics](../../hadoop-project-dist/hadoop-common/ Router Federation Rename ------- -Enable Router to rename across namespaces. Currently it is implemented based on [HDFS Federation Balance](../../../hadoop-federation-balance/HDFSFederationBalance.md) and has some limits comparing with normal rename. +Enable Router to rename across namespaces. Currently it is implemented based on [HDFS Federation Balance](../../hadoop-federation-balance/HDFSFederationBalance.md) and has some limits comparing with normal rename. 1. It is much slower than the normal rename so need a longer RPC timeout configuration. See `ipc.client.rpc-timeout.ms` and its description for more information about RPC timeout. 2. It doesn't support snapshot path. 3. It doesn't support to rename path with multiple destinations. diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterRefreshFairnessPolicyController.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterRefreshFairnessPolicyController.java new file mode 100644 index 0000000000000..dfda47b9a53f4 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterRefreshFairnessPolicyController.java @@ -0,0 +1,234 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.server.federation.fairness; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.mockito.Mockito; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.event.Level; + +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster; +import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; +import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster; +import org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys; +import org.apache.hadoop.hdfs.server.federation.router.RemoteMethod; +import org.apache.hadoop.hdfs.server.federation.router.RouterRpcClient; +import org.apache.hadoop.test.GenericTestUtils; + +import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_FAIR_HANDLER_COUNT_KEY_PREFIX; +import static org.junit.Assert.assertEquals; + +public class TestRouterRefreshFairnessPolicyController { + + private static final Logger LOG = + LoggerFactory.getLogger(TestRouterRefreshFairnessPolicyController.class); + private final GenericTestUtils.LogCapturer controllerLog = + GenericTestUtils.LogCapturer.captureLogs(AbstractRouterRpcFairnessPolicyController.LOG); + + private StateStoreDFSCluster cluster; + + @BeforeClass + public static void setLogLevel() { + GenericTestUtils.setLogLevel(AbstractRouterRpcFairnessPolicyController.LOG, Level.DEBUG); + } + + @After + public void cleanup() { + if (cluster != null) { + cluster.shutdown(); + cluster = null; + } + } + + @Before + public void setupCluster() throws Exception { + cluster = new StateStoreDFSCluster(false, 2); + Configuration conf = new RouterConfigBuilder().stateStore().rpc().build(); + + // Handlers concurrent:ns0 = 3:3 + conf.setClass(RBFConfigKeys.DFS_ROUTER_FAIRNESS_POLICY_CONTROLLER_CLASS, + StaticRouterRpcFairnessPolicyController.class, RouterRpcFairnessPolicyController.class); + conf.setInt(RBFConfigKeys.DFS_ROUTER_HANDLER_COUNT_KEY, 9); + // Allow metrics + conf.setBoolean(RBFConfigKeys.DFS_ROUTER_METRICS_ENABLE, true); + + // Datanodes not needed for this test. + cluster.setNumDatanodesPerNameservice(0); + + cluster.addRouterOverrides(conf); + cluster.startCluster(); + cluster.startRouters(); + cluster.waitClusterUp(); + } + + @Test + public void testRefreshNonexistentHandlerClass() { + MiniRouterDFSCluster.RouterContext routerContext = cluster.getRandomRouter(); + routerContext.getConf().set(RBFConfigKeys.DFS_ROUTER_FAIRNESS_POLICY_CONTROLLER_CLASS, + "org.apache.hadoop.hdfs.server.federation.fairness.ThisControllerDoesNotExist"); + assertEquals(StaticRouterRpcFairnessPolicyController.class.getCanonicalName(), + routerContext.getRouterRpcClient() + .refreshFairnessPolicyController(routerContext.getConf())); + } + + @Test + public void testRefreshClassDoesNotImplementControllerInterface() { + MiniRouterDFSCluster.RouterContext routerContext = cluster.getRandomRouter(); + routerContext.getConf() + .set(RBFConfigKeys.DFS_ROUTER_FAIRNESS_POLICY_CONTROLLER_CLASS, "java.lang.String"); + assertEquals(StaticRouterRpcFairnessPolicyController.class.getCanonicalName(), + routerContext.getRouterRpcClient() + .refreshFairnessPolicyController(routerContext.getConf())); + } + + @Test + public void testRefreshSuccessful() { + MiniRouterDFSCluster.RouterContext routerContext = cluster.getRandomRouter(); + + routerContext.getConf().set(RBFConfigKeys.DFS_ROUTER_FAIRNESS_POLICY_CONTROLLER_CLASS, + StaticRouterRpcFairnessPolicyController.class.getCanonicalName()); + assertEquals(StaticRouterRpcFairnessPolicyController.class.getCanonicalName(), + routerContext.getRouterRpcClient() + .refreshFairnessPolicyController(routerContext.getConf())); + + routerContext.getConf().set(RBFConfigKeys.DFS_ROUTER_FAIRNESS_POLICY_CONTROLLER_CLASS, + NoRouterRpcFairnessPolicyController.class.getCanonicalName()); + assertEquals(NoRouterRpcFairnessPolicyController.class.getCanonicalName(), + routerContext.getRouterRpcClient() + .refreshFairnessPolicyController(routerContext.getConf())); + } + + @Test + public void testConcurrentRefreshRequests() throws InterruptedException { + MiniRouterDFSCluster.RouterContext routerContext = cluster.getRandomRouter(); + RouterRpcClient client = Mockito.spy(routerContext.getRouterRpcClient()); + controllerLog.clearOutput(); + + // Spawn 100 concurrent refresh requests + Thread[] threads = new Thread[100]; + for (int i = 0; i < 100; i++) { + threads[i] = new Thread(() -> { + client.refreshFairnessPolicyController(routerContext.getConf()); + }); + } + + for (Thread thread : threads) { + thread.start(); + } + + for (Thread thread : threads) { + thread.join(); + } + + // There should be 100 controller shutdowns. All controllers created should be shut down. + assertEquals(100, StringUtils.countMatches(controllerLog.getOutput(), + "Shutting down router fairness policy controller")); + controllerLog.clearOutput(); + } + + @Test + public void testRefreshStaticChangeHandlers() throws Exception { + // Setup and mock + MiniRouterDFSCluster.RouterContext routerContext = cluster.getRandomRouter(); + RouterRpcClient client = Mockito.spy(routerContext.getRouterRpcClient()); + final long sleepTime = 3000; + Mockito.doAnswer(invocationOnMock -> { + Thread.sleep(sleepTime); + return null; + }).when(client) + .invokeMethod(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + + // No calls yet + assertEquals("{}", + routerContext.getRouterRpcServer().getRPCMetrics().getProxyOpPermitAcceptedPerNs()); + List preRefreshInvocations = makeDummyInvocations(client, 4, "ns0"); + + Thread.sleep(2000); + // 3 permits acquired, calls will take 3s to finish and release permits + // 1 invocation rejected + assertEquals("{\"ns0\":3}", + routerContext.getRouterRpcServer().getRPCMetrics().getProxyOpPermitAcceptedPerNs()); + assertEquals("{\"ns0\":1}", + routerContext.getRouterRpcServer().getRPCMetrics().getProxyOpPermitRejectedPerNs()); + + Configuration conf = routerContext.getConf(); + final int newNs0Permits = 2; + final int newNs1Permits = 4; + conf.setInt(DFS_ROUTER_FAIR_HANDLER_COUNT_KEY_PREFIX + "ns0", newNs0Permits); + conf.setInt(DFS_ROUTER_FAIR_HANDLER_COUNT_KEY_PREFIX + "ns1", newNs1Permits); + Thread threadRefreshController = new Thread(() -> { + client.refreshFairnessPolicyController(routerContext.getConf()); + }); + threadRefreshController.start(); + threadRefreshController.join(); + + // Wait for all dummy invocation threads to finish + for (Thread thread : preRefreshInvocations) { + thread.join(); + } + + // Controller should now have 2:4 handlers for ns0:ns1 + // Make 4 calls to ns0 and 6 calls to ns1 so that each will fail twice + StaticRouterRpcFairnessPolicyController controller = + (StaticRouterRpcFairnessPolicyController) client.getRouterRpcFairnessPolicyController(); + System.out.println(controller.getAvailableHandlerOnPerNs()); + List ns0Invocations = makeDummyInvocations(client, newNs0Permits + 2, "ns0"); + List ns1Invocations = makeDummyInvocations(client, newNs1Permits + 2, "ns1"); + + // Wait for these threads to finish + for (Thread thread : ns0Invocations) { + thread.join(); + } + for (Thread thread : ns1Invocations) { + thread.join(); + } + assertEquals("{\"ns0\":5,\"ns1\":4}", + routerContext.getRouterRpcServer().getRPCMetrics().getProxyOpPermitAcceptedPerNs()); + assertEquals("{\"ns0\":3,\"ns1\":2}", + routerContext.getRouterRpcServer().getRPCMetrics().getProxyOpPermitRejectedPerNs()); + } + + private List makeDummyInvocations(RouterRpcClient client, final int nThreads, + final String namespace) { + RemoteMethod dummyMethod = Mockito.mock(RemoteMethod.class); + List threadAcquirePermits = new ArrayList<>(); + for (int i = 0; i < nThreads; i++) { + Thread threadAcquirePermit = new Thread(() -> { + try { + client.invokeSingle(namespace, dummyMethod); + } catch (IOException e) { + e.printStackTrace(); + } + }); + threadAcquirePermits.add(threadAcquirePermit); + threadAcquirePermit.start(); + } + return threadAcquirePermits; + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterRpcFairnessPolicyController.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterRpcFairnessPolicyController.java index 9b301382d9596..8307f666b5d1c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterRpcFairnessPolicyController.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterRpcFairnessPolicyController.java @@ -129,6 +129,30 @@ public void testAllocationErrorForLowPreconfiguredHandlers() { verifyInstantiationError(conf, 1, 4); } + @Test + public void testHandlerAllocationConcurrentConfigured() { + Configuration conf = createConf(5); + conf.setInt(DFS_ROUTER_FAIR_HANDLER_COUNT_KEY_PREFIX + "ns1", 1); + conf.setInt(DFS_ROUTER_FAIR_HANDLER_COUNT_KEY_PREFIX + "ns2", 1); + conf.setInt(DFS_ROUTER_FAIR_HANDLER_COUNT_KEY_PREFIX + "concurrent", 1); + RouterRpcFairnessPolicyController routerRpcFairnessPolicyController = + FederationUtil.newFairnessPolicyController(conf); + + // ns1, ns2 should have 1 permit each + assertTrue(routerRpcFairnessPolicyController.acquirePermit("ns1")); + assertTrue(routerRpcFairnessPolicyController.acquirePermit("ns2")); + assertFalse(routerRpcFairnessPolicyController.acquirePermit("ns1")); + assertFalse(routerRpcFairnessPolicyController.acquirePermit("ns2")); + + // concurrent should have 3 permits + for (int i=0; i<3; i++) { + assertTrue( + routerRpcFairnessPolicyController.acquirePermit(CONCURRENT_NS)); + } + assertFalse(routerRpcFairnessPolicyController.acquirePermit(CONCURRENT_NS)); + } + + private void verifyInstantiationError(Configuration conf, int handlerCount, int totalDedicatedHandlers) { GenericTestUtils.LogCapturer logs = GenericTestUtils.LogCapturer diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRBFMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRBFMetrics.java index 25473f8df9233..c86397b511de6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRBFMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRBFMetrics.java @@ -219,6 +219,8 @@ public void testNameserviceStatsDataSource() json.getLong("numOfEnteringMaintenanceDataNodes")); assertEquals(stats.getProvidedSpace(), json.getLong("providedSpace")); + assertEquals(stats.getPendingSPSPaths(), + json.getInt("pendingSPSPaths")); nameservicesFound++; } assertEquals(getNameservices().size(), nameservicesFound); @@ -296,6 +298,7 @@ private void validateClusterStatsFederationBean(FederationMBean bean) { long highestPriorityLowRedundancyReplicatedBlocks = 0; long highestPriorityLowRedundancyECBlocks = 0; long numFiles = 0; + int pendingSPSPaths = 0; for (MembershipState mock : getActiveMemberships()) { MembershipStats stats = mock.getStats(); numBlocks += stats.getNumOfBlocks(); @@ -316,6 +319,7 @@ private void validateClusterStatsFederationBean(FederationMBean bean) { stats.getHighestPriorityLowRedundancyReplicatedBlocks(); highestPriorityLowRedundancyECBlocks += stats.getHighestPriorityLowRedundancyECBlocks(); + pendingSPSPaths += stats.getPendingSPSPaths(); } assertEquals(numBlocks, bean.getNumBlocks()); @@ -342,6 +346,7 @@ private void validateClusterStatsFederationBean(FederationMBean bean) { bean.getHighestPriorityLowRedundancyReplicatedBlocks()); assertEquals(highestPriorityLowRedundancyECBlocks, bean.getHighestPriorityLowRedundancyECBlocks()); + assertEquals(pendingSPSPaths, bean.getPendingSPSPaths()); } private void validateClusterStatsRouterBean(RouterMBean bean) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java index 44c0fc7ed3095..0464877d3cd80 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java @@ -223,10 +223,14 @@ public void testRouterMetricsWhenDisabled() throws Exception { @Test public void testSwitchRouter() throws IOException { - assertRouterHeartbeater(true, true); - assertRouterHeartbeater(true, false); - assertRouterHeartbeater(false, true); - assertRouterHeartbeater(false, false); + assertRouterHeartbeater(true, true, true); + assertRouterHeartbeater(true, true, false); + assertRouterHeartbeater(true, false, true); + assertRouterHeartbeater(true, false, false); + assertRouterHeartbeater(false, true, true); + assertRouterHeartbeater(false, true, false); + assertRouterHeartbeater(false, false, true); + assertRouterHeartbeater(false, false, false); } /** @@ -235,15 +239,19 @@ public void testSwitchRouter() throws IOException { * @param expectedRouterHeartbeat expect the routerHeartbeat enable state. * @param expectedNNHeartbeat expect the nnHeartbeat enable state. */ - private void assertRouterHeartbeater(boolean expectedRouterHeartbeat, + private void assertRouterHeartbeater(boolean enableRpcServer, boolean expectedRouterHeartbeat, boolean expectedNNHeartbeat) throws IOException { final Router router = new Router(); - Configuration baseCfg = new RouterConfigBuilder(conf).rpc().build(); + Configuration baseCfg = new RouterConfigBuilder(conf).rpc(enableRpcServer).build(); baseCfg.setBoolean(RBFConfigKeys.DFS_ROUTER_HEARTBEAT_ENABLE, expectedRouterHeartbeat); baseCfg.setBoolean(RBFConfigKeys.DFS_ROUTER_NAMENODE_HEARTBEAT_ENABLE, expectedNNHeartbeat); router.init(baseCfg); + + // RouterId can not be null , used by RouterHeartbeatService.updateStateStore() + assertNotNull(router.getRouterId()); + RouterHeartbeatService routerHeartbeatService = router.getRouterHeartbeatService(); if (expectedRouterHeartbeat) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterClientRejectOverload.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterClientRejectOverload.java index cc7f5a61a22ff..71ec747af4c30 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterClientRejectOverload.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterClientRejectOverload.java @@ -50,7 +50,8 @@ import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.ipc.StandbyException; import org.apache.hadoop.test.GenericTestUtils; -import org.codehaus.jackson.map.ObjectMapper; + +import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.After; import org.junit.Rule; import org.junit.Test; diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeHeartbeat.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeHeartbeat.java index 9bf149c6490d7..94f2baeaed136 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeHeartbeat.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeHeartbeat.java @@ -211,6 +211,50 @@ public void testHearbeat() throws InterruptedException, IOException { assertEquals(NAMENODES[1], standby.getNamenodeId()); } + @Test + public void testNamenodeHeartbeatServiceHAServiceProtocolProxy(){ + testNamenodeHeartbeatServiceHAServiceProtocol( + "test-ns", "nn", 1000, -1, -1, 1003, + "host01.test:1000", "host02.test:1000"); + testNamenodeHeartbeatServiceHAServiceProtocol( + "test-ns", "nn", 1000, 1001, -1, 1003, + "host01.test:1001", "host02.test:1001"); + testNamenodeHeartbeatServiceHAServiceProtocol( + "test-ns", "nn", 1000, -1, 1002, 1003, + "host01.test:1002", "host02.test:1002"); + testNamenodeHeartbeatServiceHAServiceProtocol( + "test-ns", "nn", 1000, 1001, 1002, 1003, + "host01.test:1002", "host02.test:1002"); + } + + private void testNamenodeHeartbeatServiceHAServiceProtocol( + String nsId, String nnId, + int rpcPort, int servicePort, + int lifelinePort, int webAddressPort, + String expected0, String expected1) { + Configuration conf = generateNamenodeConfiguration(nsId, nnId, + rpcPort, servicePort, lifelinePort, webAddressPort); + + Router testRouter = new Router(); + testRouter.setConf(conf); + + Collection heartbeatServices = + testRouter.createNamenodeHeartbeatServices(); + + assertEquals(2, heartbeatServices.size()); + + Iterator iterator = heartbeatServices.iterator(); + NamenodeHeartbeatService service0 = iterator.next(); + service0.init(conf); + assertNotNull(service0.getLocalTarget()); + assertEquals(expected0, service0.getLocalTarget().getHealthMonitorAddress().toString()); + + NamenodeHeartbeatService service1 = iterator.next(); + service1.init(conf); + assertNotNull(service1.getLocalTarget()); + assertEquals(expected1, service1.getLocalTarget().getHealthMonitorAddress().toString()); + } + @Test public void testNamenodeHeartbeatServiceNNResolution() { String nsId = "test-ns"; @@ -261,10 +305,14 @@ private Configuration generateNamenodeConfiguration( conf.set(DFS_NAMENODE_RPC_ADDRESS_KEY + "." + suffix, MockDomainNameResolver.DOMAIN + ":" + rpcPort); - conf.set(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY + "." + suffix, - MockDomainNameResolver.DOMAIN + ":" + servicePort); - conf.set(DFS_NAMENODE_LIFELINE_RPC_ADDRESS_KEY + "." + suffix, - MockDomainNameResolver.DOMAIN + ":" + lifelinePort); + if (servicePort >= 0){ + conf.set(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY + "." + suffix, + MockDomainNameResolver.DOMAIN + ":" + servicePort); + } + if (lifelinePort >= 0){ + conf.set(DFS_NAMENODE_LIFELINE_RPC_ADDRESS_KEY + "." + suffix, + MockDomainNameResolver.DOMAIN + ":" + lifelinePort); + } conf.set(DFS_NAMENODE_HTTP_ADDRESS_KEY + "." + suffix, MockDomainNameResolver.DOMAIN + ":" + webAddressPort); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCMultipleDestinationMountTableResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCMultipleDestinationMountTableResolver.java index 2887c08f15a11..238d1b0301180 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCMultipleDestinationMountTableResolver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCMultipleDestinationMountTableResolver.java @@ -20,6 +20,7 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -39,6 +40,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FsServerDefaults; import org.apache.hadoop.fs.Options.Rename; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.AclEntry; @@ -74,13 +76,14 @@ * Tests router rpc with multiple destination mount table resolver. */ public class TestRouterRPCMultipleDestinationMountTableResolver { - private static final List NS_IDS = Arrays.asList("ns0", "ns1"); + private static final List NS_IDS = Arrays.asList("ns0", "ns1", "ns2"); private static StateStoreDFSCluster cluster; private static RouterContext routerContext; private static MountTableResolver resolver; private static DistributedFileSystem nnFs0; private static DistributedFileSystem nnFs1; + private static DistributedFileSystem nnFs2; private static DistributedFileSystem routerFs; private static RouterRpcServer rpcServer; @@ -88,7 +91,7 @@ public class TestRouterRPCMultipleDestinationMountTableResolver { public static void setUp() throws Exception { // Build and start a federated cluster - cluster = new StateStoreDFSCluster(false, 2, + cluster = new StateStoreDFSCluster(false, 3, MultipleDestinationMountTableResolver.class); Configuration routerConf = new RouterConfigBuilder().stateStore().admin().quota().rpc().build(); @@ -109,6 +112,8 @@ public static void setUp() throws Exception { .getNamenode(cluster.getNameservices().get(0), null).getFileSystem(); nnFs1 = (DistributedFileSystem) cluster .getNamenode(cluster.getNameservices().get(1), null).getFileSystem(); + nnFs2 = (DistributedFileSystem) cluster + .getNamenode(cluster.getNameservices().get(2), null).getFileSystem(); routerFs = (DistributedFileSystem) routerContext.getFileSystem(); rpcServer =routerContext.getRouter().getRpcServer(); } @@ -643,6 +648,42 @@ public void testContentSummaryMultipleDestWithMaxValue() assertEquals(ssQuota, cs.getSpaceQuota()); } + /** + * Test RouterRpcServer#invokeAtAvailableNs on mount point with multiple destinations + * and making a one of the destination's subcluster unavailable. + */ + @Test + public void testInvokeAtAvailableNs() throws IOException { + // Create a mount point with multiple destinations. + Path path = new Path("/testInvokeAtAvailableNs"); + Map destMap = new HashMap<>(); + destMap.put("ns0", "/testInvokeAtAvailableNs"); + destMap.put("ns1", "/testInvokeAtAvailableNs"); + nnFs0.mkdirs(path); + nnFs1.mkdirs(path); + MountTable addEntry = + MountTable.newInstance("/testInvokeAtAvailableNs", destMap); + addEntry.setQuota(new RouterQuotaUsage.Builder().build()); + addEntry.setDestOrder(DestinationOrder.RANDOM); + addEntry.setFaultTolerant(true); + assertTrue(addMountTable(addEntry)); + + // Make one subcluster unavailable. + MiniDFSCluster dfsCluster = cluster.getCluster(); + dfsCluster.shutdownNameNode(0); + dfsCluster.shutdownNameNode(1); + try { + // Verify that #invokeAtAvailableNs works by calling #getServerDefaults. + RemoteMethod method = new RemoteMethod("getServerDefaults"); + FsServerDefaults serverDefaults = + rpcServer.invokeAtAvailableNs(method, FsServerDefaults.class); + assertNotNull(serverDefaults); + } finally { + dfsCluster.restartNameNode(0); + dfsCluster.restartNameNode(1); + } + } + /** * Test write on mount point with multiple destinations * and making a one of the destination's subcluster unavailable. @@ -857,6 +898,9 @@ private static FileSystem getFileSystem(final String nsId) { if (nsId.equals("ns1")) { return nnFs1; } + if (nsId.equals("ns2")) { + return nnFs2; + } return null; } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java index 2ab40b031495b..96e77d580073b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java @@ -704,6 +704,7 @@ public void testProxyGetDatanodeReport() throws Exception { DatanodeInfo[] combinedData = routerProtocol.getDatanodeReport(DatanodeReportType.ALL); + assertEquals(0, routerProtocol.getSlowDatanodeReport().length); final Map routerDNMap = new TreeMap<>(); for (DatanodeInfo dn : combinedData) { String subcluster = dn.getNetworkLocation().split("/")[1]; @@ -1950,9 +1951,10 @@ public void testMkdirsWithCallerContext() throws IOException { FsPermission permission = new FsPermission("755"); routerProtocol.mkdirs(dirPath, permission, false); - // The audit log should contains "callerContext=clientContext,clientIp:" - assertTrue(auditlog.getOutput() - .contains("callerContext=clientContext,clientIp:")); + // The audit log should contains "callerContext=clientIp:...,clientContext" + final String logOutput = auditlog.getOutput(); + assertTrue(logOutput.contains("callerContext=clientIp:")); + assertTrue(logOutput.contains(",clientContext")); assertTrue(verifyFileExists(routerFS, dirPath)); } @@ -1997,9 +1999,9 @@ public void testAddClientIpPortToCallerContext() throws IOException { // Create a directory via the router. routerProtocol.getFileInfo(dirPath); - // The audit log should contains the original clientIp and clientPort + // The audit log should not contain the original clientIp and clientPort // set by client. - assertTrue(auditLog.getOutput().contains("clientIp:1.1.1.1")); - assertTrue(auditLog.getOutput().contains("clientPort:1234")); + assertFalse(auditLog.getOutput().contains("clientIp:1.1.1.1")); + assertFalse(auditLog.getOutput().contains("clientPort:1234")); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java index e50464c0be7b5..370a1250a7c11 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java @@ -464,9 +464,8 @@ public void testCallerContextWithMultiDestinations() throws IOException { for (String line : auditLog.getOutput().split("\n")) { if (line.contains(auditFlag)) { // assert origin caller context exist in audit log - assertTrue(line.contains("callerContext=clientContext")); - String callerContext = line.substring( - line.indexOf("callerContext=clientContext")); + String callerContext = line.substring(line.indexOf("callerContext=")); + assertTrue(callerContext.contains("clientContext")); // assert client ip info exist in caller context assertTrue(callerContext.contains(clientIpInfo)); // assert client ip info appears only once in caller context diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterWebHdfsMethods.java index 8e82d44c4ddfe..7c3643f5a511e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterWebHdfsMethods.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterWebHdfsMethods.java @@ -23,9 +23,12 @@ import static org.junit.Assert.fail; import java.io.FileNotFoundException; +import java.io.IOException; import java.net.HttpURLConnection; import java.net.URL; import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -34,6 +37,7 @@ import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster; import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder; +import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -145,4 +149,24 @@ public void testGetNsFromDataNodeNetworkLocation() { assertEquals("", RouterWebHdfsMethods .getNsFromDataNodeNetworkLocation("whatever-rack-info1")); } + + @Test + public void testWebHdfsCreateWithInvalidPath() throws Exception { + // A path name include duplicated slashes. + String path = "//tmp//file"; + assertResponse(path); + } + + private void assertResponse(String path) throws IOException { + URL url = new URL(getUri(path)); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("PUT"); + // Assert response code. + assertEquals(HttpURLConnection.HTTP_BAD_REQUEST, conn.getResponseCode()); + // Assert exception. + Map response = WebHdfsFileSystem.jsonParse(conn, true); + assertEquals("InvalidPathException", + ((LinkedHashMap) response.get("RemoteException")).get("exception")); + conn.disconnect(); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java index 0fad76de050bd..50840460a3943 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java @@ -269,6 +269,7 @@ public static MembershipState createMockRegistrationForNamenode( stats.setNumOfDecomActiveDatanodes(15); stats.setNumOfDecomDeadDatanodes(5); stats.setNumOfBlocks(10); + stats.setPendingSPSPaths(10); entry.setStats(stats); return entry; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_3.2.3.xml b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_3.2.3.xml new file mode 100644 index 0000000000000..5454f53be9122 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_3.2.3.xml @@ -0,0 +1,674 @@ + + + + + + + + + + + A distributed implementation of {@link +org.apache.hadoop.fs.FileSystem}. This is loosely modelled after +Google's GFS.

+ +

The most important difference is that unlike GFS, Hadoop DFS files +have strictly one writer at any one time. Bytes are always appended +to the end of the writer's stream. There is no notion of "record appends" +or "mutations" that are then checked or reordered. Writers simply emit +a byte stream. That byte stream is guaranteed to be stored in the +order written.

]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This method must return as quickly as possible, since it's called + in a critical section of the NameNode's operation. + + @param succeeded Whether authorization succeeded. + @param userName Name of the user executing the request. + @param addr Remote address of the request. + @param cmd The requested command. + @param src Path of affected source file. + @param dst Path of affected destination file (if any). + @param stat File information for operations that change the file's + metadata (permissions, owner, times, etc).]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_3.3.2.xml b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_3.3.2.xml new file mode 100644 index 0000000000000..b4d954cb53ebd --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_3.3.2.xml @@ -0,0 +1,835 @@ + + + + + + + + + + + A distributed implementation of {@link +org.apache.hadoop.fs.FileSystem}. This is loosely modelled after +Google's GFS.

+ +

The most important difference is that unlike GFS, Hadoop DFS files +have strictly one writer at any one time. Bytes are always appended +to the end of the writer's stream. There is no notion of "record appends" +or "mutations" that are then checked or reordered. Writers simply emit +a byte stream. That byte stream is guaranteed to be stored in the +order written.

]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This method must return as quickly as possible, since it's called + in a critical section of the NameNode's operation. + + @param succeeded Whether authorization succeeded. + @param userName Name of the user executing the request. + @param addr Remote address of the request. + @param cmd The requested command. + @param src Path of affected source file. + @param dst Path of affected destination file (if any). + @param stat File information for operations that change the file's + metadata (permissions, owner, times, etc).]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index 10d66d055ba35..b51c7154f7b92 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -422,7 +422,7 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> src/main/webapps/static/moment.min.js src/main/webapps/static/dust-full-2.0.0.min.js src/main/webapps/static/dust-helpers-1.1.1.min.js - src/main/webapps/static/jquery-3.5.1.min.js + src/main/webapps/static/jquery-3.6.0.min.js src/main/webapps/static/jquery.dataTables.min.js src/main/webapps/static/json-bignum.js src/main/webapps/static/dataTables.bootstrap.css @@ -471,7 +471,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> org.apache.maven.plugins maven-surefire-plugin - ${ignoreTestFailure} ${testsThreadCount} false ${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 0526f1e4412dc..962b0dce833c5 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -123,6 +123,14 @@ public class DFSConfigKeys extends CommonConfigurationKeys { "dfs.datanode.data.write.bandwidthPerSec"; // A value of zero indicates no limit public static final long DFS_DATANODE_DATA_WRITE_BANDWIDTHPERSEC_DEFAULT = 0; + public static final String DFS_DATANODE_EC_RECONSTRUCT_READ_BANDWIDTHPERSEC_KEY = + "dfs.datanode.ec.reconstruct.read.bandwidthPerSec"; + public static final long DFS_DATANODE_EC_RECONSTRUCT_READ_BANDWIDTHPERSEC_DEFAULT = + 0; // A value of zero indicates no limit + public static final String DFS_DATANODE_EC_RECONSTRUCT_WRITE_BANDWIDTHPERSEC_KEY = + "dfs.datanode.ec.reconstruct.write.bandwidthPerSec"; + public static final long DFS_DATANODE_EC_RECONSTRUCT_WRITE_BANDWIDTHPERSEC_DEFAULT = + 0; // A value of zero indicates no limit @Deprecated public static final String DFS_DATANODE_READAHEAD_BYTES_KEY = HdfsClientConfigKeys.DFS_DATANODE_READAHEAD_BYTES_KEY; @@ -143,6 +151,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final long DFS_DATANODE_MAX_LOCKED_MEMORY_DEFAULT = 0; public static final String DFS_DATANODE_FSDATASETCACHE_MAX_THREADS_PER_VOLUME_KEY = "dfs.datanode.fsdatasetcache.max.threads.per.volume"; public static final int DFS_DATANODE_FSDATASETCACHE_MAX_THREADS_PER_VOLUME_DEFAULT = 4; + public static final String DFS_DATANODE_FSDATASETASYNCDISK_MAX_THREADS_PER_VOLUME_KEY = + "dfs.datanode.fsdatasetasyncdisk.max.threads.per.volume"; + public static final int DFS_DATANODE_FSDATASETASYNCDISK_MAX_THREADS_PER_VOLUME_DEFAULT = 4; public static final String DFS_DATANODE_LAZY_WRITER_INTERVAL_SEC = "dfs.datanode.lazywriter.interval.sec"; public static final int DFS_DATANODE_LAZY_WRITER_INTERVAL_DEFAULT_SEC = 60; public static final String DFS_DATANODE_RAM_DISK_REPLICA_TRACKER_KEY = "dfs.datanode.ram.disk.replica.tracker"; @@ -473,10 +484,16 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY = "dfs.namenode.startup.delay.block.deletion.sec"; public static final long DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_DEFAULT = 0L; - /** Block deletion increment. */ - public static final String DFS_NAMENODE_BLOCK_DELETION_INCREMENT_KEY = - "dfs.namenode.block.deletion.increment"; - public static final int DFS_NAMENODE_BLOCK_DELETION_INCREMENT_DEFAULT = 1000; + /** The limit of single lock holding duration.*/ + public static final String DFS_NAMENODE_BLOCK_DELETION_LOCK_THRESHOLD_MS = + "dfs.namenode.block.deletion.lock.threshold.ms"; + public static final int DFS_NAMENODE_BLOCK_DELETION_LOCK_THRESHOLD_MS_DEFAULT = + 50; + /** The sleep interval for releasing lock.*/ + public static final String DFS_NAMENODE_BLOCK_DELETION_UNLOCK_INTERVAL_MS = + "dfs.namenode.block.deletion.unlock.interval.ms"; + public static final int DFS_NAMENODE_BLOCK_DELETION_UNLOCK_INTERVAL_MS_DEFAULT = + 10; public static final String DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES = HdfsClientConfigKeys.DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES; @@ -698,6 +715,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys { "dfs.datanode.max.disks.to.report"; public static final int DFS_DATANODE_MAX_DISKS_TO_REPORT_DEFAULT = 5; + public static final String DFS_DATANODE_MAX_SLOWDISKS_TO_EXCLUDE_KEY = + "dfs.datanode.max.slowdisks.to.exclude"; + public static final int DFS_DATANODE_MAX_SLOWDISKS_TO_EXCLUDE_DEFAULT = + 0; public static final String DFS_DATANODE_HOST_NAME_KEY = HdfsClientConfigKeys.DeprecatedKeys.DFS_DATANODE_HOST_NAME_KEY; public static final String DFS_NAMENODE_CHECKPOINT_DIR_KEY = @@ -739,6 +760,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys { */ public static final String DFS_NAMENODE_GETBLOCKS_MAX_QPS_KEY = "dfs.namenode.get-blocks.max-qps"; public static final int DFS_NAMENODE_GETBLOCKS_MAX_QPS_DEFAULT = 20; + public static final String DFS_NAMENODE_GETBLOCKS_CHECK_OPERATION_KEY + = "dfs.namenode.get-blocks.check.operation"; + public static final boolean DFS_NAMENODE_GETBLOCKS_CHECK_OPERATION_DEFAULT + = true; public static final String DFS_BALANCER_MOVEDWINWIDTH_KEY = "dfs.balancer.movedWinWidth"; public static final long DFS_BALANCER_MOVEDWINWIDTH_DEFAULT = 5400*1000L; @@ -816,6 +841,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys { "dfs.storage.policy.satisfier.retry.max.attempts"; public static final int DFS_STORAGE_POLICY_SATISFIER_MAX_RETRY_ATTEMPTS_DEFAULT = 3; + public static final String DFS_STORAGE_POLICY_SATISFIER_MOVE_TASK_MAX_RETRY_ATTEMPTS_KEY = + "dfs.storage.policy.satisfier.move.task.retry.max.attempts"; + public static final int DFS_STORAGE_POLICY_SATISFIER_MOVE_TASK_MAX_RETRY_ATTEMPTS_DEFAULT = + 3; public static final String DFS_STORAGE_DEFAULT_POLICY = "dfs.storage.default.policy"; public static final HdfsConstants.StoragePolicy @@ -980,6 +1009,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys { "dfs.namenode.lifeline.handler.count"; public static final String DFS_NAMENODE_SERVICE_HANDLER_COUNT_KEY = "dfs.namenode.service.handler.count"; public static final int DFS_NAMENODE_SERVICE_HANDLER_COUNT_DEFAULT = 10; + // List of users that can override their client ip + public static final String DFS_NAMENODE_IP_PROXY_USERS = "dfs.namenode.ip-proxy-users"; public static final String DFS_HTTP_POLICY_KEY = "dfs.http.policy"; public static final String DFS_HTTP_POLICY_DEFAULT = HttpConfig.Policy.HTTP_ONLY.name(); public static final String DFS_DATANODE_HTTPSERVER_FILTER_HANDLERS = "dfs.datanode.httpserver.filter.handlers"; @@ -1523,6 +1554,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String[] NNTOP_WINDOWS_MINUTES_DEFAULT = {"1", "5", "25"}; public static final String DFS_PIPELINE_ECN_ENABLED = "dfs.pipeline.ecn"; public static final boolean DFS_PIPELINE_ECN_ENABLED_DEFAULT = false; + public static final String DFS_PIPELINE_SLOWNODE_ENABLED = "dfs.pipeline.slownode"; + public static final boolean DFS_PIPELINE_SLOWNODE_ENABLED_DEFAULT = false; // Key Provider Cache Expiry public static final String DFS_DATANODE_BLOCK_PINNING_ENABLED = @@ -1649,6 +1682,13 @@ public class DFSConfigKeys extends CommonConfigurationKeys { DFS_NAMESERVICES_RESOLVER_IMPL = "dfs.datanode.nameservices.resolver.impl"; + public static final String + DFS_DATANODE_LOCKMANAGER_TRACE = + "dfs.datanode.lockmanager.trace"; + + public static final boolean + DFS_DATANODE_LOCKMANAGER_TRACE_DEFAULT = false; + // dfs.client.retry confs are moved to HdfsClientConfigKeys.Retry @Deprecated public static final String DFS_CLIENT_RETRY_POLICY_ENABLED_KEY diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferServer.java index 1e10fbbd3dd03..fcb6b7d7bc575 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferServer.java @@ -52,10 +52,12 @@ import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.DataTransferEncryptorMessageProto.DataTransferEncryptorStatus; import org.apache.hadoop.hdfs.security.token.block.BlockPoolTokenSecretManager; import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; +import org.apache.hadoop.hdfs.security.token.block.InvalidBlockTokenException; import org.apache.hadoop.hdfs.server.datanode.DNConf; import org.apache.hadoop.security.SaslPropertiesResolver; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.SecretManager; import org.apache.hadoop.util.Lists; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -441,6 +443,14 @@ private IOStreamPair doSaslHandshake(Peer peer, OutputStream underlyingOut, // error, the client will get a new encryption key from the NN and retry // connecting to this DN. sendInvalidKeySaslErrorMessage(out, ioe.getCause().getMessage()); + } else if (ioe instanceof SaslException && + ioe.getCause() != null && + (ioe.getCause() instanceof InvalidBlockTokenException || + ioe.getCause() instanceof SecretManager.InvalidToken)) { + // This could be because the client is long-lived and block token is expired + // The client will get new block token from the NN, upon receiving this error + // and retry connecting to this DN + sendInvalidTokenSaslErrorMessage(out, ioe.getCause().getMessage()); } else { sendGenericSaslErrorMessage(out, ioe.getMessage()); } @@ -460,4 +470,16 @@ private static void sendInvalidKeySaslErrorMessage(DataOutputStream out, sendSaslMessage(out, DataTransferEncryptorStatus.ERROR_UNKNOWN_KEY, null, message); } + + /** + * Sends a SASL negotiation message indicating an invalid token error. + * + * @param out stream to receive message + * @param message to send + * @throws IOException for any error + */ + private static void sendInvalidTokenSaslErrorMessage(DataOutputStream out, + String message) throws IOException { + sendSaslMessage(out, DataTransferEncryptorStatus.ERROR, null, message, null, true); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java index 5132afaa4b15c..0164f25460dc6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java @@ -156,6 +156,8 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetQuotaUsageResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetServerDefaultsRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetServerDefaultsResponseProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSlowDatanodeReportRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSlowDatanodeReportResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSnapshotDiffReportRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSnapshotDiffReportResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSnapshotDiffReportListingRequestProto; @@ -2058,4 +2060,18 @@ public HAServiceStateResponseProto getHAServiceState( throw new ServiceException(e); } } + + @Override + public GetSlowDatanodeReportResponseProto getSlowDatanodeReport(RpcController controller, + GetSlowDatanodeReportRequestProto request) throws ServiceException { + try { + List result = + PBHelperClient.convert(server.getSlowDatanodeReport()); + return GetSlowDatanodeReportResponseProto.newBuilder() + .addAllDatanodeInfoProto(result) + .build(); + } catch (IOException e) { + throw new ServiceException(e); + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java index 695a945ec712c..fd58c0c7ca289 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java @@ -183,7 +183,8 @@ public HeartbeatResponse sendHeartbeat(DatanodeRegistration registration, rollingUpdateStatus = PBHelperClient.convert(resp.getRollingUpgradeStatus()); } return new HeartbeatResponse(cmds, PBHelper.convert(resp.getHaStatus()), - rollingUpdateStatus, resp.getFullBlockReportLeaseId()); + rollingUpdateStatus, resp.getFullBlockReportLeaseId(), + resp.getIsSlownode()); } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java index 89cc6c79a1214..08a777b9a5181 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java @@ -152,6 +152,7 @@ public HeartbeatResponseProto sendHeartbeat(RpcController controller, } builder.setFullBlockReportLeaseId(response.getFullBlockReportLeaseId()); + builder.setIsSlownode(response.getIsSlownode()); return builder.build(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java index 970078eeea9a1..df1314ac1eb56 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java @@ -45,6 +45,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_JOURNALNODE_HTTP_BIND_HOST_KEY; import static org.apache.hadoop.util.ExitUtil.terminate; +import static org.apache.hadoop.util.Time.now; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; @@ -83,6 +84,7 @@ public class JournalNode implements Tool, Configurable, JournalNodeMXBean { private String httpServerURI; private final ArrayList localDir = Lists.newArrayList(); Tracer tracer; + private long startTime = 0; static { HdfsConfiguration.init(); @@ -241,6 +243,7 @@ public void start() throws IOException { rpcServer = new JournalNodeRpcServer(conf, this); rpcServer.start(); + startTime = now(); } catch (IOException ioe) { //Shutdown JournalNode of JournalNodeRpcServer fails to start LOG.error("Failed to start JournalNode.", ioe); @@ -415,6 +418,19 @@ public String getVersion() { return VersionInfo.getVersion() + ", r" + VersionInfo.getRevision(); } + @Override // JournalNodeMXBean + public long getJNStartedTimeInMillis() { + return this.startTime; + } + + @Override + // JournalNodeMXBean + public List getStorageInfos() { + return journalsById.values().stream() + .map(journal -> journal.getStorage().toMapString()) + .collect(Collectors.toList()); + } + /** * Register JournalNodeMXBean */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeMXBean.java index f265c31a34aea..0cce05f5820d4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeMXBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeMXBean.java @@ -57,4 +57,20 @@ public interface JournalNodeMXBean { * @return the version of Hadoop. */ String getVersion(); + + /** + * Get the start time of the JournalNode. + * + * @return the start time of the JournalNode. + */ + long getJNStartedTimeInMillis(); + + /** + * Get the list of the storage infos of JournalNode's journals. Storage infos + * include layout version, namespace id, cluster id and creation time of the + * File system state. + * + * @return the list of storage infos associated with journals. + */ + List getStorageInfos(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java index f97c2f2c80b71..b89998d989586 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java @@ -383,7 +383,7 @@ protected void logUpdateMasterKey(DelegationKey key) namesystem.logUpdateMasterKey(key); } } finally { - namesystem.readUnlock(); + namesystem.readUnlock("logUpdateMasterKey"); } } catch (InterruptedException ie) { // AbstractDelegationTokenManager may crash if an exception is thrown. @@ -412,7 +412,7 @@ protected void logExpireToken(final DelegationTokenIdentifier dtId) namesystem.logExpireDelegationToken(dtId); } } finally { - namesystem.readUnlock(); + namesystem.readUnlock("logExpireToken"); } } catch (InterruptedException ie) { // AbstractDelegationTokenManager may crash if an exception is thrown. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java index bc7cad54166f9..612f4c028eff5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java @@ -495,7 +495,7 @@ public long getNumBytes(StorageGroup storage) { public static class DBlockStriped extends DBlock { - final byte[] indices; + private byte[] indices; final short dataBlockNum; final int cellSize; @@ -532,6 +532,29 @@ public long getNumBytes(StorageGroup storage) { } return block.getNumBytes(); } + + public void setIndices(byte[] indices) { + this.indices = indices; + } + + /** + * Adjust EC block indices，it will remove the element of adjustList from indices. + * @param adjustList the list will be removed from indices + */ + public void adjustIndices(List adjustList) { + if (adjustList.isEmpty()) { + return; + } + + byte[] newIndices = new byte[indices.length - adjustList.size()]; + for (int i = 0, j = 0; i < indices.length; ++i) { + if (!adjustList.contains(i)) { + newIndices[j] = indices[i]; + ++j; + } + } + this.indices = newIndices; + } } /** The class represents a desired move. */ @@ -810,7 +833,7 @@ Iterator getBlockIterator() { * * @return the total size of the received blocks in the number of bytes. */ - private long getBlockList() throws IOException { + private long getBlockList() throws IOException, IllegalArgumentException { final long size = Math.min(getBlocksSize, blocksToReceive); final BlocksWithLocations newBlksLocs = nnc.getBlocks(getDatanodeInfo(), size, getBlocksMinBlockSize, @@ -848,7 +871,14 @@ private long getBlockList() throws IOException { synchronized (block) { block.clearLocations(); + if (blkLocs instanceof StripedBlockWithLocations) { + // EC block may adjust indices before, avoid repeated adjustments + ((DBlockStriped) block).setIndices( + ((StripedBlockWithLocations) blkLocs).getIndices()); + } + // update locations + List adjustList = new ArrayList<>(); final String[] datanodeUuids = blkLocs.getDatanodeUuids(); final StorageType[] storageTypes = blkLocs.getStorageTypes(); for (int i = 0; i < datanodeUuids.length; i++) { @@ -856,8 +886,20 @@ private long getBlockList() throws IOException { datanodeUuids[i], storageTypes[i]); if (g != null) { // not unknown block.addLocation(g); + } else if (blkLocs instanceof StripedBlockWithLocations) { + // some datanode may not in storageGroupMap due to decommission operation + // or balancer cli with "-exclude" parameter + adjustList.add(i); } } + + if (!adjustList.isEmpty()) { + // block.locations mismatch with block.indices + // adjust indices to get correct internalBlock for Datanode in #getInternalBlock + ((DBlockStriped) block).adjustIndices(adjustList); + Preconditions.checkArgument(((DBlockStriped) block).indices.length + == block.locations.size()); + } } if (!srcBlocks.contains(block) && isGoodBlockCandidate(block)) { if (LOG.isTraceEnabled()) { @@ -977,7 +1019,7 @@ private void dispatchBlocks() { } blocksToReceive -= received; continue; - } catch (IOException e) { + } catch (IOException | IllegalArgumentException e) { LOG.warn("Exception while getting reportedBlock list", e); return; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java index ce0fb968bbb9f..238457bcb867a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java @@ -147,11 +147,11 @@ public static void checkOtherInstanceRunning(boolean toCheck) { private final BalancerProtocols namenode; /** - * If set requestToStandby true, Balancer will getBlocks from + * If set getBlocksToStandby true, Balancer will getBlocks from * Standby NameNode only and it can reduce the performance impact of Active * NameNode, especially in a busy HA mode cluster. */ - private boolean requestToStandby; + private boolean getBlocksToStandby; private String nsId; private Configuration config; private final KeyManager keyManager; @@ -191,9 +191,9 @@ public NameNodeConnector(String name, URI nameNodeUri, Path idPath, this.namenode = NameNodeProxies.createProxy(conf, nameNodeUri, BalancerProtocols.class, fallbackToSimpleAuth).getProxy(); - this.requestToStandby = conf.getBoolean( - DFSConfigKeys.DFS_HA_ALLOW_STALE_READ_KEY, - DFSConfigKeys.DFS_HA_ALLOW_STALE_READ_DEFAULT); + this.getBlocksToStandby = !conf.getBoolean( + DFSConfigKeys.DFS_NAMENODE_GETBLOCKS_CHECK_OPERATION_KEY, + DFSConfigKeys.DFS_NAMENODE_GETBLOCKS_CHECK_OPERATION_DEFAULT); this.config = conf; this.fs = (DistributedFileSystem)FileSystem.get(nameNodeUri, conf); @@ -318,7 +318,7 @@ public DatanodeStorageReport[] getLiveDatanodeStorageReport() private ProxyPair getProxy() throws IOException { boolean isRequestStandby = false; ClientProtocol clientProtocol = null; - if (requestToStandby && nsId != null + if (getBlocksToStandby && nsId != null && HAUtil.isHAEnabled(config, nsId)) { List namenodes = HAUtil.getProxiesForAllNameNodesInNameservice(config, nsId); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java index 81a559e0f04c8..659f218437539 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java @@ -86,7 +86,7 @@ public BlockInfo(short size) { public BlockInfo(Block blk, short size) { super(blk); - this.triplets = new Object[3*size]; + this.triplets = new Object[3 * size]; this.bcId = INVALID_INODE_ID; this.replication = isStriped() ? 0 : size; } @@ -126,34 +126,34 @@ public DatanodeDescriptor getDatanode(int index) { DatanodeStorageInfo getStorageInfo(int index) { assert this.triplets != null : "BlockInfo is not initialized"; - assert index >= 0 && index*3 < triplets.length : "Index is out of bound"; - return (DatanodeStorageInfo)triplets[index*3]; + assert index >= 0 && index * 3 < triplets.length : "Index is out of bound"; + return (DatanodeStorageInfo)triplets[index * 3]; } BlockInfo getPrevious(int index) { assert this.triplets != null : "BlockInfo is not initialized"; - assert index >= 0 && index*3+1 < triplets.length : "Index is out of bound"; - BlockInfo info = (BlockInfo)triplets[index*3+1]; + assert index >= 0 && index * 3 + 1 < triplets.length : "Index is out of bound"; + BlockInfo info = (BlockInfo)triplets[index * 3 + 1]; assert info == null || info.getClass().getName().startsWith(BlockInfo.class.getName()) : - "BlockInfo is expected at " + index*3; + "BlockInfo is expected at " + (index * 3 + 1); return info; } BlockInfo getNext(int index) { assert this.triplets != null : "BlockInfo is not initialized"; - assert index >= 0 && index*3+2 < triplets.length : "Index is out of bound"; - BlockInfo info = (BlockInfo)triplets[index*3+2]; + assert index >= 0 && index * 3 + 2 < triplets.length : "Index is out of bound"; + BlockInfo info = (BlockInfo)triplets[index * 3 + 2]; assert info == null || info.getClass().getName().startsWith( BlockInfo.class.getName()) : - "BlockInfo is expected at " + index*3; + "BlockInfo is expected at " + (index * 3 + 2); return info; } void setStorageInfo(int index, DatanodeStorageInfo storage) { assert this.triplets != null : "BlockInfo is not initialized"; - assert index >= 0 && index*3 < triplets.length : "Index is out of bound"; - triplets[index*3] = storage; + assert index >= 0 && index * 3 < triplets.length : "Index is out of bound"; + triplets[index * 3] = storage; } /** @@ -166,9 +166,9 @@ void setStorageInfo(int index, DatanodeStorageInfo storage) { */ BlockInfo setPrevious(int index, BlockInfo to) { assert this.triplets != null : "BlockInfo is not initialized"; - assert index >= 0 && index*3+1 < triplets.length : "Index is out of bound"; - BlockInfo info = (BlockInfo) triplets[index*3+1]; - triplets[index*3+1] = to; + assert index >= 0 && index * 3 + 1 < triplets.length : "Index is out of bound"; + BlockInfo info = (BlockInfo) triplets[index * 3 + 1]; + triplets[index * 3 + 1] = to; return info; } @@ -182,9 +182,9 @@ BlockInfo setPrevious(int index, BlockInfo to) { */ BlockInfo setNext(int index, BlockInfo to) { assert this.triplets != null : "BlockInfo is not initialized"; - assert index >= 0 && index*3+2 < triplets.length : "Index is out of bound"; - BlockInfo info = (BlockInfo) triplets[index*3+2]; - triplets[index*3+2] = to; + assert index >= 0 && index * 3 + 2 < triplets.length : "Index is out of bound"; + BlockInfo info = (BlockInfo) triplets[index * 3 + 2]; + triplets[index * 3 + 2] = to; return info; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 1d4e7d19f6259..c522e2604e70f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -47,6 +47,7 @@ import java.util.concurrent.FutureTask; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicLong; import javax.management.ObjectName; @@ -191,6 +192,9 @@ public class BlockManager implements BlockStatsMXBean { private volatile long lowRedundancyBlocksCount = 0L; private volatile long scheduledReplicationBlocksCount = 0L; + private final long deleteBlockLockTimeMs; + private final long deleteBlockUnlockIntervalTimeMs; + /** flag indicating whether replication queues have been initialized */ private boolean initializedReplQueues; @@ -294,6 +298,14 @@ public long getTotalECBlockGroups() { return blocksMap.getECBlockGroups(); } + /** Used by metrics. */ + public int getPendingSPSPaths() { + if (spsManager != null) { + return spsManager.getPendingSPSPaths(); + } + return 0; + } + /** * redundancyRecheckInterval is how often namenode checks for new * reconstruction work. @@ -324,6 +336,12 @@ public long getTotalECBlockGroups() { * {@link #redundancyThread} has run at least one full iteration. */ private final AtomicLong lastRedundancyCycleTS = new AtomicLong(-1); + /** + * markedDeleteBlockScrubber thread for handling async delete blocks. + */ + private final Daemon markedDeleteBlockScrubberThread = + new Daemon(new MarkedDeleteBlockScrubber()); + /** Block report thread for handling async reports. */ private final BlockReportProcessingThread blockReportThread; @@ -422,6 +440,12 @@ public long getTotalECBlockGroups() { */ private int numBlocksPerIteration; + /** + * The blocks of deleted files are put into the queue, + * and the cleanup thread processes these blocks periodically. + */ + private final ConcurrentLinkedQueue> markedDeleteQueue; + /** * Progress of the Reconstruction queues initialisation. */ @@ -461,66 +485,54 @@ public long getTotalECBlockGroups() { public BlockManager(final Namesystem namesystem, boolean haEnabled, final Configuration conf) throws IOException { this.namesystem = namesystem; - datanodeManager = new DatanodeManager(this, namesystem, conf); - heartbeatManager = datanodeManager.getHeartbeatManager(); + this.datanodeManager = new DatanodeManager(this, namesystem, conf); + this.heartbeatManager = datanodeManager.getHeartbeatManager(); this.blockIdManager = new BlockIdManager(this); - blocksPerPostpondedRescan = (int)Math.min(Integer.MAX_VALUE, + this.blocksPerPostpondedRescan = (int)Math.min(Integer.MAX_VALUE, datanodeManager.getBlocksPerPostponedMisreplicatedBlocksRescan()); - rescannedMisreplicatedBlocks = + this.rescannedMisreplicatedBlocks = new ArrayList(blocksPerPostpondedRescan); - startupDelayBlockDeletionInMs = conf.getLong( + this.startupDelayBlockDeletionInMs = conf.getLong( DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY, - DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_DEFAULT) * 1000L; - invalidateBlocks = new InvalidateBlocks( + DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_DEFAULT) + * 1000L; + this.deleteBlockLockTimeMs = conf.getLong( + DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_LOCK_THRESHOLD_MS, + DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_LOCK_THRESHOLD_MS_DEFAULT); + this.deleteBlockUnlockIntervalTimeMs = conf.getLong( + DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_UNLOCK_INTERVAL_MS, + DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_UNLOCK_INTERVAL_MS_DEFAULT); + this.invalidateBlocks = new InvalidateBlocks( datanodeManager.getBlockInvalidateLimit(), startupDelayBlockDeletionInMs, blockIdManager); - + this.markedDeleteQueue = new ConcurrentLinkedQueue<>(); // Compute the map capacity by allocating 2% of total memory - blocksMap = new BlocksMap( + this.blocksMap = new BlocksMap( LightWeightGSet.computeCapacity(2.0, "BlocksMap")); - placementPolicies = new BlockPlacementPolicies( - conf, datanodeManager.getFSClusterStats(), - datanodeManager.getNetworkTopology(), - datanodeManager.getHost2DatanodeMap()); - storagePolicySuite = BlockStoragePolicySuite.createDefaultSuite(conf); - pendingReconstruction = new PendingReconstructionBlocks(conf.getInt( + this.placementPolicies = new BlockPlacementPolicies( + conf, datanodeManager.getFSClusterStats(), + datanodeManager.getNetworkTopology(), + datanodeManager.getHost2DatanodeMap()); + this.storagePolicySuite = BlockStoragePolicySuite.createDefaultSuite(conf); + this.pendingReconstruction = new PendingReconstructionBlocks(conf.getInt( DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY, DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_DEFAULT) * 1000L); createSPSManager(conf); - blockTokenSecretManager = createBlockTokenSecretManager(conf); - - providedStorageMap = new ProvidedStorageMap(namesystem, this, conf); + this.blockTokenSecretManager = createBlockTokenSecretManager(conf); + this.providedStorageMap = new ProvidedStorageMap(namesystem, this, conf); this.maxCorruptFilesReturned = conf.getInt( - DFSConfigKeys.DFS_DEFAULT_MAX_CORRUPT_FILES_RETURNED_KEY, - DFSConfigKeys.DFS_DEFAULT_MAX_CORRUPT_FILES_RETURNED); + DFSConfigKeys.DFS_DEFAULT_MAX_CORRUPT_FILES_RETURNED_KEY, + DFSConfigKeys.DFS_DEFAULT_MAX_CORRUPT_FILES_RETURNED); this.defaultReplication = conf.getInt(DFSConfigKeys.DFS_REPLICATION_KEY, DFSConfigKeys.DFS_REPLICATION_DEFAULT); - final int maxR = conf.getInt(DFSConfigKeys.DFS_REPLICATION_MAX_KEY, - DFSConfigKeys.DFS_REPLICATION_MAX_DEFAULT); - final int minR = conf.getInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY, - DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_DEFAULT); - if (minR <= 0) - throw new IOException("Unexpected configuration parameters: " - + DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY - + " = " + minR + " <= 0"); - if (maxR > Short.MAX_VALUE) - throw new IOException("Unexpected configuration parameters: " - + DFSConfigKeys.DFS_REPLICATION_MAX_KEY - + " = " + maxR + " > " + Short.MAX_VALUE); - if (minR > maxR) - throw new IOException("Unexpected configuration parameters: " - + DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY - + " = " + minR + " > " - + DFSConfigKeys.DFS_REPLICATION_MAX_KEY - + " = " + maxR); - this.minReplication = (short)minR; - this.maxReplication = (short)maxR; + this.minReplication = (short) initMinReplication(conf); + this.maxReplication = (short) initMaxReplication(conf); this.maxReplicationStreams = conf.getInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, @@ -552,6 +564,66 @@ public BlockManager(final Namesystem namesystem, boolean haEnabled, DFSConfigKeys.DFS_BLOCK_MISREPLICATION_PROCESSING_LIMIT, DFSConfigKeys.DFS_BLOCK_MISREPLICATION_PROCESSING_LIMIT_DEFAULT); + this.minReplicationToBeInMaintenance = + (short) initMinReplicationToBeInMaintenance(conf); + this.replQueueResetToHeadThreshold = + initReplQueueResetToHeadThreshold(conf); + + long heartbeatIntervalSecs = conf.getTimeDuration( + DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, + DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS); + long blockRecoveryTimeout = getBlockRecoveryTimeout(heartbeatIntervalSecs); + this.pendingRecoveryBlocks = new PendingRecoveryBlocks(blockRecoveryTimeout); + + this.blockReportLeaseManager = new BlockReportLeaseManager(conf); + + this.bmSafeMode = new BlockManagerSafeMode(this, namesystem, haEnabled, + conf); + + int queueSize = conf.getInt( + DFSConfigKeys.DFS_NAMENODE_BLOCKREPORT_QUEUE_SIZE_KEY, + DFSConfigKeys.DFS_NAMENODE_BLOCKREPORT_QUEUE_SIZE_DEFAULT); + this.blockReportThread = new BlockReportProcessingThread(queueSize); + + this.deleteCorruptReplicaImmediately = + conf.getBoolean(DFS_NAMENODE_CORRUPT_BLOCK_DELETE_IMMEDIATELY_ENABLED, + DFS_NAMENODE_CORRUPT_BLOCK_DELETE_IMMEDIATELY_ENABLED_DEFAULT); + + printInitialConfigs(); + } + + private int initMinReplication(Configuration conf) throws IOException { + final int minR = conf.getInt( + DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY, + DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_DEFAULT); + if (minR <= 0) { + throw new IOException("Unexpected configuration parameters: " + + DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY + + " = " + minR + " <= 0"); + } + return minR; + } + + private int initMaxReplication(Configuration conf) throws IOException { + final int maxR = conf.getInt(DFSConfigKeys.DFS_REPLICATION_MAX_KEY, + DFSConfigKeys.DFS_REPLICATION_MAX_DEFAULT); + if (maxR > Short.MAX_VALUE) { + throw new IOException("Unexpected configuration parameters: " + + DFSConfigKeys.DFS_REPLICATION_MAX_KEY + + " = " + maxR + " > " + Short.MAX_VALUE); + } + if (minReplication > maxR) { + throw new IOException("Unexpected configuration parameters: " + + DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY + + " = " + minReplication + " > " + + DFSConfigKeys.DFS_REPLICATION_MAX_KEY + + " = " + maxR); + } + return maxR; + } + + private int initMinReplicationToBeInMaintenance(Configuration conf) + throws IOException { final int minMaintenanceR = conf.getInt( DFSConfigKeys.DFS_NAMENODE_MAINTENANCE_REPLICATION_MIN_KEY, DFSConfigKeys.DFS_NAMENODE_MAINTENANCE_REPLICATION_MIN_DEFAULT); @@ -568,39 +640,25 @@ public BlockManager(final Namesystem namesystem, boolean haEnabled, + DFSConfigKeys.DFS_REPLICATION_KEY + " = " + defaultReplication); } - this.minReplicationToBeInMaintenance = (short)minMaintenanceR; + return minMaintenanceR; + } - replQueueResetToHeadThreshold = conf.getInt( + private int initReplQueueResetToHeadThreshold(Configuration conf) { + int threshold = conf.getInt( DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_QUEUE_RESTART_ITERATIONS, DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_QUEUE_RESTART_ITERATIONS_DEFAULT); - if (replQueueResetToHeadThreshold < 0) { + if (threshold < 0) { LOG.warn("{} is set to {} and it must be >= 0. Resetting to default {}", DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_QUEUE_RESTART_ITERATIONS, - replQueueResetToHeadThreshold, DFSConfigKeys. + threshold, DFSConfigKeys. DFS_NAMENODE_REDUNDANCY_QUEUE_RESTART_ITERATIONS_DEFAULT); - replQueueResetToHeadThreshold = DFSConfigKeys. + threshold = DFSConfigKeys. DFS_NAMENODE_REDUNDANCY_QUEUE_RESTART_ITERATIONS_DEFAULT; } + return threshold; + } - long heartbeatIntervalSecs = conf.getTimeDuration( - DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, - DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS); - long blockRecoveryTimeout = getBlockRecoveryTimeout(heartbeatIntervalSecs); - pendingRecoveryBlocks = new PendingRecoveryBlocks(blockRecoveryTimeout); - - this.blockReportLeaseManager = new BlockReportLeaseManager(conf); - - bmSafeMode = new BlockManagerSafeMode(this, namesystem, haEnabled, conf); - - int queueSize = conf.getInt( - DFSConfigKeys.DFS_NAMENODE_BLOCKREPORT_QUEUE_SIZE_KEY, - DFSConfigKeys.DFS_NAMENODE_BLOCKREPORT_QUEUE_SIZE_DEFAULT); - blockReportThread = new BlockReportProcessingThread(queueSize); - - this.deleteCorruptReplicaImmediately = - conf.getBoolean(DFS_NAMENODE_CORRUPT_BLOCK_DELETE_IMMEDIATELY_ENABLED, - DFS_NAMENODE_CORRUPT_BLOCK_DELETE_IMMEDIATELY_ENABLED_DEFAULT); - + private void printInitialConfigs() { LOG.info("defaultReplication = {}", defaultReplication); LOG.info("maxReplication = {}", maxReplication); LOG.info("minReplication = {}", minReplication); @@ -725,6 +783,9 @@ public void activate(Configuration conf, long blockTotal) { datanodeManager.activate(conf); this.redundancyThread.setName("RedundancyMonitor"); this.redundancyThread.start(); + this.markedDeleteBlockScrubberThread. + setName("MarkedDeleteBlockScrubberThread"); + this.markedDeleteBlockScrubberThread.start(); this.blockReportThread.start(); mxBeanName = MBeans.register("NameNode", "BlockStats", this); bmSafeMode.activate(blockTotal); @@ -738,8 +799,10 @@ public void close() { try { redundancyThread.interrupt(); blockReportThread.interrupt(); + markedDeleteBlockScrubberThread.interrupt(); redundancyThread.join(3000); blockReportThread.join(3000); + markedDeleteBlockScrubberThread.join(3000); } catch (InterruptedException ie) { } datanodeManager.close(); @@ -757,6 +820,11 @@ public BlockPlacementPolicy getBlockPlacementPolicy() { return placementPolicies.getPolicy(CONTIGUOUS); } + @VisibleForTesting + public BlockPlacementPolicy getStriptedBlockPlacementPolicy() { + return placementPolicies.getPolicy(STRIPED); + } + public void refreshBlockPlacementPolicy(Configuration conf) { BlockPlacementPolicies bpp = new BlockPlacementPolicies(conf, datanodeManager.getFSClusterStats(), @@ -1638,9 +1706,16 @@ public BlocksWithLocations getBlocksWithLocations(final DatanodeID datanode, if(numBlocks == 0) { return new BlocksWithLocations(new BlockWithLocations[0]); } + + // skip stale storage + DatanodeStorageInfo[] storageInfos = Arrays + .stream(node.getStorageInfos()) + .filter(s -> !s.areBlockContentsStale()) + .toArray(DatanodeStorageInfo[]::new); + // starting from a random block int startBlock = ThreadLocalRandom.current().nextInt(numBlocks); - Iterator iter = node.getBlockIterator(startBlock); + Iterator iter = node.getBlockIterator(startBlock, storageInfos); List results = new ArrayList(); List pending = new ArrayList(); long totalSize = 0; @@ -1659,8 +1734,8 @@ public BlocksWithLocations getBlocksWithLocations(final DatanodeID datanode, } } if(totalSize srcNodes.length) { + LOG.debug("Block {} cannot be reconstructed due to shortage of source datanodes ", block); + NameNode.getNameNodeMetrics().incNumTimesReReplicationNotScheduled(); + return null; + } + } + // liveReplicaNodes can include READ_ONLY_SHARED replicas which are // not included in the numReplicas.liveReplicas() count assert liveReplicaNodes.size() >= numReplicas.liveReplicas(); @@ -2402,6 +2487,10 @@ private DatanodeDescriptor getDatanodeDescriptorFromStorage( * replicas of the given block. * @param liveBlockIndices List to be populated with indices of healthy * blocks in a striped block group + * @param liveBusyBlockIndices List to be populated with indices of healthy + * blocks in a striped block group in busy DN, + * which the recovery work have reached their + * replication limits * @param priority integer representing replication priority of the given * block * @return the array of DatanodeDescriptor of the chosen nodes from which to @@ -2544,7 +2633,7 @@ void processPendingReconstructions() { } } } finally { - namesystem.writeUnlock(); + namesystem.writeUnlock("processPendingReconstructions"); } /* If we know the target datanodes where the replication timedout, * we could invoke decBlocksScheduled() on it. Its ok for now. @@ -2658,7 +2747,7 @@ public long getProvidedCapacity() { void updateHeartbeat(DatanodeDescriptor node, StorageReport[] reports, long cacheCapacity, long cacheUsed, int xceiverCount, int failedVolumes, VolumeFailureSummary volumeFailureSummary) { - + BlockManagerFaultInjector.getInstance().mockAnException(); for (StorageReport report: reports) { providedStorageMap.updateStorage(node, report.getStorage()); } @@ -2670,6 +2759,7 @@ void updateHeartbeatState(DatanodeDescriptor node, StorageReport[] reports, long cacheCapacity, long cacheUsed, int xceiverCount, int failedVolumes, VolumeFailureSummary volumeFailureSummary) { + BlockManagerFaultInjector.getInstance().mockAnException(); for (StorageReport report: reports) { providedStorageMap.updateStorage(node, report.getStorage()); } @@ -2717,6 +2807,9 @@ public boolean checkBlockReportLease(BlockReportContext context, return true; } DatanodeDescriptor node = datanodeManager.getDatanode(nodeID); + if (node == null) { + throw new UnregisteredNodeException(nodeID, null); + } final long startTime = Time.monotonicNow(); return blockReportLeaseManager.checkLease(node, startTime, context.getLeaseId()); @@ -2741,6 +2834,8 @@ public boolean processReport(final DatanodeID nodeID, Collection invalidatedBlocks = Collections.emptyList(); String strBlockReportId = context != null ? Long.toHexString(context.getReportId()) : ""; + String fullBrLeaseId = + context != null ? Long.toHexString(context.getLeaseId()) : ""; try { node = datanodeManager.getDatanode(nodeID); @@ -2763,10 +2858,10 @@ public boolean processReport(final DatanodeID nodeID, if (namesystem.isInStartupSafeMode() && !StorageType.PROVIDED.equals(storageInfo.getStorageType()) && storageInfo.getBlockReportCount() > 0) { - blockLog.info("BLOCK* processReport 0x{}: " + blockLog.info("BLOCK* processReport 0x{} with lease ID 0x{}: " + "discarded non-initial block report from {}" + " because namenode still in startup phase", - strBlockReportId, nodeID); + strBlockReportId, fullBrLeaseId, nodeID); blockReportLeaseManager.removeLease(node); return !node.hasStaleStorages(); } @@ -2774,9 +2869,9 @@ public boolean processReport(final DatanodeID nodeID, if (storageInfo.getBlockReportCount() == 0) { // The first block report can be processed a lot more efficiently than // ordinary block reports. This shortens restart times. - blockLog.info("BLOCK* processReport 0x{}: Processing first " + blockLog.info("BLOCK* processReport 0x{} with lease ID 0x{}: Processing first " + "storage report for {} from datanode {}", - strBlockReportId, + strBlockReportId, fullBrLeaseId, storageInfo.getStorageID(), nodeID); processFirstBlockReport(storageInfo, newReport); @@ -2790,13 +2885,13 @@ public boolean processReport(final DatanodeID nodeID, storageInfo.receivedBlockReport(); } finally { endTime = Time.monotonicNow(); - namesystem.writeUnlock(); + namesystem.writeUnlock("processReport"); } if(blockLog.isDebugEnabled()) { for (Block b : invalidatedBlocks) { - blockLog.debug("BLOCK* processReport 0x{}: {} on node {} size {} " + - "does not belong to any file.", strBlockReportId, b, + blockLog.debug("BLOCK* processReport 0x{} with lease ID 0x{}: {} on node {} size {} " + + "does not belong to any file.", strBlockReportId, fullBrLeaseId, b, node, b.getNumBytes()); } } @@ -2806,9 +2901,9 @@ public boolean processReport(final DatanodeID nodeID, if (metrics != null) { metrics.addStorageBlockReport((int) (endTime - startTime)); } - blockLog.info("BLOCK* processReport 0x{}: from storage {} node {}, " + + blockLog.info("BLOCK* processReport 0x{} with lease ID 0x{}: from storage {} node {}, " + "blocks: {}, hasStaleStorage: {}, processing time: {} msecs, " + - "invalidatedBlocks: {}", strBlockReportId, storage.getStorageID(), + "invalidatedBlocks: {}", strBlockReportId, fullBrLeaseId, storage.getStorageID(), nodeID, newReport.getNumberOfBlocks(), node.hasStaleStorages(), (endTime - startTime), invalidatedBlocks.size()); @@ -2834,7 +2929,7 @@ public void removeBRLeaseIfNeeded(final DatanodeID nodeID, context.getTotalRpcs(), Long.toHexString(context.getReportId())); } } finally { - namesystem.writeUnlock(); + namesystem.writeUnlock("removeBRLeaseIfNeeded"); } } @@ -2872,7 +2967,7 @@ void rescanPostponedMisreplicatedBlocks() { postponedMisreplicatedBlocks.addAll(rescannedMisreplicatedBlocks); rescannedMisreplicatedBlocks.clear(); long endSize = postponedMisreplicatedBlocks.size(); - namesystem.writeUnlock(); + namesystem.writeUnlock("rescanPostponedMisreplicatedBlocks"); LOG.info("Rescan of postponedMisreplicatedBlocks completed in {}" + " msecs. {} blocks are left. {} blocks were removed.", (Time.monotonicNow() - startTime), endSize, (startSize - endSize)); @@ -3739,7 +3834,7 @@ private void processMisReplicatesAsync() throws InterruptedException { break; } } finally { - namesystem.writeUnlock(); + namesystem.writeUnlock("processMisReplicatesAsync"); // Make sure it is out of the write lock for sufficiently long time. Thread.sleep(sleepDuration); } @@ -3794,7 +3889,7 @@ public int processMisReplicatedBlocks(List blocks) { "Re-scanned block {}, result is {}", blk, r); } } finally { - namesystem.writeUnlock(); + namesystem.writeUnlock("processMisReplicatedBlocks"); } } } catch (InterruptedException ex) { @@ -4024,6 +4119,14 @@ private void chooseExcessRedundancyStriped(BlockCollection bc, List replicasToDelete = placementPolicy .chooseReplicasToDelete(nonExcess, candidates, (short) 1, excessTypes, null, null); + if (LOG.isDebugEnabled()) { + LOG.debug("Choose redundant EC replicas to delete from blk_{} which is located in {}", + sblk.getBlockId(), storage2index); + LOG.debug("Storages with candidate blocks to be deleted: {}", candidates); + LOG.debug("Storages with blocks to be deleted: {}", replicasToDelete); + } + Preconditions.checkArgument(candidates.containsAll(replicasToDelete), + "The EC replicas to be deleted are not in the candidate list"); for (DatanodeStorageInfo chosen : replicasToDelete) { processChosenExcessRedundancy(nonExcess, chosen, storedBlock); candidates.remove(chosen); @@ -4509,7 +4612,7 @@ void processExtraRedundancyBlocksOnInService( // testPlacementWithLocalRackNodesDecommissioned, it is not protected by // lock, only when called by DatanodeManager.refreshNodes have writeLock if (namesystem.hasWriteLock()) { - namesystem.writeUnlock(); + namesystem.writeUnlock("processExtraRedundancyBlocksOnInService"); try { Thread.sleep(1); } catch (InterruptedException e) { @@ -4641,7 +4744,7 @@ private void updateNeededReconstructions(final BlockInfo block, repl.outOfServiceReplicas(), oldExpectedReplicas); } } finally { - namesystem.writeUnlock(); + namesystem.writeUnlock("updateNeededReconstructions"); } } @@ -4698,7 +4801,7 @@ private int invalidateWorkForOneNode(DatanodeInfo dn) { return 0; } } finally { - namesystem.writeUnlock(); + namesystem.writeUnlock("invalidateWorkForOneNode"); } blockLog.debug("BLOCK* {}: ask {} to delete {}", getClass().getSimpleName(), dn, toInvalidate); @@ -4909,6 +5012,77 @@ public long getLastRedundancyMonitorTS() { return lastRedundancyCycleTS.get(); } + /** + * Periodically deletes the marked block. + */ + private class MarkedDeleteBlockScrubber implements Runnable { + private Iterator toDeleteIterator = null; + private boolean isSleep; + private NameNodeMetrics metrics; + + private void remove(long time) { + if (checkToDeleteIterator()) { + namesystem.writeLock(); + try { + while (toDeleteIterator.hasNext()) { + removeBlock(toDeleteIterator.next()); + metrics.decrPendingDeleteBlocksCount(); + if (Time.monotonicNow() - time > deleteBlockLockTimeMs) { + isSleep = true; + break; + } + } + } finally { + namesystem.writeUnlock("markedDeleteBlockScrubberThread"); + } + } + } + + private boolean checkToDeleteIterator() { + return toDeleteIterator != null && toDeleteIterator.hasNext(); + } + + @Override + public void run() { + LOG.info("Start MarkedDeleteBlockScrubber thread"); + while (namesystem.isRunning() && + !Thread.currentThread().isInterrupted()) { + if (!markedDeleteQueue.isEmpty() || checkToDeleteIterator()) { + try { + metrics = NameNode.getNameNodeMetrics(); + metrics.setDeleteBlocksQueued(markedDeleteQueue.size()); + isSleep = false; + long startTime = Time.monotonicNow(); + remove(startTime); + while (!isSleep && !markedDeleteQueue.isEmpty() && + !Thread.currentThread().isInterrupted()) { + List markedDeleteList = markedDeleteQueue.poll(); + if (markedDeleteList != null) { + toDeleteIterator = markedDeleteList.listIterator(); + } + remove(startTime); + } + } catch (Exception e){ + LOG.warn("MarkedDeleteBlockScrubber encountered an exception" + + " during the block deletion process, " + + " the deletion of the block will retry in {} millisecond.", + deleteBlockUnlockIntervalTimeMs, e); + } + } + if (isSleep) { + LOG.debug("Clear markedDeleteQueue over {}" + + " millisecond to release the write lock", deleteBlockLockTimeMs); + } + try { + Thread.sleep(deleteBlockUnlockIntervalTimeMs); + } catch (InterruptedException e) { + LOG.info("Stopping MarkedDeleteBlockScrubber."); + break; + } + } + } + } + /** * Periodically calls computeBlockRecoveryWork(). */ @@ -4977,7 +5151,7 @@ int computeDatanodeWork() { this.updateState(); this.scheduledReplicationBlocksCount = workFound; } finally { - namesystem.writeUnlock(); + namesystem.writeUnlock("computeDatanodeWork"); } workFound += this.computeInvalidateWork(nodesToProcess); return workFound; @@ -5217,7 +5391,7 @@ private void processQueue() { action = queue.poll(); } while (action != null); } finally { - namesystem.writeUnlock(); + namesystem.writeUnlock("processQueue"); metrics.addBlockOpsBatched(processed - 1); } } catch (InterruptedException e) { @@ -5257,6 +5431,17 @@ public BlockIdManager getBlockIdManager() { return blockIdManager; } + @VisibleForTesting + public ConcurrentLinkedQueue> getMarkedDeleteQueue() { + return markedDeleteQueue; + } + + public void addBLocksToMarkedDeleteQueue(List blockInfos) { + markedDeleteQueue.add(blockInfos); + NameNode.getNameNodeMetrics(). + incrPendingDeleteBlocksCount(blockInfos.size()); + } + public long nextGenerationStamp(boolean legacyBlock) throws IOException { return blockIdManager.nextGenerationStamp(legacyBlock); } @@ -5360,4 +5545,14 @@ public void disableSPS() { public StoragePolicySatisfyManager getSPSManager() { return spsManager; } + + public void setExcludeSlowNodesEnabled(boolean enable) { + placementPolicies.getPolicy(CONTIGUOUS).setExcludeSlowNodesEnabled(enable); + placementPolicies.getPolicy(STRIPED).setExcludeSlowNodesEnabled(enable); + } + + @VisibleForTesting + public boolean getExcludeSlowNodesEnabled(BlockType blockType) { + return placementPolicies.getPolicy(blockType).getExcludeSlowNodesEnabled(); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerFaultInjector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerFaultInjector.java index 7b1f04a64db9c..652cb0439a56b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerFaultInjector.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerFaultInjector.java @@ -49,4 +49,8 @@ public void requestBlockReportLease(DatanodeDescriptor node, long leaseId) { @VisibleForTesting public void removeBlockReportLease(DatanodeDescriptor node, long leaseId) { } + + @VisibleForTesting + public void mockAnException() { + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java index d825c5ecb31e8..4349ba01401d3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java @@ -649,8 +649,8 @@ private SafeModeMonitor(Configuration conf) { DFSConfigKeys.DFS_NAMENODE_SAFEMODE_RECHECK_INTERVAL_DEFAULT); if (recheckInterval < 1) { LOG.warn("Invalid value for " + - DFSConfigKeys.DFS_NAMENODE_SAFEMODE_RECHECK_INTERVAL_DEFAULT + - ".Should be greater than 0, but is {}", recheckInterval); + DFSConfigKeys.DFS_NAMENODE_SAFEMODE_RECHECK_INTERVAL_KEY + + ". Should be greater than 0, but is {}", recheckInterval); recheckInterval = DFSConfigKeys.DFS_NAMENODE_SAFEMODE_RECHECK_INTERVAL_DEFAULT; } LOG.info("Using {} as SafeModeMonitor Interval", recheckInterval); @@ -670,7 +670,7 @@ public void run() { break; } } finally { - namesystem.writeUnlock(); + namesystem.writeUnlock("leaveSafeMode"); } try { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java index 2a212d6261598..85468a57bde47 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java @@ -196,8 +196,9 @@ public void adjustSetsWithChosenReplica( if (moreThanOne.remove(cur)) { if (storages.size() == 1) { final DatanodeStorageInfo remaining = storages.get(0); - moreThanOne.remove(remaining); - exactlyOne.add(remaining); + if (moreThanOne.remove(remaining)) { + exactlyOne.add(remaining); + } } } else { exactlyOne.remove(cur); @@ -261,4 +262,16 @@ public void splitNodesWithRack( } } } + + /** + * Updates the value used for excludeSlowNodesEnabled, which is set by + * {@code DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY} + * initially. + * + * @param enable true, we will filter out slow nodes + * when choosing targets for blocks, otherwise false not filter. + */ + public abstract void setExcludeSlowNodesEnabled(boolean enable); + + public abstract boolean getExcludeSlowNodesEnabled(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java index 9d2bb786836e0..39a40f52ae5f7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java @@ -103,7 +103,7 @@ private String getText() { protected double considerLoadFactor; private boolean preferLocalNode; private boolean dataNodePeerStatsEnabled; - private boolean excludeSlowNodesEnabled; + private volatile boolean excludeSlowNodesEnabled; protected NetworkTopology clusterMap; protected Host2NodesMap host2datanodeMap; private FSClusterStats stats; @@ -304,7 +304,7 @@ private DatanodeStorageInfo[] chooseTarget(int numOfReplicas, && stats.isAvoidingStaleDataNodesForWrite()); boolean avoidLocalRack = (addBlockFlags != null && addBlockFlags.contains(AddBlockFlag.NO_LOCAL_RACK) && writer != null - && clusterMap.getNumOfRacks() > 2); + && clusterMap.getNumOfNonEmptyRacks() > 2); boolean avoidLocalNode = (addBlockFlags != null && addBlockFlags.contains(AddBlockFlag.NO_LOCAL_WRITE) && writer != null @@ -385,7 +385,7 @@ protected int[] getMaxNodesPerRack(int numOfChosen, int numOfReplicas) { totalNumOfReplicas = clusterSize; } // No calculation needed when there is only one rack or picking one node. - int numOfRacks = clusterMap.getNumOfRacks(); + int numOfRacks = clusterMap.getNumOfNonEmptyRacks(); // HDFS-14527 return default when numOfRacks = 0 to avoid // ArithmeticException when calc maxNodesPerRack at following logic. if (numOfRacks <= 1 || totalNumOfReplicas <= 1) { @@ -1173,7 +1173,7 @@ public BlockPlacementStatus verifyBlockPlacement(DatanodeInfo[] locs, .map(dn -> dn.getNetworkLocation()).distinct().count(); return new BlockPlacementStatusDefault(Math.toIntExact(rackCount), - minRacks, clusterMap.getNumOfRacks()); + minRacks, clusterMap.getNumOfNonEmptyRacks()); } /** @@ -1359,5 +1359,14 @@ protected Collection pickupReplicaSet( void setPreferLocalNode(boolean prefer) { this.preferLocalNode = prefer; } -} + @Override + public void setExcludeSlowNodesEnabled(boolean enable) { + this.excludeSlowNodesEnabled = enable; + } + + @Override + public boolean getExcludeSlowNodesEnabled() { + return excludeSlowNodesEnabled; + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRackFaultTolerant.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRackFaultTolerant.java index dad877fdc76fe..a3b3f482e8c23 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRackFaultTolerant.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRackFaultTolerant.java @@ -42,7 +42,7 @@ protected int[] getMaxNodesPerRack(int numOfChosen, int numOfReplicas) { totalNumOfReplicas = clusterSize; } // No calculation needed when there is only one rack or picking one node. - int numOfRacks = clusterMap.getNumOfRacks(); + int numOfRacks = clusterMap.getNumOfNonEmptyRacks(); // HDFS-14527 return default when numOfRacks = 0 to avoid // ArithmeticException when calc maxNodesPerRack at following logic. if (numOfRacks <= 1 || totalNumOfReplicas <= 1) { @@ -90,38 +90,39 @@ protected Node chooseTargetInOrder(int numOfReplicas, EnumMap storageTypes) throws NotEnoughReplicasException { int totalReplicaExpected = results.size() + numOfReplicas; - int numOfRacks = clusterMap.getNumOfRacks(); - if (totalReplicaExpected < numOfRacks || - totalReplicaExpected % numOfRacks == 0) { - writer = chooseOnce(numOfReplicas, writer, excludedNodes, blocksize, - maxNodesPerRack, results, avoidStaleNodes, storageTypes); - return writer; - } + int numOfRacks = clusterMap.getNumOfNonEmptyRacks(); - assert totalReplicaExpected > (maxNodesPerRack -1) * numOfRacks; + try { + if (totalReplicaExpected < numOfRacks || + totalReplicaExpected % numOfRacks == 0) { + writer = chooseOnce(numOfReplicas, writer, excludedNodes, blocksize, + maxNodesPerRack, results, avoidStaleNodes, storageTypes); + return writer; + } - // Calculate numOfReplicas for filling each rack exactly (maxNodesPerRack-1) - // replicas. - HashMap rackCounts = new HashMap<>(); - for (DatanodeStorageInfo dsInfo : results) { - String rack = dsInfo.getDatanodeDescriptor().getNetworkLocation(); - Integer count = rackCounts.get(rack); - if (count != null) { - rackCounts.put(rack, count + 1); - } else { - rackCounts.put(rack, 1); + assert totalReplicaExpected > (maxNodesPerRack -1) * numOfRacks; + + // Calculate numOfReplicas for filling each rack exactly (maxNodesPerRack-1) + // replicas. + HashMap rackCounts = new HashMap<>(); + for (DatanodeStorageInfo dsInfo : results) { + String rack = dsInfo.getDatanodeDescriptor().getNetworkLocation(); + Integer count = rackCounts.get(rack); + if (count != null) { + rackCounts.put(rack, count + 1); + } else { + rackCounts.put(rack, 1); + } } - } - int excess = 0; // Sum of the above (maxNodesPerRack-1) part of nodes in results - for (int count : rackCounts.values()) { - if (count > maxNodesPerRack -1) { - excess += count - (maxNodesPerRack -1); + int excess = 0; // Sum of the above (maxNodesPerRack-1) part of nodes in results + for (int count : rackCounts.values()) { + if (count > maxNodesPerRack -1) { + excess += count - (maxNodesPerRack -1); + } } - } - numOfReplicas = Math.min(totalReplicaExpected - results.size(), - (maxNodesPerRack -1) * numOfRacks - (results.size() - excess)); + numOfReplicas = Math.min(totalReplicaExpected - results.size(), + (maxNodesPerRack -1) * numOfRacks - (results.size() - excess)); - try { // Try to spread the replicas as evenly as possible across racks. // This is done by first placing with (maxNodesPerRack-1), then spreading // the remainder by calling again with maxNodesPerRack. @@ -243,7 +244,7 @@ public BlockPlacementStatus verifyBlockPlacement(DatanodeInfo[] locs, racks.add(dn.getNetworkLocation()); } return new BlockPlacementStatusDefault(racks.size(), numberOfReplicas, - clusterMap.getNumOfRacks()); + clusterMap.getNumOfNonEmptyRacks()); } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReportLeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReportLeaseManager.java index f45daac142c86..2e7e78d14e2c9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReportLeaseManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReportLeaseManager.java @@ -190,8 +190,8 @@ public synchronized void register(DatanodeDescriptor dn) { private synchronized NodeData registerNode(DatanodeDescriptor dn) { if (nodes.containsKey(dn.getDatanodeUuid())) { - LOG.info("Can't register DN {} because it is already registered.", - dn.getDatanodeUuid()); + LOG.info("Can't register DN {} ({}) because it is already registered.", + dn.getDatanodeUuid(), dn.getXferAddr()); return null; } NodeData node = new NodeData(dn.getDatanodeUuid()); @@ -213,8 +213,8 @@ private synchronized void remove(NodeData node) { public synchronized void unregister(DatanodeDescriptor dn) { NodeData node = nodes.remove(dn.getDatanodeUuid()); if (node == null) { - LOG.info("Can't unregister DN {} because it is not currently " + - "registered.", dn.getDatanodeUuid()); + LOG.info("Can't unregister DN {} ({}) because it is not currently " + + "registered.", dn.getDatanodeUuid(), dn.getXferAddr()); return; } remove(node); @@ -224,7 +224,7 @@ public synchronized long requestLease(DatanodeDescriptor dn) { NodeData node = nodes.get(dn.getDatanodeUuid()); if (node == null) { LOG.warn("DN {} ({}) requested a lease even though it wasn't yet " + - "registered. Registering now.", dn.getDatanodeUuid(), + "registered. Registering now.", dn.getDatanodeUuid(), dn.getXferAddr()); node = registerNode(dn); } @@ -232,9 +232,9 @@ public synchronized long requestLease(DatanodeDescriptor dn) { // The DataNode wants a new lease, even though it already has one. // This can happen if the DataNode is restarted in between requesting // a lease and using it. - LOG.debug("Removing existing BR lease 0x{} for DN {} in order to " + + LOG.debug("Removing existing BR lease 0x{} for DN {} ({}) in order to " + "issue a new one.", Long.toHexString(node.leaseId), - dn.getDatanodeUuid()); + dn.getDatanodeUuid(), dn.getXferAddr()); } remove(node); long monotonicNowMs = Time.monotonicNow(); @@ -248,9 +248,9 @@ public synchronized long requestLease(DatanodeDescriptor dn) { allLeases.append(prefix).append(cur.datanodeUuid); prefix = ", "; } - LOG.debug("Can't create a new BR lease for DN {}, because " + - "numPending equals maxPending at {}. Current leases: {}", - dn.getDatanodeUuid(), numPending, allLeases.toString()); + LOG.debug("Can't create a new BR lease for DN {} ({}), because " + + "numPending equals maxPending at {}. Current leases: {}", + dn.getDatanodeUuid(), dn.getXferAddr(), numPending, allLeases); } return 0; } @@ -259,8 +259,8 @@ public synchronized long requestLease(DatanodeDescriptor dn) { node.leaseTimeMs = monotonicNowMs; pendingHead.addToEnd(node); if (LOG.isDebugEnabled()) { - LOG.debug("Created a new BR lease 0x{} for DN {}. numPending = {}", - Long.toHexString(node.leaseId), dn.getDatanodeUuid(), numPending); + LOG.debug("Created a new BR lease 0x{} for DN {} ({}). numPending = {}", + Long.toHexString(node.leaseId), dn.getDatanodeUuid(), dn.getXferAddr(), numPending); } return node.leaseId; } @@ -293,36 +293,36 @@ private synchronized void pruneExpiredPending(long monotonicNowMs) { public synchronized boolean checkLease(DatanodeDescriptor dn, long monotonicNowMs, long id) { if (id == 0) { - LOG.debug("Datanode {} is using BR lease id 0x0 to bypass " + - "rate-limiting.", dn.getDatanodeUuid()); + LOG.debug("Datanode {} ({}) is using BR lease id 0x0 to bypass " + + "rate-limiting.", dn.getDatanodeUuid(), dn.getXferAddr()); return true; } NodeData node = nodes.get(dn.getDatanodeUuid()); if (node == null) { - LOG.info("BR lease 0x{} is not valid for unknown datanode {}", - Long.toHexString(id), dn.getDatanodeUuid()); + LOG.info("BR lease 0x{} is not valid for unknown datanode {} ({})", + Long.toHexString(id), dn.getDatanodeUuid(), dn.getXferAddr()); return false; } if (node.leaseId == 0) { - LOG.warn("BR lease 0x{} is not valid for DN {}, because the DN " + + LOG.warn("BR lease 0x{} is not valid for DN {} ({}), because the DN " + "is not in the pending set.", - Long.toHexString(id), dn.getDatanodeUuid()); + Long.toHexString(id), dn.getDatanodeUuid(), dn.getXferAddr()); return false; } if (pruneIfExpired(monotonicNowMs, node)) { - LOG.warn("BR lease 0x{} is not valid for DN {}, because the lease " + - "has expired.", Long.toHexString(id), dn.getDatanodeUuid()); + LOG.warn("BR lease 0x{} is not valid for DN {} ({}), because the lease " + + "has expired.", Long.toHexString(id), dn.getDatanodeUuid(), dn.getXferAddr()); return false; } if (id != node.leaseId) { - LOG.warn("BR lease 0x{} is not valid for DN {}. Expected BR lease 0x{}.", - Long.toHexString(id), dn.getDatanodeUuid(), + LOG.warn("BR lease 0x{} is not valid for DN {} ({}). Expected BR lease 0x{}.", + Long.toHexString(id), dn.getDatanodeUuid(), dn.getXferAddr(), Long.toHexString(node.leaseId)); return false; } if (LOG.isTraceEnabled()) { - LOG.trace("BR lease 0x{} is valid for DN {}.", - Long.toHexString(id), dn.getDatanodeUuid()); + LOG.trace("BR lease 0x{} is valid for DN {} ({}).", + Long.toHexString(id), dn.getDatanodeUuid(), dn.getXferAddr()); } return true; } @@ -330,20 +330,20 @@ public synchronized boolean checkLease(DatanodeDescriptor dn, public synchronized long removeLease(DatanodeDescriptor dn) { NodeData node = nodes.get(dn.getDatanodeUuid()); if (node == null) { - LOG.info("Can't remove lease for unknown datanode {}", - dn.getDatanodeUuid()); + LOG.info("Can't remove lease for unknown datanode {} ({})", + dn.getDatanodeUuid(), dn.getXferAddr()); return 0; } long id = node.leaseId; if (id == 0) { - LOG.debug("DN {} has no lease to remove.", dn.getDatanodeUuid()); + LOG.debug("DN {} ({}) has no lease to remove.", dn.getDatanodeUuid(), dn.getXferAddr()); return 0; } remove(node); deferredHead.addToEnd(node); if (LOG.isTraceEnabled()) { - LOG.trace("Removed BR lease 0x{} for DN {}. numPending = {}", - Long.toHexString(id), dn.getDatanodeUuid(), numPending); + LOG.trace("Removed BR lease 0x{} for DN {} ({}). numPending = {}", + Long.toHexString(id), dn.getDatanodeUuid(), dn.getXferAddr(), numPending); } return id; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java index 4cc404f55c5d6..1e5f952040d53 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java @@ -302,7 +302,7 @@ private void rescan() throws InterruptedException { rescanCachedBlockMap(); blockManager.getDatanodeManager().resetLastCachingDirectiveSentTime(); } finally { - namesystem.writeUnlock(); + namesystem.writeUnlock("cacheReplicationMonitorRescan"); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminBackoffMonitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminBackoffMonitor.java index c04f3daabf70e..9b38f2353fbd7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminBackoffMonitor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminBackoffMonitor.java @@ -34,6 +34,7 @@ import java.util.LinkedList; import java.util.ArrayDeque; import java.util.Queue; +import java.util.stream.Collectors; /** * This class implements the logic to track decommissioning and entering @@ -149,7 +150,7 @@ protected void processConf() { */ @Override public void stopTrackingNode(DatanodeDescriptor dn) { - pendingNodes.remove(dn); + getPendingNodes().remove(dn); cancelledNodes.add(dn); } @@ -189,9 +190,32 @@ public void run() { * node will be removed from tracking by the pending cancel. */ processCancelledNodes(); + + // Having more nodes decommissioning than can be tracked will impact decommissioning + // performance due to queueing delay + int numTrackedNodes = outOfServiceNodeBlocks.size(); + int numQueuedNodes = getPendingNodes().size(); + int numDecommissioningNodes = numTrackedNodes + numQueuedNodes; + if (numDecommissioningNodes > maxConcurrentTrackedNodes) { + LOG.warn( + "{} nodes are decommissioning but only {} nodes will be tracked at a time. " + + "{} nodes are currently queued waiting to be decommissioned.", + numDecommissioningNodes, maxConcurrentTrackedNodes, numQueuedNodes); + + // Re-queue unhealthy nodes to make space for decommissioning healthy nodes + final List unhealthyDns = outOfServiceNodeBlocks.keySet().stream() + .filter(dn -> !blockManager.isNodeHealthyForDecommissionOrMaintenance(dn)) + .collect(Collectors.toList()); + getUnhealthyNodesToRequeue(unhealthyDns, numDecommissioningNodes).forEach(dn -> { + getPendingNodes().add(dn); + outOfServiceNodeBlocks.remove(dn); + pendingRep.remove(dn); + }); + } + processPendingNodes(); } finally { - namesystem.writeUnlock(); + namesystem.writeUnlock("DatanodeAdminMonitorV2Thread"); } // After processing the above, various parts of the check() method will // take and drop the read / write lock as needed. Aside from the @@ -207,7 +231,7 @@ public void run() { LOG.info("Checked {} blocks this tick. {} nodes are now " + "in maintenance or transitioning state. {} nodes pending. {} " + "nodes waiting to be cancelled.", - numBlocksChecked, outOfServiceNodeBlocks.size(), pendingNodes.size(), + numBlocksChecked, outOfServiceNodeBlocks.size(), getPendingNodes().size(), cancelledNodes.size()); } } @@ -220,10 +244,10 @@ public void run() { * the pendingNodes list from being modified externally. */ private void processPendingNodes() { - while (!pendingNodes.isEmpty() && + while (!getPendingNodes().isEmpty() && (maxConcurrentTrackedNodes == 0 || outOfServiceNodeBlocks.size() < maxConcurrentTrackedNodes)) { - outOfServiceNodeBlocks.put(pendingNodes.poll(), null); + outOfServiceNodeBlocks.put(getPendingNodes().poll(), null); } } @@ -321,12 +345,12 @@ private void processMaintenanceNodes() { // which added the node to the cancelled list. Therefore expired // maintenance nodes do not need to be added to the toRemove list. dnAdmin.stopMaintenance(dn); - namesystem.writeUnlock(); + namesystem.writeUnlock("processMaintenanceNodes"); namesystem.writeLock(); } } } finally { - namesystem.writeUnlock(); + namesystem.writeUnlock("processMaintenanceNodes"); } } @@ -385,7 +409,7 @@ private void processCompletedNodes(List toRemove) { } } } finally { - namesystem.writeUnlock(); + namesystem.writeUnlock("processCompletedNodes"); } } @@ -507,7 +531,7 @@ private void moveBlocksToPending() { // replication if (blocksProcessed >= blocksPerLock) { blocksProcessed = 0; - namesystem.writeUnlock(); + namesystem.writeUnlock("moveBlocksToPending"); namesystem.writeLock(); } blocksProcessed++; @@ -529,7 +553,7 @@ private void moveBlocksToPending() { } } } finally { - namesystem.writeUnlock(); + namesystem.writeUnlock("moveBlocksToPending"); } LOG.debug("{} blocks are now pending replication", pendingCount); } @@ -613,7 +637,7 @@ private void scanDatanodeStorage(DatanodeDescriptor dn, try { storage = dn.getStorageInfos(); } finally { - namesystem.readUnlock(); + namesystem.readUnlock("scanDatanodeStorage"); } for (DatanodeStorageInfo s : storage) { @@ -643,7 +667,7 @@ private void scanDatanodeStorage(DatanodeDescriptor dn, numBlocksChecked++; } } finally { - namesystem.readUnlock(); + namesystem.readUnlock("scanDatanodeStorage"); } } } @@ -698,7 +722,7 @@ private void processPendingReplication() { suspectBlocks.getOutOfServiceBlockCount()); } } finally { - namesystem.writeUnlock(); + namesystem.writeUnlock("processPendingReplication"); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java index a217c9978e822..4149d198c72f8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java @@ -123,7 +123,7 @@ private boolean exceededNumBlocksPerCheck() { @Override public void stopTrackingNode(DatanodeDescriptor dn) { - pendingNodes.remove(dn); + getPendingNodes().remove(dn); outOfServiceNodeBlocks.remove(dn); } @@ -158,25 +158,25 @@ public void run() { LOG.warn("DatanodeAdminMonitor caught exception when processing node.", e); } finally { - namesystem.writeUnlock(); + namesystem.writeUnlock("DatanodeAdminMonitorThread"); } if (numBlocksChecked + numNodesChecked > 0) { LOG.info("Checked {} blocks and {} nodes this tick. {} nodes are now " + "in maintenance or transitioning state. {} nodes pending.", numBlocksChecked, numNodesChecked, outOfServiceNodeBlocks.size(), - pendingNodes.size()); + getPendingNodes().size()); } } /** - * Pop datanodes off the pending list and into decomNodeBlocks, + * Pop datanodes off the pending priority queue and into decomNodeBlocks, * subject to the maxConcurrentTrackedNodes limit. */ private void processPendingNodes() { - while (!pendingNodes.isEmpty() && + while (!getPendingNodes().isEmpty() && (maxConcurrentTrackedNodes == 0 || outOfServiceNodeBlocks.size() < maxConcurrentTrackedNodes)) { - outOfServiceNodeBlocks.put(pendingNodes.poll(), null); + outOfServiceNodeBlocks.put(getPendingNodes().poll(), null); } } @@ -185,6 +185,7 @@ private void check() { it = new CyclicIteration<>(outOfServiceNodeBlocks, iterkey).iterator(); final List toRemove = new ArrayList<>(); + final List unhealthyDns = new ArrayList<>(); while (it.hasNext() && !exceededNumBlocksPerCheck() && namesystem .isRunning()) { @@ -221,6 +222,10 @@ private void check() { LOG.debug("Processing {} node {}", dn.getAdminState(), dn); pruneReliableBlocks(dn, blocks); } + final boolean isHealthy = blockManager.isNodeHealthyForDecommissionOrMaintenance(dn); + if (!isHealthy) { + unhealthyDns.add(dn); + } if (blocks.size() == 0) { if (!fullScan) { // If we didn't just do a full scan, need to re-check with the @@ -236,8 +241,6 @@ private void check() { } // If the full scan is clean AND the node liveness is okay, // we can finally mark as DECOMMISSIONED or IN_MAINTENANCE. - final boolean isHealthy = - blockManager.isNodeHealthyForDecommissionOrMaintenance(dn); if (blocks.size() == 0 && isHealthy) { if (dn.isDecommissionInProgress()) { dnAdmin.setDecommissioned(dn); @@ -270,12 +273,32 @@ private void check() { // an invalid state. LOG.warn("DatanodeAdminMonitor caught exception when processing node " + "{}.", dn, e); - pendingNodes.add(dn); + getPendingNodes().add(dn); toRemove.add(dn); + unhealthyDns.remove(dn); } finally { iterkey = dn; } } + + // Having more nodes decommissioning than can be tracked will impact decommissioning + // performance due to queueing delay + int numTrackedNodes = outOfServiceNodeBlocks.size() - toRemove.size(); + int numQueuedNodes = getPendingNodes().size(); + int numDecommissioningNodes = numTrackedNodes + numQueuedNodes; + if (numDecommissioningNodes > maxConcurrentTrackedNodes) { + LOG.warn( + "{} nodes are decommissioning but only {} nodes will be tracked at a time. " + + "{} nodes are currently queued waiting to be decommissioned.", + numDecommissioningNodes, maxConcurrentTrackedNodes, numQueuedNodes); + + // Re-queue unhealthy nodes to make space for decommissioning healthy nodes + getUnhealthyNodesToRequeue(unhealthyDns, numDecommissioningNodes).forEach(dn -> { + getPendingNodes().add(dn); + outOfServiceNodeBlocks.remove(dn); + }); + } + // Remove the datanodes that are DECOMMISSIONED or in service after // maintenance expiration. for (DatanodeDescriptor dn : toRemove) { @@ -350,7 +373,7 @@ private void processBlocksInternal( // lock. // Yielding is required in case of block number is greater than the // configured per-iteration-limit. - namesystem.writeUnlock(); + namesystem.writeUnlock("processBlocksInternal"); try { LOG.debug("Yielded lock during decommission/maintenance check"); Thread.sleep(0, 500); @@ -367,8 +390,10 @@ private void processBlocksInternal( // Remove the block from the list if it's no longer in the block map, // e.g. the containing file has been deleted if (blockManager.blocksMap.getStoredBlock(block) == null) { - LOG.trace("Removing unknown block {}", block); - it.remove(); + if (pruneReliableBlocks) { + LOG.trace("Removing unknown block {}", block); + it.remove(); + } continue; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminManager.java index 42b6ddd8c78fb..1c95c26a190c3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminManager.java @@ -177,6 +177,8 @@ public void startDecommission(DatanodeDescriptor node) { if (!node.isDecommissionInProgress() && !node.isDecommissioned()) { // Update DN stats maintained by HeartbeatManager hbManager.startDecommission(node); + // Update cluster's emptyRack + blockManager.getDatanodeManager().getNetworkTopology().decommissionNode(node); // hbManager.startDecommission will set dead node to decommissioned. if (node.isDecommissionInProgress()) { for (DatanodeStorageInfo storage : node.getStorageInfos()) { @@ -201,6 +203,8 @@ public void stopDecommission(DatanodeDescriptor node) { if (node.isDecommissionInProgress() || node.isDecommissioned()) { // Update DN stats maintained by HeartbeatManager hbManager.stopDecommission(node); + // Update cluster's emptyRack + blockManager.getDatanodeManager().getNetworkTopology().recommissionNode(node); // extra redundancy blocks will be detected and processed when // the dead node comes back and send in its full block report. if (node.isAlive()) { @@ -413,4 +417,4 @@ void runMonitorForTest() throws ExecutionException, InterruptedException { executor.submit(monitor).get(); } -} \ No newline at end of file +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminMonitorBase.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminMonitorBase.java index 9eee241edddf8..f9761c2dfcb4e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminMonitorBase.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminMonitorBase.java @@ -24,8 +24,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayDeque; +import java.util.Comparator; +import java.util.List; +import java.util.PriorityQueue; import java.util.Queue; +import java.util.stream.Stream; /** * This abstract class provides some base methods which are inherited by @@ -35,12 +38,20 @@ public abstract class DatanodeAdminMonitorBase implements DatanodeAdminMonitorInterface, Configurable { + /** + * Sort by lastUpdate time descending order, such that unhealthy + * nodes are de-prioritized given they cannot be decommissioned. + */ + static final Comparator PENDING_NODES_QUEUE_COMPARATOR = + (dn1, dn2) -> Long.compare(dn2.getLastUpdate(), dn1.getLastUpdate()); + protected BlockManager blockManager; protected Namesystem namesystem; protected DatanodeAdminManager dnAdmin; protected Configuration conf; - protected final Queue pendingNodes = new ArrayDeque<>(); + private final PriorityQueue pendingNodes = new PriorityQueue<>( + PENDING_NODES_QUEUE_COMPARATOR); /** * The maximum number of nodes to track in outOfServiceNodeBlocks. @@ -151,4 +162,34 @@ public int getPendingNodeCount() { public Queue getPendingNodes() { return pendingNodes; } + + /** + * If node "is dead while in Decommission In Progress", it cannot be decommissioned + * until it becomes healthy again. If there are more pendingNodes than can be tracked + * & some unhealthy tracked nodes, then re-queue the unhealthy tracked nodes + * to avoid blocking decommissioning of healthy nodes. + * + * @param unhealthyDns The unhealthy datanodes which may be re-queued + * @param numDecommissioningNodes The total number of nodes being decommissioned + * @return Stream of unhealthy nodes to be re-queued + */ + Stream getUnhealthyNodesToRequeue( + final List unhealthyDns, int numDecommissioningNodes) { + if (!unhealthyDns.isEmpty()) { + // Compute the number of unhealthy nodes to re-queue + final int numUnhealthyNodesToRequeue = + Math.min(numDecommissioningNodes - maxConcurrentTrackedNodes, unhealthyDns.size()); + + LOG.warn("{} limit has been reached, re-queueing {} " + + "nodes which are dead while in Decommission In Progress.", + DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_MAX_CONCURRENT_TRACKED_NODES, + numUnhealthyNodesToRequeue); + + // Order unhealthy nodes by lastUpdate descending such that nodes + // which have been unhealthy the longest are preferred to be re-queued + return unhealthyDns.stream().sorted(PENDING_NODES_QUEUE_COMPARATOR.reversed()) + .limit(numUnhealthyNodesToRequeue); + } + return Stream.empty(); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java index f4ee01d4271a8..a6ca697fa63f7 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java @@ -647,6 +647,17 @@ Iterator getBlockIterator(final int startBlock) { return new BlockIterator(startBlock, getStorageInfos()); } + /** + * Get iterator, which starts iterating from the specified block and storages. + * + * @param startBlock on which blocks are start iterating + * @param storageInfos specified storages + */ + Iterator getBlockIterator( + final int startBlock, final DatanodeStorageInfo[] storageInfos) { + return new BlockIterator(startBlock, storageInfos); + } + @VisibleForTesting public void incrementPendingReplicationWithoutTargets() { pendingReplicationWithoutTargets++; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java index 91cd68d06547d..b1dbbdde3b9f8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java @@ -23,6 +23,8 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.util.Preconditions; + +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList; import org.apache.hadoop.thirdparty.com.google.common.net.InetAddresses; import org.apache.hadoop.fs.StorageType; @@ -193,10 +195,6 @@ public class DatanodeManager { private final HashMap datanodesSoftwareVersions = new HashMap<>(4, 0.75f); - /** - * True if we should process latency metrics from downstream peers. - */ - private final boolean dataNodePeerStatsEnabled; /** * True if we should process latency metrics from individual DN disks. */ @@ -210,11 +208,11 @@ public class DatanodeManager { private static final String IP_PORT_SEPARATOR = ":"; @Nullable - private final SlowPeerTracker slowPeerTracker; + private SlowPeerTracker slowPeerTracker; private static Set slowNodesUuidSet = Sets.newConcurrentHashSet(); private Daemon slowPeerCollectorDaemon; private final long slowPeerCollectionInterval; - private final int maxSlowPeerReportNodes; + private volatile int maxSlowPeerReportNodes; @Nullable private final SlowDiskTracker slowDiskTracker; @@ -247,16 +245,15 @@ public class DatanodeManager { this.datanodeAdminManager = new DatanodeAdminManager(namesystem, blockManager, heartbeatManager); this.fsClusterStats = newFSClusterStats(); - this.dataNodePeerStatsEnabled = conf.getBoolean( - DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_KEY, - DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_DEFAULT); this.dataNodeDiskStatsEnabled = Util.isDiskStatsEnabled(conf.getInt( DFSConfigKeys.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_PERCENTAGE_KEY, DFSConfigKeys. DFS_DATANODE_FILEIO_PROFILING_SAMPLING_PERCENTAGE_DEFAULT)); final Timer timer = new Timer(); - this.slowPeerTracker = dataNodePeerStatsEnabled ? - new SlowPeerTracker(conf, timer) : null; + final boolean dataNodePeerStatsEnabledVal = + conf.getBoolean(DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_KEY, + DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_DEFAULT); + initSlowPeerTracker(conf, timer, dataNodePeerStatsEnabledVal); this.maxSlowPeerReportNodes = conf.getInt( DFSConfigKeys.DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_KEY, DFSConfigKeys.DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_DEFAULT); @@ -264,7 +261,7 @@ public class DatanodeManager { DFSConfigKeys.DFS_NAMENODE_SLOWPEER_COLLECT_INTERVAL_KEY, DFSConfigKeys.DFS_NAMENODE_SLOWPEER_COLLECT_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS); - if (slowPeerTracker != null) { + if (slowPeerTracker.isSlowPeerTrackerEnabled()) { startSlowPeerCollector(); } this.slowDiskTracker = dataNodeDiskStatsEnabled ? @@ -314,18 +311,12 @@ public class DatanodeManager { DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_DEFAULT); // 5 minutes this.heartbeatExpireInterval = 2 * heartbeatRecheckInterval + 10 * 1000 * heartbeatIntervalSeconds; - // Effected block invalidate limit is the bigger value between - // value configured in hdfs-site.xml, and 20 * HB interval. final int configuredBlockInvalidateLimit = conf.getInt( DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_KEY, DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_DEFAULT); - final int countedBlockInvalidateLimit = 20*(int)(heartbeatIntervalSeconds); - this.blockInvalidateLimit = Math.max(countedBlockInvalidateLimit, - configuredBlockInvalidateLimit); - LOG.info(DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_KEY - + ": configured=" + configuredBlockInvalidateLimit - + ", counted=" + countedBlockInvalidateLimit - + ", effected=" + blockInvalidateLimit); + // Block invalidate limit also has some dependency on heartbeat interval. + // Check setBlockInvalidateLimit(). + setBlockInvalidateLimit(configuredBlockInvalidateLimit); this.checkIpHostnameInRegistration = conf.getBoolean( DFSConfigKeys.DFS_NAMENODE_DATANODE_REGISTRATION_IP_HOSTNAME_CHECK_KEY, DFSConfigKeys.DFS_NAMENODE_DATANODE_REGISTRATION_IP_HOSTNAME_CHECK_DEFAULT); @@ -372,6 +363,21 @@ public class DatanodeManager { DFSConfigKeys.DFS_NAMENODE_BLOCKS_PER_POSTPONEDBLOCKS_RESCAN_KEY_DEFAULT); } + /** + * Determines whether slow peer tracker should be enabled. If dataNodePeerStatsEnabledVal is + * true, slow peer tracker is initialized. + * + * @param conf The configuration to use while initializing slowPeerTracker. + * @param timer Timer object for slowPeerTracker. + * @param dataNodePeerStatsEnabled To determine whether slow peer tracking should be enabled. + */ + public void initSlowPeerTracker(Configuration conf, Timer timer, + boolean dataNodePeerStatsEnabled) { + this.slowPeerTracker = dataNodePeerStatsEnabled ? + new SlowPeerTracker(conf, timer) : + new SlowPeerDisabledTracker(conf, timer); + } + private void startSlowPeerCollector() { if (slowPeerCollectorDaemon != null) { return; @@ -515,6 +521,15 @@ public boolean getEnableAvoidSlowDataNodesForRead() { return this.avoidSlowDataNodesForRead; } + public void setMaxSlowpeerCollectNodes(int maxNodes) { + this.maxSlowPeerReportNodes = maxNodes; + } + + @VisibleForTesting + public int getMaxSlowpeerCollectNodes() { + return this.maxSlowPeerReportNodes; + } + /** * Sort the non-striped located blocks by the distance to the target host. * @@ -849,7 +864,7 @@ public void removeDatanode(final DatanodeID node) + node + " does not exist"); } } finally { - namesystem.writeUnlock(); + namesystem.writeUnlock("removeDatanode"); } } @@ -1168,6 +1183,7 @@ public void registerDatanode(DatanodeRegistration nodeReg) nodeN = null; } + boolean updateHost2DatanodeMap = false; if (nodeS != null) { if (nodeN == nodeS) { // The same datanode has been just restarted to serve the same data @@ -1186,7 +1202,11 @@ public void registerDatanode(DatanodeRegistration nodeReg) nodes with its data cleared (or user can just remove the StorageID value in "VERSION" file under the data directory of the datanode, but this is might not work if VERSION file format has changed - */ + */ + // Check if nodeS's host information is same as nodeReg's, if not, + // it needs to update host2DatanodeMap accordringly. + updateHost2DatanodeMap = !nodeS.getXferAddr().equals(nodeReg.getXferAddr()); + NameNode.stateChangeLog.info("BLOCK* registerDatanode: " + nodeS + " is replaced by " + nodeReg + " with the same storageID " + nodeReg.getDatanodeUuid()); @@ -1196,6 +1216,11 @@ nodes with its data cleared (or user can just remove the StorageID try { // update cluster map getNetworkTopology().remove(nodeS); + + // Update Host2DatanodeMap + if (updateHost2DatanodeMap) { + getHost2DatanodeMap().remove(nodeS); + } if(shouldCountVersion(nodeS)) { decrementVersionCount(nodeS.getSoftwareVersion()); } @@ -1214,6 +1239,11 @@ nodes with its data cleared (or user can just remove the StorageID nodeS.setDependentHostNames( getNetworkDependenciesWithDefault(nodeS)); } + + if (updateHost2DatanodeMap) { + getHost2DatanodeMap().add(nodeS); + } + getNetworkTopology().add(nodeS); resolveUpgradeDomain(nodeS); @@ -1293,7 +1323,7 @@ public void refreshNodes(final Configuration conf) throws IOException { refreshDatanodes(); countSoftwareVersions(); } finally { - namesystem.writeUnlock(); + namesystem.writeUnlock("refreshNodes"); } } @@ -1637,7 +1667,17 @@ public List getDatanodeListForReport( } return nodes; } - + + public List getAllSlowDataNodes() { + if (slowPeerTracker == null) { + LOG.debug("{} is disabled. Try enabling it first to capture slow peer outliers.", + DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_KEY); + return ImmutableList.of(); + } + List slowNodes = slowPeerTracker.getSlowNodes(getNumOfDataNodes()); + return getDnDescriptorsFromIpAddr(slowNodes); + } + /** * Checks if name resolution was successful for the given address. If IP * address and host name are the same, then it means name resolution has @@ -1853,12 +1893,13 @@ public DatanodeCommand[] handleHeartbeat(DatanodeRegistration nodeReg, nodeinfo.setBalancerBandwidth(0); } - if (slowPeerTracker != null) { + Preconditions.checkNotNull(slowPeerTracker, "slowPeerTracker should not be un-assigned"); + + if (slowPeerTracker.isSlowPeerTrackerEnabled()) { final Map slowPeersMap = slowPeers.getSlowPeers(); if (!slowPeersMap.isEmpty()) { if (LOG.isDebugEnabled()) { - LOG.debug("DataNode " + nodeReg + " reported slow peers: " + - slowPeersMap); + LOG.debug("DataNode " + nodeReg + " reported slow peers: " + slowPeersMap); } for (String slowNodeId : slowPeersMap.keySet()) { slowPeerTracker.addReport(slowNodeId, nodeReg.getIpcAddr(false)); @@ -2079,8 +2120,25 @@ private void setHeartbeatInterval(long intervalSeconds, this.heartbeatRecheckInterval = recheckInterval; this.heartbeatExpireInterval = 2L * recheckInterval + 10 * 1000 * intervalSeconds; - this.blockInvalidateLimit = Math.max(20 * (int) (intervalSeconds), - blockInvalidateLimit); + this.blockInvalidateLimit = getBlockInvalidateLimit(blockInvalidateLimit); + } + + private int getBlockInvalidateLimitFromHBInterval() { + return 20 * (int) heartbeatIntervalSeconds; + } + + private int getBlockInvalidateLimit(int configuredBlockInvalidateLimit) { + return Math.max(getBlockInvalidateLimitFromHBInterval(), configuredBlockInvalidateLimit); + } + + public void setBlockInvalidateLimit(int configuredBlockInvalidateLimit) { + final int countedBlockInvalidateLimit = getBlockInvalidateLimitFromHBInterval(); + // Effected block invalidate limit is the bigger value between + // value configured in hdfs-site.xml, and 20 * HB interval. + this.blockInvalidateLimit = getBlockInvalidateLimit(configuredBlockInvalidateLimit); + LOG.info("{} : configured={}, counted={}, effected={}", + DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_KEY, configuredBlockInvalidateLimit, + countedBlockInvalidateLimit, this.blockInvalidateLimit); } /** @@ -2089,7 +2147,8 @@ private void setHeartbeatInterval(long intervalSeconds, * @return */ public String getSlowPeersReport() { - return slowPeerTracker != null ? slowPeerTracker.getJson() : null; + Preconditions.checkNotNull(slowPeerTracker, "slowPeerTracker should not be un-assigned"); + return slowPeerTracker.getJson(); } /** @@ -2098,24 +2157,29 @@ public String getSlowPeersReport() { */ public Set getSlowPeersUuidSet() { Set slowPeersUuidSet = Sets.newConcurrentHashSet(); - if (slowPeerTracker == null) { - return slowPeersUuidSet; - } - ArrayList slowNodes = - slowPeerTracker.getSlowNodes(maxSlowPeerReportNodes); - for (String slowNode : slowNodes) { - if (StringUtils.isBlank(slowNode) - || !slowNode.contains(IP_PORT_SEPARATOR)) { + List slowNodes; + Preconditions.checkNotNull(slowPeerTracker, "slowPeerTracker should not be un-assigned"); + slowNodes = slowPeerTracker.getSlowNodes(maxSlowPeerReportNodes); + List datanodeDescriptors = getDnDescriptorsFromIpAddr(slowNodes); + datanodeDescriptors.forEach( + datanodeDescriptor -> slowPeersUuidSet.add(datanodeDescriptor.getDatanodeUuid())); + return slowPeersUuidSet; + } + + private List getDnDescriptorsFromIpAddr(List nodes) { + List datanodeDescriptors = new ArrayList<>(); + for (String node : nodes) { + if (StringUtils.isBlank(node) || !node.contains(IP_PORT_SEPARATOR)) { continue; } - String ipAddr = slowNode.split(IP_PORT_SEPARATOR)[0]; + String ipAddr = node.split(IP_PORT_SEPARATOR)[0]; DatanodeDescriptor datanodeByHost = host2DatanodeMap.getDatanodeByHost(ipAddr); if (datanodeByHost != null) { - slowPeersUuidSet.add(datanodeByHost.getDatanodeUuid()); + datanodeDescriptors.add(datanodeByHost); } } - return slowPeersUuidSet; + return datanodeDescriptors; } /** @@ -2173,9 +2237,15 @@ public DatanodeStorageReport[] getDatanodeStorageReport( for (int i = 0; i < reports.length; i++) { final DatanodeDescriptor d = datanodes.get(i); reports[i] = new DatanodeStorageReport( - new DatanodeInfoBuilder().setFrom(d).build(), d.getStorageReports()); + new DatanodeInfoBuilder().setFrom(d).setNumBlocks(d.numBlocks()).build(), + d.getStorageReports()); } return reports; } + + @VisibleForTesting + public Map getDatanodeMap() { + return datanodeMap; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java index 01ef5dca4e6b6..df9f6e00a39df 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java @@ -168,6 +168,11 @@ public boolean areBlockContentsStale() { return blockContentsStale; } + @VisibleForTesting + public void setBlockContentsStale(boolean value) { + blockContentsStale = value; + } + void markStaleAfterFailover() { heartbeatedSinceFailover = false; blockContentsStale = true; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java index 8de3f381ddffe..6158677654b94 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java @@ -30,8 +30,8 @@ import java.util.Set; class ErasureCodingWork extends BlockReconstructionWork { - private final byte[] liveBlockIndicies; - private final byte[] liveBusyBlockIndicies; + private final byte[] liveBlockIndices; + private final byte[] liveBusyBlockIndices; private final String blockPoolId; public ErasureCodingWork(String blockPoolId, BlockInfo block, @@ -40,18 +40,18 @@ public ErasureCodingWork(String blockPoolId, BlockInfo block, List containingNodes, List liveReplicaStorages, int additionalReplRequired, int priority, - byte[] liveBlockIndicies, byte[] liveBusyBlockIndicies) { + byte[] liveBlockIndices, byte[] liveBusyBlockIndices) { super(block, bc, srcNodes, containingNodes, liveReplicaStorages, additionalReplRequired, priority); this.blockPoolId = blockPoolId; - this.liveBlockIndicies = liveBlockIndicies; - this.liveBusyBlockIndicies = liveBusyBlockIndicies; + this.liveBlockIndices = liveBlockIndices; + this.liveBusyBlockIndices = liveBusyBlockIndices; LOG.debug("Creating an ErasureCodingWork to {} reconstruct ", block); } - byte[] getLiveBlockIndicies() { - return liveBlockIndicies; + byte[] getLiveBlockIndices() { + return liveBlockIndices; } @Override @@ -72,15 +72,15 @@ void chooseTargets(BlockPlacementPolicy blockplacement, */ private boolean hasAllInternalBlocks() { final BlockInfoStriped block = (BlockInfoStriped) getBlock(); - if (liveBlockIndicies.length - + liveBusyBlockIndicies.length < block.getRealTotalBlockNum()) { + if (liveBlockIndices.length + + liveBusyBlockIndices.length < block.getRealTotalBlockNum()) { return false; } BitSet bitSet = new BitSet(block.getTotalBlockNum()); - for (byte index : liveBlockIndicies) { + for (byte index : liveBlockIndices) { bitSet.set(index); } - for (byte busyIndex: liveBusyBlockIndicies) { + for (byte busyIndex: liveBusyBlockIndices) { bitSet.set(busyIndex); } for (int i = 0; i < block.getRealDataBlockNum(); i++) { @@ -147,14 +147,14 @@ void addTaskToDatanode(NumberReplicas numberReplicas) { } else { targets[0].getDatanodeDescriptor().addBlockToBeErasureCoded( new ExtendedBlock(blockPoolId, stripedBlk), getSrcNodes(), targets, - getLiveBlockIndicies(), stripedBlk.getErasureCodingPolicy()); + getLiveBlockIndices(), stripedBlk.getErasureCodingPolicy()); } } private void createReplicationWork(int sourceIndex, DatanodeStorageInfo target) { BlockInfoStriped stripedBlk = (BlockInfoStriped) getBlock(); - final byte blockIndex = liveBlockIndicies[sourceIndex]; + final byte blockIndex = liveBlockIndices[sourceIndex]; final DatanodeDescriptor source = getSrcNodes()[sourceIndex]; final long internBlkLen = StripedBlockUtil.getInternalBlockLength( stripedBlk.getNumBytes(), stripedBlk.getCellSize(), @@ -173,7 +173,7 @@ private List findLeavingServiceSources() { BitSet bitSet = new BitSet(block.getRealTotalBlockNum()); for (int i = 0; i < getSrcNodes().length; i++) { if (getSrcNodes()[i].isInService()) { - bitSet.set(liveBlockIndicies[i]); + bitSet.set(liveBlockIndices[i]); } } // If the block is on the node which is decommissioning or @@ -184,7 +184,7 @@ private List findLeavingServiceSources() { if ((getSrcNodes()[i].isDecommissionInProgress() || (getSrcNodes()[i].isEnteringMaintenance() && getSrcNodes()[i].isAlive())) && - !bitSet.get(liveBlockIndicies[i])) { + !bitSet.get(liveBlockIndices[i])) { srcIndices.add(i); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java index 96d83f29ae271..b923ba3a65590 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java @@ -256,9 +256,12 @@ synchronized void updateHeartbeat(final DatanodeDescriptor node, int xceiverCount, int failedVolumes, VolumeFailureSummary volumeFailureSummary) { stats.subtract(node); - blockManager.updateHeartbeat(node, reports, cacheCapacity, cacheUsed, - xceiverCount, failedVolumes, volumeFailureSummary); - stats.add(node); + try { + blockManager.updateHeartbeat(node, reports, cacheCapacity, cacheUsed, + xceiverCount, failedVolumes, volumeFailureSummary); + } finally { + stats.add(node); + } } synchronized void updateLifeline(final DatanodeDescriptor node, @@ -266,13 +269,16 @@ synchronized void updateLifeline(final DatanodeDescriptor node, int xceiverCount, int failedVolumes, VolumeFailureSummary volumeFailureSummary) { stats.subtract(node); - // This intentionally calls updateHeartbeatState instead of - // updateHeartbeat, because we don't want to modify the - // heartbeatedSinceRegistration flag. Arrival of a lifeline message does - // not count as arrival of the first heartbeat. - blockManager.updateHeartbeatState(node, reports, cacheCapacity, cacheUsed, - xceiverCount, failedVolumes, volumeFailureSummary); - stats.add(node); + try { + // This intentionally calls updateHeartbeatState instead of + // updateHeartbeat, because we don't want to modify the + // heartbeatedSinceRegistration flag. Arrival of a lifeline message does + // not count as arrival of the first heartbeat. + blockManager.updateHeartbeatState(node, reports, cacheCapacity, cacheUsed, + xceiverCount, failedVolumes, volumeFailureSummary); + } finally { + stats.add(node); + } } synchronized void startDecommission(final DatanodeDescriptor node) { @@ -497,7 +503,7 @@ void heartbeatCheck() { try { dm.removeDeadDatanode(dead, !dead.isMaintenance()); } finally { - namesystem.writeUnlock(); + namesystem.writeUnlock("removeDeadDatanode"); } } if (failedStorage != null) { @@ -506,7 +512,7 @@ void heartbeatCheck() { try { blockManager.removeBlocksAssociatedTo(failedStorage); } finally { - namesystem.writeUnlock(); + namesystem.writeUnlock("removeBlocksAssociatedTo"); } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/LowRedundancyBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/LowRedundancyBlocks.java index a73266795ea6e..ddaa2fec5ded3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/LowRedundancyBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/LowRedundancyBlocks.java @@ -248,8 +248,8 @@ private int getPriorityContiguous(int curReplicas, int readOnlyReplicas, // highest priority return QUEUE_HIGHEST_PRIORITY; } else if ((curReplicas * 3) < expectedReplicas) { - //can only afford one replica loss - //this is considered very insufficiently redundant blocks. + //there is less than a third as many blocks as requested; + //this is considered very under-replicated. return QUEUE_VERY_LOW_REDUNDANCY; } else { //add to the normal queue for insufficiently redundant blocks diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerDisabledTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerDisabledTracker.java new file mode 100644 index 0000000000000..2f6be2f5a7b39 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerDisabledTracker.java @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.server.blockmanagement; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedSet; + +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.util.Preconditions; +import org.apache.hadoop.util.Timer; + +/** + * Disabled tracker for slow peers. To be used when dfs.datanode.peer.stats.enabled is disabled. + */ +@InterfaceAudience.Private +public class SlowPeerDisabledTracker extends SlowPeerTracker { + + private static final Logger LOG = LoggerFactory.getLogger(SlowPeerDisabledTracker.class); + + public SlowPeerDisabledTracker(Configuration conf, Timer timer) { + super(conf, timer); + final boolean dataNodePeerStatsEnabledVal = + conf.getBoolean(DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_KEY, + DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_DEFAULT); + Preconditions.checkArgument(!dataNodePeerStatsEnabledVal, + "SlowPeerDisabledTracker should only be used for disabled slow peer stats."); + } + + @Override + public boolean isSlowPeerTrackerEnabled() { + return false; + } + + @Override + public void addReport(String slowNode, String reportingNode) { + LOG.trace("Adding slow peer report is disabled. To enable it, please enable config {}.", + DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_KEY); + } + + @Override + public Set getReportsForNode(String slowNode) { + LOG.trace("Retrieval of slow peer report is disabled. To enable it, please enable config {}.", + DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_KEY); + return ImmutableSet.of(); + } + + @Override + public Map> getReportsForAllDataNodes() { + LOG.trace("Retrieval of slow peer report for all nodes is disabled. " + + "To enable it, please enable config {}.", + DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_KEY); + return ImmutableMap.of(); + } + + @Override + public String getJson() { + LOG.trace("Retrieval of slow peer reports as json string is disabled. " + + "To enable it, please enable config {}.", + DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_KEY); + return null; + } + + @Override + public List getSlowNodes(int numNodes) { + return ImmutableList.of(); + } + +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerTracker.java index e54934e8813a7..c674bc19f8a29 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerTracker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerTracker.java @@ -39,6 +39,7 @@ import java.util.Collections; import java.util.Comparator; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.PriorityQueue; import java.util.Set; @@ -109,6 +110,15 @@ public SlowPeerTracker(Configuration conf, Timer timer) { DFSConfigKeys.DFS_DATANODE_MAX_NODES_TO_REPORT_DEFAULT); } + /** + * If SlowPeerTracker is enabled, return true, else returns false. + * + * @return true if slow peer tracking is enabled, else false. + */ + public boolean isSlowPeerTrackerEnabled() { + return true; + } + /** * Add a new report. DatanodeIds can be the DataNodeIds or addresses * We don't care as long as the caller is consistent. @@ -239,7 +249,7 @@ public SortedSet getReportingNodes() { * @param numNodes * @return */ - public ArrayList getSlowNodes(int numNodes) { + public List getSlowNodes(int numNodes) { Collection jsonReports = getJsonReports(numNodes); ArrayList slowNodes = new ArrayList<>(); for (ReportForJson jsonReport : jsonReports) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/AutoCloseDataSetLock.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/AutoCloseDataSetLock.java new file mode 100644 index 0000000000000..bf6edda7abd06 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/AutoCloseDataSetLock.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.server.common; + +import org.apache.hadoop.util.AutoCloseableLock; +import org.apache.hadoop.util.StringUtils; + +import java.util.concurrent.locks.Lock; + +import static org.apache.hadoop.hdfs.server.datanode.DataSetLockManager.LOG; + +/** + * Extending AutoCloseableLock such that the users can + * use a try-with-resource syntax. + */ +public class AutoCloseDataSetLock extends AutoCloseableLock { + private Lock lock; + private AutoCloseDataSetLock parentLock; + private DataNodeLockManager dataNodeLockManager; + + public AutoCloseDataSetLock(Lock lock) { + this.lock = lock; + } + + @Override + public void close() { + if (lock != null) { + lock.unlock(); + if (dataNodeLockManager != null) { + dataNodeLockManager.hook(); + } + } else { + LOG.error("Try to unlock null lock" + + StringUtils.getStackTrace(Thread.currentThread())); + } + if (parentLock != null) { + parentLock.close(); + } + } + + /** + * Actually acquire the lock. + */ + public void lock() { + if (lock != null) { + lock.lock(); + return; + } + LOG.error("Try to lock null lock" + + StringUtils.getStackTrace(Thread.currentThread())); + } + + public void setParentLock(AutoCloseDataSetLock parent) { + if (parentLock == null) { + this.parentLock = parent; + } + } + + public void setDataNodeLockManager(DataNodeLockManager + dataNodeLockManager) { + this.dataNodeLockManager = dataNodeLockManager; + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/DataNodeLockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/DataNodeLockManager.java new file mode 100644 index 0000000000000..e7a3b38357ac9 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/DataNodeLockManager.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.server.common; + +/** + * Use for manage a set of lock for datanode. + */ +public interface DataNodeLockManager { + + /** + * Acquire block pool level first if you want to Acquire volume lock. + * Or only acquire block pool level lock. + */ + enum LockLevel { + BLOCK_POOl, + VOLUME + } + + /** + * Acquire readLock and then lock. + */ + T readLock(LockLevel level, String... resources); + + /** + * Acquire writeLock and then lock. + */ + T writeLock(LockLevel level, String... resources); + + /** + * Add a lock to LockManager. + */ + void addLock(LockLevel level, String... resources); + + /** + * Remove a lock from LockManager. + */ + void removeLock(LockLevel level, String... resources); + + /** + * LockManager may need to back hook. + */ + void hook(); +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/NoLockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/NoLockManager.java new file mode 100644 index 0000000000000..848495cc4e018 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/NoLockManager.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.server.common; + +import java.util.concurrent.locks.Lock; + +/** + * Some ut or temp replicaMap not need to lock with DataSetLockManager. + */ +public class NoLockManager implements DataNodeLockManager { + private final NoDataSetLock lock = new NoDataSetLock(null); + + private static final class NoDataSetLock extends AutoCloseDataSetLock { + + private NoDataSetLock(Lock lock) { + super(lock); + } + + @Override + public void lock() { + } + + @Override + public void close() { + } + } + + public NoLockManager() { + } + + @Override + public AutoCloseDataSetLock readLock(LockLevel level, String... resources) { + return lock; + } + + @Override + public AutoCloseDataSetLock writeLock(LockLevel level, String... resources) { + return lock; + } + + @Override + public void addLock(LockLevel level, String... resources) { + } + + @Override + public void removeLock(LockLevel level, String... resources) { + } + + @Override + public void hook() { + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java index 8643821c86993..4a06588258abd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java @@ -21,6 +21,7 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.RandomAccessFile; +import java.util.HashMap; import java.util.Map; import java.util.Properties; import java.util.SortedSet; @@ -112,6 +113,20 @@ public String toString() { .append(";nsid=").append(namespaceID).append(";c=").append(cTime); return sb.toString(); } + + /** + * Returns string representation of Storage info attributes stored in Map. + * + * @return string representation of storage info attributes. + */ + public String toMapString() { + Map storageInfo = new HashMap<>(); + storageInfo.put("LayoutVersion", layoutVersion); + storageInfo.put("ClusterId", clusterID); + storageInfo.put("NamespaceId", namespaceID); + storageInfo.put("CreationTime", cTime); + return storageInfo.toString(); + } public String toColonSeparatedString() { return Joiner.on(":").join( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/sps/BlockDispatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/sps/BlockDispatcher.java index f87fcaef054c0..f7756c74851a6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/sps/BlockDispatcher.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/sps/BlockDispatcher.java @@ -101,7 +101,7 @@ public BlockDispatcher(int sockTimeout, int ioFileBuffSize, */ public BlockMovementStatus moveBlock(BlockMovingInfo blkMovingInfo, SaslDataTransferClient saslClient, ExtendedBlock eb, Socket sock, - DataEncryptionKeyFactory km, Token accessToken) { + DataEncryptionKeyFactory km, Token accessToken) throws IOException { LOG.info("Start moving block:{} from src:{} to destin:{} to satisfy " + "storageType, sourceStoragetype:{} and destinStoragetype:{}", blkMovingInfo.getBlock(), blkMovingInfo.getSource(), @@ -149,14 +149,6 @@ public BlockMovementStatus moveBlock(BlockMovingInfo blkMovingInfo, LOG.debug("Pinned block can't be moved, so skipping block:{}", blkMovingInfo.getBlock(), e); return BlockMovementStatus.DN_BLK_STORAGE_MOVEMENT_SUCCESS; - } catch (IOException e) { - // TODO: handle failure retries - LOG.warn( - "Failed to move block:{} from src:{} to destin:{} to satisfy " - + "storageType:{}", - blkMovingInfo.getBlock(), blkMovingInfo.getSource(), - blkMovingInfo.getTarget(), blkMovingInfo.getTargetStorageType(), e); - return BlockMovementStatus.DN_BLK_STORAGE_MOVEMENT_FAILURE; } finally { IOUtils.closeStream(out); IOUtils.closeStream(in); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java index 3c63160f28362..51a6f115ccdea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java @@ -853,4 +853,12 @@ boolean shouldRetryInit() { return isAlive(); } + boolean isSlownode() { + for (BPServiceActor actor : bpServices) { + if (actor.isSlownode()) { + return true; + } + } + return false; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java index 718520f7273de..838259d7f6a0b 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hdfs.server.datanode; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY; import static org.apache.hadoop.util.Time.monotonicNow; import java.io.Closeable; @@ -53,6 +55,7 @@ import org.apache.hadoop.hdfs.protocolPB.DatanodeLifelineProtocolClientSideTranslatorPB; import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB; import org.apache.hadoop.hdfs.server.common.IncorrectVersionException; +import org.apache.hadoop.hdfs.server.common.DataNodeLockManager.LockLevel; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.protocol.BlockReportContext; import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand; @@ -70,6 +73,7 @@ import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.util.Time; import org.apache.hadoop.util.VersionInfo; import org.apache.hadoop.util.VersionUtil; @@ -112,6 +116,7 @@ enum RunningState { private String nnId = null; private volatile RunningState runningState = RunningState.CONNECTING; private volatile boolean shouldServiceRun = true; + private volatile boolean isSlownode = false; private final DataNode dn; private final DNConf dnConf; private long prevBlockReportId; @@ -205,6 +210,7 @@ Map getActorInfoMap() { String.valueOf(getScheduler().getLastBlockReportTime())); info.put("maxBlockReportSize", String.valueOf(getMaxBlockReportSize())); info.put("maxDataLength", String.valueOf(maxDataLength)); + info.put("isSlownode", String.valueOf(isSlownode)); return info; } @@ -304,6 +310,10 @@ private void connectToNNAndHandshake() throws IOException { // info. NamespaceInfo nsInfo = retrieveNamespaceInfo(); + // init block pool lock when init. + dn.getDataSetLockManager().addLock(LockLevel.BLOCK_POOl, + nsInfo.getBlockPoolID()); + // Verify that this matches the other NN in this HA pair. // This also initializes our block pool in the DN if we are // the first NN connection for this BP. @@ -452,8 +462,8 @@ List blockReport(long fullBrLeaseId) throws IOException { dn.getMetrics().addBlockReport(brSendCost, getRpcMetricSuffix()); final int nCmds = cmds.size(); LOG.info((success ? "S" : "Uns") + - "uccessfully sent block report 0x" + - Long.toHexString(reportId) + " to namenode: " + nnAddr + + "uccessfully sent block report 0x" + Long.toHexString(reportId) + + " with lease ID 0x" + Long.toHexString(fullBrLeaseId) + " to namenode: " + nnAddr + ", containing " + reports.length + " storage report(s), of which we sent " + numReportsSent + "." + " The reports had " + totalBlockCount + @@ -549,11 +559,11 @@ HeartbeatResponse sendHeartBeat(boolean requestBlockReportLease) volumeFailureSummary.getFailedStorageLocations().length : 0; final boolean outliersReportDue = scheduler.isOutliersReportDue(now); final SlowPeerReports slowPeers = - outliersReportDue && dn.getPeerMetrics() != null ? + outliersReportDue && dnConf.peerStatsEnabled && dn.getPeerMetrics() != null ? SlowPeerReports.create(dn.getPeerMetrics().getOutliers()) : SlowPeerReports.EMPTY_REPORT; final SlowDiskReports slowDisks = - outliersReportDue && dn.getDiskMetrics() != null ? + outliersReportDue && dnConf.diskStatsEnabled && dn.getDiskMetrics() != null ? SlowDiskReports.create(dn.getDiskMetrics().getDiskOutliersStats()) : SlowDiskReports.EMPTY_REPORT; @@ -729,6 +739,7 @@ private void offerService() throws Exception { handleRollingUpgradeStatus(resp); } commandProcessingThread.enqueue(resp.getCommands()); + isSlownode = resp.getIsSlownode(); } } @@ -1190,8 +1201,8 @@ static class Scheduler { private final long heartbeatIntervalMs; private final long lifelineIntervalMs; - private final long blockReportIntervalMs; - private final long outliersReportIntervalMs; + private volatile long blockReportIntervalMs; + private volatile long outliersReportIntervalMs; Scheduler(long heartbeatIntervalMs, long lifelineIntervalMs, long blockReportIntervalMs, long outliersReportIntervalMs) { @@ -1267,6 +1278,7 @@ boolean isOutliersReportDue(long curTime) { void forceFullBlockReportNow() { forceFullBlockReport.set(true); + resetBlockReportTime = true; } /** @@ -1346,6 +1358,27 @@ void setNextBlockReportTime(long nextBlockReportTime) { this.nextBlockReportTime.getAndSet(nextBlockReportTime); } + long getBlockReportIntervalMs() { + return this.blockReportIntervalMs; + } + + void setBlockReportIntervalMs(long intervalMs) { + Preconditions.checkArgument(intervalMs > 0, + DFS_BLOCKREPORT_INTERVAL_MSEC_KEY + " should be larger than 0"); + this.blockReportIntervalMs = intervalMs; + } + + void setOutliersReportIntervalMs(long intervalMs) { + Preconditions.checkArgument(intervalMs > 0, + DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY + " should be larger than 0"); + this.outliersReportIntervalMs = intervalMs; + } + + @VisibleForTesting + long getOutliersReportIntervalMs() { + return this.outliersReportIntervalMs; + } + /** * Wrapped for testing. * @return @@ -1474,4 +1507,8 @@ void stopCommandProcessingThread() { commandProcessingThread.interrupt(); } } + + boolean isSlownode() { + return isSlownode; + } } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolManager.java index 1139500c22a19..2ce81f593e33a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolManager.java @@ -307,4 +307,27 @@ protected BPOfferService createBPOS( Map getBpByNameserviceId() { return bpByNameserviceId; } + + boolean isSlownodeByNameserviceId(String nsId) { + if (bpByNameserviceId.containsKey(nsId)) { + return bpByNameserviceId.get(nsId).isSlownode(); + } + return false; + } + + boolean isSlownodeByBlockPoolId(String bpId) { + if (bpByBlockPoolId.containsKey(bpId)) { + return bpByBlockPoolId.get(bpId).isSlownode(); + } + return false; + } + + boolean isSlownode() { + for (BPOfferService bpOfferService : bpByBlockPoolId.values()) { + if (bpOfferService.isSlownode()) { + return true; + } + } + return false; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java index cf1f688c11d44..9b3a899323642 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java @@ -412,6 +412,7 @@ boolean packetSentInTime() { void flushOrSync(boolean isSync, long seqno) throws IOException { long flushTotalNanos = 0; long begin = Time.monotonicNow(); + DataNodeFaultInjector.get().delay(); if (checksumOut != null) { long flushStartNanos = System.nanoTime(); checksumOut.flush(); @@ -445,6 +446,7 @@ void flushOrSync(boolean isSync, long seqno) throws IOException { } long duration = Time.monotonicNow() - begin; if (duration > datanodeSlowLogThresholdMs && LOG.isWarnEnabled()) { + datanode.metrics.incrSlowFlushOrSyncCount(); LOG.warn("Slow flushOrSync took " + duration + "ms (threshold=" + datanodeSlowLogThresholdMs + "ms), isSync:" + isSync + ", flushTotalNanos=" + flushTotalNanos + "ns, volume=" + getVolumeBaseUri() @@ -887,7 +889,7 @@ duration, datanodeSlowLogThresholdMs, getVolumeBaseUri(), */ private void trackSendPacketToLastNodeInPipeline(final long elapsedMs) { final DataNodePeerMetrics peerMetrics = datanode.getPeerMetrics(); - if (peerMetrics != null && isPenultimateNode) { + if (datanode.getDnConf().peerStatsEnabled && peerMetrics != null && isPenultimateNode) { peerMetrics.addSendPacketDownstream(mirrorNameForMetrics, elapsedMs); } } @@ -1107,7 +1109,7 @@ private void initPerfMonitoring(DatanodeInfo[] downstreams) { if (downstreams != null && downstreams.length > 0) { downstreamDNs = downstreams; isPenultimateNode = (downstreams.length == 1); - if (isPenultimateNode && datanode.getPeerMetrics() != null) { + if (isPenultimateNode && datanode.getDnConf().peerStatsEnabled) { mirrorNameForMetrics = (downstreams[0].getInfoSecurePort() != 0 ? downstreams[0].getInfoSecureAddr() : downstreams[0].getInfoAddr()); LOG.debug("Will collect peer metrics for downstream node {}", @@ -1327,7 +1329,8 @@ void sendOOBResponse(final Status ackStatus) throws IOException, LOG.info("Sending an out of band ack of type " + ackStatus); try { sendAckUpstreamUnprotected(null, PipelineAck.UNKOWN_SEQNO, 0L, 0L, - PipelineAck.combineHeader(datanode.getECN(), ackStatus)); + PipelineAck.combineHeader(datanode.getECN(), ackStatus, + datanode.getSLOWByBlockPoolId(block.getBlockPoolId()))); } finally { // Let others send ack. Unless there are miltiple OOB send // calls, there can be only one waiter, the responder thread. @@ -1409,7 +1412,8 @@ public void run() { LOG.info("Relaying an out of band ack of type " + oobStatus); sendAckUpstream(ack, PipelineAck.UNKOWN_SEQNO, 0L, 0L, PipelineAck.combineHeader(datanode.getECN(), - Status.SUCCESS)); + Status.SUCCESS, + datanode.getSLOWByBlockPoolId(block.getBlockPoolId()))); continue; } seqno = ack.getSeqno(); @@ -1499,7 +1503,8 @@ public void run() { Status myStatus = pkt != null ? pkt.ackStatus : Status.SUCCESS; sendAckUpstream(ack, expected, totalAckTimeNanos, (pkt != null ? pkt.offsetInBlock : 0), - PipelineAck.combineHeader(datanode.getECN(), myStatus)); + PipelineAck.combineHeader(datanode.getECN(), myStatus, + datanode.getSLOWByBlockPoolId(block.getBlockPoolId()))); if (pkt != null) { // remove the packet from the ack queue removeAckHead(); @@ -1620,8 +1625,10 @@ private void sendAckUpstreamUnprotected(PipelineAck ack, long seqno, // downstream nodes, reply should contain one reply. replies = new int[] { myHeader }; } else if (mirrorError) { // ack read error - int h = PipelineAck.combineHeader(datanode.getECN(), Status.SUCCESS); - int h1 = PipelineAck.combineHeader(datanode.getECN(), Status.ERROR); + int h = PipelineAck.combineHeader(datanode.getECN(), Status.SUCCESS, + datanode.getSLOWByBlockPoolId(block.getBlockPoolId())); + int h1 = PipelineAck.combineHeader(datanode.getECN(), Status.ERROR, + datanode.getSLOWByBlockPoolId(block.getBlockPoolId())); replies = new int[] {h, h1}; } else { short ackLen = type == PacketResponderType.LAST_IN_PIPELINE ? 0 : ack @@ -1631,6 +1638,7 @@ private void sendAckUpstreamUnprotected(PipelineAck ack, long seqno, for (int i = 0; i < ackLen; ++i) { replies[i + 1] = ack.getHeaderFlag(i); } + DataNodeFaultInjector.get().markSlow(mirrorAddr, replies); // If the mirror has reported that it received a corrupt packet, // do self-destruct to mark myself bad, instead of making the // mirror node bad. The mirror is guaranteed to be good without @@ -1650,6 +1658,7 @@ private void sendAckUpstreamUnprotected(PipelineAck ack, long seqno, } // send my ack back to upstream datanode long begin = Time.monotonicNow(); + DataNodeFaultInjector.get().delay(); /* for test only, no-op in production system */ DataNodeFaultInjector.get().delaySendingAckToUpstream(inAddr); replyAck.write(upstreamOut); @@ -1659,6 +1668,7 @@ private void sendAckUpstreamUnprotected(PipelineAck ack, long seqno, inAddr, duration); if (duration > datanodeSlowLogThresholdMs) { + datanode.metrics.incrSlowAckToUpstreamCount(); LOG.warn("Slow PacketResponder send ack to upstream took " + duration + "ms (threshold=" + datanodeSlowLogThresholdMs + "ms), " + myString + ", replyAck=" + replyAck diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java index 172a245b3525a..5c4212fea537f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java @@ -41,6 +41,7 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.datatransfer.PacketHeader; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState; +import org.apache.hadoop.hdfs.server.common.DataNodeLockManager.LockLevel; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference; import org.apache.hadoop.hdfs.server.datanode.fsdataset.LengthInputStream; import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaInputStreams; @@ -256,7 +257,8 @@ class BlockSender implements java.io.Closeable { // the append write. ChunkChecksum chunkChecksum = null; final long replicaVisibleLength; - try(AutoCloseableLock lock = datanode.data.acquireDatasetReadLock()) { + try (AutoCloseableLock lock = datanode.getDataSetLockManager().readLock( + LockLevel.BLOCK_POOl, block.getBlockPoolId())) { replica = getReplica(block, datanode); replicaVisibleLength = replica.getVisibleLength(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java index 7f86668fbb93d..9b5343321d30b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java @@ -74,6 +74,7 @@ import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.DataTransferSaslUtil; import org.apache.hadoop.hdfs.server.common.Util; import org.apache.hadoop.security.SaslPropertiesResolver; +import org.apache.hadoop.util.Preconditions; import java.util.concurrent.TimeUnit; @@ -105,14 +106,14 @@ public class DNConf { final long readaheadLength; final long heartBeatInterval; private final long lifelineIntervalMs; - final long blockReportInterval; - final long blockReportSplitThreshold; - final boolean peerStatsEnabled; - final boolean diskStatsEnabled; - final long outliersReportIntervalMs; + volatile long blockReportInterval; + volatile long blockReportSplitThreshold; + volatile boolean peerStatsEnabled; + volatile boolean diskStatsEnabled; + volatile long outliersReportIntervalMs; final long ibrInterval; - final long initialBlockReportDelayMs; - final long cacheReportInterval; + volatile long initialBlockReportDelayMs; + volatile long cacheReportInterval; final long datanodeSlowIoWarningThresholdMs; final String minimumNameNodeVersion; @@ -214,19 +215,7 @@ public DNConf(final Configurable dn) { this.datanodeSlowIoWarningThresholdMs = getConf().getLong( DFSConfigKeys.DFS_DATANODE_SLOW_IO_WARNING_THRESHOLD_KEY, DFSConfigKeys.DFS_DATANODE_SLOW_IO_WARNING_THRESHOLD_DEFAULT); - - long initBRDelay = getConf().getTimeDuration( - DFS_BLOCKREPORT_INITIAL_DELAY_KEY, - DFS_BLOCKREPORT_INITIAL_DELAY_DEFAULT, - TimeUnit.SECONDS, TimeUnit.MILLISECONDS); - if (initBRDelay >= blockReportInterval) { - initBRDelay = 0; - DataNode.LOG.info(DFS_BLOCKREPORT_INITIAL_DELAY_KEY + " is " - + "greater than or equal to" + DFS_BLOCKREPORT_INTERVAL_MSEC_KEY - + ". Setting initial delay to 0 msec:"); - } - initialBlockReportDelayMs = initBRDelay; - + initBlockReportDelay(); heartBeatInterval = getConf().getTimeDuration(DFS_HEARTBEAT_INTERVAL_KEY, DFS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS, TimeUnit.MILLISECONDS); @@ -310,6 +299,19 @@ public DNConf(final Configurable dn) { ); } + private void initBlockReportDelay() { + long initBRDelay = getConf().getTimeDuration( + DFS_BLOCKREPORT_INITIAL_DELAY_KEY, + DFS_BLOCKREPORT_INITIAL_DELAY_DEFAULT, TimeUnit.SECONDS, TimeUnit.MILLISECONDS); + if (initBRDelay >= blockReportInterval || initBRDelay < 0) { + initBRDelay = 0; + DataNode.LOG.info(DFS_BLOCKREPORT_INITIAL_DELAY_KEY + + " is greater than or equal to " + DFS_BLOCKREPORT_INTERVAL_MSEC_KEY + + ". Setting initial delay to 0 msec."); + } + initialBlockReportDelayMs = initBRDelay; + } + // We get minimumNameNodeVersion via a method so it can be mocked out in tests. String getMinimumNameNodeVersion() { return this.minimumNameNodeVersion; @@ -475,7 +477,49 @@ public long getProcessCommandsThresholdMs() { return processCommandsThresholdMs; } + void setBlockReportInterval(long intervalMs) { + Preconditions.checkArgument(intervalMs > 0, + DFS_BLOCKREPORT_INTERVAL_MSEC_KEY + " should be larger than 0"); + blockReportInterval = intervalMs; + } + public long getBlockReportInterval() { return blockReportInterval; } + + void setCacheReportInterval(long intervalMs) { + Preconditions.checkArgument(intervalMs > 0, + DFS_CACHEREPORT_INTERVAL_MSEC_KEY + " should be larger than 0"); + cacheReportInterval = intervalMs; + } + + public long getCacheReportInterval() { + return cacheReportInterval; + } + + void setBlockReportSplitThreshold(long threshold) { + Preconditions.checkArgument(threshold >= 0, + DFS_BLOCKREPORT_SPLIT_THRESHOLD_KEY + " should be larger than or equal to 0"); + blockReportSplitThreshold = threshold; + } + + void setInitBRDelayMs(String delayMs) { + dn.getConf().set(DFS_BLOCKREPORT_INITIAL_DELAY_KEY, delayMs); + initBlockReportDelay(); + } + + void setPeerStatsEnabled(boolean enablePeerStats) { + peerStatsEnabled = enablePeerStats; + } + + public void setFileIoProfilingSamplingPercentage(int samplingPercentage) { + diskStatsEnabled = Util.isDiskStatsEnabled(samplingPercentage); + } + + public void setOutliersReportIntervalMs(String reportIntervalMs) { + dn.getConf().set(DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY, reportIntervalMs); + outliersReportIntervalMs = getConf().getTimeDuration( + DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY, + DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 0bb59de95ae5e..10438cd79e39c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -18,6 +18,19 @@ package org.apache.hadoop.hdfs.server.datanode; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_INITIAL_DELAY_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_INITIAL_DELAY_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_GETSPACEUSED_CLASSNAME; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DU_INTERVAL_DEFAULT; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DU_INTERVAL_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_GETSPACEUSED_JITTER_DEFAULT; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_GETSPACEUSED_JITTER_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CACHEREPORT_INTERVAL_MSEC_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CACHEREPORT_INTERVAL_MSEC_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_SPLIT_THRESHOLD_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_SPLIT_THRESHOLD_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_ADDRESS_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_ALLOW_SAME_DISK_TIERING; @@ -27,6 +40,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DNS_INTERFACE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DNS_NAMESERVER_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_PERCENTAGE_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_PERCENTAGE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HANDLER_COUNT_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HANDLER_COUNT_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY; @@ -36,11 +51,27 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_KEYTAB_FILE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MAX_LOCKED_MEMORY_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MAX_RECEIVER_THREADS_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MAX_RECEIVER_THREADS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_NETWORK_COUNTS_CACHE_MAX_SIZE_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_NETWORK_COUNTS_CACHE_MAX_SIZE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_OOB_TIMEOUT_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_OOB_TIMEOUT_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_PEER_METRICS_MIN_OUTLIER_DETECTION_SAMPLES_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_PEER_METRICS_MIN_OUTLIER_DETECTION_SAMPLES_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_PLUGINS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_STARTUP_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_DEFAULT; @@ -52,9 +83,14 @@ import static org.apache.hadoop.hdfs.protocol.datatransfer.BlockConstructionStage.PIPELINE_SETUP_CREATE; import static org.apache.hadoop.hdfs.protocol.datatransfer.BlockConstructionStage.PIPELINE_SETUP_STREAMING_RECOVERY; import static org.apache.hadoop.util.ExitUtil.terminate; +import static org.apache.hadoop.util.Preconditions.checkNotNull; +import static org.apache.hadoop.util.Time.now; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.DF; +import org.apache.hadoop.fs.DU; +import org.apache.hadoop.fs.GetSpaceUsed; +import org.apache.hadoop.fs.WindowsGetSpaceUsed; import org.apache.hadoop.hdfs.protocol.proto.ReconfigurationProtocolProtos.ReconfigurationProtocolService; import java.io.BufferedOutputStream; @@ -118,8 +154,11 @@ import org.apache.hadoop.hdfs.DFSUtilClient; import org.apache.hadoop.hdfs.HDFSPolicyProvider; import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.server.common.DataNodeLockManager.LockLevel; import org.apache.hadoop.hdfs.server.datanode.checker.DatasetVolumeChecker; import org.apache.hadoop.hdfs.server.datanode.checker.StorageLocationChecker; +import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.BlockPoolSlice; +import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl; import org.apache.hadoop.hdfs.util.DataTransferThrottler; import org.apache.hadoop.util.*; import org.apache.hadoop.hdfs.client.BlockReportOptions; @@ -299,7 +338,23 @@ public class DataNode extends ReconfigurableBase Collections.unmodifiableList( Arrays.asList( DFS_DATANODE_DATA_DIR_KEY, - DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_KEY)); + DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_KEY, + DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, + DFS_BLOCKREPORT_SPLIT_THRESHOLD_KEY, + DFS_BLOCKREPORT_INITIAL_DELAY_KEY, + DFS_DATANODE_MAX_RECEIVER_THREADS_KEY, + DFS_CACHEREPORT_INTERVAL_MSEC_KEY, + DFS_DATANODE_PEER_STATS_ENABLED_KEY, + DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_KEY, + DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_KEY, + DFS_DATANODE_PEER_METRICS_MIN_OUTLIER_DETECTION_SAMPLES_KEY, + DFS_DATANODE_FILEIO_PROFILING_SAMPLING_PERCENTAGE_KEY, + DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY, + DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_KEY, + DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY, + FS_DU_INTERVAL_KEY, + FS_GETSPACEUSED_JITTER_KEY, + FS_GETSPACEUSED_CLASSNAME)); public static final Log METRICS_LOG = LogFactory.getLog("DataNodeMetricsLog"); @@ -341,8 +396,8 @@ public static InetSocketAddress createSocketAddr(String target) { DataNodeMetrics metrics; @Nullable - private DataNodePeerMetrics peerMetrics; - private DataNodeDiskMetrics diskMetrics; + private volatile DataNodePeerMetrics peerMetrics; + private volatile DataNodeDiskMetrics diskMetrics; private InetSocketAddress streamingAddr; private LoadingCache> datanodeNetworkCounts; @@ -372,6 +427,7 @@ public static InetSocketAddress createSocketAddr(String target) { private final String confVersion; private final long maxNumberOfBlocksToLog; private final boolean pipelineSupportECN; + private final boolean pipelineSupportSlownode; private final List usersWithLocalPathAccess; private final boolean connectToDnViaHostname; @@ -391,6 +447,7 @@ public static InetSocketAddress createSocketAddr(String target) { .availableProcessors(); private static final double CONGESTION_RATIO = 1.5; private DiskBalancer diskBalancer; + private DataSetLockManager dataSetLockManager; private final ExecutorService xferService; @@ -411,6 +468,11 @@ private static Tracer createTracer(Configuration conf) { private ScheduledThreadPoolExecutor metricsLoggerTimer; + private long startTime = 0; + + private DataTransferThrottler ecReconstuctReadThrottler; + private DataTransferThrottler ecReconstuctWriteThrottler; + /** * Creates a dummy DataNode for testing purpose. */ @@ -427,8 +489,10 @@ private static Tracer createTracer(Configuration conf) { this.connectToDnViaHostname = false; this.blockScanner = new BlockScanner(this, this.getConf()); this.pipelineSupportECN = false; + this.pipelineSupportSlownode = false; this.socketFactory = NetUtils.getDefaultSocketFactory(conf); this.dnConf = new DNConf(this); + this.dataSetLockManager = new DataSetLockManager(conf); initOOBTimeout(); storageLocationChecker = null; volumeChecker = new DatasetVolumeChecker(conf, new Timer()); @@ -447,6 +511,7 @@ private static Tracer createTracer(Configuration conf) { super(conf); this.tracer = createTracer(conf); this.fileIoProvider = new FileIoProvider(conf, this); + this.dataSetLockManager = new DataSetLockManager(conf); this.blockScanner = new BlockScanner(this); this.lastDiskErrorCheck = 0; this.maxNumberOfBlocksToLog = conf.getLong(DFS_MAX_NUM_BLOCKS_TO_LOG_KEY, @@ -465,6 +530,9 @@ private static Tracer createTracer(Configuration conf) { this.pipelineSupportECN = conf.getBoolean( DFSConfigKeys.DFS_PIPELINE_ECN_ENABLED, DFSConfigKeys.DFS_PIPELINE_ECN_ENABLED_DEFAULT); + this.pipelineSupportSlownode = conf.getBoolean( + DFSConfigKeys.DFS_PIPELINE_SLOWNODE_ENABLED, + DFSConfigKeys.DFS_PIPELINE_SLOWNODE_ENABLED_DEFAULT); confVersion = "core-" + conf.get("hadoop.common.configuration.version", "UNSPECIFIED") + @@ -519,6 +587,16 @@ public Map load(String key) { initOOBTimeout(); this.storageLocationChecker = storageLocationChecker; + long ecReconstuctReadBandwidth = conf.getLongBytes( + DFSConfigKeys.DFS_DATANODE_EC_RECONSTRUCT_READ_BANDWIDTHPERSEC_KEY, + DFSConfigKeys.DFS_DATANODE_EC_RECONSTRUCT_READ_BANDWIDTHPERSEC_DEFAULT); + long ecReconstuctWriteBandwidth = conf.getLongBytes( + DFSConfigKeys.DFS_DATANODE_EC_RECONSTRUCT_WRITE_BANDWIDTHPERSEC_KEY, + DFSConfigKeys.DFS_DATANODE_EC_RECONSTRUCT_WRITE_BANDWIDTHPERSEC_DEFAULT); + this.ecReconstuctReadThrottler = ecReconstuctReadBandwidth > 0 ? + new DataTransferThrottler(100, ecReconstuctReadBandwidth) : null; + this.ecReconstuctWriteThrottler = ecReconstuctWriteBandwidth > 0 ? + new DataTransferThrottler(100, ecReconstuctWriteBandwidth) : null; } @Override // ReconfigurableBase @@ -533,83 +611,336 @@ protected Configuration getNewConf() { public String reconfigurePropertyImpl(String property, String newVal) throws ReconfigurationException { switch (property) { - case DFS_DATANODE_DATA_DIR_KEY: { - IOException rootException = null; + case DFS_DATANODE_DATA_DIR_KEY: { + IOException rootException = null; + try { + LOG.info("Reconfiguring {} to {}", property, newVal); + this.refreshVolumes(newVal); + return getConf().get(DFS_DATANODE_DATA_DIR_KEY); + } catch (IOException e) { + rootException = e; + } finally { + // Send a full block report to let NN acknowledge the volume changes. try { - LOG.info("Reconfiguring {} to {}", property, newVal); - this.refreshVolumes(newVal); - return getConf().get(DFS_DATANODE_DATA_DIR_KEY); + triggerBlockReport( + new BlockReportOptions.Factory().setIncremental(false).build()); } catch (IOException e) { - rootException = e; + LOG.warn("Exception while sending the block report after refreshing" + + " volumes {} to {}", property, newVal, e); + if (rootException == null) { + rootException = e; + } } finally { - // Send a full block report to let NN acknowledge the volume changes. - try { - triggerBlockReport( - new BlockReportOptions.Factory().setIncremental(false).build()); - } catch (IOException e) { - LOG.warn("Exception while sending the block report after refreshing" - + " volumes {} to {}", property, newVal, e); - if (rootException == null) { - rootException = e; - } - } finally { - if (rootException != null) { - throw new ReconfigurationException(property, newVal, - getConf().get(property), rootException); - } + if (rootException != null) { + throw new ReconfigurationException(property, newVal, + getConf().get(property), rootException); } } - break; } - case DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_KEY: { - ReconfigurationException rootException = null; - try { - LOG.info("Reconfiguring {} to {}", property, newVal); - int movers; - if (newVal == null) { - // set to default - movers = DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_DEFAULT; - } else { - movers = Integer.parseInt(newVal); - if (movers <= 0) { - rootException = new ReconfigurationException( - property, - newVal, - getConf().get(property), - new IllegalArgumentException( - "balancer max concurrent movers must be larger than 0")); - } - } - boolean success = xserver.updateBalancerMaxConcurrentMovers(movers); - if (!success) { + break; + } + case DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_KEY: { + ReconfigurationException rootException = null; + try { + LOG.info("Reconfiguring {} to {}", property, newVal); + int movers; + if (newVal == null) { + // set to default + movers = DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_DEFAULT; + } else { + movers = Integer.parseInt(newVal); + if (movers <= 0) { rootException = new ReconfigurationException( property, newVal, getConf().get(property), new IllegalArgumentException( - "Could not modify concurrent moves thread count")); + "balancer max concurrent movers must be larger than 0")); } - return Integer.toString(movers); - } catch (NumberFormatException nfe) { + } + boolean success = xserver.updateBalancerMaxConcurrentMovers(movers); + if (!success) { rootException = new ReconfigurationException( - property, newVal, getConf().get(property), nfe); - } finally { - if (rootException != null) { - LOG.warn(String.format( - "Exception in updating balancer max concurrent movers %s to %s", - property, newVal), rootException); - throw rootException; - } + property, + newVal, + getConf().get(property), + new IllegalArgumentException( + "Could not modify concurrent moves thread count")); + } + return Integer.toString(movers); + } catch (NumberFormatException nfe) { + rootException = new ReconfigurationException( + property, newVal, getConf().get(property), nfe); + } finally { + if (rootException != null) { + LOG.warn(String.format( + "Exception in updating balancer max concurrent movers %s to %s", + property, newVal), rootException); + throw rootException; } - break; } - default: - break; + break; + } + case DFS_BLOCKREPORT_INTERVAL_MSEC_KEY: + case DFS_BLOCKREPORT_SPLIT_THRESHOLD_KEY: + case DFS_BLOCKREPORT_INITIAL_DELAY_KEY: + return reconfBlockReportParameters(property, newVal); + case DFS_DATANODE_MAX_RECEIVER_THREADS_KEY: + return reconfDataXceiverParameters(property, newVal); + case DFS_CACHEREPORT_INTERVAL_MSEC_KEY: + return reconfCacheReportParameters(property, newVal); + case DFS_DATANODE_PEER_STATS_ENABLED_KEY: + case DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_KEY: + case DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_KEY: + case DFS_DATANODE_PEER_METRICS_MIN_OUTLIER_DETECTION_SAMPLES_KEY: + return reconfSlowPeerParameters(property, newVal); + case DFS_DATANODE_FILEIO_PROFILING_SAMPLING_PERCENTAGE_KEY: + case DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY: + case DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_KEY: + case DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY: + return reconfSlowDiskParameters(property, newVal); + case FS_DU_INTERVAL_KEY: + case FS_GETSPACEUSED_JITTER_KEY: + case FS_GETSPACEUSED_CLASSNAME: + return reconfDfsUsageParameters(property, newVal); + default: + break; } throw new ReconfigurationException( property, newVal, getConf().get(property)); } + private String reconfDataXceiverParameters(String property, String newVal) + throws ReconfigurationException { + String result; + try { + LOG.info("Reconfiguring {} to {}", property, newVal); + Preconditions.checkNotNull(getXferServer(), "DataXceiverServer has not been initialized."); + int threads = (newVal == null ? DFS_DATANODE_MAX_RECEIVER_THREADS_DEFAULT : + Integer.parseInt(newVal)); + result = Integer.toString(threads); + getXferServer().setMaxXceiverCount(threads); + LOG.info("RECONFIGURE* changed {} to {}", property, newVal); + return result; + } catch (IllegalArgumentException e) { + throw new ReconfigurationException(property, newVal, getConf().get(property), e); + } + } + + private String reconfCacheReportParameters(String property, String newVal) + throws ReconfigurationException { + String result; + try { + LOG.info("Reconfiguring {} to {}", property, newVal); + Preconditions.checkNotNull(dnConf, "DNConf has not been initialized."); + long reportInterval = (newVal == null ? DFS_CACHEREPORT_INTERVAL_MSEC_DEFAULT : + Long.parseLong(newVal)); + result = Long.toString(reportInterval); + dnConf.setCacheReportInterval(reportInterval); + LOG.info("RECONFIGURE* changed {} to {}", property, newVal); + return result; + } catch (IllegalArgumentException e) { + throw new ReconfigurationException(property, newVal, getConf().get(property), e); + } + } + + private String reconfBlockReportParameters(String property, String newVal) + throws ReconfigurationException { + String result = null; + try { + LOG.info("Reconfiguring {} to {}", property, newVal); + if (property.equals(DFS_BLOCKREPORT_INTERVAL_MSEC_KEY)) { + Preconditions.checkNotNull(dnConf, "DNConf has not been initialized."); + long intervalMs = newVal == null ? DFS_BLOCKREPORT_INTERVAL_MSEC_DEFAULT : + Long.parseLong(newVal); + result = Long.toString(intervalMs); + dnConf.setBlockReportInterval(intervalMs); + for (BPOfferService bpos : blockPoolManager.getAllNamenodeThreads()) { + if (bpos != null) { + for (BPServiceActor actor : bpos.getBPServiceActors()) { + actor.getScheduler().setBlockReportIntervalMs(intervalMs); + } + } + } + } else if (property.equals(DFS_BLOCKREPORT_SPLIT_THRESHOLD_KEY)) { + Preconditions.checkNotNull(dnConf, "DNConf has not been initialized."); + long threshold = newVal == null ? DFS_BLOCKREPORT_SPLIT_THRESHOLD_DEFAULT : + Long.parseLong(newVal); + result = Long.toString(threshold); + dnConf.setBlockReportSplitThreshold(threshold); + } else if (property.equals(DFS_BLOCKREPORT_INITIAL_DELAY_KEY)) { + Preconditions.checkNotNull(dnConf, "DNConf has not been initialized."); + int initialDelay = newVal == null ? DFS_BLOCKREPORT_INITIAL_DELAY_DEFAULT : + Integer.parseInt(newVal); + result = Integer.toString(initialDelay); + dnConf.setInitBRDelayMs(result); + } + LOG.info("RECONFIGURE* changed {} to {}", property, newVal); + return result; + } catch (IllegalArgumentException e) { + throw new ReconfigurationException(property, newVal, getConf().get(property), e); + } + } + + private String reconfSlowPeerParameters(String property, String newVal) + throws ReconfigurationException { + String result = null; + try { + LOG.info("Reconfiguring {} to {}", property, newVal); + if (property.equals(DFS_DATANODE_PEER_STATS_ENABLED_KEY)) { + Preconditions.checkNotNull(dnConf, "DNConf has not been initialized."); + if (newVal != null && !newVal.equalsIgnoreCase("true") + && !newVal.equalsIgnoreCase("false")) { + throw new IllegalArgumentException("Not a valid Boolean value for " + property + + " in reconfSlowPeerParameters"); + } + boolean enable = (newVal == null ? DFS_DATANODE_PEER_STATS_ENABLED_DEFAULT : + Boolean.parseBoolean(newVal)); + result = Boolean.toString(enable); + dnConf.setPeerStatsEnabled(enable); + if (enable) { + // Create if it doesn't exist, overwrite if it does. + peerMetrics = DataNodePeerMetrics.create(getDisplayName(), getConf()); + } + } else if (property.equals(DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_KEY)) { + Preconditions.checkNotNull(peerMetrics, "DataNode peer stats may be disabled."); + long minNodes = (newVal == null ? DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_DEFAULT : + Long.parseLong(newVal)); + result = Long.toString(minNodes); + peerMetrics.setMinOutlierDetectionNodes(minNodes); + } else if (property.equals(DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_KEY)) { + Preconditions.checkNotNull(peerMetrics, "DataNode peer stats may be disabled."); + long threshold = (newVal == null ? DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_DEFAULT : + Long.parseLong(newVal)); + result = Long.toString(threshold); + peerMetrics.setLowThresholdMs(threshold); + } else if (property.equals(DFS_DATANODE_PEER_METRICS_MIN_OUTLIER_DETECTION_SAMPLES_KEY)) { + Preconditions.checkNotNull(peerMetrics, "DataNode peer stats may be disabled."); + long minSamples = (newVal == null ? + DFS_DATANODE_PEER_METRICS_MIN_OUTLIER_DETECTION_SAMPLES_DEFAULT : + Long.parseLong(newVal)); + result = Long.toString(minSamples); + peerMetrics.setMinOutlierDetectionSamples(minSamples); + } + LOG.info("RECONFIGURE* changed {} to {}", property, newVal); + return result; + } catch (IllegalArgumentException e) { + throw new ReconfigurationException(property, newVal, getConf().get(property), e); + } + } + + private String reconfSlowDiskParameters(String property, String newVal) + throws ReconfigurationException { + String result = null; + try { + LOG.info("Reconfiguring {} to {}", property, newVal); + if (property.equals(DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY)) { + checkNotNull(dnConf, "DNConf has not been initialized."); + String reportInterval = (newVal == null ? DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_DEFAULT : + newVal); + result = reportInterval; + dnConf.setOutliersReportIntervalMs(reportInterval); + for (BPOfferService bpos : blockPoolManager.getAllNamenodeThreads()) { + if (bpos != null) { + for (BPServiceActor actor : bpos.getBPServiceActors()) { + actor.getScheduler().setOutliersReportIntervalMs( + dnConf.outliersReportIntervalMs); + } + } + } + } else if (property.equals(DFS_DATANODE_FILEIO_PROFILING_SAMPLING_PERCENTAGE_KEY)) { + checkNotNull(dnConf, "DNConf has not been initialized."); + int samplingPercentage = (newVal == null ? + DFS_DATANODE_FILEIO_PROFILING_SAMPLING_PERCENTAGE_DEFAULT : + Integer.parseInt(newVal)); + result = Integer.toString(samplingPercentage); + dnConf.setFileIoProfilingSamplingPercentage(samplingPercentage); + if (fileIoProvider != null) { + fileIoProvider.getProfilingEventHook().setSampleRangeMax(samplingPercentage); + } + if (samplingPercentage > 0 && diskMetrics == null) { + diskMetrics = new DataNodeDiskMetrics(this, + dnConf.outliersReportIntervalMs, getConf()); + } else if (samplingPercentage <= 0 && diskMetrics != null) { + diskMetrics.shutdownAndWait(); + } + } else if (property.equals(DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_KEY)) { + checkNotNull(diskMetrics, "DataNode disk stats may be disabled."); + long minDisks = (newVal == null ? DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_DEFAULT : + Long.parseLong(newVal)); + result = Long.toString(minDisks); + diskMetrics.setMinOutlierDetectionDisks(minDisks); + } else if (property.equals(DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY)) { + checkNotNull(diskMetrics, "DataNode disk stats may be disabled."); + long threshold = (newVal == null ? DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_DEFAULT : + Long.parseLong(newVal)); + result = Long.toString(threshold); + diskMetrics.setLowThresholdMs(threshold); + } + LOG.info("RECONFIGURE* changed {} to {}", property, newVal); + return result; + } catch (IllegalArgumentException e) { + throw new ReconfigurationException(property, newVal, getConf().get(property), e); + } + } + + private String reconfDfsUsageParameters(String property, String newVal) + throws ReconfigurationException { + String result = null; + try { + LOG.info("Reconfiguring {} to {}", property, newVal); + if (property.equals(FS_DU_INTERVAL_KEY)) { + Preconditions.checkNotNull(data, "FsDatasetSpi has not been initialized."); + long interval = (newVal == null ? FS_DU_INTERVAL_DEFAULT : + Long.parseLong(newVal)); + result = Long.toString(interval); + List volumeList = data.getVolumeList(); + for (FsVolumeImpl fsVolume : volumeList) { + Map blockPoolSlices = fsVolume.getBlockPoolSlices(); + for (BlockPoolSlice value : blockPoolSlices.values()) { + value.updateDfsUsageConfig(interval, null, null); + } + } + } else if (property.equals(FS_GETSPACEUSED_JITTER_KEY)) { + Preconditions.checkNotNull(data, "FsDatasetSpi has not been initialized."); + long jitter = (newVal == null ? FS_GETSPACEUSED_JITTER_DEFAULT : + Long.parseLong(newVal)); + result = Long.toString(jitter); + List volumeList = data.getVolumeList(); + for (FsVolumeImpl fsVolume : volumeList) { + Map blockPoolSlices = fsVolume.getBlockPoolSlices(); + for (BlockPoolSlice value : blockPoolSlices.values()) { + value.updateDfsUsageConfig(null, jitter, null); + } + } + } else if (property.equals(FS_GETSPACEUSED_CLASSNAME)) { + Preconditions.checkNotNull(data, "FsDatasetSpi has not been initialized."); + Class klass; + if (newVal == null) { + if (Shell.WINDOWS) { + klass = DU.class; + } else { + klass = WindowsGetSpaceUsed.class; + } + } else { + klass = Class.forName(newVal).asSubclass(GetSpaceUsed.class); + } + result = klass.getName(); + List volumeList = data.getVolumeList(); + for (FsVolumeImpl fsVolume : volumeList) { + Map blockPoolSlices = fsVolume.getBlockPoolSlices(); + for (BlockPoolSlice value : blockPoolSlices.values()) { + value.updateDfsUsageConfig(null, null, klass); + } + } + } + LOG.info("RECONFIGURE* changed {} to {}", property, newVal); + return result; + } catch (IllegalArgumentException | IOException | ClassNotFoundException e) { + throw new ReconfigurationException(property, newVal, getConf().get(property), e); + } + } + /** * Get a list of the keys of the re-configurable properties in configuration. */ @@ -636,6 +967,23 @@ public PipelineAck.ECN getECN() { PipelineAck.ECN.SUPPORTED; } + /** + * The SLOW bit for the DataNode of the specific BlockPool. + * The DataNode should return: + *

    + *
  • SLOW.DISABLED when SLOW is disabled + *
  • SLOW.NORMAL when SLOW is enabled and DN is not slownode.
    • + *
  • SLOW.SLOW when SLOW is enabled and DN is slownode.
    • + *
+ */ + public PipelineAck.SLOW getSLOWByBlockPoolId(String bpId) { + if (!pipelineSupportSlownode) { + return PipelineAck.SLOW.DISABLED; + } + return isSlownodeByBlockPoolId(bpId) ? PipelineAck.SLOW.SLOW : + PipelineAck.SLOW.NORMAL; + } + public FileIoProvider getFileIoProvider() { return fileIoProvider; } @@ -826,6 +1174,7 @@ private void refreshVolumes(String newVolumes) throws IOException { .newFixedThreadPool(changedVolumes.newLocations.size()); List> exceptions = Lists.newArrayList(); + Preconditions.checkNotNull(data, "Storage not yet initialized"); for (final StorageLocation location : changedVolumes.newLocations) { exceptions.add(service.submit(new Callable() { @Override @@ -925,6 +1274,7 @@ private synchronized void removeVolumes( clearFailure, Joiner.on(",").join(storageLocations))); IOException ioe = null; + Preconditions.checkNotNull(data, "Storage not yet initialized"); // Remove volumes and block infos from FsDataset. data.removeVolumes(storageLocations, clearFailure); @@ -2001,6 +2351,7 @@ FileInputStream[] requestShortCircuitFdsForRead(final ExtendedBlock blk, FileInputStream fis[] = new FileInputStream[2]; try { + Preconditions.checkNotNull(data, "Storage not yet initialized"); fis[0] = (FileInputStream)data.getBlockInputStream(blk, 0); fis[1] = DatanodeUtil.getMetaDataInputStream(blk, data); } catch (ClassCastException e) { @@ -2185,7 +2536,7 @@ public void shutdown() { if (metrics != null) { metrics.shutdown(); } - if (diskMetrics != null) { + if (dnConf.diskStatsEnabled && diskMetrics != null) { diskMetrics.shutdownAndWait(); } if (dataNodeInfoBeanName != null) { @@ -2201,6 +2552,7 @@ public void shutdown() { notifyAll(); } tracer.close(); + dataSetLockManager.lockLeakCheck(); } /** @@ -2718,6 +3070,7 @@ public void runDatanodeDaemon() throws IOException { } ipcServer.setTracer(tracer); ipcServer.start(); + startTime = now(); startPlugins(getConf()); } @@ -3029,6 +3382,7 @@ public static void main(String args[]) { @Override // InterDatanodeProtocol public ReplicaRecoveryInfo initReplicaRecovery(RecoveringBlock rBlock) throws IOException { + Preconditions.checkNotNull(data, "Storage not yet initialized"); return data.initReplicaRecovery(rBlock); } @@ -3039,6 +3393,7 @@ public ReplicaRecoveryInfo initReplicaRecovery(RecoveringBlock rBlock) public String updateReplicaUnderRecovery(final ExtendedBlock oldBlock, final long recoveryId, final long newBlockId, final long newLength) throws IOException { + Preconditions.checkNotNull(data, "Storage not yet initialized"); final Replica r = data.updateReplicaUnderRecovery(oldBlock, recoveryId, newBlockId, newLength); // Notify the namenode of the updated block info. This is important @@ -3104,7 +3459,8 @@ void transferReplicaForPipelineRecovery(final ExtendedBlock b, final BlockConstructionStage stage; //get replica information - try(AutoCloseableLock lock = data.acquireDatasetReadLock()) { + try (AutoCloseableLock lock = dataSetLockManager.readLock( + LockLevel.BLOCK_POOl, b.getBlockPoolId())) { Block storedBlock = data.getStoredBlock(b.getBlockPoolId(), b.getBlockId()); if (null == storedBlock) { @@ -3195,6 +3551,11 @@ public String getHttpPort(){ return this.getConf().get("dfs.datanode.info.port"); } + @Override // DataNodeMXBean + public long getDNStartedTimeInMillis() { + return this.startTime; + } + public String getRevision() { return VersionInfo.getRevision(); } @@ -3315,7 +3676,7 @@ public void deleteBlockPool(String blockPoolId, boolean force) "The block pool is still running. First do a refreshNamenodes to " + "shutdown the block pool service"); } - + Preconditions.checkNotNull(data, "Storage not yet initialized"); data.deleteBlockPool(blockPoolId, force); } @@ -3482,6 +3843,14 @@ public ShortCircuitRegistry getShortCircuitRegistry() { return shortCircuitRegistry; } + public DataTransferThrottler getEcReconstuctReadThrottler() { + return ecReconstuctReadThrottler; + } + + public DataTransferThrottler getEcReconstuctWriteThrottler() { + return ecReconstuctWriteThrottler; + } + /** * Check the disk error synchronously. */ @@ -3741,13 +4110,13 @@ void setBlockScanner(BlockScanner blockScanner) { @Override // DataNodeMXBean public String getSendPacketDownstreamAvgInfo() { - return peerMetrics != null ? + return dnConf.peerStatsEnabled && peerMetrics != null ? peerMetrics.dumpSendPacketDownstreamAvgInfoAsJson() : null; } @Override // DataNodeMXBean public String getSlowDisks() { - if (diskMetrics == null) { + if (!dnConf.diskStatsEnabled || diskMetrics == null) { //Disk Stats not enabled return null; } @@ -3759,6 +4128,7 @@ public String getSlowDisks() { @Override public List getVolumeReport() throws IOException { checkSuperuserPrivilege(); + Preconditions.checkNotNull(data, "Storage not yet initialized"); Map volumeInfoMap = data.getVolumeInfoMap(); if (volumeInfoMap == null) { LOG.warn("DataNode volume info not available."); @@ -3814,4 +4184,25 @@ private static boolean isWrite(BlockConstructionStage stage) { return (stage == PIPELINE_SETUP_STREAMING_RECOVERY || stage == PIPELINE_SETUP_APPEND_RECOVERY); } + + public DataSetLockManager getDataSetLockManager() { + return dataSetLockManager; + } + + boolean isSlownodeByNameserviceId(String nsId) { + return blockPoolManager.isSlownodeByNameserviceId(nsId); + } + + boolean isSlownodeByBlockPoolId(String bpId) { + return blockPoolManager.isSlownodeByBlockPoolId(bpId); + } + + boolean isSlownode() { + return blockPoolManager.isSlownode(); + } + + @VisibleForTesting + public BlockPoolManager getBlockPoolManager() { + return blockPoolManager; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeFaultInjector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeFaultInjector.java index 1d7bcf1d50f9c..e9a9c69016347 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeFaultInjector.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeFaultInjector.java @@ -155,4 +155,6 @@ public void delay() {} * into an erasure coding reconstruction. */ public void badDecoding(ByteBuffer[] outputs) {} + + public void markSlow(String dnAddr, int[] replies) {} } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeMXBean.java index 7a8f59bb667a5..65537754741cb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeMXBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeMXBean.java @@ -157,4 +157,11 @@ public interface DataNodeMXBean { * @return true, if security is enabled. */ boolean isSecurityEnabled(); + + /** + * Get the start time of the DataNode. + * + * @return Start time of the DataNode. + */ + long getDNStartedTimeInMillis(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataSetLockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataSetLockManager.java new file mode 100644 index 0000000000000..1d59f87ab2b82 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataSetLockManager.java @@ -0,0 +1,297 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.server.datanode; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.server.common.AutoCloseDataSetLock; +import org.apache.hadoop.hdfs.server.common.DataNodeLockManager; + +import java.util.HashMap; +import java.util.Stack; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +/** + * Class for maintain a set of lock for fsDataSetImpl. + */ +public class DataSetLockManager implements DataNodeLockManager { + public static final Log LOG = LogFactory.getLog(DataSetLockManager.class); + private final HashMap threadCountMap = new HashMap<>(); + private final LockMap lockMap = new LockMap(); + private boolean isFair = true; + private final boolean openLockTrace; + private Exception lastException; + + /** + * Class for maintain lockMap and is thread safe. + */ + private class LockMap { + private final HashMap readlockMap = new HashMap<>(); + private final HashMap writeLockMap = new HashMap<>(); + + public synchronized void addLock(String name, ReentrantReadWriteLock lock) { + AutoCloseDataSetLock readLock = new AutoCloseDataSetLock(lock.readLock()); + AutoCloseDataSetLock writeLock = new AutoCloseDataSetLock(lock.writeLock()); + if (openLockTrace) { + readLock.setDataNodeLockManager(DataSetLockManager.this); + writeLock.setDataNodeLockManager(DataSetLockManager.this); + } + readlockMap.putIfAbsent(name, readLock); + writeLockMap.putIfAbsent(name, writeLock); + } + + public synchronized void removeLock(String name) { + if (!readlockMap.containsKey(name) || !writeLockMap.containsKey(name)) { + LOG.error("The lock " + name + " is not in LockMap"); + } + readlockMap.remove(name); + writeLockMap.remove(name); + } + + public synchronized AutoCloseDataSetLock getReadLock(String name) { + return readlockMap.get(name); + } + + public synchronized AutoCloseDataSetLock getWriteLock(String name) { + return writeLockMap.get(name); + } + } + + /** + * Generate lock order string concatenates with lock name. + * @param level which level lock want to acquire. + * @param resources lock name by lock order. + * @return lock order string concatenates with lock name. + */ + private String generateLockName(LockLevel level, String... resources) { + if (resources.length == 1 && level == LockLevel.BLOCK_POOl) { + if (resources[0] == null) { + throw new IllegalArgumentException("acquire a null block pool lock"); + } + return resources[0]; + } else if (resources.length == 2 && level == LockLevel.VOLUME) { + if (resources[0] == null || resources[1] == null) { + throw new IllegalArgumentException("acquire a null bp lock : " + + resources[0] + "volume lock :" + resources[1]); + } + return resources[0] + resources[1]; + } else { + throw new IllegalArgumentException("lock level do not match resource"); + } + } + + /** + * Class for record thread acquire lock stack trace and count. + */ + private static class TrackLog { + private final Stack logStack = new Stack<>(); + private int lockCount = 0; + private final String threadName; + + TrackLog(String threadName) { + this.threadName = threadName; + incrLockCount(); + } + + public void incrLockCount() { + logStack.push(new Exception("lock stack trace")); + lockCount += 1; + } + + public void decrLockCount() { + logStack.pop(); + lockCount -= 1; + } + + public void showLockMessage() { + LOG.error("hold lock thread name is:" + threadName + + " hold count is:" + lockCount); + while (!logStack.isEmpty()) { + Exception e = logStack.pop(); + LOG.error("lock stack ", e); + } + } + + public boolean shouldClear() { + return lockCount == 1; + } + } + + public DataSetLockManager(Configuration conf) { + this.isFair = conf.getBoolean( + DFSConfigKeys.DFS_DATANODE_LOCK_FAIR_KEY, + DFSConfigKeys.DFS_DATANODE_LOCK_FAIR_DEFAULT); + this.openLockTrace = conf.getBoolean( + DFSConfigKeys.DFS_DATANODE_LOCKMANAGER_TRACE, + DFSConfigKeys.DFS_DATANODE_LOCKMANAGER_TRACE_DEFAULT); + } + + public DataSetLockManager() { + this.openLockTrace = true; + } + + @Override + public AutoCloseDataSetLock readLock(LockLevel level, String... resources) { + if (level == LockLevel.BLOCK_POOl) { + return getReadLock(level, resources[0]); + } else { + AutoCloseDataSetLock bpLock = getReadLock(LockLevel.BLOCK_POOl, resources[0]); + AutoCloseDataSetLock volLock = getReadLock(level, resources); + volLock.setParentLock(bpLock); + if (openLockTrace) { + LOG.info("Sub lock " + resources[0] + resources[1] + " parent lock " + + resources[0]); + } + return volLock; + } + } + + @Override + public AutoCloseDataSetLock writeLock(LockLevel level, String... resources) { + if (level == LockLevel.BLOCK_POOl) { + return getWriteLock(level, resources[0]); + } else { + AutoCloseDataSetLock bpLock = getReadLock(LockLevel.BLOCK_POOl, resources[0]); + AutoCloseDataSetLock volLock = getWriteLock(level, resources); + volLock.setParentLock(bpLock); + if (openLockTrace) { + LOG.info("Sub lock " + resources[0] + resources[1] + " parent lock " + + resources[0]); + } + return volLock; + } + } + + /** + * Return a not null ReadLock. + */ + private AutoCloseDataSetLock getReadLock(LockLevel level, String... resources) { + String lockName = generateLockName(level, resources); + AutoCloseDataSetLock lock = lockMap.getReadLock(lockName); + if (lock == null) { + LOG.warn("Ignore this error during dn restart: Not existing readLock " + + lockName); + lockMap.addLock(lockName, new ReentrantReadWriteLock(isFair)); + lock = lockMap.getReadLock(lockName); + } + lock.lock(); + if (openLockTrace) { + putThreadName(getThreadName()); + } + return lock; + } + + /** + * Return a not null WriteLock. + */ + private AutoCloseDataSetLock getWriteLock(LockLevel level, String... resources) { + String lockName = generateLockName(level, resources); + AutoCloseDataSetLock lock = lockMap.getWriteLock(lockName); + if (lock == null) { + LOG.warn("Ignore this error during dn restart: Not existing writeLock" + + lockName); + lockMap.addLock(lockName, new ReentrantReadWriteLock(isFair)); + lock = lockMap.getWriteLock(lockName); + } + lock.lock(); + if (openLockTrace) { + putThreadName(getThreadName()); + } + return lock; + } + + @Override + public void addLock(LockLevel level, String... resources) { + String lockName = generateLockName(level, resources); + if (level == LockLevel.BLOCK_POOl) { + lockMap.addLock(lockName, new ReentrantReadWriteLock(isFair)); + } else { + lockMap.addLock(resources[0], new ReentrantReadWriteLock(isFair)); + lockMap.addLock(lockName, new ReentrantReadWriteLock(isFair)); + } + } + + @Override + public void removeLock(LockLevel level, String... resources) { + String lockName = generateLockName(level, resources); + try (AutoCloseDataSetLock lock = writeLock(level, resources)) { + lock.lock(); + lockMap.removeLock(lockName); + } + } + + @Override + public void hook() { + if (openLockTrace) { + removeThreadName(getThreadName()); + } + } + + /** + * Add thread name when lock a lock. + */ + private synchronized void putThreadName(String thread) { + if (threadCountMap.containsKey(thread)) { + TrackLog trackLog = threadCountMap.get(thread); + trackLog.incrLockCount(); + } + threadCountMap.putIfAbsent(thread, new TrackLog(thread)); + } + + public void lockLeakCheck() { + if (!openLockTrace) { + LOG.warn("not open lock leak check func"); + return; + } + if (threadCountMap.isEmpty()) { + LOG.warn("all lock has release"); + return; + } + setLastException(new Exception("lock Leak")); + threadCountMap.forEach((name, trackLog) -> trackLog.showLockMessage()); + } + + /** + * Remove thread name when unlock a lock. + */ + private synchronized void removeThreadName(String thread) { + if (threadCountMap.containsKey(thread)) { + TrackLog trackLog = threadCountMap.get(thread); + if (trackLog.shouldClear()) { + threadCountMap.remove(thread); + return; + } + trackLog.decrLockCount(); + } + } + + private void setLastException(Exception e) { + this.lastException = e; + } + + public Exception getLastException() { + return lastException; + } + + private String getThreadName() { + return Thread.currentThread().getName() + Thread.currentThread().getId(); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java index 8150b7974a159..9ad3e7cf32613 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java @@ -341,7 +341,7 @@ public void run() { * the thread dies away. */ private void collectThreadLocalStates() { - if (datanode.getPeerMetrics() != null) { + if (datanode.getDnConf().peerStatsEnabled && datanode.getPeerMetrics() != null) { datanode.getPeerMetrics().collectThreadLocalStates(); } } @@ -415,6 +415,9 @@ public void requestShortCircuitFds(final ExtendedBlock blk, "Not verifying {}", slotId); } success = true; + // update metrics + datanode.metrics.addReadBlockOp(elapsed()); + datanode.metrics.incrReadsFromClient(true, blk.getNumBytes()); } } finally { if ((!success) && (registeredSlotId != null)) { @@ -1093,7 +1096,7 @@ public void copyBlock(final ExtendedBlock block, if (!dataXceiverServer.balanceThrottler.acquire()) { // not able to start String msg = "Not able to copy block " + block.getBlockId() + " " + "to " + peer.getRemoteAddressString() + " because threads " + - "quota is exceeded."; + "quota=" + dataXceiverServer.balanceThrottler.getMaxConcurrentMovers() + " is exceeded."; LOG.info(msg); sendResponse(ERROR, msg); return; @@ -1167,7 +1170,7 @@ public void replaceBlock(final ExtendedBlock block, if (!dataXceiverServer.balanceThrottler.acquire()) { // not able to start String msg = "Not able to receive block " + block.getBlockId() + " from " + peer.getRemoteAddressString() + " because threads " + - "quota is exceeded."; + "quota=" + dataXceiverServer.balanceThrottler.getMaxConcurrentMovers() + " is exceeded."; LOG.warn(msg); sendResponse(ERROR, msg); return; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java index 77fa70e3cc54f..3a67b76e43467 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java @@ -68,8 +68,7 @@ class DataXceiverServer implements Runnable { * Enforcing the limit is required in order to avoid data-node * running out of memory. */ - int maxXceiverCount = - DFSConfigKeys.DFS_DATANODE_MAX_RECEIVER_THREADS_DEFAULT; + volatile int maxXceiverCount; /** * A manager to make sure that cluster balancing does not take too much @@ -514,4 +513,15 @@ public boolean updateBalancerMaxConcurrentMovers(final int movers) { void setMaxReconfigureWaitTime(int max) { this.maxReconfigureWaitTime = max; } + + public void setMaxXceiverCount(int xceiverCount) { + Preconditions.checkArgument(xceiverCount > 0, + "dfs.datanode.max.transfer.threads should be larger than 0"); + maxXceiverCount = xceiverCount; + } + + @VisibleForTesting + public int getMaxXceiverCount() { + return maxXceiverCount; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java index 58276e5ccc00e..f97dbbfcd6e8a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java @@ -41,6 +41,7 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; @@ -141,7 +142,8 @@ public String toString() { + ", missing metadata files: " + missingMetaFile + ", missing block files: " + missingBlockFile + ", missing blocks in memory: " + missingMemoryBlocks - + ", mismatched blocks: " + mismatchBlocks; + + ", mismatched blocks: " + mismatchBlocks + + ", duplicated blocks: " + duplicateBlocks; } } @@ -391,7 +393,7 @@ private void clear() { } /** - * Main program loop for DirectoryScanner. Runs {@link reconcile()} and + * Main program loop for DirectoryScanner. Runs {@link #reconcile()} and * handles any exceptions. */ @Override @@ -539,21 +541,30 @@ private void scan() { m++; continue; } - // Block file and/or metadata file exists on the disk - // Block exists in memory - if (info.getBlockFile() == null) { - // Block metadata file exits and block file is missing - addDifference(diffRecord, statsRecord, info); - } else if (info.getGenStamp() != memBlock.getGenerationStamp() - || info.getBlockLength() != memBlock.getNumBytes()) { - // Block metadata file is missing or has wrong generation stamp, - // or block file length is different than expected + + // Block and meta must be regular file + boolean isRegular = FileUtil.isRegularFile(info.getBlockFile(), false) && + FileUtil.isRegularFile(info.getMetaFile(), false); + if (!isRegular) { statsRecord.mismatchBlocks++; addDifference(diffRecord, statsRecord, info); - } else if (memBlock.compareWith(info) != 0) { - // volumeMap record and on-disk files do not match. - statsRecord.duplicateBlocks++; - addDifference(diffRecord, statsRecord, info); + } else { + // Block file and/or metadata file exists on the disk + // Block exists in memory + if (info.getBlockFile() == null) { + // Block metadata file exits and block file is missing + addDifference(diffRecord, statsRecord, info); + } else if (info.getGenStamp() != memBlock.getGenerationStamp() + || info.getBlockLength() != memBlock.getNumBytes()) { + // Block metadata file is missing or has wrong generation stamp, + // or block file length is different than expected + statsRecord.mismatchBlocks++; + addDifference(diffRecord, statsRecord, info); + } else if (memBlock.compareWith(info) != 0) { + // volumeMap record and on-disk files do not match. + statsRecord.duplicateBlocks++; + addDifference(diffRecord, statsRecord, info); + } } d++; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java index 0f710a143ad87..4126140678759 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java @@ -25,7 +25,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi .FsVolumeReferences; -import org.apache.hadoop.util.AutoCloseableLock; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.server.datanode.DiskBalancerWorkStatus @@ -502,16 +501,13 @@ private void createWorkPlan(NodePlan plan) throws DiskBalancerException { private Map getStorageIDToVolumeBasePathMap() throws DiskBalancerException { Map storageIDToVolBasePathMap = new HashMap<>(); - FsDatasetSpi.FsVolumeReferences references; - try { - try(AutoCloseableLock lock = this.dataset.acquireDatasetReadLock()) { - references = this.dataset.getFsVolumeReferences(); - for (int ndx = 0; ndx < references.size(); ndx++) { - FsVolumeSpi vol = references.get(ndx); - storageIDToVolBasePathMap.put(vol.getStorageID(), - vol.getBaseURI().getPath()); - } - references.close(); + // Get volumes snapshot so no need to acquire dataset lock. + try (FsDatasetSpi.FsVolumeReferences references = dataset. + getFsVolumeReferences()) { + for (int ndx = 0; ndx < references.size(); ndx++) { + FsVolumeSpi vol = references.get(ndx); + storageIDToVolBasePathMap.put(vol.getStorageID(), + vol.getBaseURI().getPath()); } } catch (IOException ex) { LOG.error("Disk Balancer - Internal Error.", ex); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java index 7d6435ea3f2c7..552f5199f1d3a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java @@ -1071,4 +1071,8 @@ private void onFailure(@Nullable FsVolumeSpi volume, long begin) { } profilingEventHook.onFailure(volume, begin); } + + public ProfilingFileIoEvents getProfilingEventHook() { + return profilingEventHook; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java index e97873341e423..c22401b645f14 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdfs.server.datanode; +import org.apache.hadoop.classification.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -40,8 +41,8 @@ class ProfilingFileIoEvents { static final Logger LOG = LoggerFactory.getLogger(ProfilingFileIoEvents.class); - private final boolean isEnabled; - private final int sampleRangeMax; + private volatile boolean isEnabled; + private volatile int sampleRangeMax; public ProfilingFileIoEvents(@Nullable Configuration conf) { if (conf != null) { @@ -49,15 +50,7 @@ public ProfilingFileIoEvents(@Nullable Configuration conf) { DFSConfigKeys.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_PERCENTAGE_KEY, DFSConfigKeys .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_PERCENTAGE_DEFAULT); - isEnabled = Util.isDiskStatsEnabled(fileIOSamplingPercentage); - if (fileIOSamplingPercentage > 100) { - LOG.warn(DFSConfigKeys - .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_PERCENTAGE_KEY + - " value cannot be more than 100. Setting value to 100"); - fileIOSamplingPercentage = 100; - } - sampleRangeMax = (int) ((double) fileIOSamplingPercentage / 100 * - Integer.MAX_VALUE); + setSampleRangeMax(fileIOSamplingPercentage); } else { isEnabled = false; sampleRangeMax = 0; @@ -145,4 +138,26 @@ private DataNodeVolumeMetrics getVolumeMetrics(final FsVolumeSpi volume) { } return null; } + + public void setSampleRangeMax(int fileIOSamplingPercentage) { + isEnabled = Util.isDiskStatsEnabled(fileIOSamplingPercentage); + if (fileIOSamplingPercentage > 100) { + LOG.warn(DFSConfigKeys + .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_PERCENTAGE_KEY + + " value cannot be more than 100. Setting value to 100"); + fileIOSamplingPercentage = 100; + } + sampleRangeMax = (int) ((double) fileIOSamplingPercentage / 100 * + Integer.MAX_VALUE); + } + + @VisibleForTesting + public boolean getDiskStatsEnabled() { + return isEnabled; + } + + @VisibleForTesting + public int getSampleRangeMax() { + return sampleRangeMax; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java index 5ab5b49e1a8aa..d8f1e23ec379b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java @@ -293,7 +293,7 @@ public void handle(ExtendedBlock block, IOException e) { volume, block); return; } - LOG.warn("Reporting bad {} on {}", block, volume); + LOG.warn("Reporting bad {} on {}", block, volume, e); scanner.datanode.handleBadBlock(block, e, true); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java index f8f6450ea5221..da6ae95918d9f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java @@ -77,7 +77,6 @@ public class DatasetVolumeChecker { private final AtomicLong numVolumeChecks = new AtomicLong(0); private final AtomicLong numSyncDatasetChecks = new AtomicLong(0); - private final AtomicLong numAsyncDatasetChecks = new AtomicLong(0); private final AtomicLong numSkippedChecks = new AtomicLong(0); /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java index 3ead793542c7b..ecd6351b46f64 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java @@ -95,6 +95,10 @@ void reconstruct() throws IOException { (int) Math.min(getStripedReader().getBufferSize(), remaining); long start = Time.monotonicNow(); + long bytesToRead = (long) toReconstructLen * getStripedReader().getMinRequiredSources(); + if (getDatanode().getEcReconstuctReadThrottler() != null) { + getDatanode().getEcReconstuctReadThrottler().throttle(bytesToRead); + } // step1: read from minimum source DNs required for reconstruction. // The returned success list is the source DNs we do real read from getStripedReader().readMinimumSources(toReconstructLen); @@ -105,6 +109,10 @@ void reconstruct() throws IOException { long decodeEnd = Time.monotonicNow(); // step3: transfer data + long bytesToWrite = (long) toReconstructLen * stripedWriter.getTargets(); + if (getDatanode().getEcReconstuctWriteThrottler() != null) { + getDatanode().getEcReconstuctWriteThrottler().throttle(bytesToWrite); + } if (stripedWriter.transferData2Targets() == 0) { String error = "Transfer failed for all targets."; throw new IOException(error); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReader.java index 940cb71172805..20d5c6f44fb41 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReader.java @@ -508,4 +508,9 @@ CachingStrategy getCachingStrategy() { int getXmits() { return xmits; } + + public int getMinRequiredSources() { + return minRequiredSources; + } + } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/AvailableSpaceVolumeChoosingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/AvailableSpaceVolumeChoosingPolicy.java index 72ed47c67fa93..5d12fa72bb165 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/AvailableSpaceVolumeChoosingPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/AvailableSpaceVolumeChoosingPolicy.java @@ -63,7 +63,6 @@ public class AvailableSpaceVolumeChoosingPolicy public AvailableSpaceVolumeChoosingPolicy() { this(new Random()); - initLocks(); } private void initLocks() { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java index f162ea1b3ae15..8d1d10bccd2fc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java @@ -35,8 +35,10 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.hdfs.server.common.AutoCloseDataSetLock; +import org.apache.hadoop.hdfs.server.common.DataNodeLockManager; +import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.MountVolumeMap; -import org.apache.hadoop.util.AutoCloseableLock; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; @@ -241,7 +243,7 @@ StorageReport[] getStorageReports(String bpid) * Gets a list of references to the finalized blocks for the given block pool. *

* Callers of this function should call - * {@link FsDatasetSpi#acquireDatasetLock} to avoid blocks' status being + * {@link FsDatasetSpi#acquireDatasetLockManager} to avoid blocks' status being * changed during list iteration. *

* @return a list of references to the finalized blocks for the given block @@ -657,21 +659,12 @@ ReplicaInfo moveBlockAcrossStorage(final ExtendedBlock block, ReplicaInfo moveBlockAcrossVolumes(final ExtendedBlock block, FsVolumeSpi destination) throws IOException; - /** - * Acquire the lock of the data set. This prevents other threads from - * modifying the volume map structure inside the datanode, but other changes - * are still possible. For example modifying the genStamp of a block instance. - */ - AutoCloseableLock acquireDatasetLock(); - /*** - * Acquire the read lock of the data set. This prevents other threads from - * modifying the volume map structure inside the datanode, but other changes - * are still possible. For example modifying the genStamp of a block instance. + * Acquire lock Manager for the data set. This prevents other threads from + * modifying the volume map structure inside the datanode. * @return The AutoClosable read lock instance. */ - AutoCloseableLock acquireDatasetReadLock(); - + DataNodeLockManager acquireDatasetLockManager(); /** * Deep copy the replica info belonging to given block pool. @@ -687,4 +680,9 @@ ReplicaInfo moveBlockAcrossVolumes(final ExtendedBlock block, * @throws IOException */ MountVolumeMap getMountVolumeMap() throws IOException; + + /** + * Get the volume list. + */ + List getVolumeList(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java index f2d4e08a45bdb..23f3602a456c8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java @@ -43,10 +43,10 @@ import java.util.concurrent.ForkJoinTask; import java.util.concurrent.RecursiveAction; import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.concurrent.TimeUnit; import org.apache.hadoop.hdfs.server.datanode.FSCachingGetSpaceUsed; +import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -78,6 +78,10 @@ import org.apache.hadoop.classification.VisibleForTesting; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DU_INTERVAL_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_GETSPACEUSED_JITTER_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_GETSPACEUSED_CLASSNAME; + /** * A block pool slice represents a portion of a block pool stored on a volume. * Taken together, all BlockPoolSlices sharing a block pool ID across a @@ -85,7 +89,7 @@ * * This class is synchronized by {@link FsVolumeImpl}. */ -class BlockPoolSlice { +public class BlockPoolSlice { static final Logger LOG = LoggerFactory.getLogger(BlockPoolSlice.class); private final String bpid; @@ -116,6 +120,8 @@ class BlockPoolSlice { private final Timer timer; private final int maxDataLength; private final FileIoProvider fileIoProvider; + private final Configuration config; + private final File bpDir; private static ForkJoinPool addReplicaThreadPool = null; private static final int VOLUMES_REPLICA_ADD_THREADPOOL_SIZE = Runtime @@ -129,7 +135,7 @@ public int compare(File f1, File f2) { }; // TODO:FEDERATION scalability issue - a thread per DU is needed - private final GetSpaceUsed dfsUsage; + private volatile GetSpaceUsed dfsUsage; /** * Create a blook pool slice @@ -142,6 +148,8 @@ public int compare(File f1, File f2) { */ BlockPoolSlice(String bpid, FsVolumeImpl volume, File bpDir, Configuration conf, Timer timer) throws IOException { + this.config = conf; + this.bpDir = bpDir; this.bpid = bpid; this.volume = volume; this.fileIoProvider = volume.getFileIoProvider(); @@ -233,6 +241,40 @@ public void run() { SHUTDOWN_HOOK_PRIORITY); } + public void updateDfsUsageConfig(Long interval, Long jitter, Class klass) + throws IOException { + // Close the old dfsUsage if it is CachingGetSpaceUsed. + if (dfsUsage instanceof CachingGetSpaceUsed) { + ((CachingGetSpaceUsed) dfsUsage).close(); + } + if (interval != null) { + Preconditions.checkArgument(interval > 0, + FS_DU_INTERVAL_KEY + " should be larger than 0"); + config.setLong(FS_DU_INTERVAL_KEY, interval); + } + if (jitter != null) { + Preconditions.checkArgument(jitter >= 0, + FS_GETSPACEUSED_JITTER_KEY + " should be larger than or equal to 0"); + config.setLong(FS_GETSPACEUSED_JITTER_KEY, jitter); + } + + if (klass != null) { + config.setClass(FS_GETSPACEUSED_CLASSNAME, klass, CachingGetSpaceUsed.class); + } + // Start new dfsUsage. + this.dfsUsage = new FSCachingGetSpaceUsed.Builder().setBpid(bpid) + .setVolume(volume) + .setPath(bpDir) + .setConf(config) + .setInitialUsed(loadDfsUsed()) + .build(); + } + + @VisibleForTesting + public GetSpaceUsed getDfsUsage() { + return dfsUsage; + } + private synchronized static void initializeAddReplicaPool(Configuration conf, FsDatasetImpl dataset) { if (addReplicaThreadPool == null) { @@ -298,9 +340,13 @@ long loadDfsUsed() { long mtime; Scanner sc; + File duCacheFile = new File(currentDir, DU_CACHE_FILE); try { - sc = new Scanner(new File(currentDir, DU_CACHE_FILE), "UTF-8"); + sc = new Scanner(duCacheFile, "UTF-8"); } catch (FileNotFoundException fnfe) { + FsDatasetImpl.LOG.warn("{} file missing in {}, will proceed with Du " + + "for space computation calculation, ", + DU_CACHE_FILE, currentDir); return -1; } @@ -309,21 +355,31 @@ long loadDfsUsed() { if (sc.hasNextLong()) { cachedDfsUsed = sc.nextLong(); } else { + FsDatasetImpl.LOG.warn("cachedDfsUsed not found in file:{}, will " + + "proceed with Du for space computation calculation, ", + duCacheFile); return -1; } // Get the recorded mtime from the file. if (sc.hasNextLong()) { mtime = sc.nextLong(); } else { + FsDatasetImpl.LOG.warn("mtime not found in file:{}, will proceed" + + " with Du for space computation calculation, ", duCacheFile); return -1; } + long elapsedTime = timer.now() - mtime; // Return the cached value if mtime is okay. - if (mtime > 0 && (timer.now() - mtime < cachedDfsUsedCheckTime)) { + if (mtime > 0 && (elapsedTime < cachedDfsUsedCheckTime)) { FsDatasetImpl.LOG.info("Cached dfsUsed found for " + currentDir + ": " + cachedDfsUsed); return cachedDfsUsed; } + FsDatasetImpl.LOG.warn("elapsed time:{} is greater than threshold:{}," + + " mtime:{} in file:{}, will proceed with Du for space" + + " computation calculation", + elapsedTime, cachedDfsUsedCheckTime, mtime, duCacheFile); return -1; } finally { sc.close(); @@ -900,7 +956,7 @@ void shutdown(BlockListAsLongs blocksListToPersist) { private boolean readReplicasFromCache(ReplicaMap volumeMap, final RamDiskReplicaTracker lazyWriteReplicaMap) { - ReplicaMap tmpReplicaMap = new ReplicaMap(new ReentrantReadWriteLock()); + ReplicaMap tmpReplicaMap = new ReplicaMap(); File replicaFile = new File(replicaCacheDir, REPLICA_CACHE_FILE); // Check whether the file exists or not. if (!replicaFile.exists()) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java index 7d5f33b8b8cf8..db4987e25ac98 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java @@ -30,6 +30,8 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; @@ -65,7 +67,7 @@ class FsDatasetAsyncDiskService { // ThreadPool core pool size private static final int CORE_THREADS_PER_VOLUME = 1; // ThreadPool maximum pool size - private static final int MAXIMUM_THREADS_PER_VOLUME = 4; + private final int maxNumThreadsPerVolume; // ThreadPool keep-alive time for threads over core pool size private static final long THREADS_KEEP_ALIVE_SECONDS = 60; @@ -90,6 +92,12 @@ class FsDatasetAsyncDiskService { this.datanode = datanode; this.fsdatasetImpl = fsdatasetImpl; this.threadGroup = new ThreadGroup(getClass().getSimpleName()); + maxNumThreadsPerVolume = datanode.getConf().getInt( + DFSConfigKeys.DFS_DATANODE_FSDATASETASYNCDISK_MAX_THREADS_PER_VOLUME_KEY, + DFSConfigKeys.DFS_DATANODE_FSDATASETASYNCDISK_MAX_THREADS_PER_VOLUME_DEFAULT); + Preconditions.checkArgument(maxNumThreadsPerVolume > 0, + DFSConfigKeys.DFS_DATANODE_FSDATASETASYNCDISK_MAX_THREADS_PER_VOLUME_KEY + + " must be a positive integer."); } private void addExecutorForVolume(final FsVolumeImpl volume) { @@ -110,7 +118,7 @@ public Thread newThread(Runnable r) { }; ThreadPoolExecutor executor = new ThreadPoolExecutor( - CORE_THREADS_PER_VOLUME, MAXIMUM_THREADS_PER_VOLUME, + CORE_THREADS_PER_VOLUME, maxNumThreadsPerVolume, THREADS_KEEP_ALIVE_SECONDS, TimeUnit.SECONDS, new LinkedBlockingQueue(), threadFactory); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java index 8d37bda166091..df24f9890db04 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java @@ -32,7 +32,6 @@ import java.nio.file.Files; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -41,14 +40,12 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executor; -import java.util.concurrent.locks.Condition; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReentrantReadWriteLock; import javax.management.NotCompliantMBeanException; import javax.management.ObjectName; import javax.management.StandardMBean; +import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.HardLink; import org.apache.hadoop.classification.VisibleForTesting; @@ -63,6 +60,10 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtilClient; import org.apache.hadoop.hdfs.ExtendedBlockId; +import org.apache.hadoop.hdfs.server.common.AutoCloseDataSetLock; +import org.apache.hadoop.hdfs.server.common.DataNodeLockManager; +import org.apache.hadoop.hdfs.server.common.DataNodeLockManager.LockLevel; +import org.apache.hadoop.hdfs.server.datanode.DataSetLockManager; import org.apache.hadoop.hdfs.server.datanode.FileIoProvider; import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica; import org.apache.hadoop.hdfs.server.datanode.LocalReplica; @@ -118,7 +119,6 @@ import org.apache.hadoop.util.DataChecksum; import org.apache.hadoop.util.DiskChecker.DiskErrorException; import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException; -import org.apache.hadoop.util.InstrumentedReadWriteLock; import org.apache.hadoop.util.Lists; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.Sets; @@ -187,7 +187,8 @@ public StorageReport[] getStorageReports(String bpid) @Override public FsVolumeImpl getVolume(final ExtendedBlock b) { - try (AutoCloseableLock lock = datasetReadLock.acquire()) { + try (AutoCloseableLock lock = lockManager.readLock(LockLevel.BLOCK_POOl, + b.getBlockPoolId())) { final ReplicaInfo r = volumeMap.get(b.getBlockPoolId(), b.getLocalBlock()); return r != null ? (FsVolumeImpl) r.getVolume() : null; @@ -197,7 +198,8 @@ public FsVolumeImpl getVolume(final ExtendedBlock b) { @Override // FsDatasetSpi public Block getStoredBlock(String bpid, long blkid) throws IOException { - try (AutoCloseableLock lock = datasetReadLock.acquire()) { + try (AutoCloseableLock lock = lockManager.readLock(LockLevel.BLOCK_POOl, + bpid)) { ReplicaInfo r = volumeMap.get(bpid, blkid); if (r == null) { return null; @@ -209,12 +211,16 @@ public Block getStoredBlock(String bpid, long blkid) @Override public Set deepCopyReplica(String bpid) throws IOException { - Set replicas = null; - try (AutoCloseableLock lock = datasetReadLock.acquire()) { - replicas = new HashSet<>(volumeMap.replicas(bpid) == null ? Collections. - EMPTY_SET : volumeMap.replicas(bpid)); + try (AutoCloseDataSetLock l = lockManager.readLock(LockLevel.BLOCK_POOl, bpid)) { + Set replicas = new HashSet<>(); + volumeMap.replicas(bpid, (iterator) -> { + while (iterator.hasNext()) { + ReplicaInfo b = iterator.next(); + replicas.add(b); + } + }); + return replicas; } - return Collections.unmodifiableSet(replicas); } /** @@ -274,13 +280,7 @@ public LengthInputStream getMetaDataInputStream(ExtendedBlock b) private boolean blockPinningEnabled; private final int maxDataLength; - @VisibleForTesting - final AutoCloseableLock datasetWriteLock; - @VisibleForTesting - final AutoCloseableLock datasetReadLock; - @VisibleForTesting - final InstrumentedReadWriteLock datasetRWLock; - private final Condition datasetWriteLockCondition; + private final DataSetLockManager lockManager; private static String blockPoolId = ""; // Make limited notify times from DirectoryScanner to NameNode. @@ -299,33 +299,7 @@ public LengthInputStream getMetaDataInputStream(ExtendedBlock b) this.dataStorage = storage; this.conf = conf; this.smallBufferSize = DFSUtilClient.getSmallBufferSize(conf); - this.datasetRWLock = new InstrumentedReadWriteLock( - conf.getBoolean(DFSConfigKeys.DFS_DATANODE_LOCK_FAIR_KEY, - DFSConfigKeys.DFS_DATANODE_LOCK_FAIR_DEFAULT), - "FsDatasetRWLock", LOG, conf.getTimeDuration( - DFSConfigKeys.DFS_LOCK_SUPPRESS_WARNING_INTERVAL_KEY, - DFSConfigKeys.DFS_LOCK_SUPPRESS_WARNING_INTERVAL_DEFAULT, - TimeUnit.MILLISECONDS), - conf.getTimeDuration( - DFSConfigKeys.DFS_DATANODE_LOCK_REPORTING_THRESHOLD_MS_KEY, - DFSConfigKeys.DFS_DATANODE_LOCK_REPORTING_THRESHOLD_MS_DEFAULT, - TimeUnit.MILLISECONDS)); - this.datasetWriteLock = new AutoCloseableLock(datasetRWLock.writeLock()); - boolean enableRL = conf.getBoolean( - DFSConfigKeys.DFS_DATANODE_LOCK_READ_WRITE_ENABLED_KEY, - DFSConfigKeys.DFS_DATANODE_LOCK_READ_WRITE_ENABLED_DEFAULT); - // The read lock can be disabled by the above config key. If it is disabled - // then we simply make the both the read and write lock variables hold - // the write lock. All accesses to the lock are via these variables, so that - // effectively disables the read lock. - if (enableRL) { - LOG.info("The datanode lock is a read write lock"); - this.datasetReadLock = new AutoCloseableLock(datasetRWLock.readLock()); - } else { - LOG.info("The datanode lock is an exclusive write lock"); - this.datasetReadLock = this.datasetWriteLock; - } - this.datasetWriteLockCondition = datasetWriteLock.newCondition(); + this.lockManager = datanode.getDataSetLockManager(); // The number of volumes required for operation is the total number // of volumes minus the number of failed volumes we can tolerate. @@ -364,7 +338,7 @@ public LengthInputStream getMetaDataInputStream(ExtendedBlock b) } storageMap = new ConcurrentHashMap(); - volumeMap = new ReplicaMap(datasetReadLock, datasetWriteLock); + volumeMap = new ReplicaMap(lockManager); ramDiskReplicaTracker = RamDiskReplicaTracker.getInstance(conf, this); @SuppressWarnings("unchecked") @@ -374,7 +348,7 @@ public LengthInputStream getMetaDataInputStream(ExtendedBlock b) RoundRobinVolumeChoosingPolicy.class, VolumeChoosingPolicy.class), conf); volumes = new FsVolumeList(volumeFailureInfos, datanode.getBlockScanner(), - blockChooserImpl, conf); + blockChooserImpl, conf, datanode.getDiskMetrics()); asyncDiskService = new FsDatasetAsyncDiskService(datanode, this); asyncLazyPersistService = new RamDiskAsyncLazyPersistService(datanode, conf); deletingBlock = new HashMap>(); @@ -420,16 +394,6 @@ public LengthInputStream getMetaDataInputStream(ExtendedBlock b) lastDirScannerNotifyTime = System.currentTimeMillis(); } - @Override - public AutoCloseableLock acquireDatasetLock() { - return datasetWriteLock.acquire(); - } - - @Override - public AutoCloseableLock acquireDatasetReadLock() { - return datasetReadLock.acquire(); - } - /** * Gets initial volume failure information for all volumes that failed * immediately at startup. The method works by determining the set difference @@ -464,42 +428,43 @@ private static List getInitialVolumeFailureInfos( * Activate a volume to serve requests. * @throws IOException if the storage UUID already exists. */ - private void activateVolume( + private synchronized void activateVolume( ReplicaMap replicaMap, Storage.StorageDirectory sd, StorageType storageType, FsVolumeReference ref) throws IOException { - try (AutoCloseableLock lock = datasetWriteLock.acquire()) { - DatanodeStorage dnStorage = storageMap.get(sd.getStorageUuid()); - if (dnStorage != null) { - final String errorMsg = String.format( - "Found duplicated storage UUID: %s in %s.", - sd.getStorageUuid(), sd.getVersionFile()); - LOG.error(errorMsg); - throw new IOException(errorMsg); - } - // Check if there is same storage type on the mount. - // Only useful when same disk tiering is turned on. - FsVolumeImpl volumeImpl = (FsVolumeImpl) ref.getVolume(); - FsVolumeReference checkRef = volumes - .getMountVolumeMap() - .getVolumeRefByMountAndStorageType( - volumeImpl.getMount(), volumeImpl.getStorageType()); - if (checkRef != null) { - final String errorMsg = String.format( - "Storage type %s already exists on same mount: %s.", - volumeImpl.getStorageType(), volumeImpl.getMount()); - checkRef.close(); - LOG.error(errorMsg); - throw new IOException(errorMsg); - } - volumeMap.mergeAll(replicaMap); - storageMap.put(sd.getStorageUuid(), - new DatanodeStorage(sd.getStorageUuid(), - DatanodeStorage.State.NORMAL, - storageType)); - asyncDiskService.addVolume(volumeImpl); - volumes.addVolume(ref); - } + for (String bp : volumeMap.getBlockPoolList()) { + lockManager.addLock(LockLevel.VOLUME, bp, ref.getVolume().getStorageID()); + } + DatanodeStorage dnStorage = storageMap.get(sd.getStorageUuid()); + if (dnStorage != null) { + final String errorMsg = String.format( + "Found duplicated storage UUID: %s in %s.", + sd.getStorageUuid(), sd.getVersionFile()); + LOG.error(errorMsg); + throw new IOException(errorMsg); + } + // Check if there is same storage type on the mount. + // Only useful when same disk tiering is turned on. + FsVolumeImpl volumeImpl = (FsVolumeImpl) ref.getVolume(); + FsVolumeReference checkRef = volumes + .getMountVolumeMap() + .getVolumeRefByMountAndStorageType( + volumeImpl.getMount(), volumeImpl.getStorageType()); + if (checkRef != null) { + final String errorMsg = String.format( + "Storage type %s already exists on same mount: %s.", + volumeImpl.getStorageType(), volumeImpl.getMount()); + checkRef.close(); + LOG.error(errorMsg); + throw new IOException(errorMsg); + } + volumeMap.mergeAll(replicaMap); + storageMap.put(sd.getStorageUuid(), + new DatanodeStorage(sd.getStorageUuid(), + DatanodeStorage.State.NORMAL, + storageType)); + asyncDiskService.addVolume(volumeImpl); + volumes.addVolume(ref); } private void addVolume(Storage.StorageDirectory sd) throws IOException { @@ -516,8 +481,8 @@ private void addVolume(Storage.StorageDirectory sd) throws IOException { .setConf(this.conf) .build(); FsVolumeReference ref = fsVolume.obtainReference(); - ReplicaMap tempVolumeMap = - new ReplicaMap(datasetReadLock, datasetWriteLock); + // no need to acquire lock. + ReplicaMap tempVolumeMap = new ReplicaMap(); fsVolume.getVolumeMap(tempVolumeMap, ramDiskReplicaTracker); activateVolume(tempVolumeMap, sd, storageLocation.getStorageType(), ref); @@ -556,13 +521,13 @@ public void addVolume(final StorageLocation location, StorageType storageType = location.getStorageType(); final FsVolumeImpl fsVolume = createFsVolume(sd.getStorageUuid(), sd, location); - final ReplicaMap tempVolumeMap = - new ReplicaMap(new ReentrantReadWriteLock()); + // no need to add lock + final ReplicaMap tempVolumeMap = new ReplicaMap(); ArrayList exceptions = Lists.newArrayList(); for (final NamespaceInfo nsInfo : nsInfos) { String bpid = nsInfo.getBlockPoolID(); - try { + try (AutoCloseDataSetLock l = lockManager.writeLock(LockLevel.BLOCK_POOl, bpid)) { fsVolume.addBlockPool(bpid, this.conf, this.timer); fsVolume.getVolumeMap(bpid, tempVolumeMap, ramDiskReplicaTracker); } catch (IOException e) { @@ -602,7 +567,9 @@ public void removeVolumes( new ArrayList<>(storageLocsToRemove); Map> blkToInvalidate = new HashMap<>(); List storageToRemove = new ArrayList<>(); - try (AutoCloseableLock lock = datasetWriteLock.acquire()) { + // This object lock is protect data structure related volumes like add and + // remove.This will obtain volumeMap lock again if access replicaInfo. + synchronized (this) { for (int idx = 0; idx < dataStorage.getNumStorageDirs(); idx++) { Storage.StorageDirectory sd = dataStorage.getStorageDir(idx); final StorageLocation sdLocation = sd.getStorageLocation(); @@ -614,7 +581,7 @@ public void removeVolumes( // Disable the volume from the service. asyncDiskService.removeVolume(sd.getStorageUuid()); volumes.removeVolume(sdLocation, clearFailure); - volumes.waitVolumeRemoved(5000, datasetWriteLockCondition); + volumes.waitVolumeRemoved(5000, this); // Removed all replica information for the blocks on the volume. // Unlike updating the volumeMap in addVolume(), this operation does @@ -622,18 +589,19 @@ public void removeVolumes( for (String bpid : volumeMap.getBlockPoolList()) { List blocks = blkToInvalidate .computeIfAbsent(bpid, (k) -> new ArrayList<>()); - for (Iterator it = - volumeMap.replicas(bpid).iterator(); it.hasNext();) { - ReplicaInfo block = it.next(); - final StorageLocation blockStorageLocation = - block.getVolume().getStorageLocation(); - LOG.trace("checking for block " + block.getBlockId() + - " with storageLocation " + blockStorageLocation); - if (blockStorageLocation.equals(sdLocation)) { - blocks.add(block); - it.remove(); + volumeMap.replicas(bpid, (iterator) -> { + while (iterator.hasNext()) { + ReplicaInfo block = iterator.next(); + final StorageLocation blockStorageLocation = + block.getVolume().getStorageLocation(); + LOG.trace("checking for block " + block.getBlockId() + + " with storageLocation " + blockStorageLocation); + if (blockStorageLocation.equals(sdLocation)) { + blocks.add(block); + iterator.remove(); + } } - } + }); } storageToRemove.add(sd.getStorageUuid()); storageLocationsToRemove.remove(sdLocation); @@ -661,9 +629,12 @@ public void removeVolumes( } } - try (AutoCloseableLock lock = datasetWriteLock.acquire()) { - for(String storageUuid : storageToRemove) { + synchronized (this) { + for (String storageUuid : storageToRemove) { storageMap.remove(storageUuid); + for (String bp : volumeMap.getBlockPoolList()) { + lockManager.removeLock(LockLevel.VOLUME, bp, storageUuid); + } } } } @@ -852,7 +823,8 @@ public InputStream getBlockInputStream(ExtendedBlock b, long seekOffset) throws IOException { ReplicaInfo info; - try (AutoCloseableLock lock = datasetReadLock.acquire()) { + try (AutoCloseableLock lock = lockManager.readLock(LockLevel.BLOCK_POOl, + b.getBlockPoolId())) { info = volumeMap.get(b.getBlockPoolId(), b.getLocalBlock()); } @@ -940,7 +912,8 @@ ReplicaInfo getReplicaInfo(String bpid, long blkid) @Override // FsDatasetSpi public ReplicaInputStreams getTmpInputStreams(ExtendedBlock b, long blkOffset, long metaOffset) throws IOException { - try (AutoCloseableLock lock = datasetReadLock.acquire()) { + try (AutoCloseDataSetLock l = lockManager.readLock(LockLevel.VOLUME, + b.getBlockPoolId(), getReplicaInfo(b).getStorageUuid())) { ReplicaInfo info = getReplicaInfo(b); FsVolumeReference ref = info.getVolume().obtainReference(); try { @@ -1116,7 +1089,8 @@ public ReplicaInfo moveBlockAcrossStorage(ExtendedBlock block, targetStorageType, targetStorageId); boolean useVolumeOnSameMount = false; - try (AutoCloseableLock lock = datasetReadLock.acquire()) { + try (AutoCloseableLock lock = lockManager.readLock(LockLevel.BLOCK_POOl, + block.getBlockPoolId())) { if (shouldConsiderSameMountVolume) { volumeRef = volumes.getVolumeByMount(targetStorageType, ((FsVolumeImpl) replicaInfo.getVolume()).getMount(), @@ -1310,7 +1284,8 @@ public ReplicaInfo moveBlockAcrossVolumes(ExtendedBlock block, FsVolumeSpi FsVolumeReference volumeRef = null; - try (AutoCloseableLock lock = datasetReadLock.acquire()) { + try (AutoCloseableLock lock = lockManager.readLock(LockLevel.BLOCK_POOl, + block.getBlockPoolId())) { volumeRef = destination.obtainReference(); } @@ -1324,6 +1299,11 @@ public ReplicaInfo moveBlockAcrossVolumes(ExtendedBlock block, FsVolumeSpi return replicaInfo; } + @Override + public DataNodeLockManager acquireDatasetLockManager() { + return lockManager; + } + /** * Compute and store the checksum for a block file that does not already have * its checksum computed. @@ -1398,7 +1378,8 @@ static void computeChecksum(ReplicaInfo srcReplica, File dstMeta, @Override // FsDatasetSpi public ReplicaHandler append(ExtendedBlock b, long newGS, long expectedBlockLen) throws IOException { - try (AutoCloseableLock lock = datasetWriteLock.acquire()) { + try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.VOLUME, + b.getBlockPoolId(), getReplicaInfo(b).getStorageUuid())) { // If the block was successfully finalized because all packets // were successfully processed at the Datanode but the ack for // some of the packets were not received by the client. The client @@ -1450,7 +1431,8 @@ public ReplicaHandler append(ExtendedBlock b, private ReplicaInPipeline append(String bpid, ReplicaInfo replicaInfo, long newGS, long estimateBlockLen) throws IOException { - try (AutoCloseableLock lock = datasetWriteLock.acquire()) { + try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.VOLUME, + bpid, replicaInfo.getStorageUuid())) { // If the block is cached, start uncaching it. if (replicaInfo.getState() != ReplicaState.FINALIZED) { throw new IOException("Only a Finalized replica can be appended to; " @@ -1546,7 +1528,8 @@ public ReplicaHandler recoverAppend( while (true) { try { - try (AutoCloseableLock lock = datasetWriteLock.acquire()) { + try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.BLOCK_POOl, + b.getBlockPoolId())) { ReplicaInfo replicaInfo = recoverCheck(b, newGS, expectedBlockLen); FsVolumeReference ref = replicaInfo.getVolume().obtainReference(); ReplicaInPipeline replica; @@ -1578,7 +1561,8 @@ public Replica recoverClose(ExtendedBlock b, long newGS, LOG.info("Recover failed close " + b); while (true) { try { - try (AutoCloseableLock lock = datasetWriteLock.acquire()) { + try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.VOLUME, + b.getBlockPoolId(), getReplicaInfo(b).getStorageUuid())) { // check replica's state ReplicaInfo replicaInfo = recoverCheck(b, newGS, expectedBlockLen); // bump the replica's GS @@ -1601,7 +1585,8 @@ public ReplicaHandler createRbw( StorageType storageType, String storageId, ExtendedBlock b, boolean allowLazyPersist) throws IOException { long startTimeMs = Time.monotonicNow(); - try (AutoCloseableLock lock = datasetWriteLock.acquire()) { + try (AutoCloseableLock lock = lockManager.readLock(LockLevel.BLOCK_POOl, + b.getBlockPoolId())) { ReplicaInfo replicaInfo = volumeMap.get(b.getBlockPoolId(), b.getBlockId()); if (replicaInfo != null) { @@ -1648,20 +1633,20 @@ public ReplicaHandler createRbw( } ReplicaInPipeline newReplicaInfo; - try { + try (AutoCloseableLock l = lockManager.writeLock(LockLevel.VOLUME, + b.getBlockPoolId(), v.getStorageID())) { newReplicaInfo = v.createRbw(b); if (newReplicaInfo.getReplicaInfo().getState() != ReplicaState.RBW) { throw new IOException("CreateRBW returned a replica of state " + newReplicaInfo.getReplicaInfo().getState() + " for block " + b.getBlockId()); } + volumeMap.add(b.getBlockPoolId(), newReplicaInfo.getReplicaInfo()); + return new ReplicaHandler(newReplicaInfo, ref); } catch (IOException e) { IOUtils.cleanupWithLogger(null, ref); throw e; } - - volumeMap.add(b.getBlockPoolId(), newReplicaInfo.getReplicaInfo()); - return new ReplicaHandler(newReplicaInfo, ref); } finally { if (dataNodeMetrics != null) { long createRbwMs = Time.monotonicNow() - startTimeMs; @@ -1679,7 +1664,8 @@ public ReplicaHandler recoverRbw( try { while (true) { try { - try (AutoCloseableLock lock = datasetWriteLock.acquire()) { + try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.VOLUME, + b.getBlockPoolId(), getReplicaInfo(b).getStorageUuid())) { ReplicaInfo replicaInfo = getReplicaInfo(b.getBlockPoolId(), b.getBlockId()); // check the replica's state @@ -1710,7 +1696,8 @@ public ReplicaHandler recoverRbw( private ReplicaHandler recoverRbwImpl(ReplicaInPipeline rbw, ExtendedBlock b, long newGS, long minBytesRcvd, long maxBytesRcvd) throws IOException { - try (AutoCloseableLock lock = datasetWriteLock.acquire()) { + try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.VOLUME, + b.getBlockPoolId(), getReplicaInfo(b).getStorageUuid())) { // check generation stamp long replicaGenerationStamp = rbw.getGenerationStamp(); if (replicaGenerationStamp < b.getGenerationStamp() || @@ -1771,7 +1758,8 @@ private ReplicaHandler recoverRbwImpl(ReplicaInPipeline rbw, public ReplicaInPipeline convertTemporaryToRbw( final ExtendedBlock b) throws IOException { long startTimeMs = Time.monotonicNow(); - try (AutoCloseableLock lock = datasetWriteLock.acquire()) { + try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.VOLUME, + b.getBlockPoolId(), getReplicaInfo(b).getStorageUuid())) { final long blockId = b.getBlockId(); final long expectedGs = b.getGenerationStamp(); final long visible = b.getNumBytes(); @@ -1850,7 +1838,8 @@ public ReplicaHandler createTemporary(StorageType storageType, ReplicaInfo lastFoundReplicaInfo = null; boolean isInPipeline = false; do { - try (AutoCloseableLock lock = datasetWriteLock.acquire()) { + try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.BLOCK_POOl, + b.getBlockPoolId())) { ReplicaInfo currentReplicaInfo = volumeMap.get(b.getBlockPoolId(), b.getBlockId()); if (currentReplicaInfo == lastFoundReplicaInfo) { @@ -1905,11 +1894,12 @@ public ReplicaHandler createTemporary(StorageType storageType, false); } long startHoldLockTimeMs = Time.monotonicNow(); - try (AutoCloseableLock lock = datasetWriteLock.acquire()) { - FsVolumeReference ref = volumes.getNextVolume(storageType, storageId, b - .getNumBytes()); - FsVolumeImpl v = (FsVolumeImpl) ref.getVolume(); - ReplicaInPipeline newReplicaInfo; + FsVolumeReference ref = volumes.getNextVolume(storageType, storageId, b + .getNumBytes()); + FsVolumeImpl v = (FsVolumeImpl) ref.getVolume(); + ReplicaInPipeline newReplicaInfo; + try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.VOLUME, + b.getBlockPoolId(), v.getStorageID())) { try { newReplicaInfo = v.createTemporary(b); LOG.debug("creating temporary for block: {} on volume: {}", @@ -1966,7 +1956,8 @@ public void finalizeBlock(ExtendedBlock b, boolean fsyncDir) ReplicaInfo replicaInfo = null; ReplicaInfo finalizedReplicaInfo = null; long startTimeMs = Time.monotonicNow(); - try (AutoCloseableLock lock = datasetWriteLock.acquire()) { + try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.VOLUME, + b.getBlockPoolId(), getReplicaInfo(b).getStorageUuid())) { if (Thread.interrupted()) { // Don't allow data modifications from interrupted threads throw new IOException("Cannot finalize block from Interrupted Thread"); @@ -2002,7 +1993,8 @@ public void finalizeBlock(ExtendedBlock b, boolean fsyncDir) private ReplicaInfo finalizeReplica(String bpid, ReplicaInfo replicaInfo) throws IOException { - try (AutoCloseableLock lock = datasetWriteLock.acquire()) { + try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.VOLUME, + bpid, replicaInfo.getStorageUuid())) { // Compare generation stamp of old and new replica before finalizing if (volumeMap.get(bpid, replicaInfo.getBlockId()).getGenerationStamp() > replicaInfo.getGenerationStamp()) { @@ -2048,7 +2040,8 @@ private ReplicaInfo finalizeReplica(String bpid, ReplicaInfo replicaInfo) @Override // FsDatasetSpi public void unfinalizeBlock(ExtendedBlock b) throws IOException { long startTimeMs = Time.monotonicNow(); - try (AutoCloseableLock lock = datasetWriteLock.acquire()) { + try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.VOLUME, + b.getBlockPoolId(), getReplicaInfo(b).getStorageUuid())) { ReplicaInfo replicaInfo = volumeMap.get(b.getBlockPoolId(), b.getLocalBlock()); if (replicaInfo != null && @@ -2106,47 +2099,50 @@ public Map getBlockReports(String bpid) { new HashMap(); List curVolumes = null; - try (AutoCloseableLock lock = datasetReadLock.acquire()) { + try (AutoCloseableLock lock = lockManager.readLock(LockLevel.BLOCK_POOl, bpid)) { curVolumes = volumes.getVolumes(); for (FsVolumeSpi v : curVolumes) { builders.put(v.getStorageID(), BlockListAsLongs.builder(maxDataLength)); } Set missingVolumesReported = new HashSet<>(); - for (ReplicaInfo b : volumeMap.replicas(bpid)) { - // skip PROVIDED replicas. - if (b.getVolume().getStorageType() == StorageType.PROVIDED) { - continue; - } - String volStorageID = b.getVolume().getStorageID(); - switch(b.getState()) { - case FINALIZED: - case RBW: - case RWR: - break; - case RUR: - // use the original replica. - b = b.getOriginalReplica(); - break; - case TEMPORARY: - continue; - default: - assert false : "Illegal ReplicaInfo state."; - continue; - } - BlockListAsLongs.Builder storageBuilder = builders.get(volStorageID); - // a storage in the process of failing will not be in the volumes list - // but will be in the replica map. - if (storageBuilder != null) { - storageBuilder.add(b); - } else { - if (!missingVolumesReported.contains(volStorageID)) { - LOG.warn("Storage volume: " + volStorageID + " missing for the" - + " replica block: " + b + ". Probably being removed!"); - missingVolumesReported.add(volStorageID); + volumeMap.replicas(bpid, (iterator) -> { + while (iterator.hasNext()) { + ReplicaInfo b = iterator.next(); + // skip PROVIDED replicas. + if (b.getVolume().getStorageType() == StorageType.PROVIDED) { + continue; + } + String volStorageID = b.getVolume().getStorageID(); + switch(b.getState()) { + case FINALIZED: + case RBW: + case RWR: + break; + case RUR: + // use the original replica. + b = b.getOriginalReplica(); + break; + case TEMPORARY: + continue; + default: + assert false : "Illegal ReplicaInfo state."; + continue; + } + BlockListAsLongs.Builder storageBuilder = builders.get(volStorageID); + // a storage in the process of failing will not be in the volumes list + // but will be in the replica map. + if (storageBuilder != null) { + storageBuilder.add(b); + } else { + if (!missingVolumesReported.contains(volStorageID)) { + LOG.warn("Storage volume: " + volStorageID + " missing for the" + + " replica block: " + b + ". Probably being removed!"); + missingVolumesReported.add(volStorageID); + } } } - } + }); } for (FsVolumeImpl v : curVolumes) { @@ -2161,7 +2157,7 @@ public Map getBlockReports(String bpid) { * Gets a list of references to the finalized blocks for the given block pool. *

* Callers of this function should call - * {@link FsDatasetSpi#acquireDatasetLock()} to avoid blocks' status being + * {@link FsDatasetSpi#acquireDatasetLockManager()} ()} to avoid blocks' status being * changed during list iteration. *

* @return a list of references to the finalized blocks for the given block @@ -2169,14 +2165,17 @@ public Map getBlockReports(String bpid) { */ @Override public List getFinalizedBlocks(String bpid) { - try (AutoCloseableLock lock = datasetReadLock.acquire()) { - final List finalized = new ArrayList( - volumeMap.size(bpid)); - for (ReplicaInfo b : volumeMap.replicas(bpid)) { - if (b.getState() == ReplicaState.FINALIZED) { - finalized.add(b); + try (AutoCloseDataSetLock l = lockManager.readLock(LockLevel.BLOCK_POOl, bpid)) { + ArrayList finalized = + new ArrayList<>(volumeMap.size(bpid)); + volumeMap.replicas(bpid, (iterator) -> { + while (iterator.hasNext()) { + ReplicaInfo b = iterator.next(); + if (b.getState() == ReplicaState.FINALIZED) { + finalized.add(new FinalizedReplica((FinalizedReplica)b)); + } } - } + }); return finalized; } } @@ -2309,7 +2308,7 @@ private void invalidate(String bpid, Block[] invalidBlks, boolean async) for (int i = 0; i < invalidBlks.length; i++) { final ReplicaInfo removing; final FsVolumeImpl v; - try (AutoCloseableLock lock = datasetWriteLock.acquire()) { + try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.BLOCK_POOl, bpid)) { final ReplicaInfo info = volumeMap.get(bpid, invalidBlks[i]); if (info == null) { ReplicaInfo infoByBlockId = @@ -2432,10 +2431,17 @@ private void cacheBlock(String bpid, long blockId) { long length, genstamp; Executor volumeExecutor; - try (AutoCloseableLock lock = datasetWriteLock.acquire()) { - ReplicaInfo info = volumeMap.get(bpid, blockId); + ReplicaInfo info = volumeMap.get(bpid, blockId); + if (info == null) { + LOG.warn("Failed to cache block with id " + blockId + ", pool " + + bpid + ": ReplicaInfo not found."); + return; + } + try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.VOLUME, bpid, + info.getStorageUuid())) { boolean success = false; try { + info = volumeMap.get(bpid, blockId); if (info == null) { LOG.warn("Failed to cache block with id " + blockId + ", pool " + bpid + ": ReplicaInfo not found."); @@ -2500,7 +2506,8 @@ public boolean isCached(String bpid, long blockId) { @Override // FsDatasetSpi public boolean contains(final ExtendedBlock block) { - try (AutoCloseableLock lock = datasetReadLock.acquire()) { + try (AutoCloseableLock lock = lockManager.readLock(LockLevel.BLOCK_POOl, + block.getBlockPoolId())) { final long blockId = block.getLocalBlock().getBlockId(); final String bpid = block.getBlockPoolId(); final ReplicaInfo r = volumeMap.get(bpid, blockId); @@ -2627,7 +2634,8 @@ public void checkAndUpdate(String bpid, ScanInfo scanInfo) curDirScannerNotifyCount = 0; lastDirScannerNotifyTime = startTimeMs; } - try (AutoCloseableLock lock = datasetWriteLock.acquire()) { + try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.VOLUME, bpid, + vol.getStorageID())) { memBlockInfo = volumeMap.get(bpid, blockId); if (memBlockInfo != null && memBlockInfo.getState() != ReplicaState.FINALIZED) { @@ -2645,6 +2653,9 @@ public void checkAndUpdate(String bpid, ScanInfo scanInfo) Block.getGenerationStamp(diskMetaFile.getName()) : HdfsConstants.GRANDFATHER_GENERATION_STAMP; + final boolean isRegular = FileUtil.isRegularFile(diskMetaFile, false) && + FileUtil.isRegularFile(diskFile, false); + if (vol.getStorageType() == StorageType.PROVIDED) { if (memBlockInfo == null) { // replica exists on provided store but not in memory @@ -2812,6 +2823,9 @@ public void checkAndUpdate(String bpid, ScanInfo scanInfo) + memBlockInfo.getNumBytes() + " to " + memBlockInfo.getBlockDataLength()); memBlockInfo.setNumBytes(memBlockInfo.getBlockDataLength()); + } else if (!isRegular) { + corruptBlock = new Block(memBlockInfo); + LOG.warn("Block:{} is not a regular file.", corruptBlock.getBlockId()); } } finally { if (dataNodeMetrics != null) { @@ -2844,7 +2858,7 @@ public ReplicaInfo getReplica(String bpid, long blockId) { @Override public String getReplicaString(String bpid, long blockId) { - try (AutoCloseableLock lock = datasetReadLock.acquire()) { + try (AutoCloseableLock lock = lockManager.readLock(LockLevel.BLOCK_POOl, bpid)) { final Replica r = volumeMap.get(bpid, blockId); return r == null ? "null" : r.toString(); } @@ -2858,12 +2872,40 @@ public ReplicaRecoveryInfo initReplicaRecovery(RecoveringBlock rBlock) datanode.getDnConf().getXceiverStopTimeout()); } + ReplicaRecoveryInfo initReplicaRecovery(String bpid, ReplicaMap map, + Block block, long recoveryId, long xceiverStopTimeout) throws IOException { + while (true) { + try { + ReplicaInfo replica = map.get(bpid, block.getBlockId()); + if (replica == null) { + return null; + } + LOG.info("initReplicaRecovery: " + block + ", recoveryId=" + recoveryId + + ", replica=" + replica); + try (AutoCloseDataSetLock l = lockManager.writeLock(LockLevel.VOLUME, bpid, + replica.getStorageUuid())) { + return initReplicaRecoveryImpl(bpid, map, block, recoveryId); + } + } catch (MustStopExistingWriter e) { + e.getReplicaInPipeline().stopWriter(xceiverStopTimeout); + } + } + } + /** static version of {@link #initReplicaRecovery(RecoveringBlock)}. */ static ReplicaRecoveryInfo initReplicaRecovery(String bpid, ReplicaMap map, - Block block, long recoveryId, long xceiverStopTimeout) throws IOException { + Block block, long recoveryId, long xceiverStopTimeout, DataSetLockManager + lockManager) throws IOException { while (true) { try { - try (AutoCloseableLock lock = map.getLock().acquire()) { + ReplicaInfo replica = map.get(bpid, block.getBlockId()); + if (replica == null) { + return null; + } + LOG.info("initReplicaRecovery: " + block + ", recoveryId=" + recoveryId + + ", replica=" + replica); + try (AutoCloseDataSetLock l = lockManager.writeLock(LockLevel.VOLUME, bpid, + replica.getStorageUuid())) { return initReplicaRecoveryImpl(bpid, map, block, recoveryId); } } catch (MustStopExistingWriter e) { @@ -2876,9 +2918,6 @@ static ReplicaRecoveryInfo initReplicaRecoveryImpl(String bpid, ReplicaMap map, Block block, long recoveryId) throws IOException, MustStopExistingWriter { final ReplicaInfo replica = map.get(bpid, block.getBlockId()); - LOG.info("initReplicaRecovery: " + block + ", recoveryId=" + recoveryId - + ", replica=" + replica); - //check replica if (replica == null) { return null; @@ -2952,7 +2991,8 @@ public Replica updateReplicaUnderRecovery( final long newBlockId, final long newlength) throws IOException { long startTimeMs = Time.monotonicNow(); - try (AutoCloseableLock lock = datasetWriteLock.acquire()) { + try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.VOLUME, + oldBlock.getBlockPoolId(), getReplicaInfo(oldBlock).getStorageUuid())) { //get replica final String bpid = oldBlock.getBlockPoolId(); final ReplicaInfo replica = volumeMap.get(bpid, oldBlock.getBlockId()); @@ -3071,7 +3111,8 @@ private ReplicaInfo updateReplicaUnderRecovery( @Override // FsDatasetSpi public long getReplicaVisibleLength(final ExtendedBlock block) throws IOException { - try (AutoCloseableLock lock = datasetReadLock.acquire()) { + try (AutoCloseableLock lock = lockManager.readLock(LockLevel.BLOCK_POOl, + block.getBlockPoolId())) { final Replica replica = getReplicaInfo(block.getBlockPoolId(), block.getBlockId()); if (replica.getGenerationStamp() < block.getGenerationStamp()) { @@ -3088,13 +3129,17 @@ public void addBlockPool(String bpid, Configuration conf) throws IOException { LOG.info("Adding block pool " + bpid); AddBlockPoolException volumeExceptions = new AddBlockPoolException(); - try (AutoCloseableLock lock = datasetWriteLock.acquire()) { + try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.BLOCK_POOl, bpid)) { try { volumes.addBlockPool(bpid, conf); } catch (AddBlockPoolException e) { volumeExceptions.mergeException(e); } volumeMap.initBlockPool(bpid); + Set vols = storageMap.keySet(); + for (String v : vols) { + lockManager.addLock(LockLevel.VOLUME, bpid, v); + } } try { volumes.getAllVolumesMap(bpid, volumeMap, ramDiskReplicaTracker); @@ -3118,7 +3163,7 @@ public static void setBlockPoolId(String bpid) { @Override public void shutdownBlockPool(String bpid) { - try (AutoCloseableLock lock = datasetWriteLock.acquire()) { + try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.BLOCK_POOl, bpid)) { LOG.info("Removing block pool " + bpid); Map blocksPerVolume = getBlockReports(bpid); @@ -3192,7 +3237,7 @@ public Map getVolumeInfoMap() { @Override //FsDatasetSpi public void deleteBlockPool(String bpid, boolean force) throws IOException { - try (AutoCloseableLock lock = datasetWriteLock.acquire()) { + try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.BLOCK_POOl, bpid)) { List curVolumes = volumes.getVolumes(); if (!force) { for (FsVolumeImpl volume : curVolumes) { @@ -3221,7 +3266,8 @@ public void deleteBlockPool(String bpid, boolean force) @Override // FsDatasetSpi public BlockLocalPathInfo getBlockLocalPathInfo(ExtendedBlock block) throws IOException { - try (AutoCloseableLock lock = datasetReadLock.acquire()) { + try (AutoCloseableLock lock = lockManager.readLock(LockLevel.BLOCK_POOl, + block.getBlockPoolId())) { final Replica replica = volumeMap.get(block.getBlockPoolId(), block.getBlockId()); if (replica == null) { @@ -3275,7 +3321,7 @@ public void clearRollingUpgradeMarker(String bpid) throws IOException { @Override public void onCompleteLazyPersist(String bpId, long blockId, long creationTime, File[] savedFiles, FsVolumeImpl targetVolume) { - try (AutoCloseableLock lock = datasetWriteLock.acquire()) { + try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.BLOCK_POOl, bpId)) { ramDiskReplicaTracker.recordEndLazyPersist(bpId, blockId, savedFiles); targetVolume.incDfsUsedAndNumBlocks(bpId, savedFiles[0].length() @@ -3409,7 +3455,8 @@ private boolean saveNextReplica() { try { block = ramDiskReplicaTracker.dequeueNextReplicaToPersist(); if (block != null) { - try (AutoCloseableLock lock = datasetWriteLock.acquire()) { + try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.BLOCK_POOl, + block.getBlockPoolId())) { replicaInfo = volumeMap.get(block.getBlockPoolId(), block.getBlockId()); // If replicaInfo is null, the block was either deleted before @@ -3476,7 +3523,7 @@ public void evictBlocks(long bytesNeeded) throws IOException { ReplicaInfo replicaInfo, newReplicaInfo; final String bpid = replicaState.getBlockPoolId(); - try (AutoCloseableLock lock = datasetWriteLock.acquire()) { + try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.BLOCK_POOl, bpid)) { replicaInfo = getReplicaInfo(replicaState.getBlockPoolId(), replicaState.getBlockId()); Preconditions.checkState(replicaInfo.getVolume().isTransientStorage()); @@ -3654,20 +3701,28 @@ public int getVolumeCount() { } void stopAllDataxceiverThreads(FsVolumeImpl volume) { - try (AutoCloseableLock lock = datasetWriteLock.acquire()) { - for (String bpid : volumeMap.getBlockPoolList()) { - Collection replicas = volumeMap.replicas(bpid); - for (ReplicaInfo replicaInfo : replicas) { - if ((replicaInfo.getState() == ReplicaState.TEMPORARY - || replicaInfo.getState() == ReplicaState.RBW) - && replicaInfo.getVolume().equals(volume)) { - ReplicaInPipeline replicaInPipeline = - (ReplicaInPipeline) replicaInfo; - replicaInPipeline.interruptThread(); + for (String bpid : volumeMap.getBlockPoolList()) { + try (AutoCloseDataSetLock lock = lockManager + .writeLock(LockLevel.BLOCK_POOl, bpid)) { + volumeMap.replicas(bpid, (iterator) -> { + while (iterator.hasNext()) { + ReplicaInfo replicaInfo = iterator.next(); + if ((replicaInfo.getState() == ReplicaState.TEMPORARY + || replicaInfo.getState() == ReplicaState.RBW) + && replicaInfo.getVolume().equals(volume)) { + ReplicaInPipeline replicaInPipeline = + (ReplicaInPipeline) replicaInfo; + replicaInPipeline.interruptThread(); + } } - } + }); } } } + + @Override + public List getVolumeList() { + return volumes.getVolumes(); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java index 8f15d8a70932e..806afbdb2d115 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java @@ -548,6 +548,10 @@ long getRecentReserved() { return recentReserved; } + public Map getBlockPoolSlices() { + return bpSlices; + } + long getReserved(){ return reserved != null ? reserved.getReserved() : 0; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java index 9400c7c7f4ca1..262a24bd3aa45 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java @@ -33,6 +33,7 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Condition; +import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.StorageType; @@ -43,6 +44,7 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.VolumeChoosingPolicy; import org.apache.hadoop.hdfs.server.datanode.BlockScanner; import org.apache.hadoop.hdfs.server.datanode.StorageLocation; +import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeDiskMetrics; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.util.AutoCloseableLock; @@ -67,15 +69,17 @@ class FsVolumeList { private final boolean enableSameDiskTiering; private final MountVolumeMap mountVolumeMap; private Map capacityRatioMap; + private final DataNodeDiskMetrics diskMetrics; FsVolumeList(List initialVolumeFailureInfos, BlockScanner blockScanner, VolumeChoosingPolicy blockChooser, - Configuration config) { + Configuration config, DataNodeDiskMetrics dataNodeDiskMetrics) { this.blockChooser = blockChooser; this.blockScanner = blockScanner; this.checkDirsLock = new AutoCloseableLock(); this.checkDirsLockCondition = checkDirsLock.newCondition(); + this.diskMetrics = dataNodeDiskMetrics; for (VolumeFailureInfo volumeFailureInfo: initialVolumeFailureInfos) { volumeFailureInfos.put(volumeFailureInfo.getFailedStorageLocation(), volumeFailureInfo); @@ -100,6 +104,15 @@ List getVolumes() { private FsVolumeReference chooseVolume(List list, long blockSize, String storageId) throws IOException { + + // Exclude slow disks when choosing volume. + if (diskMetrics != null) { + List slowDisksToExclude = diskMetrics.getSlowDisksToExclude(); + list = list.stream() + .filter(volume -> !slowDisksToExclude.contains(volume.getBaseURI().getPath())) + .collect(Collectors.toList()); + } + while (true) { FsVolumeImpl volume = blockChooser.chooseVolume(list, blockSize, storageId); @@ -332,6 +345,28 @@ void waitVolumeRemoved(int sleepMillis, Condition condition) { FsDatasetImpl.LOG.info("Volume reference is released."); } + /** + * Wait for the reference of the volume removed from a previous + * {@link #removeVolume(FsVolumeImpl)} call to be released. + * + * @param sleepMillis interval to recheck. + */ + void waitVolumeRemoved(int sleepMillis, Object condition) { + while (!checkVolumesRemoved()) { + if (FsDatasetImpl.LOG.isDebugEnabled()) { + FsDatasetImpl.LOG.debug("Waiting for volume reference to be released."); + } + try { + condition.wait(sleepMillis); + } catch (InterruptedException e) { + FsDatasetImpl.LOG.info("Thread interrupted when waiting for " + + "volume reference to be released."); + Thread.currentThread().interrupt(); + } + } + FsDatasetImpl.LOG.info("Volume reference is released."); + } + @Override public String toString() { return volumes.toString(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ProvidedVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ProvidedVolumeImpl.java index 7c6562f58d16e..69a46257317bf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ProvidedVolumeImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ProvidedVolumeImpl.java @@ -30,7 +30,10 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.ReentrantReadWriteLock; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; @@ -60,10 +63,6 @@ import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.Time; import org.apache.hadoop.util.Timer; -import org.codehaus.jackson.annotate.JsonProperty; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.map.ObjectReader; -import org.codehaus.jackson.map.ObjectWriter; import org.apache.hadoop.classification.VisibleForTesting; @@ -134,7 +133,7 @@ static class ProvidedBlockPoolSlice { ProvidedBlockPoolSlice(String bpid, ProvidedVolumeImpl volume, Configuration conf) { this.providedVolume = volume; - bpVolumeMap = new ReplicaMap(new ReentrantReadWriteLock()); + bpVolumeMap = new ReplicaMap(); Class fmt = conf.getClass(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_CLASS, TextFileRegionAliasMap.class, BlockAliasMap.class); @@ -219,7 +218,7 @@ private void incrNumBlocks() { } public boolean isEmpty() { - return bpVolumeMap.replicas(bpid).size() == 0; + return bpVolumeMap.size(bpid) == 0; } public void shutdown(BlockListAsLongs blocksListsAsLongs) { @@ -371,14 +370,11 @@ public void releaseReservedSpace(long bytesToRelease) { private static final ObjectWriter WRITER = new ObjectMapper().writerWithDefaultPrettyPrinter(); - private static final ObjectReader READER = - new ObjectMapper().reader(ProvidedBlockIteratorState.class); private static class ProvidedBlockIteratorState { ProvidedBlockIteratorState() { iterStartMs = Time.now(); lastSavedMs = iterStartMs; - atEnd = false; lastBlockId = -1L; } @@ -390,9 +386,6 @@ private static class ProvidedBlockIteratorState { @JsonProperty private long iterStartMs; - @JsonProperty - private boolean atEnd; - // The id of the last block read when the state of the iterator is saved. // This implementation assumes that provided blocks are returned // in sorted order of the block ids. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReplicaMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReplicaMap.java index c1d103ed50dba..6ecc48a95fd2d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReplicaMap.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReplicaMap.java @@ -18,46 +18,49 @@ package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl; import java.util.Collection; -import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; import java.util.Map; -import java.util.concurrent.locks.ReadWriteLock; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Consumer; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.server.common.AutoCloseDataSetLock; +import org.apache.hadoop.hdfs.server.common.DataNodeLockManager; +import org.apache.hadoop.hdfs.server.common.DataNodeLockManager.LockLevel; +import org.apache.hadoop.hdfs.server.common.NoLockManager; import org.apache.hadoop.hdfs.server.datanode.ReplicaInfo; import org.apache.hadoop.util.LightWeightResizableGSet; -import org.apache.hadoop.util.AutoCloseableLock; /** * Maintains the replica map. */ class ReplicaMap { // Lock object to synchronize this instance. - private final AutoCloseableLock readLock; - private final AutoCloseableLock writeLock; - + private DataNodeLockManager lockManager; + // Map of block pool Id to another map of block Id to ReplicaInfo. private final Map> map = - new HashMap<>(); + new ConcurrentHashMap<>(); - ReplicaMap(AutoCloseableLock readLock, AutoCloseableLock writeLock) { - if (readLock == null || writeLock == null) { + ReplicaMap(DataNodeLockManager manager) { + if (manager == null) { throw new HadoopIllegalArgumentException( - "Lock to synchronize on cannot be null"); + "Object to synchronize on cannot be null"); } - this.readLock = readLock; - this.writeLock = writeLock; + this.lockManager = manager; } - ReplicaMap(ReadWriteLock lock) { - this(new AutoCloseableLock(lock.readLock()), - new AutoCloseableLock(lock.writeLock())); + // Used for ut or temp replicaMap that no need to protected by lock. + ReplicaMap() { + this.lockManager = new NoLockManager(); } String[] getBlockPoolList() { - try (AutoCloseableLock l = readLock.acquire()) { - return map.keySet().toArray(new String[map.keySet().size()]); - } + Set bpset = map.keySet(); + return bpset.toArray(new String[bpset.size()]); } private void checkBlockPool(String bpid) { @@ -100,7 +103,7 @@ ReplicaInfo get(String bpid, Block block) { */ ReplicaInfo get(String bpid, long blockId) { checkBlockPool(bpid); - try (AutoCloseableLock l = readLock.acquire()) { + try (AutoCloseDataSetLock l = lockManager.readLock(LockLevel.BLOCK_POOl, bpid)) { LightWeightResizableGSet m = map.get(bpid); return m != null ? m.get(new Block(blockId)) : null; } @@ -117,12 +120,12 @@ ReplicaInfo get(String bpid, long blockId) { ReplicaInfo add(String bpid, ReplicaInfo replicaInfo) { checkBlockPool(bpid); checkBlock(replicaInfo); - try (AutoCloseableLock l = writeLock.acquire()) { + try (AutoCloseDataSetLock l = lockManager.readLock(LockLevel.BLOCK_POOl, bpid)) { LightWeightResizableGSet m = map.get(bpid); if (m == null) { // Add an entry for block pool if it does not exist already - m = new LightWeightResizableGSet(); - map.put(bpid, m); + map.putIfAbsent(bpid, new LightWeightResizableGSet()); + m = map.get(bpid); } return m.put(replicaInfo); } @@ -135,12 +138,12 @@ ReplicaInfo add(String bpid, ReplicaInfo replicaInfo) { ReplicaInfo addAndGet(String bpid, ReplicaInfo replicaInfo) { checkBlockPool(bpid); checkBlock(replicaInfo); - try (AutoCloseableLock l = writeLock.acquire()) { + try (AutoCloseDataSetLock l = lockManager.readLock(LockLevel.BLOCK_POOl, bpid)) { LightWeightResizableGSet m = map.get(bpid); if (m == null) { // Add an entry for block pool if it does not exist already - m = new LightWeightResizableGSet(); - map.put(bpid, m); + map.putIfAbsent(bpid, new LightWeightResizableGSet()); + m = map.get(bpid); } ReplicaInfo oldReplicaInfo = m.get(replicaInfo); if (oldReplicaInfo != null) { @@ -164,13 +167,28 @@ void addAll(ReplicaMap other) { * Merge all entries from the given replica map into the local replica map. */ void mergeAll(ReplicaMap other) { - other.map.forEach( - (bp, replicaInfos) -> { - replicaInfos.forEach( - replicaInfo -> add(bp, replicaInfo) - ); + Set bplist = other.map.keySet(); + for (String bp : bplist) { + checkBlockPool(bp); + try (AutoCloseDataSetLock l = lockManager.writeLock(LockLevel.BLOCK_POOl, bp)) { + LightWeightResizableGSet replicaInfos = other.map.get(bp); + LightWeightResizableGSet curSet = map.get(bp); + HashSet replicaSet = new HashSet<>(); + //Can't add to GSet while in another GSet iterator may cause endlessLoop + for (ReplicaInfo replicaInfo : replicaInfos) { + replicaSet.add(replicaInfo); } - ); + for (ReplicaInfo replicaInfo : replicaSet) { + checkBlock(replicaInfo); + if (curSet == null) { + // Add an entry for block pool if it does not exist already + curSet = new LightWeightResizableGSet<>(); + map.put(bp, curSet); + } + curSet.put(replicaInfo); + } + } + } } /** @@ -184,7 +202,7 @@ void mergeAll(ReplicaMap other) { ReplicaInfo remove(String bpid, Block block) { checkBlockPool(bpid); checkBlock(block); - try (AutoCloseableLock l = writeLock.acquire()) { + try (AutoCloseDataSetLock l = lockManager.readLock(LockLevel.BLOCK_POOl, bpid)) { LightWeightResizableGSet m = map.get(bpid); if (m != null) { ReplicaInfo replicaInfo = m.get(block); @@ -206,7 +224,7 @@ ReplicaInfo remove(String bpid, Block block) { */ ReplicaInfo remove(String bpid, long blockId) { checkBlockPool(bpid); - try (AutoCloseableLock l = writeLock.acquire()) { + try (AutoCloseDataSetLock l = lockManager.readLock(LockLevel.BLOCK_POOl, bpid)) { LightWeightResizableGSet m = map.get(bpid); if (m != null) { return m.remove(new Block(blockId)); @@ -221,7 +239,7 @@ ReplicaInfo remove(String bpid, long blockId) { * @return the number of replicas in the map */ int size(String bpid) { - try (AutoCloseableLock l = readLock.acquire()) { + try (AutoCloseDataSetLock l = lockManager.readLock(LockLevel.BLOCK_POOl, bpid)) { LightWeightResizableGSet m = map.get(bpid); return m != null ? m.size() : 0; } @@ -229,11 +247,9 @@ int size(String bpid) { /** * Get a collection of the replicas for given block pool - * This method is not synchronized. It needs to be synchronized - * externally using the lock, both for getting the replicas - * values from the map and iterating over it. Mutex can be accessed using - * {@link #getLock()} method. - * + * This method is not synchronized. If you want to keep thread safe + * Use method {@link #replicas(String, Consumer>)}. + * * @param bpid block pool id * @return a collection of the replicas belonging to the block pool */ @@ -243,9 +259,25 @@ Collection replicas(String bpid) { return m != null ? m.values() : null; } + /** + * execute function for one block pool and protect by LockManager. + * This method is synchronized. + * + * @param bpid block pool id + */ + void replicas(String bpid, Consumer> consumer) { + LightWeightResizableGSet m = null; + try (AutoCloseDataSetLock l = lockManager.readLock(LockLevel.BLOCK_POOl, bpid)) { + m = map.get(bpid); + if (m !=null) { + m.getIterator(consumer); + } + } + } + void initBlockPool(String bpid) { checkBlockPool(bpid); - try (AutoCloseableLock l = writeLock.acquire()) { + try (AutoCloseDataSetLock l = lockManager.writeLock(LockLevel.BLOCK_POOl, bpid)) { LightWeightResizableGSet m = map.get(bpid); if (m == null) { // Add an entry for block pool if it does not exist already @@ -257,26 +289,8 @@ void initBlockPool(String bpid) { void cleanUpBlockPool(String bpid) { checkBlockPool(bpid); - try (AutoCloseableLock l = writeLock.acquire()) { + try (AutoCloseDataSetLock l = lockManager.writeLock(LockLevel.BLOCK_POOl, bpid)) { map.remove(bpid); } } - - /** - * Get the lock object used for synchronizing ReplicasMap - * @return lock object - */ - AutoCloseableLock getLock() { - return writeLock; - } - - /** - * Get the lock object used for synchronizing the ReplicasMap for read only - * operations. - * @return The read lock object - */ - AutoCloseableLock getReadLock() { - return readLock; - } - } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java index cae464bfae700..409084cfe8be8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java @@ -30,13 +30,21 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports.DiskOp; import org.apache.hadoop.util.Daemon; +import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.Iterator; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; + +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY; /** * This class detects and maintains DataNode disk outliers and their @@ -64,11 +72,19 @@ public class DataNodeDiskMetrics { /** * Minimum number of disks to run outlier detection. */ - private final long minOutlierDetectionDisks; + private volatile long minOutlierDetectionDisks; /** * Threshold in milliseconds below which a disk is definitely not slow. */ - private final long lowThresholdMs; + private volatile long lowThresholdMs; + /** + * The number of slow disks that needs to be excluded. + */ + private int maxSlowDisksToExclude; + /** + * List of slow disks that need to be excluded. + */ + private List slowDisksToExclude = new ArrayList<>(); public DataNodeDiskMetrics(DataNode dn, long diskOutlierDetectionIntervalMs, Configuration conf) { @@ -80,6 +96,9 @@ public DataNodeDiskMetrics(DataNode dn, long diskOutlierDetectionIntervalMs, lowThresholdMs = conf.getLong(DFSConfigKeys.DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY, DFSConfigKeys.DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_DEFAULT); + maxSlowDisksToExclude = + conf.getInt(DFSConfigKeys.DFS_DATANODE_MAX_SLOWDISKS_TO_EXCLUDE_KEY, + DFSConfigKeys.DFS_DATANODE_MAX_SLOWDISKS_TO_EXCLUDE_DEFAULT); slowDiskDetector = new OutlierDetector(minOutlierDetectionDisks, lowThresholdMs); shouldRun = true; @@ -127,6 +146,21 @@ public void run() { detectAndUpdateDiskOutliers(metadataOpStats, readIoStats, writeIoStats); + + // Sort the slow disks by latency and extract the top n by maxSlowDisksToExclude. + if (maxSlowDisksToExclude > 0) { + ArrayList diskLatencies = new ArrayList<>(); + for (Map.Entry> diskStats : + diskOutliersStats.entrySet()) { + diskLatencies.add(new DiskLatency(diskStats.getKey(), diskStats.getValue())); + } + + Collections.sort(diskLatencies, (o1, o2) + -> Double.compare(o2.getMaxLatency(), o1.getMaxLatency())); + + slowDisksToExclude = diskLatencies.stream().limit(maxSlowDisksToExclude) + .map(DiskLatency::getSlowDisk).collect(Collectors.toList()); + } } try { @@ -171,6 +205,35 @@ private void detectAndUpdateDiskOutliers(Map metadataOpStats, } } + /** + * This structure is a wrapper over disk latencies. + */ + public static class DiskLatency { + final private String slowDisk; + final private Map latencyMap; + + public DiskLatency( + String slowDiskID, + Map latencyMap) { + this.slowDisk = slowDiskID; + this.latencyMap = latencyMap; + } + + double getMaxLatency() { + double maxLatency = 0; + for (double latency : latencyMap.values()) { + if (latency > maxLatency) { + maxLatency = latency; + } + } + return maxLatency; + } + + public String getSlowDisk() { + return slowDisk; + } + } + private void addDiskStat(Map> diskStats, String disk, DiskOp diskOp, double latency) { if (!diskStats.containsKey(disk)) { @@ -206,4 +269,35 @@ public void addSlowDiskForTesting(String slowDiskPath, diskOutliersStats.put(slowDiskPath, latencies); } } + + public List getSlowDisksToExclude() { + return slowDisksToExclude; + } + + public void setLowThresholdMs(long thresholdMs) { + Preconditions.checkArgument(thresholdMs > 0, + DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY + " should be larger than 0"); + lowThresholdMs = thresholdMs; + this.slowDiskDetector.setLowThresholdMs(thresholdMs); + } + + public long getLowThresholdMs() { + return lowThresholdMs; + } + + public void setMinOutlierDetectionDisks(long minDisks) { + Preconditions.checkArgument(minDisks > 0, + DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_KEY + " should be larger than 0"); + minOutlierDetectionDisks = minDisks; + this.slowDiskDetector.setMinNumResources(minDisks); + } + + public long getMinOutlierDetectionDisks() { + return minOutlierDetectionDisks; + } + + @VisibleForTesting + public OutlierDetector getSlowDiskDetector() { + return this.slowDiskDetector; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java index 5203d7bf87f89..649d30e91e034 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java @@ -191,6 +191,8 @@ public class DataNodeMetrics { @Metric MutableCounterLong packetsSlowWriteToMirror; @Metric MutableCounterLong packetsSlowWriteToDisk; @Metric MutableCounterLong packetsSlowWriteToOsCache; + @Metric private MutableCounterLong slowFlushOrSyncCount; + @Metric private MutableCounterLong slowAckToUpstreamCount; @Metric("Number of replaceBlock ops between" + " storage types on same host with local copy") @@ -440,6 +442,14 @@ public void incrVolumeFailures(int size) { volumeFailures.incr(size); } + public void incrSlowFlushOrSyncCount() { + slowFlushOrSyncCount.incr(); + } + + public void incrSlowAckToUpstreamCount() { + slowAckToUpstreamCount.incr(); + } + public void incrDatanodeNetworkErrors() { datanodeNetworkErrors.incr(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodePeerMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodePeerMetrics.java index 750e53db13bf7..2e456b67ca149 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodePeerMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodePeerMetrics.java @@ -21,18 +21,23 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.metrics2.MetricsJsonBuilder; import org.apache.hadoop.metrics2.lib.MutableRollingAverages; +import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Map; import java.util.concurrent.ThreadLocalRandom; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_PEER_METRICS_MIN_OUTLIER_DETECTION_SAMPLES_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_PEER_METRICS_MIN_OUTLIER_DETECTION_SAMPLES_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_KEY; /** * This class maintains DataNode peer metrics (e.g. numOps, AvgTime, etc.) for @@ -49,6 +54,8 @@ public class DataNodePeerMetrics { private final String name; + // Strictly to be used by test code only. Source code is not supposed to use this. + private Map testOutlier = null; private final OutlierDetector slowNodeDetector; @@ -57,15 +64,15 @@ public class DataNodePeerMetrics { * for outlier detection. If the number of samples is below this then * outlier detection is skipped. */ - private final long minOutlierDetectionSamples; + private volatile long minOutlierDetectionSamples; /** * Threshold in milliseconds below which a DataNode is definitely not slow. */ - private final long lowThresholdMs; + private volatile long lowThresholdMs; /** * Minimum number of nodes to run outlier detection. */ - private final long minOutlierDetectionNodes; + private volatile long minOutlierDetectionNodes; public DataNodePeerMetrics(final String name, Configuration conf) { this.name = name; @@ -73,11 +80,11 @@ public DataNodePeerMetrics(final String name, Configuration conf) { DFS_DATANODE_PEER_METRICS_MIN_OUTLIER_DETECTION_SAMPLES_KEY, DFS_DATANODE_PEER_METRICS_MIN_OUTLIER_DETECTION_SAMPLES_DEFAULT); lowThresholdMs = - conf.getLong(DFSConfigKeys.DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_KEY, - DFSConfigKeys.DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_DEFAULT); + conf.getLong(DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_KEY, + DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_DEFAULT); minOutlierDetectionNodes = - conf.getLong(DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_KEY, - DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_DEFAULT); + conf.getLong(DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_KEY, + DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_DEFAULT); this.slowNodeDetector = new OutlierDetector(minOutlierDetectionNodes, lowThresholdMs); sendPacketDownstreamRollingAverages = new MutableRollingAverages("Time"); @@ -87,7 +94,7 @@ public String name() { return name; } - long getMinOutlierDetectionSamples() { + public long getMinOutlierDetectionSamples() { return minOutlierDetectionSamples; } @@ -137,17 +144,65 @@ public void collectThreadLocalStates() { * than their peers. */ public Map getOutliers() { - // This maps the metric name to the aggregate latency. - // The metric name is the datanode ID. - final Map stats = - sendPacketDownstreamRollingAverages.getStats( - minOutlierDetectionSamples); - LOG.trace("DataNodePeerMetrics: Got stats: {}", stats); + // outlier must be null for source code. + if (testOutlier == null) { + // This maps the metric name to the aggregate latency. + // The metric name is the datanode ID. + final Map stats = + sendPacketDownstreamRollingAverages.getStats(minOutlierDetectionSamples); + LOG.trace("DataNodePeerMetrics: Got stats: {}", stats); + return slowNodeDetector.getOutliers(stats); + } else { + // this happens only for test code. + return testOutlier; + } + } - return slowNodeDetector.getOutliers(stats); + /** + * Strictly to be used by test code only. Source code is not supposed to use this. This method + * directly sets outlier mapping so that aggregate latency metrics are not calculated for tests. + * + * @param outlier outlier directly set by tests. + */ + public void setTestOutliers(Map outlier) { + this.testOutlier = outlier; } public MutableRollingAverages getSendPacketDownstreamRollingAverages() { return sendPacketDownstreamRollingAverages; } + + public void setMinOutlierDetectionNodes(long minNodes) { + Preconditions.checkArgument(minNodes > 0, + DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_KEY + " should be larger than 0"); + minOutlierDetectionNodes = minNodes; + this.slowNodeDetector.setMinNumResources(minNodes); + } + + public long getMinOutlierDetectionNodes() { + return minOutlierDetectionNodes; + } + + public void setLowThresholdMs(long thresholdMs) { + Preconditions.checkArgument(thresholdMs > 0, + DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_KEY + " should be larger than 0"); + lowThresholdMs = thresholdMs; + this.slowNodeDetector.setLowThresholdMs(thresholdMs); + } + + public long getLowThresholdMs() { + return lowThresholdMs; + } + + public void setMinOutlierDetectionSamples(long minSamples) { + Preconditions.checkArgument(minSamples > 0, + DFS_DATANODE_PEER_METRICS_MIN_OUTLIER_DETECTION_SAMPLES_KEY + + " should be larger than 0"); + minOutlierDetectionSamples = minSamples; + } + + @VisibleForTesting + public OutlierDetector getSlowNodeDetector() { + return this.slowNodeDetector; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/OutlierDetector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/OutlierDetector.java index 1e20e38fdc7be..39feca03d665e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/OutlierDetector.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/OutlierDetector.java @@ -60,7 +60,7 @@ public class OutlierDetector { /** * Minimum number of resources to run outlier detection. */ - private final long minNumResources; + private volatile long minNumResources; /** * The multiplier is from Leys, C. et al. @@ -70,7 +70,7 @@ public class OutlierDetector { /** * Threshold in milliseconds below which a node/ disk is definitely not slow. */ - private final long lowThresholdMs; + private volatile long lowThresholdMs; /** * Deviation multiplier. A sample is considered to be an outlier if it @@ -180,4 +180,20 @@ public static Double computeMedian(List sortedValues) { } return median; } + + public void setMinNumResources(long minNodes) { + minNumResources = minNodes; + } + + public long getMinOutlierDetectionNodes() { + return minNumResources; + } + + public void setLowThresholdMs(long thresholdMs) { + lowThresholdMs = thresholdMs; + } + + public long getLowThresholdMs() { + return lowThresholdMs; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/QueryCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/QueryCommand.java index 6a89e248f5967..f5d4cfacd8265 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/QueryCommand.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/QueryCommand.java @@ -84,7 +84,7 @@ public void execute(CommandLine cmd) throws Exception { System.out.printf("%s", workStatus.currentStateString()); } } catch (DiskBalancerException ex) { - LOG.error("Query plan failed. ex: {}", ex); + LOG.error("Query plan failed.", ex); throw ex; } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java index 504df6068ef3c..7bde21dcb69e9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java @@ -222,7 +222,7 @@ private synchronized void applyEdits(long firstTxId, int numTxns, byte[] data) try { getNamesystem().dir.updateCountForQuota(); } finally { - getNamesystem().writeUnlock(); + getNamesystem().writeUnlock("applyEdits"); } } finally { backupInputStream.clear(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java index d18d448ab2500..ab657fb672490 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java @@ -250,7 +250,7 @@ void doCheckpoint() throws IOException { sig.mostRecentCheckpointTxId); bnImage.reloadFromImageFile(file, backupNode.getNamesystem()); } finally { - backupNode.namesystem.writeUnlock(); + backupNode.namesystem.writeUnlock("doCheckpointByBackupNode"); } } rollForwardByApplyingLogs(manifest, bnImage, backupNode.getNamesystem()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java index 62a81f4064eec..7bf5879971615 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java @@ -192,7 +192,7 @@ public void pauseForTestingAfterNthCheckpoint(final String zone, try { iip = dir.resolvePath(pc, zone, DirOp.READ); } finally { - dir.getFSNamesystem().readUnlock(); + dir.getFSNamesystem().readUnlock("pauseForTestingAfterNthCheckpoint"); } reencryptionHandler .pauseForTestingAfterNthCheckpoint(iip.getLastINode().getId(), count); @@ -224,7 +224,7 @@ public ZoneReencryptionStatus getZoneStatus(final String zone) return getReencryptionStatus().getZoneStatus(inode.getId()); } finally { dir.readUnlock(); - dir.getFSNamesystem().readUnlock(); + dir.getFSNamesystem().readUnlock("getZoneStatus"); } } @@ -285,7 +285,7 @@ void stopReencryptThread() { try { reencryptionHandler.stopThreads(); } finally { - dir.getFSNamesystem().writeUnlock(); + dir.getFSNamesystem().writeUnlock("stopReencryptThread"); } if (reencryptHandlerExecutor != null) { reencryptHandlerExecutor.shutdownNow(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java index 8338c4fb091a8..1a7a65a0d5562 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdfs.server.namenode; import org.apache.hadoop.classification.VisibleForTesting; +import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; @@ -304,6 +305,12 @@ public synchronized ErasureCodingPolicy addPolicy( + policy.getCodecName() + " is not supported"); } + int blocksInGroup = policy.getNumDataUnits() + policy.getNumParityUnits(); + if (blocksInGroup > HdfsServerConstants.MAX_BLOCKS_IN_GROUP) { + throw new HadoopIllegalArgumentException("Number of data and parity blocks in an EC group " + + blocksInGroup + " should not exceed maximum " + HdfsServerConstants.MAX_BLOCKS_IN_GROUP); + } + if (policy.getCellSize() > maxCellSize) { throw new HadoopIllegalArgumentException("Cell size " + policy.getCellSize() + " should not exceed maximum " + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java index db1baab66b3dc..04913d1a7cee0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java @@ -415,7 +415,7 @@ static BlockInfo[] unprotectedSetReplication( bm.setReplication(oldBR, targetReplication, b); } - if (oldBR != -1) { + if (oldBR != -1 && FSDirectory.LOG.isDebugEnabled()) { if (oldBR > targetReplication) { FSDirectory.LOG.debug("Decreasing replication from {} to {} for {}", oldBR, targetReplication, iip.getPath()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java index 2971af1829809..2110a408b0877 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java @@ -656,7 +656,7 @@ static EncryptionKeyInfo getEncryptionKeyInfo(FSNamesystem fsn, Preconditions.checkNotNull(ezKeyName); // Generate EDEK while not holding the fsn lock. - fsn.writeUnlock(); + fsn.writeUnlock("getEncryptionKeyInfo"); try { EncryptionFaultInjector.getInstance().startFileBeforeGenerateKey(); return new EncryptionKeyInfo(protocolVersion, suite, ezKeyName, @@ -733,7 +733,7 @@ static String getKeyNameForZone(final FSDirectory dir, dir.ezManager.checkEncryptionZoneRoot(iip.getLastINode(), zone); return dir.ezManager.getKeyName(iip); } finally { - dir.getFSNamesystem().readUnlock(); + dir.getFSNamesystem().readUnlock("getKeyNameForZone"); } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java index cc0d29e953572..c129d1928abdd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java @@ -81,8 +81,12 @@ private static void verifyQuotaForRename(FSDirectory fsd, INodesInPath src, // Assume dstParent existence check done by callers. INode dstParent = dst.getINode(-2); // Use the destination parent's storage policy for quota delta verify. + final boolean isSrcSetSp = src.getLastINode().isSetStoragePolicy(); + final byte storagePolicyID = isSrcSetSp ? + src.getLastINode().getLocalStoragePolicyID() : + dstParent.getStoragePolicyID(); final QuotaCounts delta = src.getLastINode() - .computeQuotaUsage(bsps, dstParent.getStoragePolicyID(), false, + .computeQuotaUsage(bsps, storagePolicyID, false, Snapshot.CURRENT_STATE_ID); // Reduce the required quota by dst that is being removed diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index 4d86e52f74d52..d4fed21e98e17 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@ -1372,9 +1372,13 @@ public INodesInPath addLastINode(INodesInPath existing, INode inode, // always verify inode name verifyINodeName(inode.getLocalNameBytes()); + final boolean isSrcSetSp = inode.isSetStoragePolicy(); + final byte storagePolicyID = isSrcSetSp ? + inode.getLocalStoragePolicyID() : + parent.getStoragePolicyID(); final QuotaCounts counts = inode .computeQuotaUsage(getBlockStoragePolicySuite(), - parent.getStoragePolicyID(), false, Snapshot.CURRENT_STATE_ID); + storagePolicyID, false, Snapshot.CURRENT_STATE_ID); updateCount(existing, pos, counts, checkQuota); boolean isRename = (inode.getParent() != null); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java index 53c663307d738..850b2fc570884 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java @@ -1515,11 +1515,12 @@ public synchronized void purgeLogsOlderThan(final long minTxIdToKeep) { if (!isOpenForWrite()) { return; } - - assert curSegmentTxId == HdfsServerConstants.INVALID_TXID || // on format this is no-op - minTxIdToKeep <= curSegmentTxId : - "cannot purge logs older than txid " + minTxIdToKeep + - " when current segment starts at " + curSegmentTxId; + + Preconditions.checkArgument( + curSegmentTxId == HdfsServerConstants.INVALID_TXID || // on format this is no-op + minTxIdToKeep <= curSegmentTxId, + "cannot purge logs older than txid " + minTxIdToKeep + + " when current segment starts at " + curSegmentTxId); if (minTxIdToKeep == 0) { return; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java index b350ef1f98c3e..7302596dc6028 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java @@ -582,12 +582,12 @@ void doImportCheckpoint(FSNamesystem target) throws IOException { if (checkpointDirs == null || checkpointDirs.isEmpty()) { throw new IOException("Cannot import image from a checkpoint. " - + "\"dfs.namenode.checkpoint.dir\" is not set." ); + + "\"dfs.namenode.checkpoint.dir\" is not set."); } if (checkpointEditsDirs == null || checkpointEditsDirs.isEmpty()) { throw new IOException("Cannot import image from a checkpoint. " - + "\"dfs.namenode.checkpoint.dir\" is not set." ); + + "\"dfs.namenode.checkpoint.edits.dir\" is not set."); } FSImage realImage = target.getFSImage(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 3d9e9bb934d24..ab3c49fc2641f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -401,7 +401,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, @Metric final MutableRatesWithAggregation detailedLockHoldTimeMetrics = registry.newRatesWithAggregation("detailedLockHoldTimeMetrics"); - private static final String CLIENT_PORT_STR = "clientPort"; private final String contextFieldSeparator; boolean isAuditEnabled() { @@ -467,7 +466,7 @@ private void appendClientPortToCallerContextIfAbsent() { byte[] origSignature = ctx == null ? null : ctx.getSignature(); CallerContext.setCurrent( new CallerContext.Builder(origContext, contextFieldSeparator) - .append(CLIENT_PORT_STR, String.valueOf(Server.getRemotePort())) + .append(CallerContext.CLIENT_PORT_STR, String.valueOf(Server.getRemotePort())) .setSignature(origSignature) .build()); } @@ -475,7 +474,7 @@ private void appendClientPortToCallerContextIfAbsent() { private boolean isClientPortInfoAbsent(CallerContext ctx){ return ctx == null || ctx.getContext() == null - || !ctx.getContext().contains(CLIENT_PORT_STR); + || !ctx.getContext().contains(CallerContext.CLIENT_PORT_STR); } /** @@ -503,7 +502,12 @@ private boolean isClientPortInfoAbsent(CallerContext ctx){ private final boolean standbyShouldCheckpoint; private final boolean isSnapshotTrashRootEnabled; private final int snapshotDiffReportLimit; - private final int blockDeletionIncrement; + + /** + * Whether enable checkOperation when call getBlocks. + * It is enabled by default. + */ + private final boolean isGetBlocksCheckOperationEnabled; /** Interval between each check of lease to release. */ private final long leaseRecheckIntervalMs; @@ -1014,6 +1018,10 @@ static FSNamesystem loadFromDisk(Configuration conf) throws IOException { this.leaseRecheckIntervalMs = conf.getLong( DFS_NAMENODE_LEASE_RECHECK_INTERVAL_MS_KEY, DFS_NAMENODE_LEASE_RECHECK_INTERVAL_MS_DEFAULT); + Preconditions.checkArgument( + leaseRecheckIntervalMs > 0, + DFSConfigKeys.DFS_NAMENODE_LEASE_RECHECK_INTERVAL_MS_KEY + + " must be greater than zero"); this.maxLockHoldToReleaseLeaseMs = conf.getLong( DFS_NAMENODE_MAX_LOCK_HOLD_TO_RELEASE_LEASE_MS_KEY, DFS_NAMENODE_MAX_LOCK_HOLD_TO_RELEASE_LEASE_MS_DEFAULT); @@ -1056,12 +1064,10 @@ static FSNamesystem loadFromDisk(Configuration conf) throws IOException { this.allowOwnerSetQuota = conf.getBoolean( DFSConfigKeys.DFS_PERMISSIONS_ALLOW_OWNER_SET_QUOTA_KEY, DFSConfigKeys.DFS_PERMISSIONS_ALLOW_OWNER_SET_QUOTA_DEFAULT); - this.blockDeletionIncrement = conf.getInt( - DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_INCREMENT_KEY, - DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_INCREMENT_DEFAULT); - Preconditions.checkArgument(blockDeletionIncrement > 0, - DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_INCREMENT_KEY + - " must be a positive integer."); + this.isGetBlocksCheckOperationEnabled = conf.getBoolean( + DFSConfigKeys.DFS_NAMENODE_GETBLOCKS_CHECK_OPERATION_KEY, + DFSConfigKeys.DFS_NAMENODE_GETBLOCKS_CHECK_OPERATION_DEFAULT); + } catch(IOException e) { LOG.error(getClass().getSimpleName() + " initialization failed.", e); close(); @@ -1785,6 +1791,7 @@ public void readUnlock() { this.fsLock.readUnlock(); } + @Override public void readUnlock(String opName) { this.fsLock.readUnlock(opName); } @@ -1809,6 +1816,7 @@ public void writeUnlock() { this.fsLock.writeUnlock(); } + @Override public void writeUnlock(String opName) { this.fsLock.writeUnlock(opName); } @@ -1934,10 +1942,13 @@ public boolean isInStandbyState() { */ public BlocksWithLocations getBlocks(DatanodeID datanode, long size, long minimumBlockSize, long timeInterval) throws IOException { - checkOperation(OperationCategory.READ); + OperationCategory checkOp = + isGetBlocksCheckOperationEnabled ? OperationCategory.READ : + OperationCategory.UNCHECKED; + checkOperation(checkOp); readLock(); try { - checkOperation(OperationCategory.READ); + checkOperation(checkOp); return getBlockManager().getBlocksWithLocations(datanode, size, minimumBlockSize, timeInterval); } finally { @@ -2369,8 +2380,8 @@ boolean truncate(String src, long newLength, String clientName, } getEditLog().logSync(); if (!toRemoveBlocks.getToDeleteList().isEmpty()) { - removeBlocks(toRemoveBlocks); - toRemoveBlocks.clear(); + blockManager.addBLocksToMarkedDeleteQueue( + toRemoveBlocks.getToDeleteList()); } logAuditEvent(true, operationName, src, null, status); } catch (AccessControlException e) { @@ -2817,8 +2828,8 @@ private HdfsFileStatus startFileInt(String src, if (!skipSync) { getEditLog().logSync(); if (toRemoveBlocks != null) { - removeBlocks(toRemoveBlocks); - toRemoveBlocks.clear(); + blockManager.addBLocksToMarkedDeleteQueue( + toRemoveBlocks.getToDeleteList()); } } } @@ -3341,8 +3352,8 @@ void renameTo(final String src, final String dst, assert res != null; BlocksMapUpdateInfo collectedBlocks = res.collectedBlocks; if (!collectedBlocks.getToDeleteList().isEmpty()) { - removeBlocks(collectedBlocks); - collectedBlocks.clear(); + blockManager.addBLocksToMarkedDeleteQueue( + collectedBlocks.getToDeleteList()); } logAuditEvent(true, operationName + " (options=" + @@ -3381,7 +3392,8 @@ boolean delete(String src, boolean recursive, boolean logRetryCache) getEditLog().logSync(); logAuditEvent(ret, operationName, src); if (toRemovedBlocks != null) { - removeBlocks(toRemovedBlocks); // Incremental deletion of blocks + blockManager.addBLocksToMarkedDeleteQueue( + toRemovedBlocks.getToDeleteList()); } return ret; } @@ -3391,30 +3403,6 @@ FSPermissionChecker getPermissionChecker() return dir.getPermissionChecker(); } - /** - * From the given list, incrementally remove the blocks from blockManager - * Writelock is dropped and reacquired every blockDeletionIncrement to - * ensure that other waiters on the lock can get in. See HDFS-2938 - * - * @param blocks - * An instance of {@link BlocksMapUpdateInfo} which contains a list - * of blocks that need to be removed from blocksMap - */ - void removeBlocks(BlocksMapUpdateInfo blocks) { - List toDeleteList = blocks.getToDeleteList(); - Iterator iter = toDeleteList.iterator(); - while (iter.hasNext()) { - writeLock(); - try { - for (int i = 0; i < blockDeletionIncrement && iter.hasNext(); i++) { - blockManager.removeBlock(iter.next()); - } - } finally { - writeUnlock("removeBlocks"); - } - } - } - /** * Remove leases and inodes related to a given path * @param removedUCFiles INodes whose leases need to be released @@ -4421,8 +4409,11 @@ nodeReg, reports, getBlockPoolId(), cacheCapacity, cacheUsed, haContext.getState().getServiceState(), getFSImage().getCorrectLastAppliedOrWrittenTxId()); + Set slownodes = DatanodeManager.getSlowNodesUuidSet(); + boolean isSlownode = slownodes.contains(nodeReg.getDatanodeUuid()); + return new HeartbeatResponse(cmds, haState, rollingUpgradeInfo, - blockReportLeaseId); + blockReportLeaseId, isSlownode); } finally { readUnlock("handleHeartbeat"); } @@ -4620,7 +4611,8 @@ private void clearCorruptLazyPersistFiles() INodesInPath.fromINode((INodeFile) bc), false); changed |= toRemoveBlocks != null; if (toRemoveBlocks != null) { - removeBlocks(toRemoveBlocks); // Incremental deletion of blocks + blockManager.addBLocksToMarkedDeleteQueue( + toRemoveBlocks.getToDeleteList()); } } } finally { @@ -4876,6 +4868,12 @@ public long getCurrentTokensCount() { dtSecretManager.getCurrentTokensSize() : -1; } + @Override + @Metric({"PendingSPSPaths", "The number of paths to be processed by storage policy satisfier"}) + public int getPendingSPSPaths() { + return blockManager.getPendingSPSPaths(); + } + /** * Returns the length of the wait Queue for the FSNameSystemLock. * @@ -4915,6 +4913,32 @@ int getNumberOfDatanodes(DatanodeReportType type) { } } + DatanodeInfo[] slowDataNodesReport() throws IOException { + String operationName = "slowDataNodesReport"; + DatanodeInfo[] datanodeInfos; + checkOperation(OperationCategory.UNCHECKED); + readLock(); + try { + checkOperation(OperationCategory.UNCHECKED); + final DatanodeManager dm = getBlockManager().getDatanodeManager(); + final List results = dm.getAllSlowDataNodes(); + datanodeInfos = getDatanodeInfoFromDescriptors(results); + } finally { + readUnlock(operationName, getLockReportInfoSupplier(null)); + } + logAuditEvent(true, operationName, null); + return datanodeInfos; + } + + private DatanodeInfo[] getDatanodeInfoFromDescriptors(List results) { + DatanodeInfo[] datanodeInfos = new DatanodeInfo[results.size()]; + for (int i = 0; i < datanodeInfos.length; i++) { + datanodeInfos[i] = new DatanodeInfoBuilder().setFrom(results.get(i)).build(); + datanodeInfos[i].setNumBlocks(results.get(i).numBlocks()); + } + return datanodeInfos; + } + DatanodeInfo[] datanodeReport(final DatanodeReportType type) throws IOException { String operationName = "datanodeReport"; @@ -4926,12 +4950,7 @@ DatanodeInfo[] datanodeReport(final DatanodeReportType type) checkOperation(OperationCategory.UNCHECKED); final DatanodeManager dm = getBlockManager().getDatanodeManager(); final List results = dm.getDatanodeListForReport(type); - arr = new DatanodeInfo[results.size()]; - for (int i=0; i getStorageMovementAttemptedItems() { + return storageMovementAttemptedItems; + } + + @VisibleForTesting + public BlockingQueue getMovementFinishedBlocks() { + return movementFinishedBlocks; + } + public void clearQueues() { movementFinishedBlocks.clear(); synchronized (storageMovementAttemptedItems) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/BlockStorageMovementNeeded.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/BlockStorageMovementNeeded.java index 9a1faed952595..b6fc15d2104a0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/BlockStorageMovementNeeded.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/BlockStorageMovementNeeded.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.server.namenode.sps; +import java.io.FileNotFoundException; import java.io.IOException; import java.util.HashMap; import java.util.LinkedList; @@ -227,15 +228,18 @@ public synchronized void clearQueuesWithNotification() { * ID's to process for satisfy the policy. */ private class SPSPathIdProcessor implements Runnable { + private static final int MAX_RETRY_COUNT = 3; @Override public void run() { LOG.info("Starting SPSPathIdProcessor!."); Long startINode = null; + int retryCount = 0; while (ctxt.isRunning()) { try { if (!ctxt.isInSafeMode()) { if (startINode == null) { + retryCount = 0; startINode = ctxt.getNextSPSPath(); } // else same id will be retried if (startINode == null) { @@ -248,7 +252,12 @@ public void run() { pendingWorkForDirectory.get(startINode); if (dirPendingWorkInfo != null && dirPendingWorkInfo.isDirWorkDone()) { - ctxt.removeSPSHint(startINode); + try { + ctxt.removeSPSHint(startINode); + } catch (FileNotFoundException e) { + // ignore if the file doesn't already exist + startINode = null; + } pendingWorkForDirectory.remove(startINode); } } @@ -268,6 +277,11 @@ public void run() { LOG.info("Interrupted while waiting in SPSPathIdProcessor", t); break; } + retryCount++; + if (retryCount >= MAX_RETRY_COUNT) { + LOG.warn("Skipping this inode {} due to too many retries.", startINode); + startINode = null; + } } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/StoragePolicySatisfier.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/StoragePolicySatisfier.java index 47596019af4ed..3efe6b1cd6d86 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/StoragePolicySatisfier.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/StoragePolicySatisfier.java @@ -1077,7 +1077,7 @@ public void clearQueues() { * attempted or reported time stamp. This is used by * {@link BlockStorageMovementAttemptedItems#storageMovementAttemptedItems}. */ - final static class AttemptedItemInfo extends ItemInfo { + public final static class AttemptedItemInfo extends ItemInfo { private long lastAttemptedOrReportedTime; private final Set blocks; @@ -1095,7 +1095,7 @@ final static class AttemptedItemInfo extends ItemInfo { * @param retryCount * file retry count */ - AttemptedItemInfo(long rootId, long trackId, + public AttemptedItemInfo(long rootId, long trackId, long lastAttemptedOrReportedTime, Set blocks, int retryCount) { super(rootId, trackId, retryCount); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/StoragePolicySatisfyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/StoragePolicySatisfyManager.java index 40e3faa5550e5..2c7f36a690bd1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/StoragePolicySatisfyManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/StoragePolicySatisfyManager.java @@ -60,7 +60,7 @@ public class StoragePolicySatisfyManager { private final StoragePolicySatisfier spsService; private final boolean storagePolicyEnabled; private volatile StoragePolicySatisfierMode mode; - private final Queue pathsToBeTraveresed; + private final Queue pathsToBeTraversed; private final int outstandingPathsLimit; private final Namesystem namesystem; @@ -77,7 +77,7 @@ public StoragePolicySatisfyManager(Configuration conf, DFSConfigKeys.DFS_SPS_MAX_OUTSTANDING_PATHS_KEY, DFSConfigKeys.DFS_SPS_MAX_OUTSTANDING_PATHS_DEFAULT); mode = StoragePolicySatisfierMode.fromString(modeVal); - pathsToBeTraveresed = new LinkedList(); + pathsToBeTraversed = new LinkedList(); this.namesystem = namesystem; // instantiate SPS service by just keeps config reference and not starting // any supporting threads. @@ -218,8 +218,8 @@ public boolean isSatisfierRunning() { * storages. */ public Long getNextPathId() { - synchronized (pathsToBeTraveresed) { - return pathsToBeTraveresed.poll(); + synchronized (pathsToBeTraversed) { + return pathsToBeTraversed.poll(); } } @@ -228,7 +228,7 @@ public Long getNextPathId() { * @throws IOException */ public void verifyOutstandingPathQLimit() throws IOException { - long size = pathsToBeTraveresed.size(); + long size = pathsToBeTraversed.size(); // Checking that the SPS call Q exceeds the allowed limit. if (outstandingPathsLimit - size <= 0) { LOG.debug("Satisifer Q - outstanding limit:{}, current size:{}", @@ -244,15 +244,15 @@ public void verifyOutstandingPathQLimit() throws IOException { * @throws IOException */ private void clearPathIds(){ - synchronized (pathsToBeTraveresed) { - Iterator iterator = pathsToBeTraveresed.iterator(); + synchronized (pathsToBeTraversed) { + Iterator iterator = pathsToBeTraversed.iterator(); while (iterator.hasNext()) { Long trackId = iterator.next(); try { namesystem.removeXattr(trackId, HdfsServerConstants.XATTR_SATISFY_STORAGE_POLICY); } catch (IOException e) { - LOG.debug("Failed to remove sps xatttr!", e); + LOG.debug("Failed to remove sps xattr!", e); } iterator.remove(); } @@ -263,8 +263,8 @@ private void clearPathIds(){ * Clean up all sps path ids. */ public void removeAllPathIds() { - synchronized (pathsToBeTraveresed) { - pathsToBeTraveresed.clear(); + synchronized (pathsToBeTraversed) { + pathsToBeTraversed.clear(); } } @@ -273,8 +273,8 @@ public void removeAllPathIds() { * @param id */ public void addPathId(long id) { - synchronized (pathsToBeTraveresed) { - pathsToBeTraveresed.add(id); + synchronized (pathsToBeTraversed) { + pathsToBeTraversed.add(id); } } @@ -292,4 +292,11 @@ public boolean isEnabled() { public StoragePolicySatisfierMode getMode() { return mode; } + + /** + * @return the number of paths to be processed by storage policy satisfier. + */ + public int getPendingSPSPaths() { + return pathsToBeTraversed.size(); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java index 5910a80700ec6..a3250c213caf0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java @@ -55,6 +55,7 @@ import javax.ws.rs.core.Response.ResponseBuilder; import javax.ws.rs.core.Response.Status; +import org.apache.hadoop.fs.InvalidPathException; import org.apache.hadoop.fs.QuotaUsage; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.protocol.HdfsConstants; @@ -401,6 +402,9 @@ private URI redirectURI(ResponseBuilder rb, final NameNode namenode, final String path, final HttpOpParam.Op op, final long openOffset, final long blocksize, final String excludeDatanodes, final Param... parameters) throws URISyntaxException, IOException { + if (!DFSUtil.isValidName(path)) { + throw new InvalidPathException(path); + } final DatanodeInfo dn; final NamenodeProtocols np = getRPCServer(namenode); HdfsFileStatus status = null; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/HeartbeatResponse.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/HeartbeatResponse.java index 8d6384e700d5e..4ee930e67b724 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/HeartbeatResponse.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/HeartbeatResponse.java @@ -36,14 +36,23 @@ public class HeartbeatResponse { private final RollingUpgradeStatus rollingUpdateStatus; private final long fullBlockReportLeaseId; - + + private final boolean isSlownode; + public HeartbeatResponse(DatanodeCommand[] cmds, NNHAStatusHeartbeat haStatus, RollingUpgradeStatus rollingUpdateStatus, long fullBlockReportLeaseId) { + this(cmds, haStatus, rollingUpdateStatus, fullBlockReportLeaseId, false); + } + + public HeartbeatResponse(DatanodeCommand[] cmds, + NNHAStatusHeartbeat haStatus, RollingUpgradeStatus rollingUpdateStatus, + long fullBlockReportLeaseId, boolean isSlownode) { commands = cmds; this.haStatus = haStatus; this.rollingUpdateStatus = rollingUpdateStatus; this.fullBlockReportLeaseId = fullBlockReportLeaseId; + this.isSlownode = isSlownode; } public DatanodeCommand[] getCommands() { @@ -61,4 +70,8 @@ public RollingUpgradeStatus getRollingUpdateStatus() { public long getFullBlockReportLeaseId() { return fullBlockReportLeaseId; } + + public boolean getIsSlownode() { + return isSlownode; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalSPSBlockMoveTaskHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalSPSBlockMoveTaskHandler.java index 64dec8bbc5c3c..ec3837424cc20 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalSPSBlockMoveTaskHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalSPSBlockMoveTaskHandler.java @@ -80,11 +80,15 @@ public class ExternalSPSBlockMoveTaskHandler implements BlockMoveTaskHandler { private Daemon movementTrackerThread; private final SPSService service; private final BlockDispatcher blkDispatcher; + private final int maxRetry; public ExternalSPSBlockMoveTaskHandler(Configuration conf, NameNodeConnector nnc, SPSService spsService) { int moverThreads = conf.getInt(DFSConfigKeys.DFS_MOVER_MOVERTHREADS_KEY, DFSConfigKeys.DFS_MOVER_MOVERTHREADS_DEFAULT); + maxRetry = conf.getInt( + DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_MOVE_TASK_MAX_RETRY_ATTEMPTS_KEY, + DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_MOVE_TASK_MAX_RETRY_ATTEMPTS_DEFAULT); moveExecutor = initializeBlockMoverThreadPool(moverThreads); mCompletionServ = new ExecutorCompletionService<>(moveExecutor); this.nnc = nnc; @@ -151,7 +155,7 @@ public void submitMoveTask(BlockMovingInfo blkMovingInfo) throws IOException { // during block movement assignment logic. In the internal movement, // remaining space is bookkeeping at the DatanodeDescriptor, please refer // IntraSPSNameNodeBlockMoveTaskHandler#submitMoveTask implementation and - // updating via the funcation call - + // updating via the function call - // dn.incrementBlocksScheduled(blkMovingInfo.getTargetStorageType()); LOG.debug("Received BlockMovingTask {}", blkMovingInfo); BlockMovingTask blockMovingTask = new BlockMovingTask(blkMovingInfo); @@ -195,21 +199,25 @@ private BlockMovementStatus moveBlock() { final KeyManager km = nnc.getKeyManager(); Token accessToken; - try { - accessToken = km.getAccessToken(eb, - new StorageType[]{blkMovingInfo.getTargetStorageType()}, - new String[0]); - } catch (IOException e) { - // TODO: handle failure retries - LOG.warn( - "Failed to move block:{} from src:{} to destin:{} to satisfy " - + "storageType:{}", - blkMovingInfo.getBlock(), blkMovingInfo.getSource(), - blkMovingInfo.getTarget(), blkMovingInfo.getTargetStorageType(), e); - return BlockMovementStatus.DN_BLK_STORAGE_MOVEMENT_FAILURE; + int retry = 0; + while (retry <= maxRetry) { + try { + ExternalSPSFaultInjector.getInstance().mockAnException(retry); + accessToken = km.getAccessToken(eb, + new StorageType[]{blkMovingInfo.getTargetStorageType()}, + new String[0]); + return blkDispatcher.moveBlock(blkMovingInfo, saslClient, eb, + new Socket(), km, accessToken); + } catch (IOException e) { + LOG.warn( + "Failed to move block:{} from src:{} to dest:{} to satisfy " + + "storageType:{}, retry: {}", + blkMovingInfo.getBlock(), blkMovingInfo.getSource(), + blkMovingInfo.getTarget(), blkMovingInfo.getTargetStorageType(), retry, e); + retry++; + } } - return blkDispatcher.moveBlock(blkMovingInfo, saslClient, eb, - new Socket(), km, accessToken); + return BlockMovementStatus.DN_BLK_STORAGE_MOVEMENT_FAILURE; } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalSPSContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalSPSContext.java index 8427e93a70972..78f1dc6bf3e75 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalSPSContext.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalSPSContext.java @@ -24,6 +24,7 @@ import java.util.List; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSUtilClient; import org.apache.hadoop.hdfs.protocol.Block; @@ -39,10 +40,12 @@ import org.apache.hadoop.hdfs.server.namenode.sps.Context; import org.apache.hadoop.hdfs.server.namenode.sps.FileCollector; import org.apache.hadoop.hdfs.server.namenode.sps.SPSService; +import org.apache.hadoop.hdfs.server.namenode.sps.StoragePolicySatisfier; import org.apache.hadoop.hdfs.server.namenode.sps.StoragePolicySatisfier.DatanodeMap; import org.apache.hadoop.hdfs.server.namenode.sps.StoragePolicySatisfier.DatanodeWithStorage; import org.apache.hadoop.hdfs.server.protocol.BlockStorageMovementCommand.BlockMovingInfo; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport; +import org.apache.hadoop.hdfs.server.sps.metrics.ExternalSPSBeanMetrics; import org.apache.hadoop.net.NetworkTopology; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -62,6 +65,7 @@ public class ExternalSPSContext implements Context { private final FileCollector fileCollector; private final BlockMoveTaskHandler externalHandler; private final BlockMovementListener blkMovementListener; + private ExternalSPSBeanMetrics spsBeanMetrics; public ExternalSPSContext(SPSService service, NameNodeConnector nnc) { this.service = service; @@ -208,4 +212,17 @@ public void notifyMovementTriedBlocks(Block[] moveAttemptFinishedBlks) { LOG.info("Movement attempted blocks", actualBlockMovements); } } + + public void initMetrics(StoragePolicySatisfier sps) { + spsBeanMetrics = new ExternalSPSBeanMetrics(sps); + } + + public void closeMetrics() { + spsBeanMetrics.close(); + } + + @VisibleForTesting + public ExternalSPSBeanMetrics getSpsBeanMetrics() { + return spsBeanMetrics; + } } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalSPSFaultInjector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalSPSFaultInjector.java new file mode 100644 index 0000000000000..5ddf1ee3c0f6b --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalSPSFaultInjector.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.server.sps; + +import org.apache.hadoop.classification.VisibleForTesting; + +import java.io.IOException; + +/** + * Used to inject certain faults for testing. + */ +public class ExternalSPSFaultInjector { + @VisibleForTesting + private static ExternalSPSFaultInjector instance = + new ExternalSPSFaultInjector(); + + @VisibleForTesting + public static ExternalSPSFaultInjector getInstance() { + return instance; + } + + @VisibleForTesting + public static void setInstance(ExternalSPSFaultInjector instance) { + ExternalSPSFaultInjector.instance = instance; + } + + @VisibleForTesting + public void mockAnException(int retry) throws IOException { + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalStoragePolicySatisfier.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalStoragePolicySatisfier.java index 15cdc6eb47bcf..40d01bda4a3f2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalStoragePolicySatisfier.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalStoragePolicySatisfier.java @@ -38,6 +38,7 @@ import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.util.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,8 +48,7 @@ */ @InterfaceAudience.Private public final class ExternalStoragePolicySatisfier { - public static final Logger LOG = LoggerFactory - .getLogger(ExternalStoragePolicySatisfier.class); + public static final Logger LOG = LoggerFactory.getLogger(ExternalStoragePolicySatisfier.class); private ExternalStoragePolicySatisfier() { // This is just a class to start and run external sps. @@ -59,6 +59,7 @@ private ExternalStoragePolicySatisfier() { */ public static void main(String[] args) throws Exception { NameNodeConnector nnc = null; + ExternalSPSContext context = null; try { StringUtils.startupShutdownMessage(StoragePolicySatisfier.class, args, LOG); @@ -68,9 +69,10 @@ public static void main(String[] args) throws Exception { StoragePolicySatisfier sps = new StoragePolicySatisfier(spsConf); nnc = getNameNodeConnector(spsConf); - ExternalSPSContext context = new ExternalSPSContext(sps, nnc); + context = new ExternalSPSContext(sps, nnc); sps.init(context); sps.start(StoragePolicySatisfierMode.EXTERNAL); + context.initMetrics(sps); if (sps != null) { sps.join(); } @@ -81,6 +83,11 @@ public static void main(String[] args) throws Exception { if (nnc != null) { nnc.close(); } + if (context!= null) { + if (context.getSpsBeanMetrics() != null) { + context.closeMetrics(); + } + } } } @@ -96,20 +103,25 @@ private static void secureLogin(Configuration conf) socAddr.getHostName()); } - private static NameNodeConnector getNameNodeConnector(Configuration conf) - throws IOException, InterruptedException { + public static NameNodeConnector getNameNodeConnector(Configuration conf) + throws InterruptedException { final Collection namenodes = DFSUtil.getInternalNsRpcUris(conf); final Path externalSPSPathId = HdfsServerConstants.MOVER_ID_PATH; + String serverName = ExternalStoragePolicySatisfier.class.getSimpleName(); while (true) { try { final List nncs = NameNodeConnector .newNameNodeConnectors(namenodes, - ExternalStoragePolicySatisfier.class.getSimpleName(), + serverName, externalSPSPathId, conf, NameNodeConnector.DEFAULT_MAX_IDLE_ITERATIONS); return nncs.get(0); } catch (IOException e) { LOG.warn("Failed to connect with namenode", e); + if (e.getMessage().equals("Another " + serverName + " is running.")) { + ExitUtil.terminate(-1, + "Exit immediately because another " + serverName + " is running"); + } Thread.sleep(3000); // retry the connection after few secs } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/metrics/ExternalSPSBeanMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/metrics/ExternalSPSBeanMetrics.java new file mode 100644 index 0000000000000..75546386f9da3 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/metrics/ExternalSPSBeanMetrics.java @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.sps.metrics; + +import org.apache.hadoop.classification.VisibleForTesting; +import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.server.namenode.sps.ItemInfo; +import org.apache.hadoop.hdfs.server.namenode.sps.StoragePolicySatisfier; +import org.apache.hadoop.metrics2.util.MBeans; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.management.NotCompliantMBeanException; +import javax.management.ObjectName; +import javax.management.StandardMBean; +import java.util.HashSet; + +/** + * Expose the ExternalSPS metrics. + */ +public class ExternalSPSBeanMetrics implements ExternalSPSMXBean { + + private static final Logger LOG = + LoggerFactory.getLogger(ExternalSPSBeanMetrics.class); + + /** + * ExternalSPS bean. + */ + private ObjectName externalSPSBeanName; + private StoragePolicySatisfier storagePolicySatisfier; + + public ExternalSPSBeanMetrics(StoragePolicySatisfier sps) { + try { + this.storagePolicySatisfier = sps; + StandardMBean bean = new StandardMBean(this, ExternalSPSMXBean.class); + this.externalSPSBeanName = MBeans.register("ExternalSPS", "ExternalSPS", bean); + LOG.info("Registered ExternalSPS MBean: {}", this.externalSPSBeanName); + } catch (NotCompliantMBeanException e) { + throw new RuntimeException("Bad externalSPS MBean setup", e); + } + } + + /** + * Unregister the JMX interfaces. + */ + public void close() { + if (externalSPSBeanName != null) { + MBeans.unregister(externalSPSBeanName); + externalSPSBeanName = null; + } + } + + @Override + public int getProcessingQueueSize() { + return storagePolicySatisfier.processingQueueSize(); + } + + @VisibleForTesting + public void updateProcessingQueueSize() { + storagePolicySatisfier.getStorageMovementQueue() + .add(new ItemInfo(0, 1, 1)); + } + + @Override + public int getMovementFinishedBlocksCount() { + return storagePolicySatisfier.getAttemptedItemsMonitor().getMovementFinishedBlocksCount(); + } + + @VisibleForTesting + public void updateMovementFinishedBlocksCount() { + storagePolicySatisfier.getAttemptedItemsMonitor().getMovementFinishedBlocks() + .add(new Block(1)); + } + + @Override + public int getAttemptedItemsCount() { + return storagePolicySatisfier.getAttemptedItemsMonitor().getAttemptedItemsCount(); + } + + @VisibleForTesting + public void updateAttemptedItemsCount() { + storagePolicySatisfier.getAttemptedItemsMonitor().getStorageMovementAttemptedItems() + .add(new StoragePolicySatisfier.AttemptedItemInfo(0, 1, 1, new HashSet<>(), 1)); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/metrics/ExternalSPSMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/metrics/ExternalSPSMXBean.java new file mode 100644 index 0000000000000..12492fbbf0d9a --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/metrics/ExternalSPSMXBean.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.sps.metrics; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * This is the JMX management interface for ExternalSPS information. + * End users shouldn't be implementing these interfaces, and instead + * access this information through the JMX APIs. + */ +@InterfaceAudience.Private +@InterfaceStability.Stable +public interface ExternalSPSMXBean { + + /** + * Gets the queue size of StorageMovementNeeded. + * + * @return the queue size of StorageMovementNeeded. + */ + int getProcessingQueueSize(); + + /** + * Gets the count of movement finished blocks. + * + * @return the count of movement finished blocks. + */ + int getMovementFinishedBlocksCount(); + + /** + * Gets the count of attempted items. + * + * @return the count of attempted items. + */ + int getAttemptedItemsCount(); +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/metrics/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/metrics/package-info.java new file mode 100644 index 0000000000000..a6d26f69d37f9 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/metrics/package-info.java @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This package provides the ability to expose external SPS metrics to JMX. + */ +package org.apache.hadoop.hdfs.server.sps.metrics; \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java index dec8bef24b4df..d435bb6da3ebf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java @@ -36,7 +36,10 @@ import java.util.Map; import java.util.Optional; import java.util.TreeSet; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; @@ -433,7 +436,7 @@ static int run(DistributedFileSystem dfs, String[] argv, int idx) throws IOExcep */ private static final String commonUsageSummary = "\t[-report [-live] [-dead] [-decommissioning] " + - "[-enteringmaintenance] [-inmaintenance]]\n" + + "[-enteringmaintenance] [-inmaintenance] [-slownodes]]\n" + "\t[-safemode ]\n" + "\t[-saveNamespace [-beforeShutdown]]\n" + "\t[-rollEdits]\n" + @@ -451,8 +454,7 @@ static int run(DistributedFileSystem dfs, String[] argv, int idx) throws IOExcep "\t[-refreshSuperUserGroupsConfiguration]\n" + "\t[-refreshCallQueue]\n" + "\t[-refresh [arg1..argn]\n" + - "\t[-reconfig " + - "]\n" + + "\t[-reconfig ]\n" + "\t[-printTopology]\n" + "\t[-refreshNamenodes datanode_host:ipc_port]\n" + "\t[-getVolumeReport datanode_host:ipc_port]\n" + @@ -587,11 +589,13 @@ public void report(String[] argv, int i) throws IOException { StringUtils.popOption("-enteringmaintenance", args); final boolean listInMaintenance = StringUtils.popOption("-inmaintenance", args); + final boolean listSlowNodes = + StringUtils.popOption("-slownodes", args); // If no filter flags are found, then list all DN types boolean listAll = (!listLive && !listDead && !listDecommissioning - && !listEnteringMaintenance && !listInMaintenance); + && !listEnteringMaintenance && !listInMaintenance && !listSlowNodes); if (listAll || listLive) { printDataNodeReports(dfs, DatanodeReportType.LIVE, listLive, "Live"); @@ -615,6 +619,10 @@ public void report(String[] argv, int i) throws IOException { printDataNodeReports(dfs, DatanodeReportType.IN_MAINTENANCE, listInMaintenance, "In maintenance"); } + + if (listAll || listSlowNodes) { + printSlowDataNodeReports(dfs, listSlowNodes, "Slow"); + } } private static void printDataNodeReports(DistributedFileSystem dfs, @@ -632,6 +640,20 @@ private static void printDataNodeReports(DistributedFileSystem dfs, } } + private static void printSlowDataNodeReports(DistributedFileSystem dfs, boolean listNodes, + String nodeState) throws IOException { + DatanodeInfo[] nodes = dfs.getSlowDatanodeStats(); + if (nodes.length > 0 || listNodes) { + System.out.println(nodeState + " datanodes (" + nodes.length + "):\n"); + } + if (nodes.length > 0) { + for (DatanodeInfo dn : nodes) { + System.out.println(dn.getDatanodeReport()); + System.out.println(); + } + } + } + /** * Safe mode maintenance command. * Usage: hdfs dfsadmin -safemode [enter | leave | get | wait | forceExit] @@ -1148,7 +1170,7 @@ private void printHelp(String cmd) { commonUsageSummary; String report ="-report [-live] [-dead] [-decommissioning] " - + "[-enteringmaintenance] [-inmaintenance]:\n" + + + "[-enteringmaintenance] [-inmaintenance] [-slownodes]:\n" + "\tReports basic filesystem information and statistics. \n" + "\tThe dfs usage can be different from \"du\" usage, because it\n" + "\tmeasures raw space used by replication, checksums, snapshots\n" + @@ -1224,12 +1246,14 @@ private void printHelp(String cmd) { String refreshCallQueue = "-refreshCallQueue: Reload the call queue from config\n"; - String reconfig = "-reconfig " + + String reconfig = "-reconfig " + ":\n" + "\tStarts or gets the status of a reconfiguration operation, \n" + "\tor gets a list of reconfigurable properties.\n" + - - "\tThe second parameter specifies the node type\n"; + "\tThe second parameter specifies the node type\n" + + "\tThe third parameter specifies host address. For start or status, \n" + + "\tdatanode supports livenodes as third parameter, which will start \n" + + "\tor retrieve reconfiguration on all live datanodes."; String genericRefresh = "-refresh: Arguments are " + " [arg1..argn]\n" + "\tTriggers a runtime-refresh of the resource specified by " + @@ -1877,15 +1901,15 @@ public int refreshCallQueue() throws IOException { return 0; } - public int reconfig(String[] argv, int i) throws IOException { + public int reconfig(String[] argv, int i) throws IOException, InterruptedException { String nodeType = argv[i]; String address = argv[i + 1]; String op = argv[i + 2]; if ("start".equals(op)) { - return startReconfiguration(nodeType, address, System.out, System.err); + return startReconfigurationUtil(nodeType, address, System.out, System.err); } else if ("status".equals(op)) { - return getReconfigurationStatus(nodeType, address, System.out, System.err); + return getReconfigurationStatusUtil(nodeType, address, System.out, System.err); } else if ("properties".equals(op)) { return getReconfigurableProperties(nodeType, address, System.out, System.err); @@ -1895,12 +1919,57 @@ public int reconfig(String[] argv, int i) throws IOException { } int startReconfiguration(final String nodeThpe, final String address) - throws IOException { - return startReconfiguration(nodeThpe, address, System.out, System.err); + throws IOException, InterruptedException { + return startReconfigurationUtil(nodeThpe, address, System.out, System.err); + } + + int startReconfigurationUtil(final String nodeType, final String address, final PrintStream out, + final PrintStream err) throws IOException, InterruptedException { + if (!"livenodes".equals(address)) { + return startReconfiguration(nodeType, address, out, err); + } + if (!"datanode".equals(nodeType)) { + err.println("Only datanode type supports reconfiguration in bulk."); + return 1; + } + ExecutorService executorService = Executors.newFixedThreadPool(5); + DistributedFileSystem dfs = getDFS(); + DatanodeInfo[] nodes = dfs.getDataNodeStats(DatanodeReportType.LIVE); + AtomicInteger successCount = new AtomicInteger(); + AtomicInteger failCount = new AtomicInteger(); + if (nodes != null) { + for (DatanodeInfo node : nodes) { + executorService.submit(() -> { + int status = startReconfiguration(nodeType, node.getIpcAddr(false), out, err); + if (status == 0) { + successCount.incrementAndGet(); + } else { + failCount.incrementAndGet(); + } + }); + } + while ((successCount.get() + failCount.get()) < nodes.length) { + Thread.sleep(1000); + } + executorService.shutdown(); + if (!executorService.awaitTermination(1, TimeUnit.MINUTES)) { + err.println("Executor service could not be terminated in 60s. Please wait for" + + " sometime before the system cools down."); + } + out.println("Starting of reconfiguration task successful on " + successCount.get() + + " nodes, failed on " + failCount.get() + " nodes."); + if (failCount.get() == 0) { + return 0; + } else { + return 1; + } + } + err.println("DFS datanode stats could not be retrieved."); + return 1; } int startReconfiguration(final String nodeType, final String address, - final PrintStream out, final PrintStream err) throws IOException { + final PrintStream out, final PrintStream err) { String outMsg = null; String errMsg = null; int ret = 0; @@ -1941,8 +2010,53 @@ int startReconfigurationDispatch(final String nodeType, } } - int getReconfigurationStatus(final String nodeType, final String address, - final PrintStream out, final PrintStream err) throws IOException { + int getReconfigurationStatusUtil(final String nodeType, final String address, + final PrintStream out, final PrintStream err) throws IOException, InterruptedException { + if (!"livenodes".equals(address)) { + return getReconfigurationStatus(nodeType, address, out, err); + } + if (!"datanode".equals(nodeType)) { + err.println("Only datanode type supports reconfiguration in bulk."); + return 1; + } + ExecutorService executorService = Executors.newFixedThreadPool(5); + DistributedFileSystem dfs = getDFS(); + DatanodeInfo[] nodes = dfs.getDataNodeStats(DatanodeReportType.LIVE); + AtomicInteger successCount = new AtomicInteger(); + AtomicInteger failCount = new AtomicInteger(); + if (nodes != null) { + for (DatanodeInfo node : nodes) { + executorService.submit(() -> { + int status = getReconfigurationStatus(nodeType, node.getIpcAddr(false), out, err); + if (status == 0) { + successCount.incrementAndGet(); + } else { + failCount.incrementAndGet(); + } + }); + } + while ((successCount.get() + failCount.get()) < nodes.length) { + Thread.sleep(1000); + } + executorService.shutdown(); + if (!executorService.awaitTermination(1, TimeUnit.MINUTES)) { + err.println("Executor service could not be terminated in 60s. Please wait for" + + " sometime before the system cools down."); + } + out.println("Retrieval of reconfiguration status successful on " + successCount.get() + + " nodes, failed on " + failCount.get() + " nodes."); + if (failCount.get() == 0) { + return 0; + } else { + return 1; + } + } + err.println("DFS datanode stats could not be retrieved."); + return 1; + } + + int getReconfigurationStatus(final String nodeType, final String address, final PrintStream out, + final PrintStream err) { String outMsg = null; String errMsg = null; ReconfigurationTaskStatus status = null; @@ -2126,7 +2240,7 @@ private static void printUsage(String cmd) { if ("-report".equals(cmd)) { System.err.println("Usage: hdfs dfsadmin" + " [-report] [-live] [-dead] [-decommissioning]" - + " [-enteringmaintenance] [-inmaintenance]"); + + " [-enteringmaintenance] [-inmaintenance] [-slownodes]"); } else if ("-safemode".equals(cmd)) { System.err.println("Usage: hdfs dfsadmin" + " [-safemode enter | leave | get | wait | forceExit]"); @@ -2188,7 +2302,7 @@ private static void printUsage(String cmd) { + " [-refreshCallQueue]"); } else if ("-reconfig".equals(cmd)) { System.err.println("Usage: hdfs dfsadmin" - + " [-reconfig " + + " [-reconfig " + "]"); } else if ("-refresh".equals(cmd)) { System.err.println("Usage: hdfs dfsadmin" diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsXmlLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsXmlLoader.java index 7238c58cb579b..fc5f30e883001 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsXmlLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsXmlLoader.java @@ -86,6 +86,10 @@ public OfflineEditsXmlLoader(OfflineEditsVisitor visitor, public void loadEdits() throws IOException { try { XMLReader xr = XMLReaderFactory.createXMLReader(); + xr.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); + xr.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false); + xr.setFeature("http://xml.org/sax/features/external-general-entities", false); + xr.setFeature("http://xml.org/sax/features/external-parameter-entities", false); xr.setContentHandler(this); xr.setErrorHandler(this); xr.setDTDHandler(null); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java index 1aedcaf65cf18..78a7301db0469 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java @@ -1761,6 +1761,10 @@ private void processXml() throws Exception { XMLEvent ev = expectTag("[section header]", true); if (ev.getEventType() == XMLStreamConstants.END_ELEMENT) { if (ev.asEndElement().getName().getLocalPart().equals("fsimage")) { + if(unprocessedSections.size() == 1 && unprocessedSections.contains + (SnapshotDiffSectionProcessor.NAME)){ + break; + } throw new IOException("FSImage XML ended prematurely, without " + "including section(s) " + StringUtils.join(", ", unprocessedSections)); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java index dbcb452e166aa..05e687ab97e43 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java @@ -107,6 +107,7 @@ public class OfflineImageViewerPB { + " Delimited outputs. If not set, the processor\n" + " constructs the namespace in memory \n" + " before outputting text.\n" + + "-m,--multiThread Use multiThread to process sub-sections.\n" + "-h,--help Display usage information and exit\n"; /** @@ -132,6 +133,7 @@ private static Options buildOptions() { options.addOption("delimiter", true, ""); options.addOption("sp", false, ""); options.addOption("t", "temp", true, ""); + options.addOption("m", "multiThread", true, ""); return options; } @@ -185,6 +187,7 @@ public static int run(String[] args) throws Exception { String delimiter = cmd.getOptionValue("delimiter", PBImageTextWriter.DEFAULT_DELIMITER); String tempPath = cmd.getOptionValue("t", ""); + int threads = Integer.parseInt(cmd.getOptionValue("m", "1")); Configuration conf = new Configuration(); PrintStream out = null; @@ -227,15 +230,14 @@ public static int run(String[] args) throws Exception { boolean printStoragePolicy = cmd.hasOption("sp"); try (PBImageDelimitedTextWriter writer = new PBImageDelimitedTextWriter(out, delimiter, - tempPath, printStoragePolicy); - RandomAccessFile r = new RandomAccessFile(inputFile, "r")) { - writer.visit(r); + tempPath, printStoragePolicy, threads, outputFile)) { + writer.visit(inputFile); } break; case "DETECTCORRUPTION": try (PBImageCorruptionDetector detector = new PBImageCorruptionDetector(out, delimiter, tempPath)) { - detector.visit(new RandomAccessFile(inputFile, "r")); + detector.visit(inputFile); } break; default: diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageCorruptionDetector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageCorruptionDetector.java index 28c450701b846..17593867bd642 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageCorruptionDetector.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageCorruptionDetector.java @@ -337,7 +337,7 @@ public void afterOutput() throws IOException { if (parentId != -1) { entryBuilder.setParentId(parentId); } - printIfNotEmpty(entryBuilder.build()); + printIfNotEmpty(serialOutStream(), entryBuilder.build()); } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageDelimitedTextWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageDelimitedTextWriter.java index 45d42f0396b1a..3e080ec8e65cd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageDelimitedTextWriter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageDelimitedTextWriter.java @@ -146,7 +146,13 @@ public String build() { PBImageDelimitedTextWriter(PrintStream out, String delimiter, String tempPath, boolean printStoragePolicy) throws IOException { - super(out, delimiter, tempPath); + this(out, delimiter, tempPath, printStoragePolicy, 1, "-"); + } + + PBImageDelimitedTextWriter(PrintStream out, String delimiter, + String tempPath, boolean printStoragePolicy, int threads, + String parallelOut) throws IOException { + super(out, delimiter, tempPath, threads, parallelOut); this.printStoragePolicy = printStoragePolicy; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java index 08fe7fb943c15..2dab44a036ac6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java @@ -21,17 +21,25 @@ import java.io.Closeable; import java.io.File; import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.PrintStream; import java.io.RandomAccessFile; import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import java.util.HashMap; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -455,20 +463,22 @@ public String getParentPath(long inode) throws IOException { return "/"; } long parent = getFromDirChildMap(inode); - if (!dirPathCache.containsKey(parent)) { - byte[] bytes = dirMap.get(toBytes(parent)); - if (parent != INodeId.ROOT_INODE_ID && bytes == null) { - // The parent is an INodeReference, which is generated from snapshot. - // For delimited oiv tool, no need to print out metadata in snapshots. - throw PBImageTextWriter.createIgnoredSnapshotException(inode); + byte[] bytes = dirMap.get(toBytes(parent)); + synchronized (this) { + if (!dirPathCache.containsKey(parent)) { + if (parent != INodeId.ROOT_INODE_ID && bytes == null) { + // The parent is an INodeReference, which is generated from snapshot. + // For delimited oiv tool, no need to print out metadata in snapshots. + throw PBImageTextWriter.createIgnoredSnapshotException(inode); + } + String parentName = toString(bytes); + String parentPath = + new Path(getParentPath(parent), + parentName.isEmpty() ? "/" : parentName).toString(); + dirPathCache.put(parent, parentPath); } - String parentName = toString(bytes); - String parentPath = - new Path(getParentPath(parent), - parentName.isEmpty() ? "/" : parentName).toString(); - dirPathCache.put(parent, parentPath); + return dirPathCache.get(parent); } - return dirPathCache.get(parent); } @Override @@ -493,9 +503,12 @@ public long getParentId(long id) throws IOException { } private SerialNumberManager.StringTable stringTable; - private PrintStream out; + private final PrintStream out; private MetadataMap metadataMap = null; private String delimiter; + private File filename; + private int numThreads; + private String parallelOutputFile; /** * Construct a PB FsImage writer to generate text file. @@ -503,8 +516,8 @@ public long getParentId(long id) throws IOException { * @param tempPath the path to store metadata. If it is empty, store metadata * in memory instead. */ - PBImageTextWriter(PrintStream out, String delimiter, String tempPath) - throws IOException { + PBImageTextWriter(PrintStream out, String delimiter, String tempPath, + int numThreads, String parallelOutputFile) throws IOException { this.out = out; this.delimiter = delimiter; if (tempPath.isEmpty()) { @@ -512,6 +525,17 @@ public long getParentId(long id) throws IOException { } else { metadataMap = new LevelDBMetadataMap(tempPath); } + this.numThreads = numThreads; + this.parallelOutputFile = parallelOutputFile; + } + + PBImageTextWriter(PrintStream out, String delimiter, String tempPath) + throws IOException { + this(out, delimiter, tempPath, 1, "-"); + } + + protected PrintStream serialOutStream() { + return out; } @Override @@ -562,7 +586,9 @@ void append(StringBuffer buffer, String field) { */ abstract protected void afterOutput() throws IOException; - public void visit(RandomAccessFile file) throws IOException { + public void visit(String filePath) throws IOException { + filename = new File(filePath); + RandomAccessFile file = new RandomAccessFile(filePath, "r"); Configuration conf = new Configuration(); if (!FSImageUtil.checkFileFormat(file)) { throw new IOException("Unrecognized FSImage"); @@ -642,21 +668,122 @@ long getParentId(long id) throws IOException { private void output(Configuration conf, FileSummary summary, FileInputStream fin, ArrayList sections) throws IOException { + ArrayList allINodeSubSections = + getINodeSubSections(sections); + if (numThreads > 1 && !parallelOutputFile.equals("-") && + allINodeSubSections.size() > 1) { + outputInParallel(conf, summary, allINodeSubSections); + } else { + LOG.info("Serial output due to threads num: {}, parallel output file: {}, " + + "subSections: {}.", numThreads, parallelOutputFile, allINodeSubSections.size()); + outputInSerial(conf, summary, fin, sections); + } + } + + private void outputInSerial(Configuration conf, FileSummary summary, + FileInputStream fin, ArrayList sections) + throws IOException { InputStream is; long startTime = Time.monotonicNow(); - out.println(getHeader()); + serialOutStream().println(getHeader()); for (FileSummary.Section section : sections) { if (SectionName.fromString(section.getName()) == SectionName.INODE) { fin.getChannel().position(section.getOffset()); is = FSImageUtil.wrapInputStreamForCompression(conf, summary.getCodec(), new BufferedInputStream(new LimitInputStream( fin, section.getLength()))); - outputINodes(is); + INodeSection s = INodeSection.parseDelimitedFrom(is); + LOG.info("Found {} INodes in the INode section", s.getNumInodes()); + int count = outputINodes(is, serialOutStream()); + LOG.info("Outputted {} INodes.", count); } } afterOutput(); long timeTaken = Time.monotonicNow() - startTime; - LOG.debug("Time to output inodes: {}ms", timeTaken); + LOG.debug("Time to output inodes: {} ms", timeTaken); + } + + /** + * STEP1: Multi-threaded process sub-sections. + * Given n (n>1) threads to process k (k>=n) sections, + * output parsed results of each section to tmp file in order. + * STEP2: Merge tmp files. + */ + private void outputInParallel(Configuration conf, FileSummary summary, + ArrayList subSections) + throws IOException { + int nThreads = Integer.min(numThreads, subSections.size()); + LOG.info("Outputting in parallel with {} sub-sections using {} threads", + subSections.size(), nThreads); + final CopyOnWriteArrayList exceptions = new CopyOnWriteArrayList<>(); + CountDownLatch latch = new CountDownLatch(subSections.size()); + ExecutorService executorService = Executors.newFixedThreadPool(nThreads); + AtomicLong expectedINodes = new AtomicLong(0); + AtomicLong totalParsed = new AtomicLong(0); + String codec = summary.getCodec(); + String[] paths = new String[subSections.size()]; + + for (int i = 0; i < subSections.size(); i++) { + paths[i] = parallelOutputFile + ".tmp." + i; + int index = i; + executorService.submit(() -> { + LOG.info("Output iNodes of section-{}", index); + InputStream is = null; + try (PrintStream outStream = new PrintStream(paths[index], "UTF-8")) { + long startTime = Time.monotonicNow(); + is = getInputStreamForSection(subSections.get(index), codec, conf); + if (index == 0) { + // The first iNode section has a header which must be processed first + INodeSection s = INodeSection.parseDelimitedFrom(is); + expectedINodes.set(s.getNumInodes()); + } + totalParsed.addAndGet(outputINodes(is, outStream)); + long timeTaken = Time.monotonicNow() - startTime; + LOG.info("Time to output iNodes of section-{}: {} ms", index, timeTaken); + } catch (Exception e) { + exceptions.add(new IOException(e)); + } finally { + latch.countDown(); + try { + if (is != null) { + is.close(); + } + } catch (IOException ioe) { + LOG.warn("Failed to close the input stream, ignoring", ioe); + } + } + }); + } + + try { + latch.await(); + } catch (InterruptedException e) { + LOG.error("Interrupted waiting for countdown latch", e); + throw new IOException(e); + } + + executorService.shutdown(); + if (exceptions.size() != 0) { + LOG.error("Failed to output INode sub-sections, {} exception(s) occurred.", + exceptions.size()); + throw exceptions.get(0); + } + if (totalParsed.get() != expectedINodes.get()) { + throw new IOException("Expected to parse " + expectedINodes + " in parallel, " + + "but parsed " + totalParsed.get() + ". The image may be corrupt."); + } + LOG.info("Completed outputting all INode sub-sections to {} tmp files.", + subSections.size()); + + try (PrintStream ps = new PrintStream(parallelOutputFile, "UTF-8")) { + ps.println(getHeader()); + } + + // merge tmp files + long startTime = Time.monotonicNow(); + mergeFiles(paths, parallelOutputFile); + long timeTaken = Time.monotonicNow() - startTime; + LOG.info("Completed all stages. Time to merge files: {} ms", timeTaken); } protected PermissionStatus getPermission(long perm) { @@ -763,22 +890,27 @@ protected void buildNamespace(InputStream in, List refIdList) LOG.info("Scanned {} INode directories to build namespace.", count); } - void printIfNotEmpty(String line) { + void printIfNotEmpty(PrintStream outStream, String line) { if (!line.isEmpty()) { - out.println(line); + outStream.println(line); } } - private void outputINodes(InputStream in) throws IOException { - INodeSection s = INodeSection.parseDelimitedFrom(in); - LOG.info("Found {} INodes in the INode section", s.getNumInodes()); + private int outputINodes(InputStream in, PrintStream outStream) + throws IOException { long ignored = 0; long ignoredSnapshots = 0; - for (int i = 0; i < s.getNumInodes(); ++i) { + // As the input stream is a LimitInputStream, the reading will stop when + // EOF is encountered at the end of the stream. + int count = 0; + while (true) { INode p = INode.parseDelimitedFrom(in); + if (p == null) { + break; + } try { String parentPath = metadataMap.getParentPath(p.getId()); - printIfNotEmpty(getEntry(parentPath, p)); + printIfNotEmpty(outStream, getEntry(parentPath, p)); } catch (IOException ioe) { ignored++; if (!(ioe instanceof IgnoreSnapshotException)) { @@ -790,16 +922,16 @@ private void outputINodes(InputStream in) throws IOException { } } } - - if (LOG.isDebugEnabled() && i % 100000 == 0) { - LOG.debug("Outputted {} INodes.", i); + count++; + if (LOG.isDebugEnabled() && count % 100000 == 0) { + LOG.debug("Outputted {} INodes.", count); } } if (ignored > 0) { LOG.warn("Ignored {} nodes, including {} in snapshots. Please turn on" + " debug log for details", ignored, ignoredSnapshots); } - LOG.info("Outputted {} INodes.", s.getNumInodes()); + return count; } private static IgnoreSnapshotException createIgnoredSnapshotException( @@ -822,4 +954,79 @@ public int getStoragePolicy( } return HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED; } + + private ArrayList getINodeSubSections( + ArrayList sections) { + ArrayList subSections = new ArrayList<>(); + Iterator iter = sections.iterator(); + while (iter.hasNext()) { + FileSummary.Section s = iter.next(); + if (SectionName.fromString(s.getName()) == SectionName.INODE_SUB) { + subSections.add(s); + } + } + return subSections; + } + + /** + * Given a FSImage FileSummary.section, return a LimitInput stream set to + * the starting position of the section and limited to the section length. + * @param section The FileSummary.Section containing the offset and length + * @param compressionCodec The compression codec in use, if any + * @return An InputStream for the given section + * @throws IOException + */ + private InputStream getInputStreamForSection(FileSummary.Section section, + String compressionCodec, Configuration conf) + throws IOException { + // channel of RandomAccessFile is not thread safe, use File + FileInputStream fin = new FileInputStream(filename); + try { + FileChannel channel = fin.getChannel(); + channel.position(section.getOffset()); + InputStream in = new BufferedInputStream(new LimitInputStream(fin, + section.getLength())); + + in = FSImageUtil.wrapInputStreamForCompression(conf, + compressionCodec, in); + return in; + } catch (IOException e) { + fin.close(); + throw e; + } + } + + /** + * @param srcPaths Source files of contents to be merged + * @param resultPath Merged file path + * @throws IOException + */ + public static void mergeFiles(String[] srcPaths, String resultPath) + throws IOException { + if (srcPaths == null || srcPaths.length < 1) { + LOG.warn("no source files to merge."); + return; + } + + File[] files = new File[srcPaths.length]; + for (int i = 0; i < srcPaths.length; i++) { + files[i] = new File(srcPaths[i]); + } + + File resultFile = new File(resultPath); + try (FileChannel resultChannel = + new FileOutputStream(resultFile, true).getChannel()) { + for (File file : files) { + try (FileChannel src = new FileInputStream(file).getChannel()) { + resultChannel.transferFrom(src, resultChannel.size(), src.size()); + } + } + } + + for (File file : files) { + if (!file.delete() && file.exists()) { + LOG.warn("delete tmp file: {} returned false", file); + } + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/RwLock.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/RwLock.java index deaeaa43247fd..05c1a06abda23 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/RwLock.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/RwLock.java @@ -28,6 +28,12 @@ public interface RwLock { /** Release read lock. */ public void readUnlock(); + /** + * Release read lock with operation name. + * @param opName Option name. + */ + public void readUnlock(String opName); + /** Check if the current thread holds read lock. */ public boolean hasReadLock(); @@ -40,6 +46,12 @@ public interface RwLock { /** Release write lock. */ public void writeUnlock(); + /** + * Release write lock with operation name. + * @param opName Option name. + */ + public void writeUnlock(String opName); + /** Check if the current thread holds write lock. */ public boolean hasWriteLock(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeLifelineProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeLifelineProtocol.proto index e10a8861e6153..9e436ea9b4d6a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeLifelineProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeLifelineProtocol.proto @@ -18,7 +18,7 @@ /** * These .proto interfaces are private and stable. - * Please see http://wiki.apache.org/hadoop/Compatibility + * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html * for what changes are allowed for a *stable* .proto interface. */ syntax = "proto2"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto index 4a98f2d01e9e9..48a3855c0381e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto @@ -18,7 +18,7 @@ /** * These .proto interfaces are private and stable. - * Please see http://wiki.apache.org/hadoop/Compatibility + * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html * for what changes are allowed for a *stable* .proto interface. */ @@ -223,6 +223,7 @@ message HeartbeatResponseProto { optional RollingUpgradeStatusProto rollingUpgradeStatus = 3; optional RollingUpgradeStatusProto rollingUpgradeStatusV2 = 4; optional uint64 fullBlockReportLeaseId = 5 [ default = 0 ]; + optional bool isSlownode = 6 [ default = false ]; } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HAZKInfo.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HAZKInfo.proto index 6d45a935ee4fd..ecff0e8bd8743 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HAZKInfo.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HAZKInfo.proto @@ -18,7 +18,7 @@ /** * These .proto interfaces are private and stable. - * Please see http://wiki.apache.org/hadoop/Compatibility + * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html * for what changes are allowed for a *stable* .proto interface. */ syntax = "proto2"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HdfsServer.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HdfsServer.proto index 78607efddab30..e1488258692de 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HdfsServer.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HdfsServer.proto @@ -18,7 +18,7 @@ /** * These .proto interfaces are private and stable. - * Please see http://wiki.apache.org/hadoop/Compatibility + * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html * for what changes are allowed for a *stable* .proto interface. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/InterDatanodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/InterDatanodeProtocol.proto index 47332a8817bdd..742fd82643df0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/InterDatanodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/InterDatanodeProtocol.proto @@ -18,7 +18,7 @@ /** * These .proto interfaces are private and stable. - * Please see http://wiki.apache.org/hadoop/Compatibility + * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html * for what changes are allowed for a *stable* .proto interface. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/InterQJournalProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/InterQJournalProtocol.proto index e73ca23e92f94..1c78423b40990 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/InterQJournalProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/InterQJournalProtocol.proto @@ -18,7 +18,7 @@ /** * These .proto interfaces are private and stable. - * Please see http://wiki.apache.org/hadoop/Compatibility + * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html * for what changes are allowed for a *stable* .proto interface. */ syntax = "proto2"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/JournalProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/JournalProtocol.proto index 35c401e33e5e6..bfbfc6fd3367b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/JournalProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/JournalProtocol.proto @@ -18,7 +18,7 @@ /** * These .proto interfaces are private and stable. - * Please see http://wiki.apache.org/hadoop/Compatibility + * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html * for what changes are allowed for a *stable* .proto interface. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/NamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/NamenodeProtocol.proto index 88d9fbc2e04d0..32cdade055eee 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/NamenodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/NamenodeProtocol.proto @@ -18,7 +18,7 @@ /** * These .proto interfaces are private and stable. - * Please see http://wiki.apache.org/hadoop/Compatibility + * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html * for what changes are allowed for a *stable* .proto interface. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/QJournalProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/QJournalProtocol.proto index e366d1fb8d74b..b0a5a19f8ec7a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/QJournalProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/QJournalProtocol.proto @@ -18,7 +18,7 @@ /** * These .proto interfaces are private and stable. - * Please see http://wiki.apache.org/hadoop/Compatibility + * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html * for what changes are allowed for a *stable* .proto interface. */ syntax = "proto2"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 7bcbccd81728c..bffde8cf7482a 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -565,6 +565,17 @@ + + dfs.namenode.ip-proxy-users + + A comma separated list of user names that are allowed by the + NameNode to specify a different client IP address in the caller context. + This is used by Router-Based Federation (RBF) to provide the actual client's + IP address to the NameNode, which is critical to preserve data locality when + using RBF. If you are using RBF, add the user that runs the routers. + + + dfs.namenode.acls.enabled true @@ -1640,7 +1651,7 @@ dfs.block.scanner.volume.bytes.per.second 1048576 - If this is 0, the DataNode's block scanner will be disabled. If this + If this is configured less than or equal to zero, the DataNode's block scanner will be disabled. If this is positive, this is the number of bytes per second that the DataNode's block scanner will try to scan from each volume. @@ -2483,6 +2494,15 @@ + + dfs.datanode.max.slowdisks.to.exclude + 0 + + The number of slow disks that needs to be excluded. By default, this parameter is set to 0, + which disables excluding slow disk when choosing volume. + + + hadoop.user.group.metrics.percentiles.intervals @@ -2982,6 +3002,16 @@ + + dfs.datanode.fsdatasetasyncdisk.max.threads.per.volume + 4 + + The maximum number of threads per volume used to process async disk + operations on the datanode. These threads consume I/O and CPU at the + same time. This will affect normal data node operations. + + + dfs.cachereport.intervalMsec 10000 @@ -3348,6 +3378,25 @@ + + dfs.client.refresh.read-block-locations.register-automatically + true + + Whether to auto-register all DFSInputStreams for background LocatedBlock refreshes. + If false, user must manually register using DFSClient#addLocatedBlocksRefresh(DFSInputStream) + + + + + dfs.client.refresh.read-block-locations.threads + 5 + + Number of threads to use for refreshing LocatedBlocks of registered + DFSInputStreams. If a DFSClient opens many DFSInputStreams, increasing + this may help refresh them all in a timely manner. + + + dfs.namenode.lease-recheck-interval-ms 2000 @@ -4081,6 +4130,14 @@ Mover, and StoragePolicySatisfier. + + dfs.namenode.get-blocks.check.operation + true + + Set false to disable checkOperation and getBlocks for Balancer + will route to Standby NameNode for HA mode setup. + + dfs.balancer.dispatcherThreads 200 @@ -4667,6 +4724,24 @@ + + dfs.datanode.ec.reconstruct.read.bandwidthPerSec + 0 + + Specifies the maximum amount of bandwidth that the EC reconstruction can utilize for reading. + When the bandwidth value is zero, there is no limit. + + + + + dfs.datanode.ec.reconstruct.write.bandwidthPerSec + 0 + + Specifies the maximum amount of bandwidth that the EC reconstruction can utilize for writing. + When the bandwidth value is zero, there is no limit. + + + dfs.datanode.fsdataset.factory @@ -5450,6 +5525,14 @@ + + dfs.storage.policy.satisfier.move.task.retry.max.attempts + 3 + + Max retries for moving task to satisfy the block storage policy. + + + dfs.storage.policy.satisfier.datanode.cache.refresh.interval.ms 300000 @@ -5511,6 +5594,14 @@ + + dfs.pipeline.slownode + false + + If true, allows slownode information to be replied to Client via PipelineAck. + + + dfs.qjournal.accept-recovery.timeout.ms 120000 @@ -6054,12 +6145,21 @@ - dfs.namenode.block.deletion.increment - 1000 + dfs.namenode.block.deletion.lock.threshold.ms + 50 + + The limit of single time lock holding duration for the block asynchronous + deletion thread. + + + + + dfs.namenode.block.deletion.unlock.interval.ms + 10 - The number of block deletion increment. - This setting will control the block increment deletion rate to - ensure that other waiters on the lock can get in. + The sleep interval for yield lock. + When the single time lock holding duration of the block asynchronous deletion + thread exceeds limit, sleeping to yield lock. @@ -6300,4 +6400,24 @@ Effective with dfs.nameservices.resolution-enabled on. + + + dfs.client.mark.slownode.as.badnode.threshold + 10 + + The threshold to mark a slownode as a badnode. If we get PipelineAck from + a slownode continuously for ${dfs.client.treat.slownode.as.badnode.threshold} + times, we should mark it as a badnode. + + + + + dfs.datanode.lockmanager.trace + false + + If this is true, after shut down datanode lock Manager will print all leak + thread that not release by lock Manager. Only used for test or trace dead lock + problem. In produce default set false, because it's have little performance loss. + + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html index 7301064651e2b..caab81ef686b9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html @@ -71,6 +71,7 @@

DataNode on {HostName}:{DataPort}

+
Cluster ID:{ClusterId}
Started:{DNStartedTimeInMillis|date_tostring}
Version:{Version}
{/dn} @@ -125,7 +126,7 @@ {/dn.VolumeInfo} - + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html index 72952edd4d71d..8c577001b2a68 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html @@ -479,7 +479,7 @@ - - - + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/status.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/status.html index 41a468d4cfd80..a3484fbcb6a88 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/status.html +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/status.html @@ -86,7 +86,7 @@ {/snn} -