From e7d36e75172ddb9b81d65712d934a592e6c58e7f Mon Sep 17 00:00:00 2001 From: Jiajun Mao Date: Wed, 31 Jul 2024 03:40:14 +0000 Subject: [PATCH] three datanode --- dockers/build-restart.sh | 4 +- dockers/core-site.xml | 2 +- dockers/dn1/core-site.xml | 6 +++ dockers/dn1/hadoop-id-rsa | 38 +++++++++++++++ dockers/dn1/hadoop-id-rsa.pub | 1 + dockers/dn1/hadoop-ssh-config | 2 + dockers/dn1/hdfs-site.xml | 34 ++++++++++++++ dockers/dn2/core-site.xml | 6 +++ dockers/dn2/hadoop-id-rsa | 38 +++++++++++++++ dockers/dn2/hadoop-id-rsa.pub | 1 + dockers/dn2/hadoop-ssh-config | 2 + dockers/dn2/hdfs-site.xml | 46 +++++++++++++++++++ dockers/dn3/core-site.xml | 6 +++ dockers/dn3/hadoop-id-rsa | 38 +++++++++++++++ dockers/dn3/hadoop-id-rsa.pub | 1 + dockers/dn3/hadoop-ssh-config | 2 + dockers/dn3/hdfs-site.xml | 46 +++++++++++++++++++ dockers/docker-compose.yml | 2 +- dockers/hdfs-site.xml | 2 +- .../server/blockmanagement/BlockManager.java | 2 - .../blockmanagement/ZfsBlockManagement.java | 2 + .../hadoop/hdfs/server/datanode/DataNode.java | 13 ++++++ 22 files changed, 288 insertions(+), 6 deletions(-) create mode 100644 dockers/dn1/core-site.xml create mode 100644 dockers/dn1/hadoop-id-rsa create mode 100644 dockers/dn1/hadoop-id-rsa.pub create mode 100644 dockers/dn1/hadoop-ssh-config create mode 100644 dockers/dn1/hdfs-site.xml create mode 100644 dockers/dn2/core-site.xml create mode 100644 dockers/dn2/hadoop-id-rsa create mode 100644 dockers/dn2/hadoop-id-rsa.pub create mode 100644 dockers/dn2/hadoop-ssh-config create mode 100644 dockers/dn2/hdfs-site.xml create mode 100644 dockers/dn3/core-site.xml create mode 100644 dockers/dn3/hadoop-id-rsa create mode 100644 dockers/dn3/hadoop-id-rsa.pub create mode 100644 dockers/dn3/hadoop-ssh-config create mode 100644 dockers/dn3/hdfs-site.xml diff --git a/dockers/build-restart.sh b/dockers/build-restart.sh index 28127fb8c0218..66ffa95f3a36f 100755 --- a/dockers/build-restart.sh +++ b/dockers/build-restart.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e + read -p "Do you want to build HDFS source code? (y/n) " yn case $yn in y ) @@ -13,7 +15,7 @@ esac cd dockers && docker build --no-cache -t hdfs-runtime:latest -f Dockerfile .. # Restart docker-compose -docker compose stop && docker-compose rm -f +docker compose stop && docker compose rm -f # Start the docker-compose docker compose up -d \ No newline at end of file diff --git a/dockers/core-site.xml b/dockers/core-site.xml index 77044e3c334ac..e88f92aa6fe88 100644 --- a/dockers/core-site.xml +++ b/dockers/core-site.xml @@ -1,6 +1,6 @@ fs.defaultFS - hdfs://namenode:9000 + hdfs://localhost:9000 \ No newline at end of file diff --git a/dockers/dn1/core-site.xml b/dockers/dn1/core-site.xml new file mode 100644 index 0000000000000..e88f92aa6fe88 --- /dev/null +++ b/dockers/dn1/core-site.xml @@ -0,0 +1,6 @@ + + + fs.defaultFS + hdfs://localhost:9000 + + \ No newline at end of file diff --git a/dockers/dn1/hadoop-id-rsa b/dockers/dn1/hadoop-id-rsa new file mode 100644 index 0000000000000..5d59146c37a31 --- /dev/null +++ b/dockers/dn1/hadoop-id-rsa @@ -0,0 +1,38 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn +NhAAAAAwEAAQAAAYEA21jLtr+TCNuyrjcpPURaQCM6pcpNJKRkmS3YcZRpdibS55A3KtUr +crSc2NBjXqoMm+N7CUT2T25qLH91eSQnUjTBi2muPXXBf7NfhRTjSXszfi1OxdDNYyy31v ++TkF8DHQBBahdhCCkqbbnUIddTt4p85yuEIxH3BKnMdfw0KbBdvMB4q3pRxbvWfKuWGmar +Sj7dVWDlIR8LePPIYLtpkKlHA5LNOtygXCt7Z63TijHUG7c8ALgJ8RIKWP8dky3fu10yeo +6rK1d8ZAiQpPXtgn4eCPIrLXyp73bVuXLC3Eux2uyVRZYDGB98s/xpcHvzS70XhTzeXznW +rsy1LnDBith5lzTJowQXS5kjkvx5ABA/wbP61DpuFQdcqPeH1bj0ZTsr1iC9DfUvhiSVzC +8YsJGbLxJTw/pJCl7vDwOGtzjwYMAbTXc3nZtQRPknnIjW3sRSNT719kELg/RaVWNWAXCS +/0NRB4DwPxIWJJhfjVpvQgKOzFozUbjQ2ouC6aZ5AAAFkP4rzGb+K8xmAAAAB3NzaC1yc2 +EAAAGBANtYy7a/kwjbsq43KT1EWkAjOqXKTSSkZJkt2HGUaXYm0ueQNyrVK3K0nNjQY16q +DJvjewlE9k9uaix/dXkkJ1I0wYtprj11wX+zX4UU40l7M34tTsXQzWMst9b/k5BfAx0AQW +oXYQgpKm251CHXU7eKfOcrhCMR9wSpzHX8NCmwXbzAeKt6UcW71nyrlhpmq0o+3VVg5SEf +C3jzyGC7aZCpRwOSzTrcoFwre2et04ox1Bu3PAC4CfESClj/HZMt37tdMnqOqytXfGQIkK +T17YJ+HgjyKy18qe921blywtxLsdrslUWWAxgffLP8aXB780u9F4U83l851q7MtS5wwYrY +eZc0yaMEF0uZI5L8eQAQP8Gz+tQ6bhUHXKj3h9W49GU7K9YgvQ31L4YklcwvGLCRmy8SU8 +P6SQpe7w8Dhrc48GDAG013N52bUET5J5yI1t7EUjU+9fZBC4P0WlVjVgFwkv9DUQeA8D8S +FiSYX41ab0ICjsxaM1G40NqLgummeQAAAAMBAAEAAAGALZvCCAa/kPj1p0nq9kVPKsTofe +6ZqcPQ8xTYS42bfCjIc8PF5M8cUKaDcoWqE9Xx9zwOqje/q3Z+9y+tgnnk/y/fCpj31Om2 +SVJHy0kzlWTbqSHMnk8zVarHsxnDWiKXaCFUeLXRxDujCdu44yx9yM0v+XVP7Wbf4QVZcT +L2ubBlXUj8DlPJyv6qs5OOC4FLno2Bwi36lq942DaAdLSjV6QLExAkQlf7tA6Kg11eff6s +ZvLrtA6QaqLgqe6p/J9t4lxlfkmLWCnWW74Sl+yKrpnC5HwMoaInvgt0jEza2tJ5ZE0Lyw +ydptt1DNs7/Af96IpULjkXyh1WgOxWJ9DoPf2rwOIf0IqfiLD0uYGMdqmsvyYHKOM/4wmM +MvUi0dxsTUjSExTgYYapRM7kbn93+DxJ02h49nCva/gCmHjajOW4P5z63gQvDle41fHBOY +CxLWbx06ERM0kziJqKtEcnCMksgSzMS4OUH/m46EClFM1HTk/J24bAmq5kS0EZHGoxAAAA +wQCYYcOIfYb+fs9Wz0vuu0YK/b5d7bowPK84SI9VW0P/ImL9f62oJa4FzdGaprSCev0eSH +0asrWHtab/kNL3X5aEW0BNDE52g2bzp5xjeC8009nj0c+D8VDh40zOy1PnVpfJmRcienPL +3KuIH4fRJ1bdToeU1yZF+Kv8kwogGew7y58Al/ouqhqUk0A9JknMAQQyBsewf001jGvFqL +pdlVPBjo9MD2XUnZRASrzRMzJyeCk7bzoi2Urft42fCbfclkwAAADBAPGH8mX1iBuzU0uE +tsJWcgH3MJmJNbW4RPvVVVuewPyghrr2pfszGtMnV+Z/9QumWU6MB5+RNB4dpY+mr9AqGW +j5LboazP7PIqtReD4ADCGTE9ubyR/OY2bhsp2k88mn0p0Hst/nSV3RhBvbZgdWXFCEzi0v +tGVjBcAYh2b1DyR6WyAITCd+pFuezk8Wwc5Mve731L4T/h685JzOSRKKFhdZ60BV2WV0hG +e/L2/aXF58oyrDQjQ0YS/X/BGFInKshQAAAMEA6HyjbZGvjM99ldlEC9sxRmTldxg7c9hI +KtvSURCSO7I6YiyGl9WVhqTY61A3UJogHNxcNxqPWTqwaYuFgZyaSe0XGf1v/sReRiJoic +n8HDL6GY3iVaNHCPhsK/Ktw/rF9d47jSw3u84G6gKtZ2fgzqiTOgNNueYJP0bmyupSazHC +OIXdyGoi+zkIlsIx5Pe4tjxIXWUm/RmUv+vv4aHv564RHHo0d2MqAPT9s2q1P9em8HiABm +4lNzgwCWER5B5lAAAAFGFhcm9ubWFvQEppYWp1bi00MDkwAQIDBAUG +-----END OPENSSH PRIVATE KEY----- diff --git a/dockers/dn1/hadoop-id-rsa.pub b/dockers/dn1/hadoop-id-rsa.pub new file mode 100644 index 0000000000000..8c96cd3893dca --- /dev/null +++ b/dockers/dn1/hadoop-id-rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDbWMu2v5MI27KuNyk9RFpAIzqlyk0kpGSZLdhxlGl2JtLnkDcq1StytJzY0GNeqgyb43sJRPZPbmosf3V5JCdSNMGLaa49dcF/s1+FFONJezN+LU7F0M1jLLfW/5OQXwMdAEFqF2EIKSptudQh11O3inznK4QjEfcEqcx1/DQpsF28wHirelHFu9Z8q5YaZqtKPt1VYOUhHwt488hgu2mQqUcDks063KBcK3tnrdOKMdQbtzwAuAnxEgpY/x2TLd+7XTJ6jqsrV3xkCJCk9e2Cfh4I8istfKnvdtW5csLcS7Ha7JVFlgMYH3yz/Glwe/NLvReFPN5fOdauzLUucMGK2HmXNMmjBBdLmSOS/HkAED/Bs/rUOm4VB1yo94fVuPRlOyvWIL0N9S+GJJXMLxiwkZsvElPD+kkKXu8PA4a3OPBgwBtNdzedm1BE+SeciNbexFI1PvX2QQuD9FpVY1YBcJL/Q1EHgPA/EhYkmF+NWm9CAo7MWjNRuNDai4Lppnk= aaronmao@Jiajun-4090 diff --git a/dockers/dn1/hadoop-ssh-config b/dockers/dn1/hadoop-ssh-config new file mode 100644 index 0000000000000..64779c28879f7 --- /dev/null +++ b/dockers/dn1/hadoop-ssh-config @@ -0,0 +1,2 @@ +Host * + StrictHostKeyChecking no \ No newline at end of file diff --git a/dockers/dn1/hdfs-site.xml b/dockers/dn1/hdfs-site.xml new file mode 100644 index 0000000000000..0d9049bd4ec81 --- /dev/null +++ b/dockers/dn1/hdfs-site.xml @@ -0,0 +1,34 @@ + + + dfs.namenode.name.dir + /data/nameNode + + + dfs.datanode.data.dir + [ZFS]/data/dataNode1,[ZFS_RECON_BUFFER]/data/recon_buffer1 + + + dfs.namenode.heartbeat.recheck-interval + 5000 + + + dfs.namenode.ec.system.default.policy + XOR-2-1-1024k + + + net.topology.script.file.name + /opt/rack-topo.sh + + + dfs.blocksize + 1M + + + dfs.datanode.disk.check.min.gap + 1s + + + dfs.datanode.failed.volumes.tolerated + 1 + + \ No newline at end of file diff --git a/dockers/dn2/core-site.xml b/dockers/dn2/core-site.xml new file mode 100644 index 0000000000000..e88f92aa6fe88 --- /dev/null +++ b/dockers/dn2/core-site.xml @@ -0,0 +1,6 @@ + + + fs.defaultFS + hdfs://localhost:9000 + + \ No newline at end of file diff --git a/dockers/dn2/hadoop-id-rsa b/dockers/dn2/hadoop-id-rsa new file mode 100644 index 0000000000000..5d59146c37a31 --- /dev/null +++ b/dockers/dn2/hadoop-id-rsa @@ -0,0 +1,38 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn +NhAAAAAwEAAQAAAYEA21jLtr+TCNuyrjcpPURaQCM6pcpNJKRkmS3YcZRpdibS55A3KtUr +crSc2NBjXqoMm+N7CUT2T25qLH91eSQnUjTBi2muPXXBf7NfhRTjSXszfi1OxdDNYyy31v ++TkF8DHQBBahdhCCkqbbnUIddTt4p85yuEIxH3BKnMdfw0KbBdvMB4q3pRxbvWfKuWGmar +Sj7dVWDlIR8LePPIYLtpkKlHA5LNOtygXCt7Z63TijHUG7c8ALgJ8RIKWP8dky3fu10yeo +6rK1d8ZAiQpPXtgn4eCPIrLXyp73bVuXLC3Eux2uyVRZYDGB98s/xpcHvzS70XhTzeXznW +rsy1LnDBith5lzTJowQXS5kjkvx5ABA/wbP61DpuFQdcqPeH1bj0ZTsr1iC9DfUvhiSVzC +8YsJGbLxJTw/pJCl7vDwOGtzjwYMAbTXc3nZtQRPknnIjW3sRSNT719kELg/RaVWNWAXCS +/0NRB4DwPxIWJJhfjVpvQgKOzFozUbjQ2ouC6aZ5AAAFkP4rzGb+K8xmAAAAB3NzaC1yc2 +EAAAGBANtYy7a/kwjbsq43KT1EWkAjOqXKTSSkZJkt2HGUaXYm0ueQNyrVK3K0nNjQY16q +DJvjewlE9k9uaix/dXkkJ1I0wYtprj11wX+zX4UU40l7M34tTsXQzWMst9b/k5BfAx0AQW +oXYQgpKm251CHXU7eKfOcrhCMR9wSpzHX8NCmwXbzAeKt6UcW71nyrlhpmq0o+3VVg5SEf +C3jzyGC7aZCpRwOSzTrcoFwre2et04ox1Bu3PAC4CfESClj/HZMt37tdMnqOqytXfGQIkK +T17YJ+HgjyKy18qe921blywtxLsdrslUWWAxgffLP8aXB780u9F4U83l851q7MtS5wwYrY +eZc0yaMEF0uZI5L8eQAQP8Gz+tQ6bhUHXKj3h9W49GU7K9YgvQ31L4YklcwvGLCRmy8SU8 +P6SQpe7w8Dhrc48GDAG013N52bUET5J5yI1t7EUjU+9fZBC4P0WlVjVgFwkv9DUQeA8D8S +FiSYX41ab0ICjsxaM1G40NqLgummeQAAAAMBAAEAAAGALZvCCAa/kPj1p0nq9kVPKsTofe +6ZqcPQ8xTYS42bfCjIc8PF5M8cUKaDcoWqE9Xx9zwOqje/q3Z+9y+tgnnk/y/fCpj31Om2 +SVJHy0kzlWTbqSHMnk8zVarHsxnDWiKXaCFUeLXRxDujCdu44yx9yM0v+XVP7Wbf4QVZcT +L2ubBlXUj8DlPJyv6qs5OOC4FLno2Bwi36lq942DaAdLSjV6QLExAkQlf7tA6Kg11eff6s +ZvLrtA6QaqLgqe6p/J9t4lxlfkmLWCnWW74Sl+yKrpnC5HwMoaInvgt0jEza2tJ5ZE0Lyw +ydptt1DNs7/Af96IpULjkXyh1WgOxWJ9DoPf2rwOIf0IqfiLD0uYGMdqmsvyYHKOM/4wmM +MvUi0dxsTUjSExTgYYapRM7kbn93+DxJ02h49nCva/gCmHjajOW4P5z63gQvDle41fHBOY +CxLWbx06ERM0kziJqKtEcnCMksgSzMS4OUH/m46EClFM1HTk/J24bAmq5kS0EZHGoxAAAA +wQCYYcOIfYb+fs9Wz0vuu0YK/b5d7bowPK84SI9VW0P/ImL9f62oJa4FzdGaprSCev0eSH +0asrWHtab/kNL3X5aEW0BNDE52g2bzp5xjeC8009nj0c+D8VDh40zOy1PnVpfJmRcienPL +3KuIH4fRJ1bdToeU1yZF+Kv8kwogGew7y58Al/ouqhqUk0A9JknMAQQyBsewf001jGvFqL +pdlVPBjo9MD2XUnZRASrzRMzJyeCk7bzoi2Urft42fCbfclkwAAADBAPGH8mX1iBuzU0uE +tsJWcgH3MJmJNbW4RPvVVVuewPyghrr2pfszGtMnV+Z/9QumWU6MB5+RNB4dpY+mr9AqGW +j5LboazP7PIqtReD4ADCGTE9ubyR/OY2bhsp2k88mn0p0Hst/nSV3RhBvbZgdWXFCEzi0v +tGVjBcAYh2b1DyR6WyAITCd+pFuezk8Wwc5Mve731L4T/h685JzOSRKKFhdZ60BV2WV0hG +e/L2/aXF58oyrDQjQ0YS/X/BGFInKshQAAAMEA6HyjbZGvjM99ldlEC9sxRmTldxg7c9hI +KtvSURCSO7I6YiyGl9WVhqTY61A3UJogHNxcNxqPWTqwaYuFgZyaSe0XGf1v/sReRiJoic +n8HDL6GY3iVaNHCPhsK/Ktw/rF9d47jSw3u84G6gKtZ2fgzqiTOgNNueYJP0bmyupSazHC +OIXdyGoi+zkIlsIx5Pe4tjxIXWUm/RmUv+vv4aHv564RHHo0d2MqAPT9s2q1P9em8HiABm +4lNzgwCWER5B5lAAAAFGFhcm9ubWFvQEppYWp1bi00MDkwAQIDBAUG +-----END OPENSSH PRIVATE KEY----- diff --git a/dockers/dn2/hadoop-id-rsa.pub b/dockers/dn2/hadoop-id-rsa.pub new file mode 100644 index 0000000000000..8c96cd3893dca --- /dev/null +++ b/dockers/dn2/hadoop-id-rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDbWMu2v5MI27KuNyk9RFpAIzqlyk0kpGSZLdhxlGl2JtLnkDcq1StytJzY0GNeqgyb43sJRPZPbmosf3V5JCdSNMGLaa49dcF/s1+FFONJezN+LU7F0M1jLLfW/5OQXwMdAEFqF2EIKSptudQh11O3inznK4QjEfcEqcx1/DQpsF28wHirelHFu9Z8q5YaZqtKPt1VYOUhHwt488hgu2mQqUcDks063KBcK3tnrdOKMdQbtzwAuAnxEgpY/x2TLd+7XTJ6jqsrV3xkCJCk9e2Cfh4I8istfKnvdtW5csLcS7Ha7JVFlgMYH3yz/Glwe/NLvReFPN5fOdauzLUucMGK2HmXNMmjBBdLmSOS/HkAED/Bs/rUOm4VB1yo94fVuPRlOyvWIL0N9S+GJJXMLxiwkZsvElPD+kkKXu8PA4a3OPBgwBtNdzedm1BE+SeciNbexFI1PvX2QQuD9FpVY1YBcJL/Q1EHgPA/EhYkmF+NWm9CAo7MWjNRuNDai4Lppnk= aaronmao@Jiajun-4090 diff --git a/dockers/dn2/hadoop-ssh-config b/dockers/dn2/hadoop-ssh-config new file mode 100644 index 0000000000000..64779c28879f7 --- /dev/null +++ b/dockers/dn2/hadoop-ssh-config @@ -0,0 +1,2 @@ +Host * + StrictHostKeyChecking no \ No newline at end of file diff --git a/dockers/dn2/hdfs-site.xml b/dockers/dn2/hdfs-site.xml new file mode 100644 index 0000000000000..ad40d17ee1df7 --- /dev/null +++ b/dockers/dn2/hdfs-site.xml @@ -0,0 +1,46 @@ + + + dfs.datanode.address + 0.0.0.0:9766 + + + dfs.datanode.http.address + 0.0.0.0:9764 + + + dfs.datanode.ipc.address + 0.0.0.0:9767 + + + dfs.namenode.name.dir + /data/nameNode + + + dfs.datanode.data.dir + [ZFS]/data/dataNode2,[ZFS_RECON_BUFFER]/data/recon_buffer2 + + + dfs.namenode.heartbeat.recheck-interval + 5000 + + + dfs.namenode.ec.system.default.policy + XOR-2-1-1024k + + + net.topology.script.file.name + /opt/rack-topo.sh + + + dfs.blocksize + 1M + + + dfs.datanode.disk.check.min.gap + 1s + + + dfs.datanode.failed.volumes.tolerated + 1 + + \ No newline at end of file diff --git a/dockers/dn3/core-site.xml b/dockers/dn3/core-site.xml new file mode 100644 index 0000000000000..e88f92aa6fe88 --- /dev/null +++ b/dockers/dn3/core-site.xml @@ -0,0 +1,6 @@ + + + fs.defaultFS + hdfs://localhost:9000 + + \ No newline at end of file diff --git a/dockers/dn3/hadoop-id-rsa b/dockers/dn3/hadoop-id-rsa new file mode 100644 index 0000000000000..5d59146c37a31 --- /dev/null +++ b/dockers/dn3/hadoop-id-rsa @@ -0,0 +1,38 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn +NhAAAAAwEAAQAAAYEA21jLtr+TCNuyrjcpPURaQCM6pcpNJKRkmS3YcZRpdibS55A3KtUr +crSc2NBjXqoMm+N7CUT2T25qLH91eSQnUjTBi2muPXXBf7NfhRTjSXszfi1OxdDNYyy31v ++TkF8DHQBBahdhCCkqbbnUIddTt4p85yuEIxH3BKnMdfw0KbBdvMB4q3pRxbvWfKuWGmar +Sj7dVWDlIR8LePPIYLtpkKlHA5LNOtygXCt7Z63TijHUG7c8ALgJ8RIKWP8dky3fu10yeo +6rK1d8ZAiQpPXtgn4eCPIrLXyp73bVuXLC3Eux2uyVRZYDGB98s/xpcHvzS70XhTzeXznW +rsy1LnDBith5lzTJowQXS5kjkvx5ABA/wbP61DpuFQdcqPeH1bj0ZTsr1iC9DfUvhiSVzC +8YsJGbLxJTw/pJCl7vDwOGtzjwYMAbTXc3nZtQRPknnIjW3sRSNT719kELg/RaVWNWAXCS +/0NRB4DwPxIWJJhfjVpvQgKOzFozUbjQ2ouC6aZ5AAAFkP4rzGb+K8xmAAAAB3NzaC1yc2 +EAAAGBANtYy7a/kwjbsq43KT1EWkAjOqXKTSSkZJkt2HGUaXYm0ueQNyrVK3K0nNjQY16q +DJvjewlE9k9uaix/dXkkJ1I0wYtprj11wX+zX4UU40l7M34tTsXQzWMst9b/k5BfAx0AQW +oXYQgpKm251CHXU7eKfOcrhCMR9wSpzHX8NCmwXbzAeKt6UcW71nyrlhpmq0o+3VVg5SEf +C3jzyGC7aZCpRwOSzTrcoFwre2et04ox1Bu3PAC4CfESClj/HZMt37tdMnqOqytXfGQIkK +T17YJ+HgjyKy18qe921blywtxLsdrslUWWAxgffLP8aXB780u9F4U83l851q7MtS5wwYrY +eZc0yaMEF0uZI5L8eQAQP8Gz+tQ6bhUHXKj3h9W49GU7K9YgvQ31L4YklcwvGLCRmy8SU8 +P6SQpe7w8Dhrc48GDAG013N52bUET5J5yI1t7EUjU+9fZBC4P0WlVjVgFwkv9DUQeA8D8S +FiSYX41ab0ICjsxaM1G40NqLgummeQAAAAMBAAEAAAGALZvCCAa/kPj1p0nq9kVPKsTofe +6ZqcPQ8xTYS42bfCjIc8PF5M8cUKaDcoWqE9Xx9zwOqje/q3Z+9y+tgnnk/y/fCpj31Om2 +SVJHy0kzlWTbqSHMnk8zVarHsxnDWiKXaCFUeLXRxDujCdu44yx9yM0v+XVP7Wbf4QVZcT +L2ubBlXUj8DlPJyv6qs5OOC4FLno2Bwi36lq942DaAdLSjV6QLExAkQlf7tA6Kg11eff6s +ZvLrtA6QaqLgqe6p/J9t4lxlfkmLWCnWW74Sl+yKrpnC5HwMoaInvgt0jEza2tJ5ZE0Lyw +ydptt1DNs7/Af96IpULjkXyh1WgOxWJ9DoPf2rwOIf0IqfiLD0uYGMdqmsvyYHKOM/4wmM +MvUi0dxsTUjSExTgYYapRM7kbn93+DxJ02h49nCva/gCmHjajOW4P5z63gQvDle41fHBOY +CxLWbx06ERM0kziJqKtEcnCMksgSzMS4OUH/m46EClFM1HTk/J24bAmq5kS0EZHGoxAAAA +wQCYYcOIfYb+fs9Wz0vuu0YK/b5d7bowPK84SI9VW0P/ImL9f62oJa4FzdGaprSCev0eSH +0asrWHtab/kNL3X5aEW0BNDE52g2bzp5xjeC8009nj0c+D8VDh40zOy1PnVpfJmRcienPL +3KuIH4fRJ1bdToeU1yZF+Kv8kwogGew7y58Al/ouqhqUk0A9JknMAQQyBsewf001jGvFqL +pdlVPBjo9MD2XUnZRASrzRMzJyeCk7bzoi2Urft42fCbfclkwAAADBAPGH8mX1iBuzU0uE +tsJWcgH3MJmJNbW4RPvVVVuewPyghrr2pfszGtMnV+Z/9QumWU6MB5+RNB4dpY+mr9AqGW +j5LboazP7PIqtReD4ADCGTE9ubyR/OY2bhsp2k88mn0p0Hst/nSV3RhBvbZgdWXFCEzi0v +tGVjBcAYh2b1DyR6WyAITCd+pFuezk8Wwc5Mve731L4T/h685JzOSRKKFhdZ60BV2WV0hG +e/L2/aXF58oyrDQjQ0YS/X/BGFInKshQAAAMEA6HyjbZGvjM99ldlEC9sxRmTldxg7c9hI +KtvSURCSO7I6YiyGl9WVhqTY61A3UJogHNxcNxqPWTqwaYuFgZyaSe0XGf1v/sReRiJoic +n8HDL6GY3iVaNHCPhsK/Ktw/rF9d47jSw3u84G6gKtZ2fgzqiTOgNNueYJP0bmyupSazHC +OIXdyGoi+zkIlsIx5Pe4tjxIXWUm/RmUv+vv4aHv564RHHo0d2MqAPT9s2q1P9em8HiABm +4lNzgwCWER5B5lAAAAFGFhcm9ubWFvQEppYWp1bi00MDkwAQIDBAUG +-----END OPENSSH PRIVATE KEY----- diff --git a/dockers/dn3/hadoop-id-rsa.pub b/dockers/dn3/hadoop-id-rsa.pub new file mode 100644 index 0000000000000..8c96cd3893dca --- /dev/null +++ b/dockers/dn3/hadoop-id-rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDbWMu2v5MI27KuNyk9RFpAIzqlyk0kpGSZLdhxlGl2JtLnkDcq1StytJzY0GNeqgyb43sJRPZPbmosf3V5JCdSNMGLaa49dcF/s1+FFONJezN+LU7F0M1jLLfW/5OQXwMdAEFqF2EIKSptudQh11O3inznK4QjEfcEqcx1/DQpsF28wHirelHFu9Z8q5YaZqtKPt1VYOUhHwt488hgu2mQqUcDks063KBcK3tnrdOKMdQbtzwAuAnxEgpY/x2TLd+7XTJ6jqsrV3xkCJCk9e2Cfh4I8istfKnvdtW5csLcS7Ha7JVFlgMYH3yz/Glwe/NLvReFPN5fOdauzLUucMGK2HmXNMmjBBdLmSOS/HkAED/Bs/rUOm4VB1yo94fVuPRlOyvWIL0N9S+GJJXMLxiwkZsvElPD+kkKXu8PA4a3OPBgwBtNdzedm1BE+SeciNbexFI1PvX2QQuD9FpVY1YBcJL/Q1EHgPA/EhYkmF+NWm9CAo7MWjNRuNDai4Lppnk= aaronmao@Jiajun-4090 diff --git a/dockers/dn3/hadoop-ssh-config b/dockers/dn3/hadoop-ssh-config new file mode 100644 index 0000000000000..64779c28879f7 --- /dev/null +++ b/dockers/dn3/hadoop-ssh-config @@ -0,0 +1,2 @@ +Host * + StrictHostKeyChecking no \ No newline at end of file diff --git a/dockers/dn3/hdfs-site.xml b/dockers/dn3/hdfs-site.xml new file mode 100644 index 0000000000000..1101856cf837e --- /dev/null +++ b/dockers/dn3/hdfs-site.xml @@ -0,0 +1,46 @@ + + + dfs.datanode.address + 0.0.0.0:9666 + + + dfs.datanode.http.address + 0.0.0.0:9664 + + + dfs.datanode.ipc.address + 0.0.0.0:9667 + + + dfs.namenode.name.dir + /data/nameNode + + + dfs.datanode.data.dir + [ZFS]/data/dataNode3,[ZFS_RECON_BUFFER]/data/recon_buffer3 + + + dfs.namenode.heartbeat.recheck-interval + 5000 + + + dfs.namenode.ec.system.default.policy + XOR-2-1-1024k + + + net.topology.script.file.name + /opt/rack-topo.sh + + + dfs.blocksize + 1M + + + dfs.datanode.disk.check.min.gap + 1s + + + dfs.datanode.failed.volumes.tolerated + 1 + + \ No newline at end of file diff --git a/dockers/docker-compose.yml b/dockers/docker-compose.yml index aae67c6b9e86d..bc72a0fe7a49f 100644 --- a/dockers/docker-compose.yml +++ b/dockers/docker-compose.yml @@ -37,7 +37,7 @@ services: networks: - intra deploy: - replicas: 4 + replicas: 3 networks: intra: \ No newline at end of file diff --git a/dockers/hdfs-site.xml b/dockers/hdfs-site.xml index adaddcfd99f5b..5056bab1177c5 100644 --- a/dockers/hdfs-site.xml +++ b/dockers/hdfs-site.xml @@ -5,7 +5,7 @@ dfs.datanode.data.dir - [ZFS]/data/dataNode1,[ZFS]/data/dataNode2,[ZFS_RECON_BUFFER]/data/recon_buffer + [ZFS]/data/dataNode1,[ZFS_RECON_BUFFER]/data/recon_buffer dfs.namenode.heartbeat.recheck-interval diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 432cb81231cec..5fdda9a0e4761 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -4727,8 +4727,6 @@ private void processIncrementalBlockReport(final DatanodeDescriptor node, for (ReceivedDeletedBlockInfo rdbi : srdb.getBlocks()) { switch (rdbi.getStatus()) { case DELETED_BLOCK: - LOG.info("Deleting block"); - // Get the datanode that we are deleting the block from LOG.info("Deleting block from data node {}", node.getHostName()); LOG.info("Datanode has blocks"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ZfsBlockManagement.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ZfsBlockManagement.java index abda9fd920471..857180cac40e1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ZfsBlockManagement.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ZfsBlockManagement.java @@ -1,5 +1,7 @@ package org.apache.hadoop.hdfs.server.blockmanagement; +import jni.DnodeAttributes; +import jni.Tools; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 35f8e2526a2f7..d74d622fc45ff 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -100,6 +100,8 @@ import static org.apache.hadoop.util.Preconditions.checkNotNull; import static org.apache.hadoop.util.Time.now; +import jni.DnodeAttributes; +import jni.Tools; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.DF; import org.apache.hadoop.fs.DU; @@ -2725,12 +2727,23 @@ private void handleDiskError(String failedVolumes, int failedNumber) { : DatanodeProtocol.FATAL_DISK_ERROR; metrics.incrVolumeFailures(failedNumber); + // Get MLEC failure info + Tools zfsBinding = new Tools(); + List dnodes = zfsBinding.getFailedChunks("pool"); + LOG.warn("Printing ZFS datanode info"); + for (DnodeAttributes dn : dnodes) { + LOG.info("This datanode contains following zfs dn {}", dn); + } + LOG.warn("============"); + //inform NameNodes for(BPOfferService bpos: blockPoolManager.getAllNamenodeThreads()) { + LOG.info("Try sending error report"); bpos.trySendErrorReport(dpError, failedVolumes); } if(hasEnoughResources) { + LOG.info("Scheduling all block report"); scheduleAllBlockReport(0); return; // do not shutdown }