three datanode

jiajunmao · Jul 31, 2024 · e7d36e7 · e7d36e7
1 parent 40c9d35
commit e7d36e7
Show file tree

Hide file tree

Showing 22 changed files with 288 additions and 6 deletions.
diff --git a/dockers/build-restart.sh b/dockers/build-restart.sh
@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -e
+
 read -p "Do you want to build HDFS source code? (y/n) " yn
 
 case $yn in y )
@@ -13,7 +15,7 @@ esac
 cd dockers && docker build --no-cache -t hdfs-runtime:latest -f Dockerfile ..
 
 # Restart docker-compose
-docker compose stop && docker-compose rm -f
+docker compose stop && docker compose rm -f
 
 # Start the docker-compose
 docker compose up -d
diff --git a/dockers/core-site.xml b/dockers/core-site.xml
@@ -1,6 +1,6 @@
 <configuration>
     <property>
         <name>fs.defaultFS</name>
-        <value>hdfs://namenode:9000</value>
+        <value>hdfs://localhost:9000</value>
     </property>
 </configuration>
diff --git a/dockers/dn1/core-site.xml b/dockers/dn1/core-site.xml
@@ -0,0 +1,6 @@
+<configuration>
+    <property>
+        <name>fs.defaultFS</name>
+        <value>hdfs://localhost:9000</value>
+    </property>
+</configuration>
diff --git a/dockers/dn1/hadoop-id-rsa b/dockers/dn1/hadoop-id-rsa
@@ -0,0 +1,38 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn
+NhAAAAAwEAAQAAAYEA21jLtr+TCNuyrjcpPURaQCM6pcpNJKRkmS3YcZRpdibS55A3KtUr
+crSc2NBjXqoMm+N7CUT2T25qLH91eSQnUjTBi2muPXXBf7NfhRTjSXszfi1OxdDNYyy31v
++TkF8DHQBBahdhCCkqbbnUIddTt4p85yuEIxH3BKnMdfw0KbBdvMB4q3pRxbvWfKuWGmar
+Sj7dVWDlIR8LePPIYLtpkKlHA5LNOtygXCt7Z63TijHUG7c8ALgJ8RIKWP8dky3fu10yeo
+6rK1d8ZAiQpPXtgn4eCPIrLXyp73bVuXLC3Eux2uyVRZYDGB98s/xpcHvzS70XhTzeXznW
+rsy1LnDBith5lzTJowQXS5kjkvx5ABA/wbP61DpuFQdcqPeH1bj0ZTsr1iC9DfUvhiSVzC
+8YsJGbLxJTw/pJCl7vDwOGtzjwYMAbTXc3nZtQRPknnIjW3sRSNT719kELg/RaVWNWAXCS
+/0NRB4DwPxIWJJhfjVpvQgKOzFozUbjQ2ouC6aZ5AAAFkP4rzGb+K8xmAAAAB3NzaC1yc2
+EAAAGBANtYy7a/kwjbsq43KT1EWkAjOqXKTSSkZJkt2HGUaXYm0ueQNyrVK3K0nNjQY16q
+DJvjewlE9k9uaix/dXkkJ1I0wYtprj11wX+zX4UU40l7M34tTsXQzWMst9b/k5BfAx0AQW
+oXYQgpKm251CHXU7eKfOcrhCMR9wSpzHX8NCmwXbzAeKt6UcW71nyrlhpmq0o+3VVg5SEf
+C3jzyGC7aZCpRwOSzTrcoFwre2et04ox1Bu3PAC4CfESClj/HZMt37tdMnqOqytXfGQIkK
+T17YJ+HgjyKy18qe921blywtxLsdrslUWWAxgffLP8aXB780u9F4U83l851q7MtS5wwYrY
+eZc0yaMEF0uZI5L8eQAQP8Gz+tQ6bhUHXKj3h9W49GU7K9YgvQ31L4YklcwvGLCRmy8SU8
+P6SQpe7w8Dhrc48GDAG013N52bUET5J5yI1t7EUjU+9fZBC4P0WlVjVgFwkv9DUQeA8D8S
+FiSYX41ab0ICjsxaM1G40NqLgummeQAAAAMBAAEAAAGALZvCCAa/kPj1p0nq9kVPKsTofe
+6ZqcPQ8xTYS42bfCjIc8PF5M8cUKaDcoWqE9Xx9zwOqje/q3Z+9y+tgnnk/y/fCpj31Om2
+SVJHy0kzlWTbqSHMnk8zVarHsxnDWiKXaCFUeLXRxDujCdu44yx9yM0v+XVP7Wbf4QVZcT
+L2ubBlXUj8DlPJyv6qs5OOC4FLno2Bwi36lq942DaAdLSjV6QLExAkQlf7tA6Kg11eff6s
+ZvLrtA6QaqLgqe6p/J9t4lxlfkmLWCnWW74Sl+yKrpnC5HwMoaInvgt0jEza2tJ5ZE0Lyw
+ydptt1DNs7/Af96IpULjkXyh1WgOxWJ9DoPf2rwOIf0IqfiLD0uYGMdqmsvyYHKOM/4wmM
+MvUi0dxsTUjSExTgYYapRM7kbn93+DxJ02h49nCva/gCmHjajOW4P5z63gQvDle41fHBOY
+CxLWbx06ERM0kziJqKtEcnCMksgSzMS4OUH/m46EClFM1HTk/J24bAmq5kS0EZHGoxAAAA
+wQCYYcOIfYb+fs9Wz0vuu0YK/b5d7bowPK84SI9VW0P/ImL9f62oJa4FzdGaprSCev0eSH
+0asrWHtab/kNL3X5aEW0BNDE52g2bzp5xjeC8009nj0c+D8VDh40zOy1PnVpfJmRcienPL
+3KuIH4fRJ1bdToeU1yZF+Kv8kwogGew7y58Al/ouqhqUk0A9JknMAQQyBsewf001jGvFqL
+pdlVPBjo9MD2XUnZRASrzRMzJyeCk7bzoi2Urft42fCbfclkwAAADBAPGH8mX1iBuzU0uE
+tsJWcgH3MJmJNbW4RPvVVVuewPyghrr2pfszGtMnV+Z/9QumWU6MB5+RNB4dpY+mr9AqGW
+j5LboazP7PIqtReD4ADCGTE9ubyR/OY2bhsp2k88mn0p0Hst/nSV3RhBvbZgdWXFCEzi0v
+tGVjBcAYh2b1DyR6WyAITCd+pFuezk8Wwc5Mve731L4T/h685JzOSRKKFhdZ60BV2WV0hG
+e/L2/aXF58oyrDQjQ0YS/X/BGFInKshQAAAMEA6HyjbZGvjM99ldlEC9sxRmTldxg7c9hI
+KtvSURCSO7I6YiyGl9WVhqTY61A3UJogHNxcNxqPWTqwaYuFgZyaSe0XGf1v/sReRiJoic
+n8HDL6GY3iVaNHCPhsK/Ktw/rF9d47jSw3u84G6gKtZ2fgzqiTOgNNueYJP0bmyupSazHC
+OIXdyGoi+zkIlsIx5Pe4tjxIXWUm/RmUv+vv4aHv564RHHo0d2MqAPT9s2q1P9em8HiABm
+4lNzgwCWER5B5lAAAAFGFhcm9ubWFvQEppYWp1bi00MDkwAQIDBAUG
+-----END OPENSSH PRIVATE KEY-----
diff --git a/dockers/dn1/hadoop-id-rsa.pub b/dockers/dn1/hadoop-id-rsa.pub
@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDbWMu2v5MI27KuNyk9RFpAIzqlyk0kpGSZLdhxlGl2JtLnkDcq1StytJzY0GNeqgyb43sJRPZPbmosf3V5JCdSNMGLaa49dcF/s1+FFONJezN+LU7F0M1jLLfW/5OQXwMdAEFqF2EIKSptudQh11O3inznK4QjEfcEqcx1/DQpsF28wHirelHFu9Z8q5YaZqtKPt1VYOUhHwt488hgu2mQqUcDks063KBcK3tnrdOKMdQbtzwAuAnxEgpY/x2TLd+7XTJ6jqsrV3xkCJCk9e2Cfh4I8istfKnvdtW5csLcS7Ha7JVFlgMYH3yz/Glwe/NLvReFPN5fOdauzLUucMGK2HmXNMmjBBdLmSOS/HkAED/Bs/rUOm4VB1yo94fVuPRlOyvWIL0N9S+GJJXMLxiwkZsvElPD+kkKXu8PA4a3OPBgwBtNdzedm1BE+SeciNbexFI1PvX2QQuD9FpVY1YBcJL/Q1EHgPA/EhYkmF+NWm9CAo7MWjNRuNDai4Lppnk= aaronmao@Jiajun-4090
diff --git a/dockers/dn1/hadoop-ssh-config b/dockers/dn1/hadoop-ssh-config
@@ -0,0 +1,2 @@
+Host *
+    StrictHostKeyChecking no
diff --git a/dockers/dn1/hdfs-site.xml b/dockers/dn1/hdfs-site.xml
@@ -0,0 +1,34 @@
+<configuration>
+    <property>
+        <name>dfs.namenode.name.dir</name>
+        <value>/data/nameNode</value>
+    </property>
+    <property>
+        <name>dfs.datanode.data.dir</name>
+        <value>[ZFS]/data/dataNode1,[ZFS_RECON_BUFFER]/data/recon_buffer1</value>
+    </property>
+    <property>
+        <name>dfs.namenode.heartbeat.recheck-interval</name>
+        <value>5000</value>
+    </property>
+    <property>
+        <name>dfs.namenode.ec.system.default.policy</name>
+        <value>XOR-2-1-1024k</value>
+    </property>
+    <property>
+        <name>net.topology.script.file.name</name>
+        <value>/opt/rack-topo.sh</value>
+    </property>
+    <property>
+        <name>dfs.blocksize</name>
+        <value>1M</value>
+    </property>
+    <property>
+        <name>dfs.datanode.disk.check.min.gap</name>
+        <value>1s</value>
+    </property>
+    <property>
+        <name>dfs.datanode.failed.volumes.tolerated</name>
+        <value>1</value>
+    </property>
+</configuration>
diff --git a/dockers/dn2/core-site.xml b/dockers/dn2/core-site.xml
@@ -0,0 +1,6 @@
+<configuration>
+    <property>
+        <name>fs.defaultFS</name>
+        <value>hdfs://localhost:9000</value>
+    </property>
+</configuration>
diff --git a/dockers/dn2/hadoop-id-rsa b/dockers/dn2/hadoop-id-rsa
@@ -0,0 +1,38 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn
+NhAAAAAwEAAQAAAYEA21jLtr+TCNuyrjcpPURaQCM6pcpNJKRkmS3YcZRpdibS55A3KtUr
+crSc2NBjXqoMm+N7CUT2T25qLH91eSQnUjTBi2muPXXBf7NfhRTjSXszfi1OxdDNYyy31v
++TkF8DHQBBahdhCCkqbbnUIddTt4p85yuEIxH3BKnMdfw0KbBdvMB4q3pRxbvWfKuWGmar
+Sj7dVWDlIR8LePPIYLtpkKlHA5LNOtygXCt7Z63TijHUG7c8ALgJ8RIKWP8dky3fu10yeo
+6rK1d8ZAiQpPXtgn4eCPIrLXyp73bVuXLC3Eux2uyVRZYDGB98s/xpcHvzS70XhTzeXznW
+rsy1LnDBith5lzTJowQXS5kjkvx5ABA/wbP61DpuFQdcqPeH1bj0ZTsr1iC9DfUvhiSVzC
+8YsJGbLxJTw/pJCl7vDwOGtzjwYMAbTXc3nZtQRPknnIjW3sRSNT719kELg/RaVWNWAXCS
+/0NRB4DwPxIWJJhfjVpvQgKOzFozUbjQ2ouC6aZ5AAAFkP4rzGb+K8xmAAAAB3NzaC1yc2
+EAAAGBANtYy7a/kwjbsq43KT1EWkAjOqXKTSSkZJkt2HGUaXYm0ueQNyrVK3K0nNjQY16q
+DJvjewlE9k9uaix/dXkkJ1I0wYtprj11wX+zX4UU40l7M34tTsXQzWMst9b/k5BfAx0AQW
+oXYQgpKm251CHXU7eKfOcrhCMR9wSpzHX8NCmwXbzAeKt6UcW71nyrlhpmq0o+3VVg5SEf
+C3jzyGC7aZCpRwOSzTrcoFwre2et04ox1Bu3PAC4CfESClj/HZMt37tdMnqOqytXfGQIkK
+T17YJ+HgjyKy18qe921blywtxLsdrslUWWAxgffLP8aXB780u9F4U83l851q7MtS5wwYrY
+eZc0yaMEF0uZI5L8eQAQP8Gz+tQ6bhUHXKj3h9W49GU7K9YgvQ31L4YklcwvGLCRmy8SU8
+P6SQpe7w8Dhrc48GDAG013N52bUET5J5yI1t7EUjU+9fZBC4P0WlVjVgFwkv9DUQeA8D8S
+FiSYX41ab0ICjsxaM1G40NqLgummeQAAAAMBAAEAAAGALZvCCAa/kPj1p0nq9kVPKsTofe
+6ZqcPQ8xTYS42bfCjIc8PF5M8cUKaDcoWqE9Xx9zwOqje/q3Z+9y+tgnnk/y/fCpj31Om2
+SVJHy0kzlWTbqSHMnk8zVarHsxnDWiKXaCFUeLXRxDujCdu44yx9yM0v+XVP7Wbf4QVZcT
+L2ubBlXUj8DlPJyv6qs5OOC4FLno2Bwi36lq942DaAdLSjV6QLExAkQlf7tA6Kg11eff6s
+ZvLrtA6QaqLgqe6p/J9t4lxlfkmLWCnWW74Sl+yKrpnC5HwMoaInvgt0jEza2tJ5ZE0Lyw
+ydptt1DNs7/Af96IpULjkXyh1WgOxWJ9DoPf2rwOIf0IqfiLD0uYGMdqmsvyYHKOM/4wmM
+MvUi0dxsTUjSExTgYYapRM7kbn93+DxJ02h49nCva/gCmHjajOW4P5z63gQvDle41fHBOY
+CxLWbx06ERM0kziJqKtEcnCMksgSzMS4OUH/m46EClFM1HTk/J24bAmq5kS0EZHGoxAAAA
+wQCYYcOIfYb+fs9Wz0vuu0YK/b5d7bowPK84SI9VW0P/ImL9f62oJa4FzdGaprSCev0eSH
+0asrWHtab/kNL3X5aEW0BNDE52g2bzp5xjeC8009nj0c+D8VDh40zOy1PnVpfJmRcienPL
+3KuIH4fRJ1bdToeU1yZF+Kv8kwogGew7y58Al/ouqhqUk0A9JknMAQQyBsewf001jGvFqL
+pdlVPBjo9MD2XUnZRASrzRMzJyeCk7bzoi2Urft42fCbfclkwAAADBAPGH8mX1iBuzU0uE
+tsJWcgH3MJmJNbW4RPvVVVuewPyghrr2pfszGtMnV+Z/9QumWU6MB5+RNB4dpY+mr9AqGW
+j5LboazP7PIqtReD4ADCGTE9ubyR/OY2bhsp2k88mn0p0Hst/nSV3RhBvbZgdWXFCEzi0v
+tGVjBcAYh2b1DyR6WyAITCd+pFuezk8Wwc5Mve731L4T/h685JzOSRKKFhdZ60BV2WV0hG
+e/L2/aXF58oyrDQjQ0YS/X/BGFInKshQAAAMEA6HyjbZGvjM99ldlEC9sxRmTldxg7c9hI
+KtvSURCSO7I6YiyGl9WVhqTY61A3UJogHNxcNxqPWTqwaYuFgZyaSe0XGf1v/sReRiJoic
+n8HDL6GY3iVaNHCPhsK/Ktw/rF9d47jSw3u84G6gKtZ2fgzqiTOgNNueYJP0bmyupSazHC
+OIXdyGoi+zkIlsIx5Pe4tjxIXWUm/RmUv+vv4aHv564RHHo0d2MqAPT9s2q1P9em8HiABm
+4lNzgwCWER5B5lAAAAFGFhcm9ubWFvQEppYWp1bi00MDkwAQIDBAUG
+-----END OPENSSH PRIVATE KEY-----
diff --git a/dockers/dn2/hadoop-id-rsa.pub b/dockers/dn2/hadoop-id-rsa.pub
@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDbWMu2v5MI27KuNyk9RFpAIzqlyk0kpGSZLdhxlGl2JtLnkDcq1StytJzY0GNeqgyb43sJRPZPbmosf3V5JCdSNMGLaa49dcF/s1+FFONJezN+LU7F0M1jLLfW/5OQXwMdAEFqF2EIKSptudQh11O3inznK4QjEfcEqcx1/DQpsF28wHirelHFu9Z8q5YaZqtKPt1VYOUhHwt488hgu2mQqUcDks063KBcK3tnrdOKMdQbtzwAuAnxEgpY/x2TLd+7XTJ6jqsrV3xkCJCk9e2Cfh4I8istfKnvdtW5csLcS7Ha7JVFlgMYH3yz/Glwe/NLvReFPN5fOdauzLUucMGK2HmXNMmjBBdLmSOS/HkAED/Bs/rUOm4VB1yo94fVuPRlOyvWIL0N9S+GJJXMLxiwkZsvElPD+kkKXu8PA4a3OPBgwBtNdzedm1BE+SeciNbexFI1PvX2QQuD9FpVY1YBcJL/Q1EHgPA/EhYkmF+NWm9CAo7MWjNRuNDai4Lppnk= aaronmao@Jiajun-4090
diff --git a/dockers/dn2/hadoop-ssh-config b/dockers/dn2/hadoop-ssh-config
@@ -0,0 +1,2 @@
+Host *
+    StrictHostKeyChecking no
diff --git a/dockers/dn2/hdfs-site.xml b/dockers/dn2/hdfs-site.xml
@@ -0,0 +1,46 @@
+<configuration>
+    <property>
+        <name>dfs.datanode.address</name>
+        <value>0.0.0.0:9766</value>
+    </property>
+    <property>
+        <name>dfs.datanode.http.address</name>
+        <value>0.0.0.0:9764</value>
+    </property>
+    <property>
+        <name>dfs.datanode.ipc.address</name>
+        <value>0.0.0.0:9767</value>
+    </property>
+    <property>
+        <name>dfs.namenode.name.dir</name>
+        <value>/data/nameNode</value>
+    </property>
+    <property>
+        <name>dfs.datanode.data.dir</name>
+        <value>[ZFS]/data/dataNode2,[ZFS_RECON_BUFFER]/data/recon_buffer2</value>
+    </property>
+    <property>
+        <name>dfs.namenode.heartbeat.recheck-interval</name>
+        <value>5000</value>
+    </property>
+    <property>
+        <name>dfs.namenode.ec.system.default.policy</name>
+        <value>XOR-2-1-1024k</value>
+    </property>
+    <property>
+        <name>net.topology.script.file.name</name>
+        <value>/opt/rack-topo.sh</value>
+    </property>
+    <property>
+        <name>dfs.blocksize</name>
+        <value>1M</value>
+    </property>
+    <property>
+        <name>dfs.datanode.disk.check.min.gap</name>
+        <value>1s</value>
+    </property>
+    <property>
+        <name>dfs.datanode.failed.volumes.tolerated</name>
+        <value>1</value>
+    </property>
+</configuration>
diff --git a/dockers/dn3/core-site.xml b/dockers/dn3/core-site.xml
@@ -0,0 +1,6 @@
+<configuration>
+    <property>
+        <name>fs.defaultFS</name>
+        <value>hdfs://localhost:9000</value>
+    </property>
+</configuration>
diff --git a/dockers/dn3/hadoop-id-rsa b/dockers/dn3/hadoop-id-rsa
@@ -0,0 +1,38 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn
+NhAAAAAwEAAQAAAYEA21jLtr+TCNuyrjcpPURaQCM6pcpNJKRkmS3YcZRpdibS55A3KtUr
+crSc2NBjXqoMm+N7CUT2T25qLH91eSQnUjTBi2muPXXBf7NfhRTjSXszfi1OxdDNYyy31v
++TkF8DHQBBahdhCCkqbbnUIddTt4p85yuEIxH3BKnMdfw0KbBdvMB4q3pRxbvWfKuWGmar
+Sj7dVWDlIR8LePPIYLtpkKlHA5LNOtygXCt7Z63TijHUG7c8ALgJ8RIKWP8dky3fu10yeo
+6rK1d8ZAiQpPXtgn4eCPIrLXyp73bVuXLC3Eux2uyVRZYDGB98s/xpcHvzS70XhTzeXznW
+rsy1LnDBith5lzTJowQXS5kjkvx5ABA/wbP61DpuFQdcqPeH1bj0ZTsr1iC9DfUvhiSVzC
+8YsJGbLxJTw/pJCl7vDwOGtzjwYMAbTXc3nZtQRPknnIjW3sRSNT719kELg/RaVWNWAXCS
+/0NRB4DwPxIWJJhfjVpvQgKOzFozUbjQ2ouC6aZ5AAAFkP4rzGb+K8xmAAAAB3NzaC1yc2
+EAAAGBANtYy7a/kwjbsq43KT1EWkAjOqXKTSSkZJkt2HGUaXYm0ueQNyrVK3K0nNjQY16q
+DJvjewlE9k9uaix/dXkkJ1I0wYtprj11wX+zX4UU40l7M34tTsXQzWMst9b/k5BfAx0AQW
+oXYQgpKm251CHXU7eKfOcrhCMR9wSpzHX8NCmwXbzAeKt6UcW71nyrlhpmq0o+3VVg5SEf
+C3jzyGC7aZCpRwOSzTrcoFwre2et04ox1Bu3PAC4CfESClj/HZMt37tdMnqOqytXfGQIkK
+T17YJ+HgjyKy18qe921blywtxLsdrslUWWAxgffLP8aXB780u9F4U83l851q7MtS5wwYrY
+eZc0yaMEF0uZI5L8eQAQP8Gz+tQ6bhUHXKj3h9W49GU7K9YgvQ31L4YklcwvGLCRmy8SU8
+P6SQpe7w8Dhrc48GDAG013N52bUET5J5yI1t7EUjU+9fZBC4P0WlVjVgFwkv9DUQeA8D8S
+FiSYX41ab0ICjsxaM1G40NqLgummeQAAAAMBAAEAAAGALZvCCAa/kPj1p0nq9kVPKsTofe
+6ZqcPQ8xTYS42bfCjIc8PF5M8cUKaDcoWqE9Xx9zwOqje/q3Z+9y+tgnnk/y/fCpj31Om2
+SVJHy0kzlWTbqSHMnk8zVarHsxnDWiKXaCFUeLXRxDujCdu44yx9yM0v+XVP7Wbf4QVZcT
+L2ubBlXUj8DlPJyv6qs5OOC4FLno2Bwi36lq942DaAdLSjV6QLExAkQlf7tA6Kg11eff6s
+ZvLrtA6QaqLgqe6p/J9t4lxlfkmLWCnWW74Sl+yKrpnC5HwMoaInvgt0jEza2tJ5ZE0Lyw
+ydptt1DNs7/Af96IpULjkXyh1WgOxWJ9DoPf2rwOIf0IqfiLD0uYGMdqmsvyYHKOM/4wmM
+MvUi0dxsTUjSExTgYYapRM7kbn93+DxJ02h49nCva/gCmHjajOW4P5z63gQvDle41fHBOY
+CxLWbx06ERM0kziJqKtEcnCMksgSzMS4OUH/m46EClFM1HTk/J24bAmq5kS0EZHGoxAAAA
+wQCYYcOIfYb+fs9Wz0vuu0YK/b5d7bowPK84SI9VW0P/ImL9f62oJa4FzdGaprSCev0eSH
+0asrWHtab/kNL3X5aEW0BNDE52g2bzp5xjeC8009nj0c+D8VDh40zOy1PnVpfJmRcienPL
+3KuIH4fRJ1bdToeU1yZF+Kv8kwogGew7y58Al/ouqhqUk0A9JknMAQQyBsewf001jGvFqL
+pdlVPBjo9MD2XUnZRASrzRMzJyeCk7bzoi2Urft42fCbfclkwAAADBAPGH8mX1iBuzU0uE
+tsJWcgH3MJmJNbW4RPvVVVuewPyghrr2pfszGtMnV+Z/9QumWU6MB5+RNB4dpY+mr9AqGW
+j5LboazP7PIqtReD4ADCGTE9ubyR/OY2bhsp2k88mn0p0Hst/nSV3RhBvbZgdWXFCEzi0v
+tGVjBcAYh2b1DyR6WyAITCd+pFuezk8Wwc5Mve731L4T/h685JzOSRKKFhdZ60BV2WV0hG
+e/L2/aXF58oyrDQjQ0YS/X/BGFInKshQAAAMEA6HyjbZGvjM99ldlEC9sxRmTldxg7c9hI
+KtvSURCSO7I6YiyGl9WVhqTY61A3UJogHNxcNxqPWTqwaYuFgZyaSe0XGf1v/sReRiJoic
+n8HDL6GY3iVaNHCPhsK/Ktw/rF9d47jSw3u84G6gKtZ2fgzqiTOgNNueYJP0bmyupSazHC
+OIXdyGoi+zkIlsIx5Pe4tjxIXWUm/RmUv+vv4aHv564RHHo0d2MqAPT9s2q1P9em8HiABm
+4lNzgwCWER5B5lAAAAFGFhcm9ubWFvQEppYWp1bi00MDkwAQIDBAUG
+-----END OPENSSH PRIVATE KEY-----
diff --git a/dockers/dn3/hadoop-id-rsa.pub b/dockers/dn3/hadoop-id-rsa.pub
@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDbWMu2v5MI27KuNyk9RFpAIzqlyk0kpGSZLdhxlGl2JtLnkDcq1StytJzY0GNeqgyb43sJRPZPbmosf3V5JCdSNMGLaa49dcF/s1+FFONJezN+LU7F0M1jLLfW/5OQXwMdAEFqF2EIKSptudQh11O3inznK4QjEfcEqcx1/DQpsF28wHirelHFu9Z8q5YaZqtKPt1VYOUhHwt488hgu2mQqUcDks063KBcK3tnrdOKMdQbtzwAuAnxEgpY/x2TLd+7XTJ6jqsrV3xkCJCk9e2Cfh4I8istfKnvdtW5csLcS7Ha7JVFlgMYH3yz/Glwe/NLvReFPN5fOdauzLUucMGK2HmXNMmjBBdLmSOS/HkAED/Bs/rUOm4VB1yo94fVuPRlOyvWIL0N9S+GJJXMLxiwkZsvElPD+kkKXu8PA4a3OPBgwBtNdzedm1BE+SeciNbexFI1PvX2QQuD9FpVY1YBcJL/Q1EHgPA/EhYkmF+NWm9CAo7MWjNRuNDai4Lppnk= aaronmao@Jiajun-4090
diff --git a/dockers/dn3/hadoop-ssh-config b/dockers/dn3/hadoop-ssh-config
@@ -0,0 +1,2 @@
+Host *
+    StrictHostKeyChecking no
diff --git a/dockers/dn3/hdfs-site.xml b/dockers/dn3/hdfs-site.xml
@@ -0,0 +1,46 @@
+<configuration>
+    <property>
+        <name>dfs.datanode.address</name>
+        <value>0.0.0.0:9666</value>
+    </property>
+    <property>
+        <name>dfs.datanode.http.address</name>
+        <value>0.0.0.0:9664</value>
+    </property>
+    <property>
+        <name>dfs.datanode.ipc.address</name>
+        <value>0.0.0.0:9667</value>
+    </property>
+    <property>
+        <name>dfs.namenode.name.dir</name>
+        <value>/data/nameNode</value>
+    </property>
+    <property>
+        <name>dfs.datanode.data.dir</name>
+        <value>[ZFS]/data/dataNode3,[ZFS_RECON_BUFFER]/data/recon_buffer3</value>
+    </property>
+    <property>
+        <name>dfs.namenode.heartbeat.recheck-interval</name>
+        <value>5000</value>
+    </property>
+    <property>
+        <name>dfs.namenode.ec.system.default.policy</name>
+        <value>XOR-2-1-1024k</value>
+    </property>
+    <property>
+        <name>net.topology.script.file.name</name>
+        <value>/opt/rack-topo.sh</value>
+    </property>
+    <property>
+        <name>dfs.blocksize</name>
+        <value>1M</value>
+    </property>
+    <property>
+        <name>dfs.datanode.disk.check.min.gap</name>
+        <value>1s</value>
+    </property>
+    <property>
+        <name>dfs.datanode.failed.volumes.tolerated</name>
+        <value>1</value>
+    </property>
+</configuration>
diff --git a/dockers/docker-compose.yml b/dockers/docker-compose.yml
@@ -37,7 +37,7 @@ services:
     networks:
       - intra
     deploy:
-      replicas: 4
+      replicas: 3
 
 networks:
     intra:
diff --git a/dockers/hdfs-site.xml b/dockers/hdfs-site.xml
@@ -5,7 +5,7 @@
     </property>
     <property>
         <name>dfs.datanode.data.dir</name>
-        <value>[ZFS]/data/dataNode1,[ZFS]/data/dataNode2,[ZFS_RECON_BUFFER]/data/recon_buffer</value>
+        <value>[ZFS]/data/dataNode1,[ZFS_RECON_BUFFER]/data/recon_buffer</value>
     </property>
     <property>
         <name>dfs.namenode.heartbeat.recheck-interval</name>

diff --git a/...hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/...hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -4727,8 +4727,6 @@ private void processIncrementalBlockReport(final DatanodeDescriptor node,
     for (ReceivedDeletedBlockInfo rdbi : srdb.getBlocks()) {
       switch (rdbi.getStatus()) {
       case DELETED_BLOCK:
-        LOG.info("Deleting block");
-
         // Get the datanode that we are deleting the block from
         LOG.info("Deleting block from data node {}", node.getHostName());
         LOG.info("Datanode has blocks");

diff --git a/...-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ZfsBlockManagement.java b/...-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ZfsBlockManagement.java
@@ -1,5 +1,7 @@
 package org.apache.hadoop.hdfs.server.blockmanagement;
 
+import jni.DnodeAttributes;
+import jni.Tools;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 

diff --git a/...fs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/...fs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -100,6 +100,8 @@
 import static org.apache.hadoop.util.Preconditions.checkNotNull;
 import static org.apache.hadoop.util.Time.now;
 
+import jni.DnodeAttributes;
+import jni.Tools;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.fs.DF;
 import org.apache.hadoop.fs.DU;
@@ -2725,12 +2727,23 @@ private void handleDiskError(String failedVolumes, int failedNumber) {
                                      : DatanodeProtocol.FATAL_DISK_ERROR;  
     metrics.incrVolumeFailures(failedNumber);
 
+    // Get MLEC failure info
+    Tools zfsBinding = new Tools();
+    List<DnodeAttributes> dnodes = zfsBinding.getFailedChunks("pool");
+    LOG.warn("Printing ZFS datanode info");
+    for (DnodeAttributes dn : dnodes) {
+      LOG.info("This datanode contains following zfs dn {}", dn);
+    }
+    LOG.warn("============");
+
     //inform NameNodes
     for(BPOfferService bpos: blockPoolManager.getAllNamenodeThreads()) {
+      LOG.info("Try sending error report");
       bpos.trySendErrorReport(dpError, failedVolumes);
     }
 
     if(hasEnoughResources) {
+      LOG.info("Scheduling all block report");
       scheduleAllBlockReport(0);
       return; // do not shutdown
     }