moved to 3.2.1/stable hadoop version

zwang96-dl · Nov 4, 2019 · c74435c · c74435c
1 parent bc861a6
commit c74435c
Show file tree

Hide file tree

Showing 13 changed files with 28 additions and 26 deletions.
diff --git a/Makefile b/Makefile
@@ -13,7 +13,7 @@ build:
 wordcount:
 	docker build -t hadoop-wordcount ./submit
 	docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -mkdir -p /input/
-	docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -copyFromLocal /opt/hadoop-3.1.3/README.txt /input/
+	docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -copyFromLocal -f /opt/hadoop-3.2.1/README.txt /input/
 	docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} hadoop-wordcount
 	docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -cat /output/*
 	docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -rm -r /output

diff --git a/base/Dockerfile b/base/Dockerfile
@@ -17,7 +17,7 @@ RUN curl -O https://dist.apache.org/repos/dist/release/hadoop/common/KEYS
 
 RUN gpg --import KEYS
 
-ENV HADOOP_VERSION 3.1.3
+ENV HADOOP_VERSION 3.2.1
 ENV HADOOP_URL https://www.apache.org/dist/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz
 
 RUN set -x \
@@ -33,11 +33,11 @@ RUN mkdir /opt/hadoop-$HADOOP_VERSION/logs
 
 RUN mkdir /hadoop-data
 
-ENV HADOOP_PREFIX=/opt/hadoop-$HADOOP_VERSION
+ENV HADOOP_HOME=/opt/hadoop-$HADOOP_VERSION
 ENV HADOOP_CONF_DIR=/etc/hadoop
 ENV MULTIHOMED_NETWORK=1
 ENV USER=root
-ENV PATH $HADOOP_PREFIX/bin/:$PATH
+ENV PATH $HADOOP_HOME/bin/:$PATH
 
 ADD entrypoint.sh /entrypoint.sh
 

diff --git a/datanode/Dockerfile b/datanode/Dockerfile
@@ -4,7 +4,7 @@ MAINTAINER Ivan Ermilov <ivan.s.ermilov@gmail.com>
 
 HEALTHCHECK CMD curl -f http://localhost:9864/ || exit 1
 
-ENV HDFS_CONF_dfs_datanode_data_dir=file:///hadoop/dfs/data
+ENV HDFS_CONF_dfs_datanode_data_dir=/hadoop/dfs/data
 RUN mkdir -p /hadoop/dfs/data
 VOLUME /hadoop/dfs/data
 

diff --git a/datanode/run.sh b/datanode/run.sh
@@ -6,4 +6,4 @@ if [ ! -d $datadir ]; then
   exit 2
 fi
 
-$HADOOP_PREFIX/bin/hdfs --config $HADOOP_CONF_DIR datanode
+$HADOOP_HOME/bin/hdfs --config $HADOOP_CONF_DIR datanode
diff --git a/docker-compose-v3.yml b/docker-compose-v3.yml
@@ -2,7 +2,7 @@ version: '3'
 
 services:
   namenode:
-    image: bde2020/hadoop-namenode:2.0.0-hadoop3.1.3-java8
+    image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
     networks:
       - hbase
     volumes:
@@ -24,7 +24,7 @@ services:
         traefik.port: 50070
 
   datanode:
-    image: bde2020/hadoop-datanode:2.0.0-hadoop3.1.3-java8
+    image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
     networks:
       - hbase
     volumes:
@@ -42,7 +42,7 @@ services:
         traefik.port: 50075
 
   resourcemanager:
-    image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.1.3-java8
+    image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.2.1-java8
     networks:
       - hbase
     environment:
@@ -64,7 +64,7 @@ services:
       disable: true
 
   nodemanager:
-    image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.1.3-java8
+    image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.2.1-java8
     networks:
       - hbase
     environment:
@@ -80,7 +80,7 @@ services:
         traefik.port: 8042
 
   historyserver:
-    image: bde2020/hadoop-historyserver:2.0.0-hadoop3.1.3-java8
+    image: bde2020/hadoop-historyserver:2.0.0-hadoop3.2.1-java8
     networks:
       - hbase
     volumes:

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -2,7 +2,7 @@ version: "3"
 
 services:
   namenode:
-    image: bde2020/hadoop-namenode:2.0.0-hadoop3.1.3-java8
+    image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
     container_name: namenode
     ports:
       - 9870:9870
@@ -14,7 +14,7 @@ services:
       - ./hadoop.env
 
   datanode:
-    image: bde2020/hadoop-datanode:2.0.0-hadoop3.1.3-java8
+    image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
     container_name: datanode
     volumes:
       - hadoop_datanode:/hadoop/dfs/data
@@ -24,23 +24,23 @@ services:
       - ./hadoop.env
 
   resourcemanager:
-    image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.1.3-java8
+    image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.2.1-java8
     container_name: resourcemanager
     environment:
       SERVICE_PRECONDITION: "namenode:9870 datanode:9864"
     env_file:
       - ./hadoop.env
 
   nodemanager1:
-    image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.1.3-java8
+    image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.2.1-java8
     container_name: nodemanager
     environment:
       SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088"
     env_file:
       - ./hadoop.env
 
   historyserver:
-    image: bde2020/hadoop-historyserver:2.0.0-hadoop3.1.3-java8
+    image: bde2020/hadoop-historyserver:2.0.0-hadoop3.2.1-java8
     container_name: historyserver
     environment:
       SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088"

diff --git a/hadoop.env b/hadoop.env
@@ -7,6 +7,8 @@ CORE_CONF_io_compression_codecs=org.apache.hadoop.io.compress.SnappyCodec
 HDFS_CONF_dfs_webhdfs_enabled=true
 HDFS_CONF_dfs_permissions_enabled=false
 HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false
+HDFS_CONF_dfs_namenode_name_dir=/hadoop/dfs/name
+HDFS_CONF_dfs_datanode_name_dir=/hadoop/dfs/data
 
 YARN_CONF_yarn_log___aggregation___enable=true
 YARN_CONF_yarn_log_server_url=http://historyserver:8188/applicationhistory/logs/
@@ -38,6 +40,6 @@ MAPRED_CONF_mapreduce_map_memory_mb=4096
 MAPRED_CONF_mapreduce_reduce_memory_mb=8192
 MAPRED_CONF_mapreduce_map_java_opts=-Xmx3072m
 MAPRED_CONF_mapreduce_reduce_java_opts=-Xmx6144m
-MAPRED_CONF_yarn_app_mapreduce_am_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.3/
-MAPRED_CONF_mapreduce_map_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.3/
-MAPRED_CONF_mapreduce_reduce_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.3/
+MAPRED_CONF_yarn_app_mapreduce_am_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/
+MAPRED_CONF_mapreduce_map_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/
+MAPRED_CONF_mapreduce_reduce_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/
diff --git a/historyserver/run.sh b/historyserver/run.sh
@@ -1,3 +1,3 @@
 #!/bin/bash
 
-$HADOOP_PREFIX/bin/yarn --config $HADOOP_CONF_DIR historyserver
+$HADOOP_HOME/bin/yarn --config $HADOOP_CONF_DIR historyserver
diff --git a/namenode/Dockerfile b/namenode/Dockerfile
@@ -4,7 +4,7 @@ MAINTAINER Ivan Ermilov <ivan.s.ermilov@gmail.com>
 
 HEALTHCHECK CMD curl -f http://localhost:9870/ || exit 1
 
-ENV HDFS_CONF_dfs_namenode_name_dir=file:///hadoop/dfs/name
+ENV HDFS_CONF_dfs_namenode_name_dir=/hadoop/dfs/name
 RUN mkdir -p /hadoop/dfs/name
 VOLUME /hadoop/dfs/name
 

diff --git a/namenode/run.sh b/namenode/run.sh
@@ -13,7 +13,7 @@ fi
 
 if [ "`ls -A $namedir`" == "" ]; then
   echo "Formatting namenode name directory: $namedir"
-  $HADOOP_PREFIX/bin/hdfs --config $HADOOP_CONF_DIR namenode -format $CLUSTER_NAME 
+  $HADOOP_HOME/bin/hdfs --config $HADOOP_CONF_DIR namenode -format $CLUSTER_NAME
 fi
 
-$HADOOP_PREFIX/bin/hdfs --config $HADOOP_CONF_DIR namenode
+$HADOOP_HOME/bin/hdfs --config $HADOOP_CONF_DIR namenode
diff --git a/nodemanager/run.sh b/nodemanager/run.sh
@@ -1,3 +1,3 @@
 #!/bin/bash
 
-$HADOOP_PREFIX/bin/yarn --config $HADOOP_CONF_DIR nodemanager
+$HADOOP_HOME/bin/yarn --config $HADOOP_CONF_DIR nodemanager
diff --git a/resourcemanager/run.sh b/resourcemanager/run.sh
@@ -1,3 +1,3 @@
 #!/bin/bash
 
-$HADOOP_PREFIX/bin/yarn --config $HADOOP_CONF_DIR resourcemanager
+$HADOOP_HOME/bin/yarn --config $HADOOP_CONF_DIR resourcemanager
diff --git a/submit/run.sh b/submit/run.sh
@@ -1,3 +1,3 @@
 #!/bin/bash
 
-$HADOOP_PREFIX/bin/hadoop jar $JAR_FILEPATH $CLASS_TO_RUN $PARAMS 
+$HADOOP_HOME/bin/hadoop jar $JAR_FILEPATH $CLASS_TO_RUN $PARAMS
-Original file line number
+Diff line change
@@ Expand Up / @@ -6,4 +6,4 @@ if [ ! -d $datadir ]; then @@
       exit 2
     fi
-    $HADOOP_PREFIX/bin/hdfs --config $HADOOP_CONF_DIR datanode
+    $HADOOP_HOME/bin/hdfs --config $HADOOP_CONF_DIR datanode