diff --git a/Makefile b/Makefile index f74842a5..4bdeb82f 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ build: wordcount: docker build -t hadoop-wordcount ./submit docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -mkdir -p /input/ - docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -copyFromLocal /opt/hadoop-3.1.1/README.txt /input/ + docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -copyFromLocal /opt/hadoop-3.1.2/README.txt /input/ docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} hadoop-wordcount docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -cat /output/* docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -rm -r /output diff --git a/base/Dockerfile b/base/Dockerfile index fe25d725..df68993a 100644 --- a/base/Dockerfile +++ b/base/Dockerfile @@ -17,7 +17,7 @@ RUN curl -O https://dist.apache.org/repos/dist/release/hadoop/common/KEYS RUN gpg --import KEYS -ENV HADOOP_VERSION 3.1.1 +ENV HADOOP_VERSION 3.1.2 ENV HADOOP_URL https://www.apache.org/dist/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz RUN set -x \ diff --git a/docker-compose-v3.yml b/docker-compose-v3.yml index e7718196..6e93fb86 100644 --- a/docker-compose-v3.yml +++ b/docker-compose-v3.yml @@ -2,7 +2,7 @@ version: '3' services: namenode: - image: bde2020/hadoop-namenode:2.0.0-hadoop3.1.1-java8 + image: bde2020/hadoop-namenode:2.0.0-hadoop3.1.2-java8 networks: - hbase volumes: @@ -24,7 +24,7 @@ services: traefik.port: 50070 datanode: - image: bde2020/hadoop-datanode:2.0.0-hadoop3.1.1-java8 + image: bde2020/hadoop-datanode:2.0.0-hadoop3.1.2-java8 networks: - hbase volumes: @@ -42,7 +42,7 @@ services: traefik.port: 50075 resourcemanager: - image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.1.1-java8 + image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.1.2-java8 networks: - hbase environment: @@ -64,7 +64,7 @@ services: disable: true nodemanager: - image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.1.1-java8 + image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.1.2-java8 networks: - hbase environment: @@ -80,7 +80,7 @@ services: traefik.port: 8042 historyserver: - image: bde2020/hadoop-historyserver:2.0.0-hadoop3.1.1-java8 + image: bde2020/hadoop-historyserver:2.0.0-hadoop3.1.2-java8 networks: - hbase volumes: diff --git a/docker-compose.yml b/docker-compose.yml index 4f3fa20e..95476412 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: "3" services: namenode: - image: bde2020/hadoop-namenode:2.0.0-hadoop3.1.1-java8 + image: bde2020/hadoop-namenode:2.0.0-hadoop3.1.2-java8 container_name: namenode ports: - 9870:9870 @@ -14,7 +14,7 @@ services: - ./hadoop.env datanode: - image: bde2020/hadoop-datanode:2.0.0-hadoop3.1.1-java8 + image: bde2020/hadoop-datanode:2.0.0-hadoop3.1.2-java8 container_name: datanode volumes: - hadoop_datanode:/hadoop/dfs/data @@ -24,7 +24,7 @@ services: - ./hadoop.env resourcemanager: - image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.1.1-java8 + image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.1.2-java8 container_name: resourcemanager environment: SERVICE_PRECONDITION: "namenode:9870 datanode:9864" @@ -32,7 +32,7 @@ services: - ./hadoop.env nodemanager1: - image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.1.1-java8 + image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.1.2-java8 container_name: nodemanager environment: SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088" @@ -40,7 +40,7 @@ services: - ./hadoop.env historyserver: - image: bde2020/hadoop-historyserver:2.0.0-hadoop3.1.1-java8 + image: bde2020/hadoop-historyserver:2.0.0-hadoop3.1.2-java8 container_name: historyserver environment: SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088" diff --git a/hadoop.env b/hadoop.env index 3366abf6..7076e1c8 100644 --- a/hadoop.env +++ b/hadoop.env @@ -38,6 +38,6 @@ MAPRED_CONF_mapreduce_map_memory_mb=4096 MAPRED_CONF_mapreduce_reduce_memory_mb=8192 MAPRED_CONF_mapreduce_map_java_opts=-Xmx3072m MAPRED_CONF_mapreduce_reduce_java_opts=-Xmx6144m -MAPRED_CONF_yarn_app_mapreduce_am_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.1/ -MAPRED_CONF_mapreduce_map_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.1/ -MAPRED_CONF_mapreduce_reduce_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.1/ +MAPRED_CONF_yarn_app_mapreduce_am_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.2/ +MAPRED_CONF_mapreduce_map_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.2/ +MAPRED_CONF_mapreduce_reduce_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.2/