Skip to content

Commit 72f5026

Browse files
yhtyht
committed
fix hadoop
1 parent 05ed8d2 commit 72f5026

File tree

7 files changed

+54
-69
lines changed

7 files changed

+54
-69
lines changed

‎platform/hadoop/base/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
hadoop-*.gz

‎platform/hadoop/base/Dockerfile

Lines changed: 17 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,50 +1,35 @@
11
FROM kalamangga/debian:bullseye
2-
RUN apt -qq update && \
3-
DEBIAN_FRONTEND=noninteractive apt-get -qq install --no-install-recommends \
4-
sudo \
5-
openjdk-11-jdk \
6-
curl \
7-
gnupg \
8-
procps \
9-
python3 \
10-
python3-pip \
11-
python-is-python3 \
12-
coreutils \
13-
libc6-dev && \
2+
RUN apt-get install -y gnupg procps coreutils libc6-dev && apt-get clean
3+
RUN apt-get install -y openjdk-11-jdk && apt-get clean
4+
RUN apt-get install -y python3 python3-pip python-is-python3 && \
145
rm -rf /var/lib/apt/lists/*
15-
166
ARG USERNAME=hdfs
177
ARG GROUPNAME=hdfs
188
ARG UID=1001
199
ARG GID=1001
10+
ARG WORKDIR=/opt/bigdata
2011

21-
RUN echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME && \
22-
chmod 0440 /etc/sudoers.d/$USERNAME && \
23-
groupadd -g $GID $GROUPNAME && \
24-
useradd -m -s /bin/bash -u $UID -g $GID $USERNAME
25-
26-
USER $USERNAME
12+
RUN echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \
13+
&& chmod 0440 /etc/sudoers.d/$USERNAME \
14+
&& groupadd -g $GID $GROUPNAME \
15+
&& useradd -m -s /bin/bash -u $UID -g $GID $USERNAME \
16+
&& mkdir $WORKDIR && chown $USERNAME:$GROUPNAME $WORKDIR
2717

2818
ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64/
2919

3020
# Hadoop
3121
ARG HADOOP_VER=3.3.6
3222
ARG HADOOP_URL=https://repo.kalamangga.net/apache/hadoop/hadoop-$HADOOP_VER.tar.gz
33-
ENV HADOOP_HOME=/opt/hadoop
23+
ENV HADOOP_HOME=$WORKDIR/hadoop-$HADOOP_VER
3424

35-
RUN set -x && \
36-
curl -fsSL https://archive.apache.org/dist/hadoop/common/KEYS -o /tmp/hadoop-KEYS && \
37-
gpg --import /tmp/hadoop-KEYS && sudo mkdir $HADOOP_HOME && \
38-
sudo chown $USERNAME:$GROUPNAME -R $HADOOP_HOME && \
39-
curl -fsSL $HADOOP_URL -o /tmp/hadoop.tar.gz && \
40-
curl -fsSL $HADOOP_URL.asc -o /tmp/hadoop.tar.gz.asc && \
41-
gpg --verify /tmp/hadoop.tar.gz.asc && \
42-
tar -xf /tmp/hadoop.tar.gz -C $HADOOP_HOME --strip-components 1 && \
43-
mkdir $HADOOP_HOME/logs && rm /tmp/hadoop*
25+
COPY hadoop-3.3.6.tar.gz /opt/bigdata
26+
WORKDIR $WORKDIR
27+
RUN set -x && tar -xf hadoop-$HADOOP_VER.tar.gz && mkdir $HADOOP_HOME/logs && rm $WORKDIR/hadoop*.gz \
28+
&& chown $USERNAME:$GROUPNAME -R $HADOOP_HOME
4429

4530
ENV HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
4631
ENV PATH=$HADOOP_HOME/sbin:$HADOOP_HOME/bin:$PATH
47-
ENV LD_LIBRARY_PATH=$HADOOP_HOME/lib/native:$LD_LIBRARY_PATH
32+
ENV LD_LIBRARY_PATH=$HADOOP_HOME/lib/native
4833

4934
# Config
5035
COPY --chown=$USERNAME:$GROUPNAME core-site.xml $HADOOP_CONF_DIR/
@@ -55,4 +40,6 @@ COPY --chown=$USERNAME:$GROUPNAME mapred-site.xml $HADOOP_CONF_DIR/
5540
# Entry point
5641
COPY entrypoint.sh /usr/local/sbin/entrypoint.sh
5742
RUN sudo chmod a+x /usr/local/sbin/entrypoint.sh
43+
WORKDIR /home/$USERNAME
44+
USER $USERNAME
5845
ENTRYPOINT ["entrypoint.sh"]

‎platform/hadoop/docker-compose.yml

Lines changed: 22 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,12 @@ services:
1111
restart: always
1212
networks:
1313
hadoopnet:
14-
ipv4_address: 172.28.1.1
14+
ipv4_address: 172.58.1.1
1515
extra_hosts:
16-
- "master:172.28.1.2"
17-
- "worker1:172.28.1.3"
18-
- "worker2:172.28.1.4"
19-
- "history:172.28.1.5"
16+
- "master:172.58.1.2"
17+
- "worker1:172.58.1.3"
18+
- "worker2:172.58.1.4"
19+
- "history:172.58.1.5"
2020

2121
master:
2222
image: kalamangga/hadoop-master
@@ -34,12 +34,12 @@ services:
3434
restart: always
3535
networks:
3636
hadoopnet:
37-
ipv4_address: 172.28.1.2
37+
ipv4_address: 172.58.1.2
3838
extra_hosts:
39-
- "metastore:172.28.1.1"
40-
- "worker1:172.28.1.3"
41-
- "worker2:172.28.1.4"
42-
- "history:172.28.1.5"
39+
- "metastore:172.58.1.1"
40+
- "worker1:172.58.1.3"
41+
- "worker2:172.58.1.4"
42+
- "history:172.58.1.5"
4343

4444
worker1:
4545
image: kalamangga/hadoop-worker
@@ -53,7 +53,7 @@ services:
5353
restart: always
5454
networks:
5555
hadoopnet:
56-
ipv4_address: 172.28.1.3
56+
ipv4_address: 172.58.1.3
5757
extra_hosts:
5858
- "metastore:172.28.1.1"
5959
- "master:172.28.1.2"
@@ -72,12 +72,12 @@ services:
7272
restart: always
7373
networks:
7474
hadoopnet:
75-
ipv4_address: 172.28.1.4
75+
ipv4_address: 172.58.1.4
7676
extra_hosts:
77-
- "metastore:172.28.1.1"
78-
- "master:172.28.1.2"
79-
- "worker1:172.28.1.3"
80-
- "history:172.28.1.5"
77+
- "metastore:172.58.1.1"
78+
- "master:172.58.1.2"
79+
- "worker1:172.58.1.3"
80+
- "history:172.58.1.5"
8181

8282
history:
8383
image: kalamangga/hadoop-history
@@ -89,12 +89,12 @@ services:
8989
restart: always
9090
networks:
9191
hadoopnet:
92-
ipv4_address: 172.28.1.5
92+
ipv4_address: 172.58.1.5
9393
extra_hosts:
94-
- "metastore:172.28.1.1"
95-
- "master:172.28.1.2"
96-
- "worker1:172.28.1.3"
97-
- "worker2:172.28.1.4"
94+
- "metastore:172.58.1.1"
95+
- "master:172.58.1.2"
96+
- "worker1:172.58.1.3"
97+
- "worker2:172.58.1.4"
9898

9999
volumes:
100100
namenode:
@@ -108,4 +108,4 @@ networks:
108108
ipam:
109109
driver: default
110110
config:
111-
- subnet: 172.28.1.0/24
111+
- subnet: 172.58.1.0/24

‎platform/hadoop/hive/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
apache-hive-*.gz

‎platform/hadoop/hive/Dockerfile

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,25 +4,20 @@ ARG USERNAME=hdfs
44
ARG GROUPNAME=hdfs
55
ARG UID=1001
66
ARG GID=1001
7-
USER $USERNAME
7+
ARG WORKDIR=/opt/bigdata
88

99
ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64/
1010

1111
# Hive
1212
ARG HIVE_VERSION=3.1.3
1313
ARG HIVE_URL=https://repo.kalamangga.net/apache/hive/apache-hive-$HIVE_VERSION-bin.tar.gz
14-
ENV HIVE_HOME=/opt/hive
14+
ENV HIVE_HOME=/opt/bigdata/apache-hive-3.1.3-bin
1515

16+
COPY --chown=$USERNAME:$GROUPNAME apache-hive-3.1.3-bin.tar.gz $WORKDIR
17+
WORKDIR $WORKDIR
1618
RUN set -x \
17-
&& curl -fsSL https://archive.apache.org/dist/hive/KEYS -o /tmp/hive-KEYS \
18-
&& gpg --import /tmp/hive-KEYS \
19-
&& sudo mkdir $HIVE_HOME \
20-
&& sudo chown $USERNAME:$GROUPNAME -R $HIVE_HOME \
21-
&& curl -fsSL $HIVE_URL -o /tmp/hive.tar.gz \
22-
&& curl -fsSL $HIVE_URL.asc -o /tmp/hive.tar.gz.asc \
23-
&& gpg --verify /tmp/hive.tar.gz.asc \
24-
&& tar -xf /tmp/hive.tar.gz -C $HIVE_HOME --strip-components 1 \
25-
&& rm /tmp/hive*
19+
&& tar -xf apache-hive-3.1.3-bin.tar.gz && rm $WORKDIR/apache-hive*.gz \
20+
&& chown $USERNAME:$GROUPNAME $HIVE_HOME
2621

2722
RUN sudo rm $HIVE_HOME/lib/guava-*.jar \
2823
&& sudo cp $HADOOP_HOME/share/hadoop/hdfs/lib/guava-*.jar $HIVE_HOME/lib/

‎platform/hadoop/master/Dockerfile

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
FROM kalamangga/hadoop-hive:3.1.3
2-
ENV NAMEDIR=/opt/hadoop/dfs/name
3-
RUN mkdir -p /opt/hadoop/dfs/name
4-
VOLUME /opt/hadoop/dfs/name
2+
ENV NAMEDIR=/opt/bigdata/hadoop-3.3.6/dfs/name
3+
RUN mkdir -p /opt/bigdata/hadoop-3.3.6/dfs/name
4+
VOLUME /opt/bigdata/hadoop-3.3.6/dfs/name
55
COPY run.sh /usr/local/sbin/run.sh
66
RUN sudo chmod a+x /usr/local/sbin/run.sh
77
CMD ["run.sh"]

‎platform/hadoop/worker/Dockerfile

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
FROM kalamangga/hadoop-hive:3.1.3
2-
ENV DATADIR=/opt/hadoop/dfs/data
3-
RUN mkdir -p /opt/hadoop/dfs/data
4-
VOLUME /opt/hadoop/dfs/data
2+
ARG WORKDIR=/opt/bigdata
3+
ENV DATADIR=$WORKDIR/data
4+
RUN mkdir -p $WORKDIR/data
5+
VOLUME $WORKDIR/data
56
COPY run.sh /usr/local/sbin/run.sh
67
RUN sudo chmod a+x /usr/local/sbin/run.sh
78
CMD ["run.sh"]

0 commit comments

Comments
 (0)