Skip to content

Commit 1ddb47f

Browse files
yhtyht
committed
fix hadoop
1 parent 5e46327 commit 1ddb47f

18 files changed

+185
-137
lines changed

‎hadoop/Makefile

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
all: 0base 1hive 2master 3worker
2+
3+
0base:
4+
cd base && $(MAKE)
5+
1hive:
6+
cd hive && $(MAKE)
7+
2master:
8+
cd master && $(MAKE)
9+
3worker:
10+
cd worker && $(MAKE)

‎hadoop/base/Dockerfile

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM kalamangga/debian:11
1+
FROM kalamangga/debian:bullseye
22
RUN apt -qq update && \
33
DEBIAN_FRONTEND=noninteractive apt-get -qq install --no-install-recommends \
44
sudo \
@@ -13,8 +13,8 @@ RUN apt -qq update && \
1313
libc6-dev && \
1414
rm -rf /var/lib/apt/lists/*
1515

16-
ARG USERNAME=jupyter
17-
ARG GROUPNAME=jupyter
16+
ARG USERNAME=hdfs
17+
ARG GROUPNAME=hdfs
1818
ARG UID=1001
1919
ARG GID=1001
2020

‎hadoop/base/Makefile

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
1-
all: build tag push
1+
all: pull build tag push
22

3+
pull:
4+
docker pull kalamangga/debian:bullseye
35
build:
46
docker build -t kalamangga/hadoop-base:latest --rm .
57
tag:

‎hadoop/base/core-site.xml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,11 @@
2323
</property>
2424
<!--To be able to run Hive via JDBC as root-->
2525
<property>
26-
<name>hadoop.proxyuser.jupyter.groups</name>
26+
<name>hadoop.proxyuser.hdfs.groups</name>
2727
<value>*</value>
2828
</property>
2929
<property>
30-
<name>hadoop.proxyuser.jupyter.hosts</name>
30+
<name>hadoop.proxyuser.hdfs.hosts</name>
3131
<value>*</value>
3232
</property>
3333
</configuration>

‎hadoop/docker-compose.yml

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
services:
2+
metastore:
3+
image: postgres:11
4+
hostname: metastore
5+
environment:
6+
POSTGRES_PASSWORD: hdfs
7+
ports:
8+
- "5432:5432"
9+
volumes:
10+
- ./init.sql:/docker-entrypoint-initdb.d/init.sql
11+
restart: always
12+
networks:
13+
hadoopnet:
14+
ipv4_address: 172.28.1.1
15+
extra_hosts:
16+
- "master:172.28.1.2"
17+
- "worker1:172.28.1.3"
18+
- "worker2:172.28.1.4"
19+
- "history:172.28.1.5"
20+
21+
master:
22+
image: kalamangga/hadoop-master
23+
hostname: master
24+
depends_on:
25+
- metastore
26+
ports:
27+
- "4040:4040"
28+
- "7077:7077"
29+
- "8020:8020"
30+
- "8080:8080"
31+
- "8088:8088"
32+
- "9870:9870"
33+
- "10000:10000"
34+
restart: always
35+
networks:
36+
hadoopnet:
37+
ipv4_address: 172.28.1.2
38+
extra_hosts:
39+
- "metastore:172.28.1.1"
40+
- "worker1:172.28.1.3"
41+
- "worker2:172.28.1.4"
42+
- "history:172.28.1.5"
43+
44+
worker1:
45+
image: kalamangga/hadoop-worker
46+
hostname: worker1
47+
depends_on:
48+
- master
49+
ports:
50+
- "8042:8042"
51+
- "8081:8081"
52+
- "9864:9864"
53+
restart: always
54+
networks:
55+
hadoopnet:
56+
ipv4_address: 172.28.1.3
57+
extra_hosts:
58+
- "metastore:172.28.1.1"
59+
- "master:172.28.1.2"
60+
- "worker2:172.28.1.4"
61+
- "history:172.28.1.5"
62+
63+
worker2:
64+
image: kalamangga/hadoop-worker
65+
hostname: worker2
66+
depends_on:
67+
- master
68+
ports:
69+
- "8043:8042"
70+
- "8082:8081"
71+
- "9865:9864"
72+
restart: always
73+
networks:
74+
hadoopnet:
75+
ipv4_address: 172.28.1.4
76+
extra_hosts:
77+
- "metastore:172.28.1.1"
78+
- "master:172.28.1.2"
79+
- "worker1:172.28.1.3"
80+
- "history:172.28.1.5"
81+
82+
volumes:
83+
namenode:
84+
namesecondary:
85+
datanode1:
86+
datanode2:
87+
metastore:
88+
89+
networks:
90+
hadoopnet:
91+
ipam:
92+
driver: default
93+
config:
94+
- subnet: 172.28.1.0/24

‎hadoop/hive/Dockerfile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
FROM kalamangga/hadoop-base:3.3.6
22

3-
ARG USERNAME=jupyter
4-
ARG GROUPNAME=jupyter
3+
ARG USERNAME=hdfs
4+
ARG GROUPNAME=hdfs
55
ARG UID=1001
66
ARG GID=1001
77
USER $USERNAME

‎hadoop/hive/hive-site.xml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,11 +30,11 @@
3030
</property>
3131
<property>
3232
<name>javax.jdo.option.ConnectionUserName</name>
33-
<value>jupyter</value>
33+
<value>hdfs</value>
3434
</property>
3535
<property>
3636
<name>javax.jdo.option.ConnectionPassword</name>
37-
<value>jupyter</value>
37+
<value>hdfs</value>
3838
</property>
3939
<property>
4040
<name>hive.metastore.schema.verification</name>

‎hadoop/init.sql

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
CREATE DATABASE "metastore";
2+
CREATE USER hdfs WITH ENCRYPTED PASSWORD 'hdfs';
3+
GRANT ALL ON DATABASE metastore TO hdfs;

‎hadoop/master/Dockerfile

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
FROM kalamangga/hadoop-hive:3.1.3
2+
ENV NAMEDIR=/opt/hadoop/dfs/name
3+
RUN mkdir -p /opt/hadoop/dfs/name
4+
VOLUME /opt/hadoop/dfs/name
5+
COPY run.sh /usr/local/sbin/run.sh
6+
RUN sudo chmod a+x /usr/local/sbin/run.sh
7+
CMD ["run.sh"]

‎hadoop/master/Makefile

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
all: build push
2+
3+
build:
4+
docker build -t kalamangga/hadoop-master:latest --rm .
5+
push:
6+
docker push kalamangga/hadoop-master:latest

0 commit comments

Comments
 (0)