fix flask-hadoop

Author: huseinzol05

18.flask-hadoop/Dockerfile

@@ -8,11 +8,12 @@ RUN apt-get update && apt-get install -y \
     python3-pip \
     python3-wheel \
     openjdk-8-jdk \
-    supervisor
+    supervisor \
+    wget
 
-RUN wget http://apache.crihan.fr/dist/hadoop/common/hadoop-3.1.0/hadoop-3.1.0.tar.gz && \
-    tar -xzf hadoop-3.1.0.tar.gz && \
-    mv hadoop-3.1.0 $HADOOP_HOME && \
+RUN wget http://www-eu.apache.org/dist/hadoop/common/hadoop-3.1.1/hadoop-3.1.1.tar.gz && \
+    tar -xzf hadoop-3.1.1.tar.gz && \
+    mv hadoop-3.1.1 $HADOOP_HOME && \
     for user in hadoop hdfs yarn mapred; do \
         useradd -U -M -d /opt/hadoop/ --shell /bin/bash ${user}; \
     done && \
@@ -27,7 +28,7 @@ RUN wget http://apache.crihan.fr/dist/hadoop/common/hadoop-3.1.0/hadoop-3.1.0.ta
     echo "export YARN_NODEMANAGER_USER=root" >> $HADOOP_HOME/etc/hadoop/yarn-env.sh && \
     echo "PATH=$PATH:$HADOOP_HOME/bin" >> ~/.bashrc
 
-RUN \
+RUN apt-get install openssh-client -y && \
     ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa && \
     cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys && \
     chmod 0600 ~/.ssh/authorized_keys
@@ -36,14 +37,13 @@ ADD *xml $HADOOP_HOME/etc/hadoop/
 
 ADD ssh_config /root/.ssh/config
 
-ADD start-all.sh start-all.sh
-
-RUN pip3 install Flask
-RUN pip3 install werkzeug
-
-ENV LC_ALL C.UTF-8
-ENV LANG C.UTF-8
+RUN pip3 install Flask werkzeug pydoop==2.0a3
 
 WORKDIR /app
 
 COPY . /app
+
+RUN apt-get install ssh -y
+
+ENV LC_ALL C.UTF-8
+ENV LANG C.UTF-8

18.flask-hadoop/app.py

@@ -1,18 +1,43 @@
 from flask import Flask, request
 from werkzeug import secure_filename
 import os
+import pydoop.hdfs as hdfs
+import json
+
 app = Flask(__name__)
 app.config['UPLOAD_FOLDER'] = os.getcwd()
 
+print(hdfs.hdfs().list_directory('/user'))
+
 @app.route('/')
 def hello_world():
     return 'Hey, we have Flask in a Docker container!'
 
-@app.route('/file', methods = ['POST'])
-def members():
+@app.route('/test', methods = ['POST'])
+def test():
     f = request.files['file']
     f.save(os.path.join(app.config['UPLOAD_FOLDER'], secure_filename(f.filename)))
     return f.filename
 
+@app.route('/wordcount', methods = ['POST'])
+def wordcount():
+    f = request.files['file']
+    f.save(os.path.join(app.config['UPLOAD_FOLDER'], secure_filename(f.filename)))
+    with open(f.filename,'r') as fopen:
+        hdfs.dump(fopen.read(), '/user/input_wordcount/text')
+    os.system('pydoop script -c combiner wordcount.py /user/input_wordcount /user/output_wordcount')
+    list_files = hdfs.hdfs().list_directory('/user/output_wordcount')
+    return json.dumps([hdfs.load(file['name'], mode='rt') for file in list_files if 'SUCCESS' not in file['name']])
+
+@app.route('/lowercase', methods = ['POST'])
+def lowercase():
+    f = request.files['file']
+    f.save(os.path.join(app.config['UPLOAD_FOLDER'], secure_filename(f.filename)))
+    with open(f.filename,'r') as fopen:
+        hdfs.dump(fopen.read(), '/user/input_lowercase/text')
+    os.system("pydoop script --num-reducers 0 -t '' lowercase.py /user/input_lowercase /user/output_lowercase")
+    list_files = hdfs.hdfs().list_directory('/user/output_lowercase')
+    return json.dumps([hdfs.load(file['name'], mode='rt') for file in list_files if 'SUCCESS' not in file['name']])
+
 if __name__ == '__main__':
     app.run(debug=True, host='0.0.0.0',port=5000)

18.flask-hadoop/compose/bash

@@ -1,4 +1,3 @@
 #!/bin/sh
 
-docker-compose -f $(dirname "$0")/docker-compose.yml exec processing bash
-
+docker-compose -f $(dirname "$0")/docker-compose.yml exec flask-hadoop bash

18.flask-hadoop/compose/develop

@@ -1,3 +1,4 @@
 #!/bin/sh
 
 bash start-all.sh
+python3 app.py

18.flask-hadoop/compose/docker-compose.yml

@@ -1,15 +1,16 @@
 version: '2.3'
 services:
-  hadoop:
+  flask-hadoop:
     build:
       context: ..
       target: base
     command: compose/develop
     environment:
       PYTHONPATH: .
     ports:
-      - '5000:5000' # flask
+      - '5000:5000'
       - '8088:8088'
+      - '9000:9000'
       - '9870:9870'
       - '9864:9864'
       - '19888:19888'

18.flask-hadoop/lowercase.py

@@ -0,0 +1,2 @@
+def mapper(_, record, writer):
+    writer.emit("", record.lower())

18.flask-hadoop/screenshot1.png

@@ -2,11 +2,12 @@
 
 /etc/init.d/ssh start
 
-# START HADOOP
-##############
-
 $HADOOP_HOME/bin/hdfs namenode -format
 
-$HADOOP_HOME/sbin/start-dfs.sh
 $HADOOP_HOME/sbin/start-yarn.sh
-$HADOOP_HOME/sbin/mr-jobhistory-daemon.sh start historyserver
+$HADOOP_HOME/sbin/start-dfs.sh
+
+$HADOOP_HOME/bin/hdfs dfs -mkdir /user
+$HADOOP_HOME/bin/hdfs dfs -mkdir /user/input_wordcount
+$HADOOP_HOME/bin/hdfs dfs -mkdir /user/input_lowercase
+$HADOOP_HOME/bin/hdfs dfs -put $HADOOP_HOME/etc/hadoop/*.xml /user