首次提交

.gitignore

@@ -0,0 +1,2 @@
+/.idea/
+/venv/

README.md

@@ -1,2 +1,40 @@
 # bigdata_practice
-大数据分析可视化实践
+![](https://img.shields.io/badge/hive-1.1-green)
+![](https://img.shields.io/badge/python-3.6%2B-brightgreen)
+![](https://img.shields.io/badge/flask-1.1%2B-orange)
+![](https://img.shields.io/badge/echarts-4.7-yellowgreen)
+
+大数据实践项目 - nginx 日志分析
+
+## 功能说明
+
+通过 hive 分析 nginx 日志，将分析结果通过 flask + echarts 进行可视化展示
+
+## 数据收集分析过程
+
+[数据收集分析过程](https://github.com/TurboWay/bigdata_practice/blob/master/process.md)
+
+
+## 运行
+
+运行 cd ironman; python app.py
+
+打开 http://127.0.0.1:5000/
+
+
+## 效果图
+
+### 24 小时访问趋势
+![image](https://github.com/TurboWay/bigdata_practice/blob/master/example/e1.jpg)
+
+### 每日访问情况
+![image](https://github.com/TurboWay/bigdata_practice/blob/master/example/e2.jpg)
+
+### 客户端设备占比
+![image](https://github.com/TurboWay/bigdata_practice/blob/master/example/e3.jpg)
+
+### 用户分布
+![image](https://github.com/TurboWay/bigdata_practice/blob/master/example/e4.jpg)
+
+### 爬虫词云
+![image](https://github.com/TurboWay/bigdata_practice/blob/master/example/e5.jpg)

ironman/app.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# @Time : 2020/8/26 14:48
+# @Author : way
+# @Site :
+# @Describe:
+
+from flask import Flask, render_template
+from ironman.data import SourceData
+
+app = Flask(__name__)
+
+source = SourceData()
+
+
+@app.route('/')
+def index():
+    return render_template('index.html')
+
+
+@app.route('/line')
+def line():
+    data = source.line
+    xAxis = data.pop('legend')
+    return render_template('line.html', title='24小时访问趋势', data=data, legend=list(data.keys()), xAxis=xAxis)
+
+
+@app.route('/bar')
+def bar():
+    data = source.bar
+    xAxis = data.pop('legend')
+    return render_template('bar.html', title='每日访问情况', data=data, legend=list(data.keys()), xAxis=xAxis)
+
+@app.route('/pie')
+def pie():
+    data = source.pie
+    return render_template('pie.html', title='客户端设备占比', data=data, legend=[i.get('name') for i in data])
+
+@app.route('/china')
+def china():
+    data = source.china
+    return render_template('china.html', title='用户分布', data=data)
+
+
+@app.route('/wordcloud')
+def wordcloud():
+    data = source.wordcloud
+    return render_template('wordcloud.html', title='爬虫词云', data=data)
+
+
+if __name__ == "__main__":
+    app.run(host='127.0.0.1', debug=True)

ironman/data.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# @Time : 2020/8/26 14:48
+# @Author : way
+# @Site : 
+# @Describe:
+
+from ironman.nginx_log_data import province_data, \
+    line_legend, line_normal, line_spider, \
+    bar_legend, bar_normal, bar_spider, \
+    client_data, spider_data
+
+class SourceData:
+
+    @property
+    def china(self):
+        return province_data
+
+    @property
+    def line(self):
+        data = {
+            '正常访问量': line_normal,
+            '爬虫访问量': line_spider,
+            'legend': line_legend
+        }
+        return data
+
+    @property
+    def bar(self):
+        data = {
+            '正常访问量': bar_normal,
+            '爬虫访问量': bar_spider,
+            'legend': bar_legend
+        }
+        return data
+
+    @property
+    def pie(self):
+        return client_data
+
+    @property
+    def wordcloud(self):
+        return spider_data

ironman/nginx_log_data.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# @Time : 2020/8/27 19:32
+# @Author : way
+# @Site : 
+# @Describe:
+
+import pandas as pd
+
+filepath = "分析结果.xlsx"
+
+df2 = pd.read_excel(filepath, sheet_name='省份', header=None)
+
+province_data = [{'name': row[0].strip(), 'value': row[1]} for row in df2.values]
+# print(province_data)
+
+df3 = pd.read_excel(filepath, sheet_name='按时', header=None)
+line_legend = [row[1] for row in df3.values if row[0]=='Normal']
+line_normal = [row[2] for row in df3.values if row[0]=='Normal']
+line_spider = [row[2] for row in df3.values if row[0]=='Spider']
+# print(line_legend)
+# print(line_normal)
+# print(line_spider)
+
+df3 = pd.read_excel(filepath, sheet_name='按天', header=None)
+bar_legend = [row[1].strftime('%Y%m%d') for row in df3.values if row[0]=='Normal']
+bar_normal = [row[2] for row in df3.values if row[0]=='Normal']
+bar_spider = [row[2] for row in df3.values if row[0]=='Spider']
+# print(bar_legend)
+# print(bar_normal)
+# print(bar_spider)
+
+df3 = pd.read_excel(filepath, sheet_name='客户端', header=None)
+client_data = [{'name': row[0].strip(), 'value': row[1]} for row in df3.values]
+# print(client_data)
+
+df3 = pd.read_excel(filepath, sheet_name='爬虫', header=None)
+spider_data = [{'name': row[0].strip(), 'value': row[1]} for row in df3.values]
+# print(spider_data)

ironman/static/css/barrager.css

@@ -0,0 +1,12 @@
+.barrage{position: fixed;bottom:70px;right:-500px;display: inline-block;width: 500px;z-index: 99999}
+.barrage_box{background-color: rgba(0,0,0,.5);padding-right: 8px; height: 40px;display: inline-block;border-radius: 25px;transition: all .3s;}
+.barrage_box .portrait{ display: inline-block;margin-top: 4px; margin-left: 4px; width: 32px;height: 32px;border-radius: 50%;overflow: hidden;}
+.barrage_box .portrait img{width: 100%;height: 100%;}
+.barrage_box div.p a{ margin-right: 2px; font-size: 14px;color: #fff;line-height: 40px;margin-left: 18px; }
+.barrage_box div.p a:hover{text-decoration: underline;}
+.barrage_box .close{visibility: hidden;opacity: 0; text-align: center; width:25px;height: 25px;margin-left: 20px;border-radius: 50%;background:rgba(255,255,255,.1);margin-top:8px; background-image: url(close.png);}
+.barrage_box:hover .close{visibility:visible;opacity: 1;}
+.barrage_box .close a{display:block;}
+.barrage_box .close .icon-close{font-size: 14px;color:rgba(255,255,255,.5);display: inline-block;margin-top: 5px; }
+.barrage .z {float: left !important;}
+.barrage  a{text-decoration:none;}

ironman/static/css/dashboard.css

@@ -0,0 +1,105 @@
+/*
+ * Base structure
+ */
+
+/* Move down content because we have a fixed navbar that is 50px tall */
+body {
+  padding-top: 50px;
+}
+
+
+/*
+ * Global add-ons
+ */
+
+.sub-header {
+  padding-bottom: 10px;
+  border-bottom: 1px solid #eee;
+}
+
+/*
+ * Top navigation
+ * Hide default border to remove 1px line.
+ */
+.navbar-fixed-top {
+  border: 0;
+}
+
+/*
+ * Sidebar
+ */
+
+/* Hide for mobile, show later */
+.sidebar {
+  display: none;
+}
+@media (min-width: 768px) {
+  .sidebar {
+    position: fixed;
+    top: 51px;
+    bottom: 0;
+    left: 0;
+    z-index: 1000;
+    display: block;
+    padding: 20px;
+    overflow-x: hidden;
+    overflow-y: auto; /* Scrollable contents if viewport is shorter than content. */
+    background-color: #f5f5f5;
+    border-right: 1px solid #eee;
+  }
+}
+
+/* Sidebar navigation */
+.nav-sidebar {
+  margin-right: -21px; /* 20px padding + 1px border */
+  margin-bottom: 20px;
+  margin-left: -20px;
+}
+.nav-sidebar > li > a {
+  padding-right: 20px;
+  padding-left: 20px;
+}
+.nav-sidebar > .active > a,
+.nav-sidebar > .active > a:hover,
+.nav-sidebar > .active > a:focus {
+  color: #fff;
+  background-color: #428bca;
+}
+
+
+/*
+ * Main content
+ */
+
+.main {
+  padding: 20px;
+}
+@media (min-width: 768px) {
+  .main {
+    padding-right: 40px;
+    padding-left: 40px;
+  }
+}
+.main .page-header {
+  margin-top: 0;
+}
+
+
+/*
+ * Placeholder dashboard ideas
+ */
+
+.placeholders {
+  margin-bottom: 30px;
+  text-align: center;
+}
+.placeholders h4 {
+  margin-bottom: 0;
+}
+.placeholder {
+  margin-bottom: 20px;
+}
+.placeholder img {
+  display: inline-block;
+  border-radius: 50%;
+}