refactor(app): refactor dictionary

app/common/__init__.py

@@ -0,0 +1 @@
+__all__ = ["common", "errorcode"]

app/config/common.py → app/common/common.py

@@ -8,6 +8,10 @@
 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
 """
 
+import traceback
+from functools import wraps
+from errorcode import *
+
 DEFAULT_WINDOW = 180
 INPUT_LEN_ENG_MAX = 32
 INPUT_LEN_CH_MAX = 64
@@ -17,3 +21,20 @@
 UPLOAD_FILE = '/tmp/tmpfile_%s.csv'
 MARK_POSITIVE = 1
 MARK_NEGATIVE = 2
+
+
+def build_ret_data(ret_code, data=""):
+    return {"code": ret_code, "msg": ERR_CODE[ret_code], "data": data}
+
+
+def exce_service(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        try:
+            ret_code, ret_data = func(*args, **kwargs)
+            return_dict = build_ret_data(ret_code, ret_data)
+        except Exception as ex:
+            traceback.print_exc()
+            return_dict = build_ret_data(THROW_EXP, str(ex))
+        return return_dict
+    return wrapper

app/config/errorcode.py → app/common/errorcode.py

@@ -32,4 +32,4 @@
     READ_FEATURE_FAILED: "读取特征数据失败",
     TRAIN_ERR: "训练出错",
     LACK_SAMPLE: "缺少正样本或负样本"
-}
+}

app/controller/api/views.py

@@ -1,16 +1,16 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 import json
+from functools import wraps
 from django.shortcuts import render
 from django.http import FileResponse
-from common.render import render_json
-from functools import wraps
+from render import render_json
 from app.service.time_series_detector.anomaly_service import *
 from app.service.time_series_detector.sample_service import *
 from app.service.time_series_detector.task_service import *
 from app.service.time_series_detector.detect_service import *
-from app.config.errorcode import *
-from app.utils.utils import *
+from app.common.errorcode import *
+from app.common.common import *
 
 
 def check_post(func):

app/dao/__init__.py

@@ -1 +1 @@
-__all__ = ["time_series_detector"]
+__all__ = ["db_common", "time_series_detector"]

app/dao/db_common/__init__.py

@@ -0,0 +1 @@
+__all__ = ["database"]

app/dao/time_series_detector/__init__.py

@@ -1 +1 @@
-__all__ = ["anomaly_op", "sample_op", "train_op"]
+__all__ = ["anomaly_op", "sample_op", "train_op"]

app/dao/time_series_detector/anomaly_op.py

@@ -8,13 +8,11 @@
 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
 """
 
-import time
-import datetime
 import MySQLdb
-from app.config import database
+from app.dao.db_common import database
 from app.dao.time_series_detector.sample_op import *
-from app.config.common import *
-from app.config.errorcode import *
+from app.common.common import *
+from app.common.errorcode import *
 
 
 class AbnormalOperation(object):

app/dao/time_series_detector/sample_op.py

@@ -13,9 +13,9 @@
 import csv
 import codecs
 import MySQLdb
-from app.config import database
-from app.config.common import *
-from app.config.errorcode import *
+from app.dao.db_common import database
+from app.common.common import *
+from app.common.errorcode import *
 
 
 class SampleOperation(object):

app/dao/time_series_detector/train_op.py

@@ -9,9 +9,9 @@
 """
 
 import MySQLdb
-from app.config import database
-from app.config.common import *
-from app.config.errorcode import *
+from app.dao.db_common import database
+from app.common.common import *
+from app.common.errorcode import *
 
 
 class TrainOperation(object):

app/service/__init__.py

@@ -1 +1 @@
-__all__ = ["time_series_detector"]
+__all__ = ["time_series_detector"]

app/service/time_series_detector/__init__.py

@@ -1 +1 @@
-__all__ = ["algorithm", "feature", "anomaly_service", "sample_service", "task_service", "detect_service"]
+__all__ = ["anomaly_service", "sample_service", "task_service", "detect_service"]

app/service/time_series_detector/anomaly_service.py

@@ -10,7 +10,6 @@
 
 import json
 from app.dao.time_series_detector.anomaly_op import *
-from app.utils.utils import *
 
 
 class AnomalyService(object):

app/service/time_series_detector/detect_service.py

@@ -14,23 +14,20 @@
 from app.dao.time_series_detector import anomaly_op
 from app.dao.time_series_detector import sample_op
 from app.dao.time_series_detector import train_op
-from app.utils.utils import *
-from app.service.time_series_detector.algorithm import isolation_forest, ewma, polynomial_interpolation, statistic, xgboosting
-from app.config.errorcode import *
-from app.config.common import *
-MODEL_PATH = os.path.join(os.path.dirname(__file__), '../../model/time_series_detector/')
+from time_series_detector.algorithm import xgboosting
+from time_series_detector import detect
+from app.common.errorcode import *
+from app.common.common import *
+from time_series_detector.common.tsd_errorcode import *
+MODEL_PATH = os.path.join(os.path.dirname(__file__), './model/')
 
 
 class DetectService(object):
 
     def __init__(self):
         self.sample_op_obj = sample_op.SampleOperation()
         self.anomaly_op_obj = anomaly_op.AbnormalOperation()
-        self.iforest_obj = isolation_forest.IForest()
-        self.ewma_obj = ewma.Ewma()
-        self.polynomial_obj = polynomial_interpolation.PolynomialInterpolation()
-        self.statistic_obj = statistic.Statistic()
-        self.supervised_obj = xgboosting.XGBoosting()
+        self.detect_obj = detect.Detect()
 
     def __generate_model(self, data, task_id):
         """
@@ -125,62 +122,14 @@ def __list_is_digit(self, data):
     def __check_param(self, data):
         if ("viewName" not in data.keys()) or ("attrId" not in data.keys()) or ("attrName" not in data.keys()) or ("time" not in data.keys()) or ("dataC" not in data.keys()) or ("dataB" not in data.keys()) or ("dataA" not in data.keys()):
             return CHECK_PARAM_FAILED, "missing parameter"
-        if not data['dataA']:
-            return CHECK_PARAM_FAILED, "dataA can not be empty"
-        if not data['dataB']:
-            return CHECK_PARAM_FAILED, "dataB can not be empty"
-        if not data['dataC']:
-            return CHECK_PARAM_FAILED, "dataC can not be empty"
-        if not self.__list_is_digit(data['dataA'].split(',')):
-            return CHECK_PARAM_FAILED, "dataA contains illegal numbers"
-        if not self.__list_is_digit(data['dataB'].split(',')):
-            return CHECK_PARAM_FAILED, "dataB contains illegal numbers"
-        if not self.__list_is_digit(data['dataC'].split(',')):
-            return CHECK_PARAM_FAILED, "dataC contains illegal numbers"
-        if "window" in data:
-            window = data["window"]
-        else:
-            window = DEFAULT_WINDOW
-        if len(data['dataC'].split(',')) != (2 * window + 1):
-            return CHECK_PARAM_FAILED, "dataC length does not match"
-        if len(data['dataB'].split(',')) != (2 * window + 1):
-            return CHECK_PARAM_FAILED, "dataB length does not match"
-        if len(data['dataA'].split(',')) != (window + 1):
-            return CHECK_PARAM_FAILED, "dataA length does not match"
         return OP_SUCCESS, ""
 
     def value_predict(self, data):
-        """
-        Predict the data
-
-        :param data: the time series to detect of
-        """
         ret_code, ret_data = self.__check_param(data)
         if ret_code != OP_SUCCESS:
             return build_ret_data(ret_code, ret_data)
-        if "taskId" in data and data["taskId"]:
-            model_name = MODEL_PATH + data["taskId"] + "_model"
-        else:
-            model_name = MODEL_PATH + "xgb_default_model"
-        combined_data = data["dataC"] + "," + data["dataB"] + "," + data["dataA"]
-        time_series = map(int, combined_data.split(','))
-        if "window" in data:
-            window = data["window"]
-        else:
-            window = DEFAULT_WINDOW
-        statistic_result = self.statistic_obj.predict(time_series)
-        ewma_result = self.ewma_obj.predict(time_series)
-        polynomial_result = self.polynomial_obj.predict(time_series, window)
-        iforest_result = self.iforest_obj.predict(time_series, window)
-        if statistic_result == 0 or ewma_result == 0 or polynomial_result == 0 or iforest_result == 0:
-            xgb_result = self.supervised_obj.predict(time_series, window, model_name)
-            res_value = xgb_result[0]
-            prob = xgb_result[1]
-        else:
-            res_value = 1
-            prob = 1
-        ret_data = {"ret": res_value, "p": str(prob)}
-        if ret_data["ret"] == 0:
+        ret_code, ret_data = self.detect_obj.value_predict(data)
+        if ret_code == TSD_OP_SUCCESS and ret_data["ret"] == 0:
             anomaly_params = {
                 "view_id": data["viewId"],
                 "view_name": data["viewName"],
@@ -192,18 +141,14 @@ def value_predict(self, data):
                 "data_a": data["dataA"]
             }
             self.anomaly_op_obj.insert_anomaly(anomaly_params)
-        return build_ret_data(OP_SUCCESS, ret_data)
+        return build_ret_data(ret_code, ret_data)
 
     def rate_predict(self, data):
-        combined_data = data["dataC"] + "," + data["dataB"] + "," + data["dataA"]
-        time_series = map(float, combined_data.split(','))
-        statistic_result = self.statistic_obj.predict(time_series)
-        if statistic_result == 0:
-            prob = 0
-        else:
-            prob = 1
-        ret_data = {"ret": statistic_result, "p": str(prob)}
-        if ret_data["ret"] == 0:
+        ret_code, ret_data = self.__check_param(data)
+        if ret_code != OP_SUCCESS:
+            return build_ret_data(ret_code, ret_data)
+        ret_data, ret_data = self.detect_obj.rate_predict(data)
+        if ret_code == TSD_OP_SUCCESS and ret_data["ret"] == 0:
             anomaly_params = {
                 "view_id": data["viewId"],
                 "view_name": data["viewName"],

app/service/time_series_detector/sample_service.py

@@ -12,9 +12,8 @@
 import traceback
 import csv
 from app.dao.time_series_detector.sample_op import *
-from app.config.errorcode import *
-from app.utils.utils import *
-from app.config.common import *
+from app.common.errorcode import *
+from app.common.common import *
 
 
 class SampleService(object):

app/service/time_series_detector/task_service.py

@@ -10,8 +10,8 @@
 
 import json
 from app.dao.time_series_detector.train_op import *
-from app.config.errorcode import *
-from app.utils.utils import *
+from app.common.errorcode import *
+from app.common.common import *
 
 
 class TrainService(object):

docs/install.md

@@ -137,7 +137,7 @@ python /data/Metis/app/controller/manage.py runserver {ip}:{port}
 
 运行npm run build
 
-将uweb目录下的custom文件夹下复制到uweb目录下生成的dist文件夹中
+将uweb目录下的custom文件夹复制到uweb目录下生成的dist文件夹中
 
 将nginx配置文件中的root定位到uweb目录下的dist文件夹
 

app/sql/time_series_detector/train_task.sql → sql/time_series_detector/train_task.sql

@@ -1,25 +1,25 @@
-SET FOREIGN_KEY_CHECKS=0;
--- ----------------------------
--- Table structure for `train_task`
--- ----------------------------
-DROP TABLE IF EXISTS `train_task`;
-CREATE TABLE `train_task` (
-  `id` int(11) NOT NULL AUTO_INCREMENT,
-  `task_id` char(255) DEFAULT NULL,
-  `sample_num` int(11) DEFAULT NULL,
-  `postive_sample_num` int(11) DEFAULT NULL,
-  `negative_sample_num` int(11) DEFAULT NULL,
-  `window` int(2) DEFAULT NULL,
-  `model_name` varchar(20) DEFAULT NULL,
-  `source` varchar(255) DEFAULT NULL,
-  `start_time` timestamp NULL DEFAULT NULL,
-  `end_time` timestamp NULL DEFAULT NULL,
-  `status` varchar(11) DEFAULT NULL,
-  PRIMARY KEY (`id`),
-  KEY `id` (`id`)
-) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
-
--- ----------------------------
--- Records of train_task
--- ----------------------------
-INSERT INTO `train_task` VALUES ('1', '1535790960079', '90675', '45228', '45447', '180', 'xgb_default_model', 'Metis', '2018-09-01 16:36:00', '2018-09-01 16:45:40', 'complete');
+SET FOREIGN_KEY_CHECKS=0;
+-- ----------------------------
+-- Table structure for `train_task`
+-- ----------------------------
+DROP TABLE IF EXISTS `train_task`;
+CREATE TABLE `train_task` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `task_id` char(255) DEFAULT NULL,
+  `sample_num` int(11) DEFAULT NULL,
+  `postive_sample_num` int(11) DEFAULT NULL,
+  `negative_sample_num` int(11) DEFAULT NULL,
+  `window` int(2) DEFAULT NULL,
+  `model_name` varchar(20) DEFAULT NULL,
+  `source` varchar(255) DEFAULT NULL,
+  `start_time` timestamp NULL DEFAULT NULL,
+  `end_time` timestamp NULL DEFAULT NULL,
+  `status` varchar(11) DEFAULT NULL,
+  PRIMARY KEY (`id`),
+  KEY `id` (`id`)
+) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
+
+-- ----------------------------
+-- Records of train_task
+-- ----------------------------
+INSERT INTO `train_task` VALUES ('1', '1535790960079', '90675', '45228', '45447', '180', 'xgb_default_model', 'Metis', '2018-09-01 16:36:00', '2018-09-01 16:45:40', 'complete');

tests/__init__.py

@@ -0,0 +1 @@
+__all__ = ["fixtures", "test_feature"]

tests/test_feature.py

@@ -9,8 +9,7 @@
 """
 
 from tests.fixtures import DataTestCase
-from app.service.time_series_detector.feature.statistical_features import *
-
+from time_series_detector.feature.statistical_features import *
 
 class FeatureTestCase(DataTestCase):
 

time_series_detector/__init__.py

@@ -0,0 +1 @@
+__all__ = ["algorithm", "feature", "common", "detect"]

app/service/time_series_detector/algorithm/ewma_and_polynomial.py → time_series_detector/algorithm/ewma_and_polynomial.py

@@ -8,9 +8,9 @@
 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
 """
 
-from app.service.time_series_detector.algorithm import ewma
-from app.service.time_series_detector.algorithm import polynomial_interpolation
-from app.config.common import *
+from time_series_detector.algorithm import ewma
+from time_series_detector.algorithm import polynomial_interpolation
+from time_series_detector.common.tsd_common import *
 
 
 class EwmaAndPolynomialInterpolation(object):