ML all the way

api_endpoints.py

@@ -2,15 +2,14 @@
 import json
 from time import time
 from urllib.parse import urlparse
-
+from requests import post
 import firebase_admin
+from dateutil.parser import parse
 from firebase_admin import credentials
 from firebase_admin import db
 from requests import post
 from tornado.web import RequestHandler
 
-from dateutil.parser import parse
-
 cred = credentials.Certificate('ids-hackathor-636a3e9f4e4c.json')
 firebase_admin.initialize_app(cred, {
     'databaseURL': 'https://ids-hackathor.firebaseio.com/'
@@ -69,19 +68,17 @@ async def get(self):
 
     async def post(self):
         print("Entered post")
-        params = json.loads(self.request.body)
         ip = self.request.remote_ip
+        params = json.loads(self.request.body)
+        response = post("http://localhost:5000/hello/hikmet", json=params)
+        response_val = json.loads(response.text)
+        for i in response_val:
+            print(i)
+            send_ref(ip, i['param'], i['val'], i['type'])
+            self.report({i['type']: {"ip": ip, "param": i['param'], "val": i['val'],
+                                     "uid": 99, "confidence": i['confidence']}})
+
         for param, val in params.items():
-            # XSS
-            for pload in XSS:
-                if pload in val:
-                    send_ref(ip, param, val, 'XSS')
-                    self.report({"XSS": {"ip": ip, "param": param, "val": val, "uid": 99}})
-                    break
-            # SQLi
-            if "'" in val and ('and' in val.lower() or 'or' in val.lower()) or '--' in val:
-                send_ref(ip, param, val, 'SQLi')
-                self.report({"SQLi": {"ip": ip, "param": param, "val": val, "uid": 99}})
 
             # CRLF
             if '%0d' in val.lower() or '%0a' in val.lower():
@@ -94,13 +91,6 @@ async def post(self):
                 send_ref(ip, param, val, 'Open Redirect')
                 self.report({"Redirect": {"ip": ip, "param": param, "val": val, "uid": 99}})
 
-            # Path Traversal
-            for pload in TRAVERS:
-                if pload in val:
-                    send_ref(ip, param, val, 'Path Traversal')
-                    self.report({"Traversal": {"ip": ip, "param": param, "val": val, "uid": 99}})
-                    break
-
         self.write({"Result": "200 Success"})
 
 

ml_model.py

@@ -0,0 +1,120 @@
+from bottle import route, run, template,post,request
+import matplotlib.pyplot as plt
+import nltk
+import pandas as pd
+import scipy
+from nltk.util import ngrams
+from scipy.sparse import coo_matrix, hstack, vstack
+from sklearn.datasets import load_iris
+from sklearn.decomposition import TruncatedSVD
+from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import f1_score
+from sklearn.model_selection import train_test_split
+import os
+import pickle
+import random
+import json
+import string
+
+def get1Grams(payload_obj):
+    '''Divides a string into 1-grams
+    
+    Example: input - payload: "<script>"
+             output- ["<","s","c","r","i","p","t",">"]
+    '''
+    payload = str(payload_obj)
+    ngrams = []
+    for i in range(0,len(payload)-1):
+        ngrams.append(payload[i:i+1])
+    return ngrams
+
+def get2Grams(payload_obj):
+    '''Divides a string into 2-grams
+    
+    Example: input - payload: "<script>"
+             output- ["<s","sc","cr","ri","ip","pt","t>"]
+    '''
+    payload = str(payload_obj)
+    ngrams = []
+    for i in range(0,len(payload)-2):
+        ngrams.append(payload[i:i+2])
+    return ngrams
+
+def get3Grams(payload_obj):
+    '''Divides a string into 3-grams
+    
+    Example: input - payload: "<script>"
+             output- ["<sc","scr","cri","rip","ipt","pt>"]
+    '''
+    payload = str(payload_obj)
+    ngrams = []
+    for i in range(0,len(payload)-3):
+        ngrams.append(payload[i:i+3])
+    return ngrams
+
+
+
+@post('/hello/<name>')
+def index(name):
+    data_params = request.json
+    print(data_params)
+    #postdata =  name = request.forms.get("query_val")
+    return_value = run_model(data_params)
+    print("return value ",return_value)
+    return str(return_value)
+
+    return "1"
+def run_model(param):
+    list_of_results = []
+    check = 0
+    for key in param:
+        content = param[key]
+        q_val = content
+        type_attack = None
+        q_key = key
+        df = pd.DataFrame([[content]],columns=['content'])
+        df['content'] = df['content'].str.strip('\n')
+        df['content'] = df['content'].str.lower()
+        X1C = count_vectorizer_1grams.transform(df["content"])
+        X2C = count_vectorizer_2grams.transform(df["content"])
+        X3C = count_vectorizer_3grams.transform(df["content"])
+        X1T = tfidf_vectorizer_1grams.transform(df["content"])
+        X2T = tfidf_vectorizer_2grams.transform(df["content"])
+        X3T = tfidf_vectorizer_3grams.transform(df["content"])
+        X = hstack([X1C,X2C,X3C,X1T,X2T,X3T])
+        predicted = logistic_model.predict(X)
+        predicted_sql = sql_model.predict_proba(X)[0][1]
+        predicted_traverse = traverse_model.predict_proba(X)[0][1]
+        predicted_xss = xss_model.predict_proba(X)[0][1]
+        max_value = max(predicted_sql,predicted_traverse,predicted_xss)
+        if max_value >= 0.5:
+            if max_value == predicted_sql:
+                type_attack = "SQLi"
+            elif max_value == predicted_traverse:
+                type_attack = "Path Traversal"
+            elif max_value == predicted_xss:
+                type_attack = "XSS"
+            dicc = {
+                    "type":type_attack,
+                    "param": q_key,
+                    "val": q_val,
+                    "confidence": max_value
+            }
+            list_of_results.append(dicc)
+    return json.dumps(list_of_results)
+
+if __name__ == '__main__':
+    count_vectorizer_1grams = pickle.load(open("c1g.vec","rb"))
+    count_vectorizer_2grams = pickle.load(open("c2g.vec","rb"))
+    count_vectorizer_3grams = pickle.load(open("c3g.vec","rb"))
+    tfidf_vectorizer_1grams = pickle.load(open("t1g.vec","rb"))
+    tfidf_vectorizer_2grams = pickle.load(open("t2g.vec","rb"))
+    tfidf_vectorizer_3grams = pickle.load(open("t3g.vec","rb"))
+    logistic_model = pickle.load(open("logistic.model","rb"))
+    sql_model = pickle.load(open("sql.model","rb"))
+    traverse_model = pickle.load(open("traverse.model","rb"))
+    xss_model = pickle.load(open("xss.model","rb"))
+
+    print("valalal")
+    run(host = 'localhost', port=5000,reloader=True)

requirements.txt

@@ -1,9 +1,11 @@
 aniso8601==8.0.0
+bottle==0.12.17
 CacheControl==0.12.5
 cachetools==3.1.1
 certifi==2019.11.28
 chardet==3.0.4
 Click==7.0
+cycler==0.10.0
 enum34==1.1.6
 firebase-admin==3.2.0
 Flask==1.1.1
@@ -25,21 +27,31 @@ httplib2==0.14.0
 idna==2.8
 itsdangerous==1.1.0
 Jinja2==2.10.3
+joblib==0.14.0
+kiwisolver==1.1.0
 MarkupSafe==1.1.1
+matplotlib==3.1.2
 msgpack==0.6.2
+nltk==3.4.5
+numpy==1.17.4
+pandas==0.25.3
 protobuf==3.11.0
 pyasn1==0.4.8
 pyasn1-modules==0.2.7
+pycryptodome==3.8.2
+pyparsing==2.4.5
+python-dateutil==2.8.1
 python-engineio==3.10.0
 python-socketio==4.4.0
 pytz==2019.3
 requests==2.22.0
 rsa==4.0
+scikit-learn==0.21.3
+scipy==1.3.3
 six==1.13.0
+sklearn==0.0
+tornado==6.0.3
 uritemplate==3.0.0
 urllib3==1.25.7
+websockets==7.0
 Werkzeug==0.16.0
-pycryptodome == 3.8.2
-tornado == 6.0.3
-websockets == 7.0
-