Enhanced classifier: RANDOM_STRING, UUID, NUMERIC, IP_ADDDRESS

Author: rscctest

stringlifier/api.py

@@ -141,7 +141,7 @@ def __call__(self, string_or_list, return_tokens=False):
         else:
             return new_strings
 
-    def _extract_tokens(self, string, pred):
+    def _extract_tokens_2class(self, string, pred):
         mask = ''
         for p in pred:
             mask += self.encodings._label_list[p]
@@ -168,3 +168,52 @@ def _extract_tokens(self, string, pred):
             tokens.append((c_tok, start, stop))
             new_str += '<RANDOM_STRING>'
         return new_str, tokens
+
+    def _extract_tokens(self, string, pred):
+        mask = ''
+        for p in pred:
+            mask += self.encodings._label_list[p]
+        start = 0
+        new_str = ''
+        tokens = []
+        c_tok = ''
+        last_label = mask[0]
+        for ii in range(len(string)):
+            # check if the label-type has changed
+            if last_label != mask[ii]:
+                if c_tok != '':
+                    if last_label == 'C':
+                        new_str += c_tok
+                    elif last_label == 'H':
+                        type = '<RANDOM_STRING>'
+                    elif last_label == 'N':
+                        type = '<NUMERIC>'
+                    elif last_label == 'I':
+                        type = '<IP_ADDR>'
+                    elif last_label == 'U':
+                        type = '<UUID>'
+
+                    if last_label != 'C':
+                        tokens.append((c_tok, start, ii, type))
+                        new_str += type
+                    c_tok = ''
+                start = ii
+
+            last_label = mask[ii]
+            c_tok += string[ii]
+
+        if c_tok != '':
+            if last_label == 'C':
+                new_str += c_tok
+            elif last_label == 'H':
+                type = '<RANDOM_STRING>'
+            elif last_label == 'N':
+                type = '<NUMERIC>'
+            elif last_label == 'I':
+                type = '<IP_ADDR>'
+            elif last_label == 'U':
+                type = '<UUID>'
+            if last_label != 'C':
+                tokens.append((c_tok, start, ii, type))
+                new_str += type
+        return new_str, tokens

stringlifier/modules/stringc2.py

@@ -239,10 +239,13 @@ def _start_train(params):
     if params.resume:
         encodings = Encodings('{0}.encodings'.format(params.output_base))
     else:
+        sys.stdout.write('Generating new random data...')
+        sys.stdout.flush()
         trainset = _generate_dataset(int(eval_at * 4 * params.batch_size))
-        devset = _generate_dataset(int(eval_at / 10 * params.batch_size))
+        sys.stdout.write('done\n')
         encodings = Encodings()
         encodings.update_encodings(trainset)
+
     print('chars={0}, types={1}'.format(len(encodings._char2int), len(encodings._label2int)))
     print(encodings._label2int)
 

stringlifier/modules/training.py

@@ -22,14 +22,26 @@ def _generate_word(known_words):
     import datetime
     import base64
     generated = []
-    ii = random.randint(0, 3)
+    ii = random.randint(0, 4)
     mask = 'H'
     if ii == 0:
         generated.append(str(uuid.uuid4()))
+        mask = 'U'
     elif ii == 1:
         generated.append(str(uuid.uuid4().hex))
+        mask = 'H'
     elif ii == 2:
-        generated.append(str(datetime.datetime.now().timestamp()))
+        c = random.randint(0, 3)
+        if c == 0:
+            generated.append(str(datetime.datetime.now().timestamp()))
+        elif c == 1:
+            generated.append(str(random.randint(0, 1000000)))
+        elif c == 2:
+            generated.append(str(random.randint(0, 999)) + '.' + str(random.randint(0, 999)))
+        else:
+            generated.append(str(random.randint(0, 999)) + '.' + str(random.randint(0, 9999)) + '.' +
+                             str(random.randint(0, 9999)))
+        mask = 'N'
     elif ii == 3:
         N = random.randint(5, 20)
         import string
@@ -40,6 +52,12 @@ def _generate_word(known_words):
         base64_bytes = base64.b64encode(message_bytes)
         base64_message = base64_bytes.decode('ascii')
         generated.append(base64_message)
+    elif ii == 4:
+        toks = []
+        for _ in range(4):
+            toks.append(str(random.randint(0, 255)))
+        generated.append('.'.join(toks))
+        mask = 'I'
     return str(generated[0]), mask[0]