Merge pull request #77 from du201/master

Add support for HTML file to extract-regexes.pl

Author: davisjam

configure

@@ -39,7 +39,7 @@ if ($INSTALL_PKG_DEPS) {
   my @rxxr2Packages_ubuntu = ("ocaml");
   my @wustholzPackages_ubuntu = ("default-jdk");
   my @shenPackages_ubuntu = ("maven");
-  my @dynamicAnalysisPackages_ubuntu = ("nodejs", "php-cli", "ruby", "cargo", "golang-go");
+  my @dynamicAnalysisPackages_ubuntu = ("nodejs", "php-cli", "ruby", "cargo", "golang-go", "python3-bs4");
   my @requiredPackages_ubuntu = (@miscPackages_ubuntu, @rxxr2Packages_ubuntu, @wustholzPackages_ubuntu, @shenPackages_ubuntu, @dynamicAnalysisPackages_ubuntu);
 
   &log("Installing dependencies");
@@ -90,7 +90,6 @@ sub configureDetectors {
   &configureWustholz();
   &configureWeideman();
   &configureShen();
-
   chdir "$ENV{VULN_REGEX_DETECTOR_ROOT}" or die "Error, chdir failed: $!\n";
   return;
 }

src/extract/extract-regexes.pl

@@ -28,6 +28,7 @@
 my %language2extractor = (
   "javascript" => "$pref/javascript/extract-regexps.js",
   "python"     => "$pref/python/python-extract-regexps-wrapper.pl",
+  "html"       => "$pref/html/extract-regexps-html.py"   
 );
 
 for my $lang (keys %language2extractor) {
@@ -64,10 +65,13 @@
 }
 
 # Invoke the appropriate extractor.
+# If HTML file, extract the js part to create a new js file and pipeline it into the js extractor
+
 my $extractor = $language2extractor{$language};
 if ($extractor and -x $extractor) {
   print STDERR "$extractor '$json->{file}'\n";
   my $result = decode_json(`$extractor '$json->{file}' 2>/dev/null`);
+
   # Add the language to the output to simplify pipelining.
   $result->{language} = $language;
   print STDOUT encode_json($result) . "\n";
@@ -95,6 +99,7 @@ sub determineLanguage {
 
   # Check the 'file' command's guess.
   my ($rc, $out) = &cmd("file $file");
+
   #print "rc $rc out $out\n";
   if ($rc eq 0) {
     if ($out =~ m/(\s|\/)node(js)?\s/i) {
@@ -103,6 +108,9 @@ sub determineLanguage {
     elsif ($out =~ m/\sPython\s/i) {
       $language = "python";
     }
+    elsif ($out =~ m/\sHTML\s/i) {
+      $language = "html";
+    }
   }
   # Did it work?
   if ($language ne $UNKNOWN_LANGUAGE) {
@@ -112,6 +120,7 @@ sub determineLanguage {
   return $language;
 }
 
+
 sub extension2language {
   my ($ext) = @_;
 
@@ -122,6 +131,9 @@ sub extension2language {
   elsif (lc $ext eq "py") {
     $language = "python";
   }
+  elsif (lc $ext eq "html") {
+    $language = "html";
+  }
 
   return $language;
 }
@@ -132,4 +144,4 @@ sub cmd {
   my $rc = $? >> 8;
 
   return ($rc, $out);
-}
+}

src/extract/src/html/extract-regexps-html.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+# Description: This file takes in a html file from extract-regexes.pl, finds all the script 
+# tags, and combine the JS in them into a temporary js file. It then sends the path of the
+# temporary js file back to extract-regexes.pl to let it pipeline the js file to the javascript
+# extractor. After extract-regexes.pl finishes extracting, The temporary JS file will be 
+# deleted by extract-regexes.pl. 
+
+from bs4 import BeautifulSoup
+import sys
+import subprocess
+import json
+import tempfile
+import os
+
+def extract_js(file_path):
+    with open(file_path) as fp:
+        soup = BeautifulSoup(fp, 'html.parser')
+
+    js_from_html = ''
+    for script in soup.find_all('script'):
+        js_from_html += script.string
+
+    return js_from_html
+
+def extract_regexes(js_from_html, file_path):
+    js_tempfile = tempfile.NamedTemporaryFile(suffix='.js', mode='w+t', delete = False)
+    js_tempfile.writelines(js_from_html)
+    js_tempfile.close()
+
+    # create temp json file to pass to the meta-program
+    json_tempfile = tempfile.NamedTemporaryFile(suffix='.json', mode='w+t', delete = False)
+    json_tempfile.writelines(json.dumps({"file": js_tempfile.name, "language": "javascript"}))
+    json_tempfile.close()
+
+    output = subprocess.run(
+        [os.path.join(os.environ['VULN_REGEX_DETECTOR_ROOT'], 'src/extract/extract-regexes.pl'), 
+            json_tempfile.name], 
+        capture_output=True, text=True)
+
+    # delete the temp js and json file
+    os.remove(js_tempfile.name)
+    os.remove(json_tempfile.name)
+    
+    output_json = json.loads(output.stdout)
+    output_json['file'] = file_path
+    return json.dumps(output_json)
+
+file_path = sys.argv[1]
+js_from_html = extract_js(file_path)
+
+# call the meta-program 
+print(extract_regexes(js_from_html, file_path), end = '')
+
+

src/extract/test/html/jsonFile.json

@@ -0,0 +1 @@
+{"file": "./test/html/t.html", "language": "html"}

src/extract/test/html/t.html

@@ -0,0 +1,25 @@
+<!DOCTYPE html>
+<html>
+<header>
+    <script>
+        let script_var = 5;
+        console.log('test');
+        'abc'.match(/def/);
+        new RegExp('aaa');
+    </script>
+
+    <script>
+        var re = /abcsdxxx/;
+    </script>
+
+</header>
+
+<body>
+    <h1>My First Heading</h1>
+    <p>My first paragraph.</p>
+    <script>
+        var re = /abcsdsdfdf/;
+    </script>
+</body>
+
+</html>

src/extract/test/javascript/jsonFile.json

@@ -1 +1 @@
-{"file":"test/js/t.js"}
+{"file": "./test/javascript/t.js", "language": "javascript"}

src/extract/test/javascript/t.js

@@ -3,3 +3,6 @@
 var re = /abc/;
 'abc'.match(/def/);
 new RegExp('aaa');
+
+var re_string = '\\w+';
+new RegExp(re_string);