Added MV3 parsing, updated extension permissions, and properly formatted code

README.md

@@ -54,4 +54,35 @@ Pulls any Chrome extension from a provided Chrome webstore link.
 * Download the original extension.
 * Download a beautified version of the extension (auto prettified HTML and JavaScript).
 * Automatic caching of scan results, running an extension scan will take a good amount of time the first time you run it. However the second time, assuming the extension hasn’t been updated, will be almost instant due to the results being cached.
-Linkable Report URLs, easily link someone else to an extension report generated by tarnish.
+Linkable Report URLs, easily link someone else to an extension report generated by tarnish.
+
+## Tarnish using local disk storage instead of S3.
+
+### Pre-requisite Steps:
+
+1.  Set host.docker.internal to 127.0.0.1 in /etc/hosts. I'd do this in a
+    separate line than localhost, otherwise it'll get overwritten.
+
+```bash
+127.0.0.1 localhost
+127.0.0.1 host.docker.internal
+```
+
+1.  Run an upload/download server on your host. The s3 client function is
+    configured to use the `multipart/form-data` upload format and has only
+    tested with [updog](https://github.com/sc0tfree/updog)
+    -   Run `python3 auto_tarnish <extension_id>` to setup folders and files in
+        the directory you are running updog on.
+    -   Update `server_dir` in docker-compose.yaml to the running directory of
+        this server. This wouldn't be needed with better upload/download server.
+2.  Run a local redis instance. (e.g. `docker run --name my-redis -p 6379:6379
+    -d redis`)
+3.  Ignore CORS errors in your browser. This can be done by
+    [chrome extension](https://addons.mozilla.org/en-US/firefox/addon/cors-everywhere/)
+    or cmdline flag (e.g. `google-chrome --disable-web-security`). (again, this
+    would be solved with a better upload/download server)
+
+To run:
+
+1.  Start your local server on port 8080
+2.  `./start.sh`

docker-compose.yaml.example

@@ -3,19 +3,26 @@ services:
     tarnishserver:
         build: ./tarnish-server/
         environment:
-            - redis_backend=redis://REDIS_HOSTNAME:6379/0
+            - redis_backend=redis://host.docker.internal:6379/0
             - aws_secret_key=REPLACE_ME_AWS_SECRET_KEY_WITH_S3_WRITE_PERMS
             - aws_access_key=REPLACE_ME_AWS_ACCESS_KEY_WITH_S3_WRITE_PERMS
             - extension_s3_bucket=REPLACE_ME_WITH_EXTENSIONS_S3
-            - debugging=false
-            - frontend_origin=https://your-web-origin.com
+            - debugging=true
+            - local=true
+            - frontend_origin=http://localhost:8001
         ports:
             - "80:80"
+        extra_hosts:
+            - "host.docker.internal:host-gateway"
     tarnishworker:
         build: ./tarnish-worker/
         environment:
-            - redis_backend=redis://REDIS_HOSTNAME:6379/0
+            - redis_backend=redis://host.docker.internal:6379/0
             - aws_secret_key=REPLACE_ME_AWS_SECRET_KEY_WITH_S3_WRITE_PERMS
             - aws_access_key=REPLACE_ME_AWS_ACCESS_KEY_WITH_S3_WRITE_PERMS
             - extension_s3_bucket=REPLACE_ME_WITH_EXTENSIONS_S3
-            - frontend_origin=https://your-web-origin.com
+            - local=true
+            - frontend_origin=http://localhost:8001
+            - server_dir=REPLACE_ME_WITH_UPDOG_DIRECTORY
+        extra_hosts:
+            - "host.docker.internal:host-gateway"

tarnish-worker/configs/permissions.json

@@ -1,97 +1,117 @@
 {
-	"version": "1.0.0",
-	"permissions_metadata": {
-		"bookmarks": {
-			"warning_text": "Read and modify your bookmarks",
-			"notes": "The <code>bookmarks</code> permission is required by the <code>chrome.bookmarks</code> module."
-		},
-		"history": {
-			"warning_text": "Read and modify your browsing history",
-			"notes": "The <code>history</code> permission is required by chrome.history.<br />The <code>topSites</code> permission is required by <code>chrome.topSites</code>."
-		},
-		"topSites": {
-			"warning_text": "Read and modify your browsing history",
-			"notes": "The <code>history</code> permission is required by chrome.history.<br />The <code>topSites</code> permission is required by <code>chrome.topSites</code>."
-		},
-		"tabs": {
-			"warning_text": "Access your browsing activity",
-			"notes": "The <code>tabs</code> permission is required by the <code>chrome.tabs</code> and <code>chrome.windows</code> modules.<br />The <code>webNavigation</code> permission is required by the <code>chrome.webNavigation</code> module."
-		},
-		"webNavigation": {
-			"warning_text": "Access your browsing activity",
-			"notes": "The <code>tabs</code> permission is required by the <code>chrome.tabs</code> and <code>chrome.windows</code> modules.<br />The <code>webNavigation</code> permission is required by the <code>chrome.webNavigation</code> module."
-		},
-		"contentSettings": {
-			"warning_text": "Manipulate settings that specify whether websites can use features such as cookies, JavaScript, plugins, geolocation, microphone, camera etc.",
-			"notes": "The <code>contentSettings</code> permission is required by <code>chrome.contentSettings</cod<code>."
-		},
-		"debugger": {
-			"warning_text": "Read and modify all your data on all websites you visit",
-			"notes": ""
-		},
-		"pageCapture": {
-			"warning_text": "Read and modify all your data on all websites you visit",
-			"notes": ""
-		},
-		"proxy": {
-			"warning_text": "Read and modify all your data on all websites you visit",
-			"notes": ""
-		},
-		"devtools_page": {
-			"warning_text": "Read and modify all your data on all websites you visit",
-			"notes": ""
-		},
-		"http://*/*": {
-			"warning_text": "Read and modify all your data on all websites you visit",
-			"notes": ""
-		},
-		"https://*/*": {
-			"warning_text": "Read and modify all your data on all websites you visit",
-			"notes": ""
-		},
-		"*://*/*": {
-			"warning_text": "Read and modify all your data on all websites you visit",
-			"notes": ""
-		},
-		"<all_urls>": {
-			"warning_text": "Read and modify all your data on all websites you visit",
-			"notes": ""
-		},
-		"http://*/": {
-			"warning_text": "Read and modify all your data on all websites you visit",
-			"notes": ""
-		},
-		"https://*/": {
-			"warning_text": "Read and modify all your data on all websites you visit",
-			"notes": ""
-		},
-		"management": {
-			"warning_text": "Manage your apps, extensions, and themes",
-			"notes": "The <code>management</code> permission is required by the <code>chrome.management</code> module."
-		},
-		"mdns": {
-			"warning_text": "Discover devices on your local network",
-			"notes": "The <code>mdns</code> permission is required by the <code>chrome.mdns</code> module."
-		},
-		"geolocation": {
-			"warning_text": "Detect your physical location",
-			"notes": "Allows the extension to use the proposed HTML5 geolocation API without prompting the user for permission."
-		},
-		"clipboardRead": {
-			"warning_text": "Access data you copy and paste",
-			"notes": "Allows the extension to use the following editing commands with <code>document.execCommand()</code>: <code>copy</code>, <code>cut</code>"
-		},
-		"privacy": {
-			"warning_text": "Manipulate privacy-related settings",
-			"notes": "The <code>privacy</code> permission is required by the <code>chrome.privacy</code> module."
-		},
-		"signedInDevices": {
-			"warning_text": "Access the list of your signed-in devices",
-			"notes": "The <code>signedInDevices</code> permission is required by the <code>chrome.signedInDevices</code> module."
-		},
-		"ttsEngine": {
-			"warning_text": "Access all text spoken using synthesized speech",
-			"notes": "The <code>ttsEngine</code> permission is required by the <code>chrome.ttsEngine</code> module."
-		}
-	}
-}
+  "version": "1.0.0",
+  "permissions_metadata": {
+    "storage": {
+      "warning_text": "Read and modify your data locally",
+      "notes": "The <code>storage</code> permission is required by the <code>chrome.storage</code> API"
+    },
+    "background": {
+      "warning_text": "Run background scripts actively without extension invocation",
+      "notes": "The <code>background</code> permission is required by the <code>chrome.background</code> API"
+    },
+    "identity": {
+      "warning_text": "Read user's identity information and OAuth tokens",
+      "notes": "The <code>identity</code> permission is required by the <code>chrome.identity</code> API"
+    },
+    "webRequest": {
+      "warning_text": "Allows interception/modification of network requests",
+      "notes": "The <code>webRequest</code> permission is required by the <code>chrome.webRequest</code> API"
+    },
+    "webRequestBlocking": {
+      "warning_text": "Allows blocking/redirection of network requests",
+      "notes": "The <code>webRequestBlocking</code> permission is required by the <code>chrome.webRequestBlocking</code> API"
+    },
+    "bookmarks": {
+      "warning_text": "Read and modify your bookmarks",
+      "notes": "The <code>bookmarks</code> permission is required by the <code>chrome.bookmarks</code> module."
+    },
+    "history": {
+      "warning_text": "Read and modify your browsing history",
+      "notes": "The <code>history</code> permission is required by chrome.history.<br />The <code>topSites</code> permission is required by <code>chrome.topSites</code>."
+    },
+    "topSites": {
+      "warning_text": "Read and modify your browsing history",
+      "notes": "The <code>history</code> permission is required by chrome.history.<br />The <code>topSites</code> permission is required by <code>chrome.topSites</code>."
+    },
+    "tabs": {
+      "warning_text": "Access your browsing activity",
+      "notes": "The <code>tabs</code> permission is required by the <code>chrome.tabs</code> and <code>chrome.windows</code> modules.<br />The <code>webNavigation</code> permission is required by the <code>chrome.webNavigation</code> module."
+    },
+    "webNavigation": {
+      "warning_text": "Access your browsing activity",
+      "notes": "The <code>tabs</code> permission is required by the <code>chrome.tabs</code> and <code>chrome.windows</code> modules.<br />The <code>webNavigation</code> permission is required by the <code>chrome.webNavigation</code> module."
+    },
+    "contentSettings": {
+      "warning_text": "Manipulate settings that specify whether websites can use features such as cookies, JavaScript, plugins, geolocation, microphone, camera etc.",
+      "notes": "The <code>contentSettings</code> permission is required by <code>chrome.contentSettings</cod<code>."
+    },
+    "debugger": {
+      "warning_text": "Read and modify all your data on all websites you visit",
+      "notes": ""
+    },
+    "pageCapture": {
+      "warning_text": "Read and modify all your data on all websites you visit",
+      "notes": ""
+    },
+    "proxy": {
+      "warning_text": "Read and modify all your data on all websites you visit",
+      "notes": ""
+    },
+    "devtools_page": {
+      "warning_text": "Read and modify all your data on all websites you visit",
+      "notes": ""
+    },
+    "http://*/*": {
+      "warning_text": "Read and modify all your data on all websites you visit",
+      "notes": ""
+    },
+    "https://*/*": {
+      "warning_text": "Read and modify all your data on all websites you visit",
+      "notes": ""
+    },
+    "*://*/*": {
+      "warning_text": "Read and modify all your data on all websites you visit",
+      "notes": ""
+    },
+    "<all_urls>": {
+      "warning_text": "Read and modify all your data on all websites you visit",
+      "notes": ""
+    },
+    "http://*/": {
+      "warning_text": "Read and modify all your data on all websites you visit",
+      "notes": ""
+    },
+    "https://*/": {
+      "warning_text": "Read and modify all your data on all websites you visit",
+      "notes": ""
+    },
+    "management": {
+      "warning_text": "Manage your apps, extensions, and themes",
+      "notes": "The <code>management</code> permission is required by the <code>chrome.management</code> module."
+    },
+    "mdns": {
+      "warning_text": "Discover devices on your local network",
+      "notes": "The <code>mdns</code> permission is required by the <code>chrome.mdns</code> module."
+    },
+    "geolocation": {
+      "warning_text": "Detect your physical location",
+      "notes": "Allows the extension to use the proposed HTML5 geolocation API without prompting the user for permission."
+    },
+    "clipboardRead": {
+      "warning_text": "Access data you copy and paste",
+      "notes": "Allows the extension to use the following editing commands with <code>document.execCommand()</code>: <code>copy</code>, <code>cut</code>"
+    },
+    "privacy": {
+      "warning_text": "Manipulate privacy-related settings",
+      "notes": "The <code>privacy</code> permission is required by the <code>chrome.privacy</code> module."
+    },
+    "signedInDevices": {
+      "warning_text": "Access the list of your signed-in devices",
+      "notes": "The <code>signedInDevices</code> permission is required by the <code>chrome.signedInDevices</code> module."
+    },
+    "ttsEngine": {
+      "warning_text": "Access all text spoken using synthesized speech",
+      "notes": "The <code>ttsEngine</code> permission is required by the <code>chrome.ttsEngine</code> module."
+    }
+  }
+}