Add Hotspots query generator

.gitignore

@@ -19,4 +19,10 @@ test-*.ql
 **/*.testproj/*
 */test-output.txt
 
+# Temporary files
+ql/hotspots/output/**
+
+# Cache
+**/__pycache__/
+
 .DS_Store

ql/hotspots/Hotspots.ql

@@ -0,0 +1,35 @@
+/**
+ * @name
+ * @id githubsecuritylab/hotspots-query-generator
+ * @description Finds all security-related TaintTracking sinks
+ * @kind problem
+ * @precision low
+ */
+
+import ql
+import utils.hotspots
+
+predicate debug_counts(int a, int b) {
+  a = count(SecurityQuery q) and
+  b = count(TaintTrackingSecurityQuery q)
+}
+
+predicate debug_missing(SecurityQuery q, string lang) {
+  not exists(TaintTrackingSecurityQuery tq | tq = q) and q.getLanguage() = lang
+}
+
+predicate supportedLanguage(string lang) {
+  lang = ["javascript", "java", "ruby", "csharp", "go", "python", "cpp"]
+}
+
+bindingset[severity]
+predicate supportedSeverity(float severity) { severity > 7.0 or severity = -1.0 }
+
+from TaintTrackingSecurityQuery q, TaintTrackingConfiguration c
+where
+  supportedLanguage(q.getLanguage()) and
+  supportedSeverity(q.getSeverity()) and
+  c = q.getTaintTrackingConfiguration()
+// 1. language, 2. query id, 3. config path, 4. config name, 5. query import stmt, 6. query pack, 7. query severity, config kind, config isStateConfig
+select q.getLanguage(), q.getId(), c.getPath(), c.getQualifiedName(), c.getImportStringFrom(q),
+  c.getQLPack(), q.getSeverity().toString(), c.getKind(), c.isStateConfig()

ql/hotspots/README.md

@@ -0,0 +1,57 @@
+# Hotspot query generator
+
+This script uses QL-4-QL to find all security related path-problem queries and extract their TaintTracking configuration and the import statement needed to run them.
+
+## Arguments
+
+| Option            | Description                                                       |
+| ----------------- | ----------------------------------------------------------------- |
+| `--ql-extractor`  | Path to the CodeQL extractor (required)                           |
+| `--ql-path`       | Path to the CodeQL repository to extract hotspots from (required) |
+| `--ql-executable` | Path to the CodeQL binary (default: "codeql")                     |
+
+## Configuration
+
+Configuration is located in `config/hotspots-config.yml` file (or where specified) and contains a configuration for each language.
+
+E.g:
+
+```yaml
+java:
+  disallowed_patterns:
+    - ".*-local"
+    - ".*-experimental"
+  disallowed_queries:
+    - java/untrusted-data-to-external-api
+    - java/log-injection
+    - java/android/intent-redirection
+    - java/improper-validation-of-array-construction
+ruby:
+  allowed_queries:
+    - rb/code-injection
+    - rb/sql-injection
+```
+
+- `allowed_queries`: List of query IDs to use to extract Hotspots from
+- `disallowed_queries`: List of queries to skip when processing TaintTracking queries to extract Hotspots from
+- `disallowed_patterns`: List of regexp patterns of queries to skip when processing TaintTracking queries to extract Hotspots from
+
+## Usage
+
+E.g:
+
+- If you havent build the extractor for QL yet, cd into the `ql` folder of your CodeQL distribution (eg: `~/src/codeql/ql`) and run `./scripts/create-extractor-pack.sh`. This will generate `~/src/codeql/ql/extractor-pack`.
+
+- Extract the hotspots info, dump it into `hotspots.csv` and create the `Hotspots.ql` queries for each language
+
+```bash
+python scripts/generate-hotspots-queries.py --ql-extractor ~/src/codeql/ql/extractor-pack --ql-path ~/src/github/codeql
+```
+
+- Create a patched version of CodeQL distro (remove private modifiers and rename files/directories to remove whitespaces and dashes)
+
+```bash
+python scripts/patch_codeql.py --hotspots hotspots.csv --ql ~/src/codeql --dest /tmp/hotspots-distro --qlpack-version 0.0.1
+```
+
+- Run Hotspots query (eg: `/tmp/hotspots-distro/java/ql/src/Hotspots.ql`)

ql/hotspots/config/hotspots-config.yml

@@ -0,0 +1,128 @@
+java:
+  disallowed_patterns:
+    - '.*-local'
+    - '.*-experimental'
+  disallowed_queries:
+    - java/untrusted-data-to-external-api
+    - java/log-injection
+    - java/android/intent-redirection
+    - java/improper-validation-of-array-construction
+    - java/improper-validation-of-array-construction-code-specified
+    - java/improper-validation-of-array-index
+    - java/improper-validation-of-array-index-code-specified
+    - java/tainted-format-string
+    - java/tainted-arithmetic
+    - java/uncontrolled-arithmetic
+    - java/extreme-value-arithmetic
+    - java/local-temp-file-or-directory-information-disclosure
+    - java/non-https-url
+    - java/weak-cryptographic-algorithm
+    - java/potentially-weak-cryptographic-algorithm
+    - java/missing-jwt-signature-check
+    - java/sensitive-log
+    - java/tainted-numeric-cast
+    - java/hardcoded-credential-api-call
+    - java/hardcoded-credential-sensitive-call
+    - java/user-controlled-bypass
+    - java/tainted-permissions-check
+    - java/android/sensitive-communication
+    - java/log4j-injection
+    - java/spring-view-manipulation
+    - java/server-side-template-injection
+    - java/static-initialization-vector
+    - java/sensitive-android-file-leak
+    - java/possible-timing-attack-against-signature
+    - java/timing-attack-against-headers-value
+    - java/timing-attack-against-signature
+    - java/disabled-certificate-revocation-checking
+    - java/unsafe-tls-version
+    - java/unvalidated-cors-origin-set
+    - java/ip-address-spoofing
+    - java/thread-resource-abuse
+    - java/sensitive-query-with-get
+    - java/uncaught-servlet-exception
+    - java/xxe
+    - java/xxe-with-experimental-sinks
+    - java/xxe-local-experimental-sinks
+    - java/android/nfe-local-android-dos
+    - java/exec-tainted-environment
+
+ruby:
+  disallowed_patterns:
+    - '.*-local'
+    - '.*-experimental'
+  disallowed_queries:
+    - rb/clear-text-logging-sensitive-data
+    - rb/clear-text-storage-sensitive-data
+    - rb/hardcoded-credentials
+    - rb/insecure-download
+    - rb/insecure-randomness
+    - rb/log-injection
+    - rb/overly-permissive-file
+    - rb/sensitive-get-query
+    - rb/stack-trace-exposure
+
+python:
+  disallowed_patterns:
+    - '.*-local'
+    - '.*-experimental'
+  disallowed_queries:
+    - py/timing-attack-against-hash
+    - py/timing-attack-sensitive-info
+    - py/timing-attack-against-header-value
+    - py/clear-text-logging-sensitive-data
+    - py/ip-address-spoofing
+    - py/log-injection
+    - py/possible-timing-attack-against-hash
+    - py/possible-timing-attack-sensitive-info
+    - py/polynomial-redos
+    - py/hardcoded-credentials
+    - py/clear-text-storage-sensitive-data
+    - py/untrusted-data-to-external-api
+    - py/azure-storage/unsafe-client-side-encryption-in-us
+    - py/xml-bomb # similar to xxe
+    - py/weak-sensitive-data-hashing
+    - py/partial-ssrf # similar to full-ssrf
+
+go:
+  disallowed_patterns:
+    - '.*-local'
+    - '.*-experimental'
+  disallowed_queries:
+    - go/log-injection
+    - go/insecure-hostkeycallback
+    - go/incomplete-hostname-regexp
+    - go/cookie-httponly-not-set
+    - go/cookie-httponly-not-set
+    - go/insecure-tls
+    - go/constant-oauth2-state
+    - go/untrusted-data-to-external-api
+    - go/timing-attack
+    - go/suspicious-character-in-regex
+    - go/divide-by-zero
+    - go/cookie-httponly-not-set
+    - go/incorrect-integer-conversion
+    - go/cookie-httponly-not-set
+    - go/untrusted-data-to-unknown-external-api
+    - go/allocation-size-overflow
+    - go/wrong-usage-of-unsafe
+
+javascript:
+  disallowed_patterns:
+    - '.*-local'
+    - '.*-experimental'
+
+cpp:
+  disallowed_patterns:
+    - '.*-local'
+    - '.*-experimental'
+
+csharp:
+  disallowed_patterns:
+    - '.*-local'
+    - '.*-experimental'
+  disallowed_queries:
+    - cs/inappropriate-encoding
+    - cs/hash-without-salt
+    - cs/stored-command-line-injection
+

ql/hotspots/qlpack.yml

@@ -0,0 +1,5 @@
+library: false
+name: githubsecuritylab/hotspots
+version: 0.0.1
+dependencies:
+  codeql/ql: '*'

ql/hotspots/requirements.txt

@@ -0,0 +1,5 @@
+pyyaml
+pandas
+Jinja2
+beautifulsoup4
+PyGithub