Add documentation for script analysis

.github/CONTRIBUTING.md

@@ -36,7 +36,7 @@ We host the capa project as three GitHub repositories:
   - [capa-rules](https://github.com/mandiant/capa-rules)
   - [capa-testfiles](https://github.com/mandiant/capa-testfiles)
 
-The command line tools, logic engine, and other Python source code are found in the `capa` repository.
+The command line tools, logic engine, scripting language analysis, and other Python source code are found in the `capa` repository.
 This is the repository to fork when you want to enhance the features, performance, or user interface of capa.
 Do *not* push rules directly to this repository, instead...
 
@@ -46,11 +46,13 @@ We keep `capa` and `capa-rules` separate to distinguish where ideas, bugs, and d
 If you're writing yaml it probably goes in `capa-rules` and if you're writing Python it probably goes in `capa`.
 Also, we encourage users to develop their own rule repositories, so we treat our default set of rules in the same way.
 
-Test fixtures, such as malware samples and analysis workspaces, are found in the `capa-testfiles` repository.
+Test fixtures, such as malware samples, script samples and analysis workspaces, are found in the `capa-testfiles` repository.
 These are files you'll need in order to run the linter (in `--thorough` mode) and full test suites;
  however, they take up a lot of space (1GB+), so by keeping `capa-testfiles` separate,
  a shallow checkout of `capa` and `capa-rules` doesn't take much bandwidth.
 
+For more information on developing and extending Script Analysis, see the [Script Analysis documentation](../doc/script-analysis.md).
+
 ### Design Decisions
 
 When we make a significant decision in how we maintain the project and what we can or cannot support,

.github/mypy/mypy.ini

@@ -86,3 +86,6 @@ ignore_missing_imports = True
 
 [mypy-ghidra.*]
 ignore_missing_imports = True
+
+[mypy-tree_sitter.*]
+ignore_missing_imports = True

.github/pyinstaller/hooks/hook-capa.features.extractors.ts.signatures.py

@@ -0,0 +1,20 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from PyInstaller.utils.hooks import collect_data_files
+
+
+# Tree-sitter signature lookups use importlib.resources, so PyInstaller must
+# bundle the JSON files alongside the package.
+datas = collect_data_files("capa.features.extractors.ts.signatures")

.github/workflows/build.yml

@@ -2,7 +2,7 @@ name: build
 
 on:
   pull_request:
-    branches: [ master ]
+    branches: [ master, feature/script-analysis ]
     paths-ignore:
       - 'web/**'
       - 'doc/**'

.github/workflows/ruff-format.yml

@@ -2,7 +2,7 @@ name: ruff auto-format
 
 on:
   pull_request:
-    branches: [ master ]
+    branches: [ master, feature/script-analysis ]
     paths-ignore:
       - 'web/**'
       - 'doc/**'

.github/workflows/tests.yml

@@ -6,13 +6,13 @@ name: CI
 
 on:
   push:
-    branches: [ master ]
+    branches: [ master, feature/script-analysis ]
     paths-ignore:
       - 'web/**'
       - 'doc/**'
       - '**.md'
   pull_request:
-    branches: [ master ]
+    branches: [ master, feature/script-analysis ]
     paths-ignore:
       - 'web/**'
       - 'doc/**'

CHANGELOG.md

@@ -148,6 +148,7 @@ This release includes Ghidra PyGhidra support, performance improvements, depende
 
 ### New Features
 
+- Tree-Sitter Script Analysis @adamstorek #1080 @EdoardoAllegrini #2931 @saniyafatima07 #3102
 - ghidra: support PyGhidra @mike-hunhoff #2788
 - vmray: extract number features from whitelisted void_ptr parameters (hKey, hKeyRoot) @adeboyedn #2835
 

README.md

@@ -246,6 +246,38 @@ $ capa 05be49819139a3fdcdbddbdefd298398779521f3d68daa25275cc77508e42310.json
 ┕━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┙
 ```
 
+# script analysis
+
+capa also supports static analysis of script-based files using [Tree-sitter](https://tree-sitter.github.io/tree-sitter/). Script analysis currently supports Python (`.py`), C# (`.cs`), HTML/ASPX (`.html`, `.aspx`) and Bash (`.sh`), allowing capa to identify capabilities in scripts.
+
+Here is an example for running a capa against a script:
+
+```
+$ capa 606dbfebdc7751ecb6cb9a845853ae1905afd4b8a2cb54e1e4a98c932e268712.aspx_
+┌──────────┬──────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐
+| md5      | a73d772a6db180ea12c7f0c7dbb8eaf2                                                                                 |
+| sha1     | acf361167ea92fff4a3d509edab7025590faa337                                                                         |
+| sha256   | 606dbfebdc7751ecb6cb9a845853ae1905afd4b8a2cb54e1e4a98c932e268712                                                 |
+| analysis | static                                                                                                           |
+| os       | any                                                                                                              |
+| format   | script                                                                                                           |
+| arch     | any                                                                                                              |
+| path     | path/606dbfebdc7751ecb6cb9a845853ae1905afd4b8a2cb54e1e4a98c932e268712.aspx_                                      |
+└──────────┴──────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
+┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+┃ MBC Objective                                    ┃ MBC Behavior                                  ┃
+┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
+│ FILE SYSTEM                                      │ Delete File [C0047]                           │
+└──────────────────────────────────────────────────┴───────────────────────────────────────────────┘
+┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+┃ Capability                          ┃ Namespace                                                  ┃
+┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
+│ delete file                         │ host-interaction/file-system/delete                        │
+└─────────────────────────────────────┴────────────────────────────────────────────────────────────┘
+```
+
+For more details, see the [Script Analysis documentation](doc/script-analysis.md).
+
 # capa rules
 capa uses a collection of rules to identify capabilities within a program.
 These rules are easy to write, even for those new to reverse engineering.

capa/capabilities/static.py

@@ -205,7 +205,7 @@ def find_static_capabilities(
                     match_count += len(matches_)
 
             logger.debug(
-                "analyzed function 0x%x and extracted %d features, %d matches in %0.02fs",
+                "analyzed function %s and extracted %d features, %d matches in %0.02fs",
                 f.address,
                 code_capabilities.feature_count,
                 match_count,

capa/features/address.py

@@ -159,6 +159,30 @@ def __hash__(self):
         return int.__hash__(self)
 
 
+class FileOffsetRangeAddress(Address):
+    """an address range relative to the start of a file"""
+
+    def __init__(self, start_byte, end_byte):
+        self.start_byte = start_byte
+        self.end_byte = end_byte
+
+    def __eq__(self, other):
+        if not isinstance(other, FileOffsetRangeAddress):
+            return NotImplemented
+        return (self.start_byte, self.end_byte) == (other.start_byte, other.end_byte)
+
+    def __lt__(self, other):
+        if not isinstance(other, FileOffsetRangeAddress):
+            return NotImplemented
+        return (self.start_byte, self.end_byte) < (other.start_byte, other.end_byte)
+
+    def __hash__(self):
+        return hash((self.start_byte, self.end_byte))
+
+    def __repr__(self):
+        return f"file(0x{self.start_byte:x}, 0x{self.end_byte:x})"
+
+
 class DNTokenAddress(int, Address):
     """a .NET token"""
 

capa/features/common.py

@@ -487,10 +487,17 @@ def evaluate(self, features: "capa.engine.FeatureSet", short_circuit=True):
         return Result(False, self, [])
 
 
+class ScriptLanguage(Feature):
+    def __init__(self, value: str, description=None):
+        super().__init__(value, description=description)
+        self.name = "script language"
+
+
 FORMAT_PE = "pe"
 FORMAT_ELF = "elf"
 FORMAT_DOTNET = "dotnet"
-VALID_FORMAT = (FORMAT_PE, FORMAT_ELF, FORMAT_DOTNET)
+FORMAT_SCRIPT = "script"
+VALID_FORMAT = (FORMAT_PE, FORMAT_ELF, FORMAT_DOTNET, FORMAT_SCRIPT)
 # internal only, not to be used in rules
 FORMAT_AUTO = "auto"
 FORMAT_SC32 = "sc32"
@@ -508,6 +515,7 @@ def evaluate(self, features: "capa.engine.FeatureSet", short_circuit=True):
     FORMAT_PE,
     FORMAT_ELF,
     FORMAT_DOTNET,
+    FORMAT_SCRIPT,
     FORMAT_FREEZE,
     FORMAT_RESULT,
     FORMAT_BINEXPORT2,

capa/features/extractors/common.py

@@ -36,13 +36,15 @@
     VALID_ARCH,
     FORMAT_FREEZE,
     FORMAT_RESULT,
+    FORMAT_SCRIPT,
     Arch,
     Format,
     String,
     Feature,
 )
 from capa.features.freeze import is_freeze
 from capa.features.address import NO_ADDRESS, Address, FileOffsetAddress
+from capa.features.extractors.ts.autodetect import is_script
 
 logger = logging.getLogger(__name__)
 
@@ -78,6 +80,8 @@ def extract_format(buf: bytes) -> Iterator[tuple[Feature, Address]]:
         # we don't know what it is exactly, but may support it (e.g. a dynamic CAPE sandbox report)
         # skip verdict here and let subsequent code analyze this further
         return
+    elif is_script(buf):
+        yield Format(FORMAT_SCRIPT), NO_ADDRESS
     else:
         # we likely end up here:
         #  1. handling a file format (e.g. macho)

capa/features/extractors/script.py

@@ -0,0 +1,55 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Tuple, Iterator
+
+from capa.features.common import OS, OS_ANY, ARCH_ANY, FORMAT_SCRIPT, Arch, Format, Feature, ScriptLanguage
+from capa.features.address import NO_ADDRESS, Address, FileOffsetRangeAddress
+
+# Can be used to instantiate tree_sitter Language objects (see ts/query.py)
+LANG_CS = "c_sharp"
+LANG_HTML = "html"
+LANG_JS = "javascript"
+LANG_PY = "python"
+LANG_TEM = "embedded_template"
+
+EXT_ASPX = (".aspx", ".aspx_")
+EXT_CS = (".cs", ".cs_")
+EXT_HTML = (".html", ".html_")
+EXT_PY = (".py", ".py_")
+
+
+LANGUAGE_FEATURE_FORMAT = {
+    LANG_CS: "C#",
+    LANG_HTML: "HTML",
+    LANG_JS: "JavaScript",
+    LANG_PY: "Python",
+    LANG_TEM: "Embedded Template",
+}
+
+
+def extract_arch() -> Iterator[Tuple[Feature, Address]]:
+    yield Arch(ARCH_ANY), NO_ADDRESS
+
+
+def extract_language(language: str, addr: FileOffsetRangeAddress) -> Iterator[Tuple[Feature, Address]]:
+    yield ScriptLanguage(LANGUAGE_FEATURE_FORMAT[language]), addr
+
+
+def extract_os() -> Iterator[Tuple[Feature, Address]]:
+    yield OS(OS_ANY), NO_ADDRESS
+
+
+def extract_format() -> Iterator[Tuple[Feature, Address]]:
+    yield Format(FORMAT_SCRIPT), NO_ADDRESS

capa/features/extractors/ts/autodetect.py

@@ -0,0 +1,80 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Optional
+from pathlib import Path
+
+from tree_sitter import Node, Tree, Query, Parser, Language, QueryCursor
+
+from capa.features.extractors.script import EXT_CS, EXT_PY, LANG_CS, LANG_PY, EXT_ASPX, EXT_HTML, LANG_TEM, LANG_HTML
+from capa.features.extractors.ts.query import TS_LANGUAGES
+
+
+def is_script(buf: bytes) -> bool:
+    try:
+        return bool(get_language_ts(buf))
+    except ValueError:
+        return False
+
+
+def _parse(ts_language: Language, buf: bytes) -> Optional[Tree]:
+    try:
+        parser = Parser(ts_language)
+        return parser.parse(buf)
+    except ValueError:
+        return None
+
+
+def _contains_errors(ts_language, node: Node) -> bool:
+    query = Query(ts_language, "(ERROR) @error")
+    return bool(QueryCursor(query).captures(node))
+
+
+def get_language_ts(buf: bytes) -> str:
+    for language, ts_language in TS_LANGUAGES.items():
+        tree = _parse(ts_language, buf)
+        if tree and not _contains_errors(ts_language, tree.root_node):
+            return language
+    raise ValueError("failed to parse the language")
+
+
+def get_template_language_ts(buf: bytes) -> str:
+    for language, ts_language in TS_LANGUAGES.items():
+        if language in [LANG_TEM, LANG_HTML]:
+            continue
+        tree = _parse(ts_language, buf)
+        if tree and not _contains_errors(ts_language, tree.root_node):
+            return language
+    raise ValueError("failed to parse the language")
+
+
+def get_language_from_ext(path: str) -> str:
+    if path.endswith(EXT_ASPX):
+        return LANG_TEM
+    if path.endswith(EXT_CS):
+        return LANG_CS
+    if path.endswith(EXT_HTML):
+        return LANG_HTML
+    if path.endswith(EXT_PY):
+        return LANG_PY
+    raise ValueError(f"{path} has an unrecognized or an unsupported extension.")
+
+
+def get_language(path: Path) -> str:
+    try:
+        return get_language_from_ext(str(path))
+    except ValueError:
+        with path.open("rb") as f:
+            buf = f.read()
+        return get_language_ts(buf)