diff --git a/src/tools/interop/idt/.gitignore b/src/tools/interop/idt/.gitignore index 55da2a806dd412..34753078b70ffe 100644 --- a/src/tools/interop/idt/.gitignore +++ b/src/tools/interop/idt/.gitignore @@ -8,3 +8,5 @@ pycache/ venv/ .zip BUILD +BUILD_LOG.txt +prober_urls.txt diff --git a/src/tools/interop/idt/README.md b/src/tools/interop/idt/README.md index acc0c252fdbe39..311feeaa91c5d9 100644 --- a/src/tools/interop/idt/README.md +++ b/src/tools/interop/idt/README.md @@ -29,11 +29,27 @@ issue uncovered via the manual test. Each ecosystem may implement an analysis that analyzes capture data, displays info to the user, probes the local environment and generates additional artifacts. +### Other functions + +#### Advertise + +Create fake advertisements for Matter devices, useful in controller testing. + +#### Probe + +Collect basic networking info from the environment, e.g. resolve and ping all matter devices. + +#### Setup + +Provides ready-made setup scripts for specific, one-time build / installation tasks, e.g. setting up a devkit as a +thread sniffer. + ## Single host installation (no Raspberry Pi) -All features of `idt` are available on macOS and Linux (tested with Debian based -systems). -If you would prefer to execute capture and discovery from a Raspberry Pi, read +`idt` can be run on both macOS and Linux (tested with Debian based systems). +See the feature map section at the bottom of this page for details on feature support by host platform. + +If you prefer to execute capture and discovery from a Raspberry Pi, read the next section instead. The machine running `idt` should be connected to the same Wi-Fi network used for @@ -56,6 +72,11 @@ Follow the steps below to execute capture and discovery without a Raspberry Pi: - Failure with a relevant stack trace in the terminal. - A prompt to allow the application access to bluetooth. +## Thread + +Thread captures require a compatible devkit. +The thread implementation in this tool currently targets the `Maker Diary nrf52840-MDK` (NOT the dongle version). + ## Raspberry Pi installation ### Environment overview @@ -148,35 +169,21 @@ This directory contains tools for use on both the admin computer and the RPi. cd ~ # Go to idt parent dir source idt/scripts/setup_shell.sh # Setup atuo aliases source idt/scripts/alias.sh # Get aliases now - idt_bootstrap # Initial configuration - idt_build # Build the container image ``` -### Install updates - -SCP may not overwrite all files. To clear the `idt` dir off of the RPi safely -between pushes, exit the container and: - -``` -idt_clean -``` - -NOTE the idt artifacts directory is contained in idt, so running this will -delete any artifacts. - -Then from the admin computer: - -``` -idt_push -``` +[TODO] Drop docker and audit native installation instructions ## User guide > **_IMPORTANT_** > `idt_` commands are shell aliases helpful for administrative commands. -> `idt` invokes the `idt` python package. +> `idt` invokes the `idt` python package. +> > Output from `idt` will generally be colorized while output from sub processes > is generally not. +> +> Each command will produce artifacts in a new directory within `idt/IDT_ARTIFACTS` and display the archive path +> at the end of execution. RPi users, as needed: @@ -185,10 +192,6 @@ RPi users, as needed: ``` idt_connect ``` -- Run the `idt` container (from the RPi): - ``` - idt_activate - ``` ### Capture @@ -197,20 +200,28 @@ RPi users, as needed: > enter to stop!" before launching the app under test. ``` -idt capture -h - -usage: idt capture [-h] [--platform {Android}] [--ecosystem {PlayServicesUser,PlayServices,ALL}] [--pcap {t,f}] [--interface {wlp0s20f3,lo,docker0,any}] +usage: idt capture [-h] [--platform {Android}] [--ecosystem {PlayServices,PlayServicesUser,ALL}] [--pcap {t,f}] [--interface {any}] [--monitor {t,f}] [--channel CHANNEL] [--band {2,5}] [--width WIDTH] [--thread {none,sniff,on_network}] options: -h, --help show this help message and exit --platform {Android}, -p {Android} Run capture for a particular platform (default Android) - --ecosystem {PlayServicesUser,PlayServices,ALL}, -e {PlayServicesUser,PlayServices,ALL} + --ecosystem {PlayServices,PlayServicesUser,ALL}, -e {PlayServices,PlayServicesUser,ALL} Run capture for a particular ecosystem or ALL ecosystems (default ALL) --pcap {t,f}, -c {t,f} Run packet capture (default t) - --interface {wlp0s20f3,lo,docker0,any}, -i {wlp0s20f3,lo,docker0,any} + --interface {any}, -i {any} Specify packet capture interface (default any) + --monitor {t,f}, -m {t,f} + Run packet capture using a monitor mode interface (default f) + --channel CHANNEL, -n CHANNEL + Use this Wi-Fi channel if in monitor mode (default 1) + --band {2,5}, -b {2,5} + Use 2 for 2.4GHz, 5 for 5GHz (default 2) + --width WIDTH, -w WIDTH + Optionally set the channel width of a monitor mode pcap (default None) + --thread {none,sniff,on_network}, -t {none,sniff,on_network} + Execute thread sniffer or join OTBR to network (Default none) ``` For packet capture interface (`-i`/`--interface`: @@ -219,41 +230,47 @@ For packet capture interface (`-i`/`--interface`: - On Linux, `idt` checks available interfaces from `/sys/class/net/` as well as allowing `any`. -#### Artifacts - -Each ecosystem and platform involved in the capture will have their own -subdirectory in the root artifact dir. - ### Discovery ``` -idt discover -h - -usage: idt discover [-h] --type {ble,b,dnssd,d} +usage: idt discover [-h] --type {ble,b,dnssd,d} [--vid VID] [--pid PID] [--v4 {t,f}] [--v6 {t,f}] options: -h, --help show this help message and exit --type {ble,b,dnssd,d}, -t {ble,b,dnssd,d} Specify the type of discovery to execute + --vid VID, -v VID Only display advertisements with this Vendor ID. Hex values (without 0x prefix) are expected. Eg FFF1 + --pid PID, -p PID If vid argument is set, filter advertisements by this Product ID as well. Hex values (without 0x prefix) are expected. Eg 8000 + --v4 {t,f}, -4 {t,f} Whether to browse on IPv4 or not (Default t) + --v6 {t,f}, -6 {t,f} Whether to browse on IPv6 or not (Default t) ``` -#### BLE +### Advertise ``` -idt discover -t b -``` +usage: idt advertise [-h] [--vid VID] [--pid PID] [--discriminator DISCRIMINATOR] [--device_name DEVICE_NAME] [--device_type {256,257,268,269,266,267,771,259,260,261,2112,772,15,21,262,263,770,773,774,775,2128,10,11,514,515,768,769,43,40,35,34,36,41,42,39}] [--port PORT] [--commissioning_open {t,f}] [--mac_address MAC_ADDRESS] [--allow_gua {t,f}] --type {ble,b,dnssd,d} -#### mDNS - -``` -idt discover -t d +options: + -h, --help show this help message and exit + --vid VID, -v VID Vendor ID to use in the advertisement (int, default: 65521) + --pid PID, -p PID Product ID to use in the advertisement (int, default: 32768) + --discriminator DISCRIMINATOR, -i DISCRIMINATOR + Discriminator to use in the advertisement (int, default: 10) + --device_name DEVICE_NAME, -n DEVICE_NAME + Device name to be used in the advertisement (str, default: IDT fake device) + --device_type {256,257,268,269,266,267,771,259,260,261,2112,772,15,21,262,263,770,773,774,775,2128,10,11,514,515,768,769,43,40,35,34,36,41,42,39}, -d {256,257,268,269,266,267,771,259,260,261,2112,772,15,21,262,263,770,773,774,775,2128,10,11,514,515,768,769,43,40,35,34,36,41,42,39} + Device type to be used in the advertisement (int, default: 770) + --port PORT, -o PORT The port that this device is reachable on (int, default: 5540) + --commissioning_open {t,f}, -c {t,f} + Whether commissioning window is open or not (default: t) + --mac_address MAC_ADDRESS, -m MAC_ADDRESS + MAC Address to use for the instance name in this advertisement (str, default: A683E7C1029A) + --allow_gua {t,f}, -g {t,f} + Whether DNS-SD advertisements should include GUAs (if they're available on the host)(default: f) + --type {ble,b,dnssd,d}, -t {ble,b,dnssd,d} + Specify the type of advertisement to create ``` -#### Artifacts - -There is a per device log in `ble` and `dnssd` subdirectory of the root artifact -dir. - ### Probe ``` @@ -263,47 +280,59 @@ options: -h, --help show this help message and exit ``` -Collect contextually relevant networking info from the local environment and -provide artifacts. +### Setup + +``` +usage: idt setup [-h] --target {Nrf52840MdkNotDongle} + +options: + -h, --help show this help message and exit + --target {Nrf52840MdkNotDongle}, -t {Nrf52840MdkNotDongle} + The target to setup +``` ## Troubleshooting - Wireless `adb` may fail to connect indefinitely depending on network configuration. Use a wired connection if wireless fails repeatedly. -- Change log level from `INFO` to `DEBUG` in root `config.py` for additional - logging. +- Set `DEBUG=True` in the root `config.py` for additional logging. - Compiling `tcpdump` for android may require additional dependencies. - - If the build script fails for you, try - `idt_go && source idt/scripts/compilers.sh`. -- You may disable colors and splash by setting `enable_color` in `config.py` + - If the build script fails for you, try `idt_linux_install_compilers_for_arm_tcpdump`. +- You may disable colors and splash by setting `ENABLE_COLOR` in `config.py` to `False`. -- `idt_clean_child` will kill any stray `tcpdump` and `adb` commands. - - `idt_check_child` will look for leftover processes. +- `idt_child_clean` will kill any stray `tcpdump` and `adb` commands. + - `idt_child_check` will look for leftover processes. - Not expected to be needed outside of development scenarios. ## Project overview -- The entry point is in `idt.py` which contains simple CLI parsing with +- The entry point of each feature is in `idt.py` which contains simple CLI parsing with `argparse`. -### `capture` +### Features + +#### `advertise` + +- `advertise` re-uses the library used in discovery to produce dns-sd advertisements mimicking matter devices. + +#### `capture` - `base` contains the base classes for ecosystems and platforms. -- `controller` contains the ecosystem and platform producer and controller -- `loader` is a generic class loader that dynamically imports classes matching - a given super class from a given directory. +- `controller` manages platform and ecosystem implementations at run time. - `/platform` and `/ecosystem` contain one package for each platform and ecosystem, which should each contain one implementation of the respective base class. +- `pcap` contains the implementation for pcaps on the idt host. +- `thread` contains the implementation for thread captures on the idt host. -### `discovery` +#### `discovery` -- `matter_ble` provides a simple ble scanner that shows matter devices being +- `ble` provides a simple ble scanner that shows matter devices being discovered and lost, as well as their VID/PID, RSSI, etc. -- `matter_dnssd` provides a simple DNS-SD browser that searches for matter +- `dnssd` provides a simple DNS-SD browser that searches for matter devices and thread border routers. -### `probe` +#### `probe` - `probe` contains the base class for (`idt`'s) host platform specific implementation. @@ -311,25 +340,36 @@ provide artifacts. - Calls platform + addr type specific probe methods for each target. - `linux` and `mac` contain `probe` implementations for each host platform. +#### `setup` + +- Contains setup implementations. The only requirement for a setup implementation is a class with a single method + (`setup`). +- The package itself (not implementations) is very minimal. It exists just to glue purpose built scripts + (the setup implementation) to the main commandline args. + ### `utils` -- `log` contains logging utilities used by everything in the project. -- `artifact` contains helper functions for managing artifacts. -- `shell` contains a simple helper class for background and foreground Bash - commands. -- `host_platform` contains helper functions for the interacting with the host - running `idt`. + +- `host` contains helper functions for interacting with the host + running `idt`. +- `analysis` contains utilities for simple causal analysis of text based logs. +- `artifact` contains helper functions for managing artifacts. +- `data` contains metadat e.g. matter device type map +- `error` contains facilities for error reporting. +- `loader` is a generic class loader that dynamically imports classes matching + a given super class from a given directory. +- `log` contains logging utilities. +- `net` contains utility functions for networking topics like ip addr type. +- `shell` contains a simple helper class for background and foreground Bash + commands. ### Conventions -- `config.py` should be used to hold development configs within the directory - where they are needed. - - It may also hold configs for flaky/cumbersome features that might need - to be disabled in an emergency. - - `config.py` **should not** be used for everyday operation. +- List host dependencies and their help text and download links in the respective array in the root `config.py`. +- Users should not need to modify source files as part of any regular operation of the tool (e.g. modifying any + `config.py` should not be required). Use commandline args for runtime options. - When needed, execute builds in a folder called `BUILD` within the source tree. - - `idt_clean_all` deletes all `BUILD` dirs and `BUILD` is in `.gitignore`. ## Extending functionality @@ -340,7 +380,7 @@ Ecosystem and Platform implementations are dynamically loaded. For each package in `capture/ecosystem`, the ecosystem loader expects a module name matching the package name. This module must contain a single class which is a subclass of -`capture.base.EcosystemCapture`. +`capture.base.EcosystemCapture` with matching function signatures and coroutines. `/capture/ecosystem/play_services_user` contains a minimal example implementation. @@ -363,3 +403,33 @@ For each package in `capture/platform`, the platform loader expects a module name matching the package name. This module must contain a single class which is a subclass of `capture.base.PlatformLogStreamer`. + +The loader is also used elsewhere in the project. + +## Feature map + +| Icon | Meaning | +|--------------------|----------------------| +| :white_check_mark: | Supported | +| :x: | Not (/yet) supported | + +| Feature | Function | Linux | MacOS | Note | +|-----------|-----------------------------------------|--------------------|--------------------|-----------------------| +| Advertise | DNS-SD | :white_check_mark: | :white_check_mark: | | +| Advertise | BLE | :x: | :x: | Not implemented | +| Capture | Android: logcat collection and analysis | :white_check_mark: | :white_check_mark: | | +| Capture | Android: screen recording | :white_check_mark: | :white_check_mark: | | +| Capture | Android: packet capture | :white_check_mark: | :white_check_mark: | Requires rooted phone | +| Capture | PCAP: Managed mode on `idt` host | :white_check_mark: | :white_check_mark: | | +| Capture | PCAP: Monitor mode on `idt` host | :white_check_mark: | :white_check_mark: | | +| Capture | Thread: Execute sniffer capture | :white_check_mark: | :x: | Requires ncp + setup | +| Capture | Thread: Execute on-network capture | :x: | :x: | Not implemented | +| Discovery | BLE | :white_check_mark: | :white_check_mark: | | +| Discovery | DNS-SD | :white_check_mark: | :white_check_mark: | | +| Probe | Resolve and ping | :white_check_mark: | :white_check_mark: | | + +### Setup support + +| Target | Supports | Linux | MacOS | Description | +|----------------------|----------------------|--------------------|-------|-------------------------------------| +| Nrf52840MdkNotDongle | capture.thread.sniff | :white_check_mark: | :x: | Setup thread sniffer on Nrf52840MDK | diff --git a/src/tools/interop/idt/__main__.py b/src/tools/interop/idt/__main__.py index b927d050b9a0a2..5d073ae988ea4b 100644 --- a/src/tools/interop/idt/__main__.py +++ b/src/tools/interop/idt/__main__.py @@ -15,9 +15,36 @@ # limitations under the License. # +import os +import sys + +import psutil + from idt import InteropDebuggingTool -from utils.host_platform import verify_py_version +from utils.host import current_platform +from utils.log import get_logger +from utils.shell import Bash + +logger = get_logger(__file__) +dirty_cleanup = True if __name__ == "__main__": - verify_py_version() - InteropDebuggingTool() + try: + current_platform.verify_py_version() + InteropDebuggingTool() + dirty_cleanup = False + finally: + logger.info("Checking for stray child processes") + psutil_proc = psutil.Process(os.getpid()) + found = False + for child_proc in psutil_proc.children(recursive=True): + found = True + command_short = Bash("").get_current_command_for_pid(child_proc.pid) + command_full = Bash("").get_current_command_for_pid_full(child_proc.pid) + logger.error(f"PID: {child_proc.pid} \nCOMMAND: {command_short}\n{command_full}") + if found: + logger.error("Stray processes detected, you might want to clean these up!") + else: + logger.info("No stray processes detected!") + if dirty_cleanup and not sys.argv[len(sys.argv)-1] == "-h": + logger.critical("Crash detected! Clean up any stray processes listed above!!!") diff --git a/src/tools/interop/idt/capture/ecosystem/play_services/play_services_analysis.py b/src/tools/interop/idt/capture/ecosystem/play_services/play_services_analysis.py deleted file mode 100644 index e6ebb8f13075d8..00000000000000 --- a/src/tools/interop/idt/capture/ecosystem/play_services/play_services_analysis.py +++ /dev/null @@ -1,102 +0,0 @@ -# -# Copyright (c) 2023 Project CHIP Authors -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import os - -from capture.platform.android import Android -from utils.artifact import create_standard_log_name, log -from utils.log import add_border, print_and_write - -logger = log.get_logger(__file__) - - -class PlayServicesAnalysis: - - def __init__(self, platform: Android, artifact_dir: str) -> None: - self.logger = logger - self.artifact_dir = artifact_dir - self.analysis_file_name = os.path.join( - self.artifact_dir, create_standard_log_name( - 'commissioning_logcat', 'txt')) - - self.platform = platform - - self.matter_commissioner_logs = '' - self.failure_stack_trace = '' - self.pake_logs = '' - self.resolver_logs = '' - self.sigma_logs = '' - self.fail_trace_line_counter = -1 - - def _log_proc_matter_commissioner(self, line: str) -> None: - """Core commissioning flow""" - if 'MatterCommissioner' in line: - self.logger.info(line) - self.matter_commissioner_logs += line - - def _log_proc_commissioning_failed(self, line: str) -> None: - parsed_stack_trace_max_depth = 15 - if self.fail_trace_line_counter > parsed_stack_trace_max_depth: - self.fail_trace_line_counter = -1 - if self.fail_trace_line_counter > -1 and 'SetupDevice' in line: - self.failure_stack_trace += line - self.fail_trace_line_counter += 1 - if 'SetupDeviceView' and 'Commissioning failed' in line: - self.logger.info(line) - self.fail_trace_line_counter = 0 - self.failure_stack_trace += line - - def _log_proc_pake(self, line: str) -> None: - """Three logs for pake 1-3 expected""" - if "Pake" in line and "chip_logging" in line: - self.logger.info(line) - self.pake_logs += line - - def _log_proc_mdns(self, line: str) -> None: - if "_matter" in line and "ServiceResolverAdapter" in line: - self.logger.info(line) - self.resolver_logs += line - - def _log_proc_sigma(self, line: str) -> None: - """Three logs expected for sigma 1-3""" - if "Sigma" in line and "chip_logging" in line: - self.logger.info(line) - self.sigma_logs += line - - def show_analysis(self) -> None: - analysis_file = open(self.analysis_file_name, mode="w+") - print_and_write(add_border('Matter commissioner logs'), analysis_file) - print_and_write(self.matter_commissioner_logs, analysis_file) - print_and_write( - add_border('Commissioning failure stack trace'), - analysis_file) - print_and_write(self.failure_stack_trace, analysis_file) - print_and_write(add_border('PASE Handshake'), analysis_file) - print_and_write(self.pake_logs, analysis_file) - print_and_write(add_border('DNS-SD resolution'), analysis_file) - print_and_write(self.resolver_logs, analysis_file) - print_and_write(add_border('CASE handshake'), analysis_file) - print_and_write(self.sigma_logs, analysis_file) - analysis_file.close() - - def process_line(self, line: str) -> None: - for line_func in [s for s in dir(self) if s.startswith('_log')]: - getattr(self, line_func)(line) - - def do_analysis(self, batch: [str]) -> None: - for line in batch: - self.process_line(line) diff --git a/src/tools/interop/idt/capture/platform/android/streams/__init__.py b/src/tools/interop/idt/capture/platform/android/streams/__init__.py deleted file mode 100644 index 9f48b1f043fcd4..00000000000000 --- a/src/tools/interop/idt/capture/platform/android/streams/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -from capture.loader import CaptureImplsLoader - -from .base import AndroidStream - -impl_loader = CaptureImplsLoader( - __path__[0], - "capture.platform.android.streams", - AndroidStream -) - -for impl_name, impl in impl_loader.impls.items(): - globals()[impl_name] = impl - -__all__ = impl_loader.impl_names diff --git a/src/tools/interop/idt/capture/platform/android/streams/pcap/pcap.py b/src/tools/interop/idt/capture/platform/android/streams/pcap/pcap.py deleted file mode 100644 index 8eab53b8ec0d03..00000000000000 --- a/src/tools/interop/idt/capture/platform/android/streams/pcap/pcap.py +++ /dev/null @@ -1,99 +0,0 @@ -# -# Copyright (c) 2023 Project CHIP Authors -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import asyncio -import os -from typing import TYPE_CHECKING - -from utils.artifact import create_standard_log_name, log, safe_mkdir -from utils.host_platform import is_mac -from utils.shell import Bash - -from ... import config -from ..base import AndroidStream - -if TYPE_CHECKING: - from capture.platform.android import Android - -logger = log.get_logger(__file__) - - -class AndroidPcap(AndroidStream): - - def __init__(self, platform: "Android"): - self.logger = logger - self.platform = platform - self.target_dir = "/sdcard/Download" - self.pcap_artifact = create_standard_log_name("android_tcpdump", "pcap", parent=self.platform.artifact_dir) - self.pcap_phone_out_path = f"{self.target_dir}/{os.path.basename(self.pcap_artifact)}" - self.pcap_phone_bin_location = "tcpdump" if platform.capabilities.c_has_tcpdump \ - else f"{self.target_dir}/tcpdump" - self.pcap_command = f"shell {self.pcap_phone_bin_location} -w {self.pcap_phone_out_path}" - self.pcap_proc = platform.get_adb_background_command(self.pcap_command) - self.pcap_pull = False - self.pcap_pull_command = f"pull {self.pcap_phone_out_path} {self.pcap_artifact}" - self.build_dir = os.path.join(os.path.dirname(__file__), "BUILD") - - async def pull_packet_capture(self) -> None: - if self.pcap_pull: - self.logger.info("Attempting to pull android pcap") - await asyncio.sleep(3) - self.platform.run_adb_command(self.pcap_pull_command) - self.pcap_pull = False - - async def start(self): - if not self.platform.capabilities.c_has_root: - self.logger.warning("Phone is not rooted, cannot take pcap!") - return - if self.platform.capabilities.c_has_tcpdump: - self.logger.info("tcpdump already available; using!") - self.pcap_proc.start_command() - self.pcap_pull = True - return - if not config.enable_build_push_tcpdump: - self.logger.critical("Android TCP Dump build and push disabled in configs!") - return - if not os.path.exists(os.path.join(self.build_dir, "tcpdump")): - self.logger.warning("tcpdump bin not found, attempting to build, please wait a few moments!") - safe_mkdir(self.build_dir) - if is_mac(): - build_script = os.path.join(os.path.dirname(__file__), "mac_build_tcpdump_64.sh") - Bash(f"{build_script} 2>&1 >> BUILD_LOG.txt", sync=True, cwd=self.build_dir).start_command() - else: - build_script = os.path.join(os.path.dirname(__file__), "linux_build_tcpdump_64.sh") - Bash(f"{build_script} 2>&1 >> BUILD_LOG.txt", sync=True, cwd=self.build_dir).start_command() - else: - self.logger.warning("Reusing existing tcpdump build") - if not self.platform.run_adb_command(f"shell ls {self.target_dir}/tcpdump").finished_success(): - self.logger.warning("Pushing tcpdump to device") - self.platform.run_adb_command(f"push {os.path.join(self.build_dir, 'tcpdump')} f{self.target_dir}") - self.platform.run_adb_command(f"chmod +x {self.target_dir}/tcpdump") - else: - self.logger.info("tcpdump already in the expected location, not pushing!") - self.logger.info("Starting Android pcap command") - self.pcap_proc.start_command() - self.pcap_pull = True - - async def run_observer(self) -> None: - while True: - # TODO: Implement, need to restart w/ new out file (no append) and keep pull manifest, much like `screen` - await asyncio.sleep(120) - - async def stop(self): - self.logger.info("Stopping android pcap proc") - self.pcap_proc.stop_command() - await self.pull_packet_capture() diff --git a/src/tools/interop/idt/config.py b/src/tools/interop/idt/config.py index f084d4918516c4..c9495820a3a67d 100644 --- a/src/tools/interop/idt/config.py +++ b/src/tools/interop/idt/config.py @@ -14,9 +14,45 @@ # See the License for the specific language governing permissions and # limitations under the License. # + import logging -enable_color = True -log_level = logging.INFO -py_major_version = 3 -py_minor_version = 11 +from res.splash import splash as _splash + +IDT_VERSION = "1.0.0" +DEBUG = False +ENABLE_COLOR = True +LOG_LEVEL = logging.DEBUG if DEBUG else logging.INFO +PY_MAJOR_VERSION = 3 +PY_MINOR_VERSION = 11 +SPLASH = _splash +# TODO: Model this differently, e.g. dataclass: help, link, reason etc.? +# TODO: Script to autodetect dependencies used in the repo to make maintaining this easier +HOST_DEPENDENCIES = { + "ALL": { + "adb": + "adb is required for interacting with Android devices: " + "https://developer.android.com/studio/command-line/adb", + "tcpdump": + "tcpdump is required for taking host packet captures. It should be available on all systems.", + "git": + "git is required for downloading build tools. It should already be present on a system using this tool.", + "tee": + "tee is required to write and view logs at the same time. It should be available on all systems." + }, + "LINUX": { + "airmon-ng": + "The aircrack suite is used to manage processes and interfaces for monitor mode pcaps" + "https://www.aircrack-ng.org/doku.php?id=install_aircrack#installing_pre-compiled_binaries", + }, + "MAC": { + "airport": + "Airport is required for taking monitor mode packet captures on the idt host." + "It is likely on your system but not on your path." + "Try looking for the binary in a path like this" + "/System/Library/PrivateFrameworks/Apple80211.framework/Versions/A/Resources/airport" + "and add it to your path!", + # "brew": + # "brew is required for installing otbrrcp build tools on MacOS: https://brew.sh/", + }, +} diff --git a/src/tools/interop/idt/discovery/dnssd.py b/src/tools/interop/idt/discovery/dnssd.py deleted file mode 100644 index 04dd8de3494eda..00000000000000 --- a/src/tools/interop/idt/discovery/dnssd.py +++ /dev/null @@ -1,354 +0,0 @@ -# -# Copyright (c) 2023 Project CHIP Authors -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import asyncio -import os -import traceback -from dataclasses import dataclass -from textwrap import dedent -from typing import Callable - -from probe.ip_utils import get_addr_type -from utils.artifact import create_standard_log_name, log -from utils.log import add_border, border_print -from zeroconf import ServiceBrowser, ServiceInfo, ServiceListener, Zeroconf - -logger = log.get_logger(__file__) - - -@dataclass() -class MdnsTypeInfo: - type: str - description: str - - -commissioner = MdnsTypeInfo( - "COMMISSIONER", - "This is a service for a Matter commissioner aka. controller" -) -commissionable = MdnsTypeInfo( - "COMMISSIONABLE / EXTENDED DISCOVERY", - "This is a service to be used in the commissioning process and provides more info about the device." -) -operational = MdnsTypeInfo( - "OPERATIONAL", - "This is a service for a commissioned Matter device. It exposes limited info about the device." -) -border_router = MdnsTypeInfo( - "THREAD BORDER ROUTER", - "This is a service for a thread border router; may be used for thread+Matter devices." -) - -_MDNS_TYPES = { - "_matterd._udp.local.": commissioner, - "_matterc._udp.local.": commissionable, - "_matter._tcp.local.": operational, - "_meshcop._udp.local.": border_router, -} - - -@dataclass() -class RecordParser: - readable_name: str - explanation: str - parse: Callable[[str], str] - - -# TODO: Meshcop parser - -class MatterTxtRecordParser: - - def __init__(self): - self.parsers = { - "D": RecordParser("Discriminator", - dedent("\ - Differentiates this instance of the device from others w/ same VID/PID that might be \n\ - in the environment."), - MatterTxtRecordParser.parse_d), # To hex - "VP": RecordParser("VID/PID", - "The Vendor ID and Product ID (each are two bytes of hex) that identify this product.", - MatterTxtRecordParser.parse_vp), # Split + to hex - "CM": RecordParser("Commissioning mode", - "Whether the device is in commissioning mode or not.", - MatterTxtRecordParser.parse_cm), # Decode - "DT": RecordParser("Device type", - "Application type for this end device.", - MatterTxtRecordParser.parse_dt), # Decode - "DN": RecordParser("Device name", - "Manufacturer provided device name. MAY match NodeLabel in Basic info cluster.", - MatterTxtRecordParser.parse_pass_through), # None - "RI": RecordParser("Rotating identifier", - "Vendor specific, non-trackable per-device ID.", - MatterTxtRecordParser.parse_pass_through), # None - "PH": RecordParser("Pairing hint", - dedent("\ - Given the current device state, follow these instructions to make the device \n\ - commissionable."), - MatterTxtRecordParser.parse_ph), # Decode - "PI": RecordParser("Pairing instructions", - dedent("\ - Used with the Pairing hint. If the Pairing hint mentions N, this is the \n\ - value of N."), - MatterTxtRecordParser.parse_pass_through), # None - # General records - "SII": RecordParser("Session idle interval", - "Message Reliability Protocol retry interval while the device is idle in milliseconds.", - MatterTxtRecordParser.parse_pass_through), # None - "SAI": RecordParser("Session active interval", - dedent("\ - Message Reliability Protocol retry interval while the device is active \n\ - in milliseconds."), - MatterTxtRecordParser.parse_pass_through), # None - "SAT": RecordParser("Session active threshold", - "Duration of time this device stays active after last activity in milliseconds.", - MatterTxtRecordParser.parse_pass_through), # None - "T": RecordParser("Supports TCP", - "Whether this device supports TCP client and or Server.", - MatterTxtRecordParser.parse_t), # Decode - } - self.unparsed_records = "" - self.parsed_records = "" - - def parse_single_record(self, key: str, value: str): - parser: RecordParser = self.parsers[key] - self.parsed_records += add_border(parser.readable_name + "\n") - self.parsed_records += parser.explanation + "\n\n" - try: - self.parsed_records += "PARSED VALUE: " + parser.parse(value) + "\n" - except Exception: - logger.error("Exception parsing TXT record, appending raw value") - logger.error(traceback.format_exc()) - self.parsed_records += f"RAW VALUE: {value}\n" - - def get_output(self) -> str: - unparsed_exp = "\nThe following TXT records were not parsed or explained:\n" - parsed_exp = "\nThe following was discovered about this device via TXT records:\n" - ret = "" - if self.unparsed_records: - ret += unparsed_exp + self.unparsed_records - if self.parsed_records: - ret += parsed_exp + self.parsed_records - return ret - - def parse_records(self, info: ServiceInfo) -> str: - if info.properties is not None: - for name, value in info.properties.items(): - try: - name = name.decode("utf-8") - except UnicodeDecodeError: - name = str(name) - try: - value = value.decode("utf-8") - except UnicodeDecodeError: - value = str(value) - if name not in self.parsers: - self.unparsed_records += f"KEY: {name} VALUE: {value}\n" - else: - self.parse_single_record(name, value) - return self.get_output() - - @staticmethod - def parse_pass_through(txt_value: str) -> str: - return txt_value - - @staticmethod - def parse_d(txt_value: str) -> str: - return hex(int(txt_value)) - - @staticmethod - def parse_vp(txt_value: str) -> str: - vid, pid = txt_value.split("+") - vid, pid = hex(int(vid)), hex(int(pid)) - return f"VID: {vid}, PID: {pid}" - - @staticmethod - def parse_cm(txt_value: str) -> str: - cm = int(txt_value) - mode_descriptions = [ - "Not in commissioning mode", - "In passcode commissioning mode (standard mode)", - "In dynamic passcode commissioning mode", - ] - return mode_descriptions[cm] - - @staticmethod - def parse_dt(txt_value: str) -> str: - application_device_types = { - # lighting - "0x100": "On/Off Light", - "0x101": "Dimmable Light", - "0x10C": "Color Temperature Light", - "0x10D": "Extended Color Light", - # smart plugs/outlets and other actuators - "0x10A": "On/Off Plug-in Unit", - "0x10B": "Dimmable Plug-In Unit", - "0x303": "Pump", - # switches and controls - "0x103": "On/Off Light Switch", - "0x104": "Dimmer Switch", - "0x105": "Color Dimmer Switch", - "0x840": "Control Bridge", - "0x304": "Pump Controller", - "0xF": "Generic Switch", - # sensors - "0x15": "Contact Sensor", - "0x106": "Light Sensor", - "0x107": "Occupancy Sensor", - "0x302": "Temperature Sensor", - "0x305": "Pressure Sensor", - "0x306": "Flow Sensor", - "0x307": "Humidity Sensor", - "0x850": "On/Off Sensor", - # closures - "0xA": "Door Lock", - "0xB": "Door Lock Controller", - "0x202": "Window Covering", - "0x203": "Window Covering Controller", - # HVAC - "0x300": "Heating/Cooling Unit", - "0x301": "Thermostat", - "0x2B": "Fan", - # media - "0x28": "Basic Video Player", - "0x23": "Casting Video Player", - "0x22": "Speaker", - "0x24": "Content App", - "0x29": "Casting Video Client", - "0x2A": "Video Remote Control", - # generic - "0x27": "Mode Select", - } - return application_device_types[hex((int(txt_value))).upper().replace("0X", "0x")] - - @staticmethod - def parse_ph(txt_value: str) -> str: - pairing_hints = [ - "Power Cycle", - "Custom commissioning flow", - "Use existing administrator (already commissioned)", - "Use settings menu on device", - "Use the PI TXT record hint", - "Read the manual", - "Press the reset button", - "Press Reset Button with application of power", - "Press Reset Button for N seconds", - "Press Reset Button until light blinks", - "Press Reset Button for N seconds with application of power", - "Press Reset Button until light blinks with application of power", - "Press Reset Button N times", - "Press Setup Button", - "Press Setup Button with application of power", - "Press Setup Button for N seconds", - "Press Setup Button until light blinks", - "Press Setup Button for N seconds with application of power", - "Press Setup Button until light blinks with application of power", - "Press Setup Button N times", - ] - ret = "\n" - b_arr = [int(b) for b in bin(int(txt_value))[2:]][::-1] - for i in range(0, len(b_arr)): - b = b_arr[i] - if b: - ret += pairing_hints[i] + "\n" - return ret - - @staticmethod - def parse_t(txt_value: str) -> str: - return "TCP supported" if int(txt_value) else "TCP not supported" - - -class MatterDnssdListener(ServiceListener): - - def __init__(self, artifact_dir: str) -> None: - super().__init__() - self.artifact_dir = artifact_dir - self.logger = logger - self.discovered_matter_devices: [str, ServiceInfo] = {} - - def write_log(self, line: str, log_name: str) -> None: - with open(self.create_device_log_name(log_name), "a+") as log_file: - log_file.write(line) - - def create_device_log_name(self, device_name) -> str: - return os.path.join( - self.artifact_dir, - create_standard_log_name(f"{device_name}_dnssd", "txt")) - - @staticmethod - def log_addr(info: ServiceInfo) -> str: - ret = add_border("This device has the following IP addresses\n") - for addr in info.parsed_scoped_addresses(): - ret += f"{get_addr_type(addr)}: {addr}\n" - return ret - - def handle_service_info( - self, - zc: Zeroconf, - type_: str, - name: str, - delta_type: str) -> None: - info = zc.get_service_info(type_, name) - self.discovered_matter_devices[name] = info - to_log = f"{name}\n" - update_str = f"\nSERVICE {delta_type}\n" - to_log += ("*" * (len(update_str) - 2)) + update_str - to_log += _MDNS_TYPES[type_].type + "\n" - to_log += _MDNS_TYPES[type_].description + "\n" - to_log += f"A/SRV TTL: {str(info.host_ttl)}\n" - to_log += f"PTR/TXT TTL: {str(info.other_ttl)}\n" - txt_parser = MatterTxtRecordParser() - to_log += txt_parser.parse_records(info) - to_log += self.log_addr(info) - self.logger.info(to_log) - self.write_log(to_log, name) - - def add_service(self, zc: Zeroconf, type_: str, name: str) -> None: - self.handle_service_info(zc, type_, name, "ADDED") - - def update_service(self, zc: Zeroconf, type_: str, name: str) -> None: - self.handle_service_info(zc, type_, name, "UPDATED") - - def remove_service(self, zc: Zeroconf, type_: str, name: str) -> None: - to_log = f"Service {name} removed\n" - to_log += _MDNS_TYPES[type_].type + "\n" - to_log += _MDNS_TYPES[type_].description - if name in self.discovered_matter_devices: - del self.discovered_matter_devices[name] - self.logger.warning(to_log) - self.write_log(to_log, name) - - def browse_interactive(self) -> None: - zc = Zeroconf() - ServiceBrowser(zc, list(_MDNS_TYPES.keys()), self) - try: - self.logger.warning( - dedent("\ - \n\ - Browsing Matter DNS-SD\n\ - DCL Lookup: https://webui.dcl.csa-iot.org/\n\ - See spec section 4.3 for details of Matter TXT records.\n")) - border_print("Press enter to stop!", important=True) - input("") - finally: - zc.close() - - async def browse_once(self, browse_time_seconds: int) -> Zeroconf: - zc = Zeroconf() - ServiceBrowser(zc, list(_MDNS_TYPES.keys()), self) - await asyncio.sleep(browse_time_seconds) - zc.close() - return zc diff --git a/src/tools/interop/idt/features/__init__.py b/src/tools/interop/idt/features/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/src/tools/interop/idt/capture/platform/android/config.py b/src/tools/interop/idt/features/advertise/__init__.py similarity index 84% rename from src/tools/interop/idt/capture/platform/android/config.py rename to src/tools/interop/idt/features/advertise/__init__.py index a0b05cd57a68cd..a927d6eb6b67c5 100644 --- a/src/tools/interop/idt/capture/platform/android/config.py +++ b/src/tools/interop/idt/features/advertise/__init__.py @@ -15,6 +15,9 @@ # limitations under the License. # -enable_build_push_tcpdump = True -enable_bug_report = True -hci_log_level = "full" +from .advertise import FakeMatterAdDnssd, FakeMatterAdBle + +__all__ = [ + "FakeMatterAdDnssd", + "FakeMatterAdBle", +] diff --git a/src/tools/interop/idt/features/advertise/advertise.py b/src/tools/interop/idt/features/advertise/advertise.py new file mode 100644 index 00000000000000..65bbbfc487516f --- /dev/null +++ b/src/tools/interop/idt/features/advertise/advertise.py @@ -0,0 +1,115 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import asyncio +import sys +from abc import ABC, abstractmethod + +from zeroconf import IPVersion +from zeroconf.asyncio import AsyncServiceInfo, AsyncZeroconf + +from utils.host import current_platform +from utils.log import get_logger + + +class FakeMatterAd(ABC): + + def __init__(self, + vid: int, + pid: int, + discriminator: int, + device_name: str, + device_type: int, + port: int, + commissioning_open: int, + mac_address: str, + allow_gua: bool, + logger=None): + self.vid = vid + self.pid = pid + self.discriminator = discriminator + self.device_name = device_name + self.device_type = device_type + self.port = port + self.commissioning_open = commissioning_open + self.mac_address = mac_address + self.allow_gua = allow_gua + self.logger = get_logger(__file__) if not logger else logger + self.logger.info(f"New fake matter ad instantiated\n{self}") + + def __repr__(self): + return f"VID {hex(self.vid)}\n" \ + f"PID {hex(self.pid)}\n" \ + f"Discriminator {hex(self.discriminator)}\n" \ + f"Device name {self.device_name}\n" \ + f"Device type {hex(self.device_type)}\n" \ + f"Port {self.port}\n" \ + f"Commissioning open {self.commissioning_open}\n" \ + f"Mac addr {self.mac_address}\n" + + @abstractmethod + async def advertise(self): + """ + Advertise until the user presses enter + """ + raise NotImplementedError + + +class FakeMatterAdDnssd(FakeMatterAd): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.service = "_matterc._udp.local." + self.instance = f"{self.mac_address}.{self.service}" + self.properties = { + "VP": f"{str(self.vid)}+{str(self.pid)}", + "DN": self.device_name, + "DT": self.device_type, + "CM": int(self.commissioning_open), + "D": self.discriminator, + } + self.aiozc = None + self.ips = current_platform.ips() + self.logger.info(f"Fake matter DNS-SD ads using the following IPs {self.ips}") + + async def advertise(self): + addrs = self.ips.v4 + self.ips.v6_link_local + self.ips.v6_unique_local + if self.allow_gua: + addrs += self.ips.v6_global + service_info = AsyncServiceInfo( + self.service, + self.instance, + addresses=addrs, + port=self.port, + properties=self.properties, + ) + # TODO: Make an option for v4 or v6 or v4 and v6 + self.aiozc = AsyncZeroconf(ip_version=IPVersion.V4Only) + await self.aiozc.async_register_service(service_info) + await asyncio.get_event_loop().run_in_executor( + None, sys.stdin.readline) + await self.aiozc.async_unregister_service(service_info) + self.logger.info("Ended advertisement!") + + +class FakeMatterAdBle(FakeMatterAd): + + def __init__(self, *args, **kwargs): + super().__init__() + + async def advertise(self): + raise NotImplementedError diff --git a/src/tools/interop/idt/features/advertise/config.py b/src/tools/interop/idt/features/advertise/config.py new file mode 100644 index 00000000000000..f54f3768612409 --- /dev/null +++ b/src/tools/interop/idt/features/advertise/config.py @@ -0,0 +1,38 @@ +# +# Copyright (c) 2024 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# MT:Y.K90EDH025-LI0IQ00 +# Version: 0 +# VendorID: 65521 +# ProductID: 32768 +# Custom flow: 0 (STANDARD) +# Discovery Bitmask: 0x00 (NONE) +# Long discriminator: 10 (0xa) +# Passcode: 32966599 +# https://project-chip.github.io/connectedhomeip/qrcode.html?data=MT:Y.K90EDH025-LI0IQ00 + +from utils.host import current_platform + +DEFAULT_VID = 0xFFF1 +DEFAULT_PID = 0x8000 +DEFAULT_DISCRIMINATOR = 0xA +DEFAULT_DEVICE_NAME = "IDT fake device" +DEFAULT_DEVICE_TYPE = 0x302 # Temperature Sensor +DEFAULT_PORT = 5540 +DEFAULT_COMMISSIONING_OPEN = "t" +DEFAULT_MAC_ADDR = current_platform.get_mac_addr() +DEFAULT_ALLOW_V6_GUA = "f" diff --git a/src/tools/interop/idt/capture/__init__.py b/src/tools/interop/idt/features/capture/__init__.py similarity index 94% rename from src/tools/interop/idt/capture/__init__.py rename to src/tools/interop/idt/features/capture/__init__.py index 18069e71ad7838..1999ce8e0bc9aa 100644 --- a/src/tools/interop/idt/capture/__init__.py +++ b/src/tools/interop/idt/features/capture/__init__.py @@ -15,7 +15,7 @@ # limitations under the License. # -from capture import ecosystem, platform +from features.capture import ecosystem, platform from .controller import EcosystemCapture, PlatformLogStreamer from .pcap import PacketCaptureRunner diff --git a/src/tools/interop/idt/capture/base.py b/src/tools/interop/idt/features/capture/base.py similarity index 98% rename from src/tools/interop/idt/capture/base.py rename to src/tools/interop/idt/features/capture/base.py index b90571020a071f..acc9583109a5ce 100644 --- a/src/tools/interop/idt/capture/base.py +++ b/src/tools/interop/idt/features/capture/base.py @@ -90,19 +90,19 @@ async def start_capture(self) -> None: raise NotImplementedError @abstractmethod - async def stop_capture(self) -> None: + async def analyze_capture(self) -> None: """ - Stop the capture and pull any artifacts from remote devices - Write artifacts to artifact_dir passed on instantiation - Platform is already stopped + Parse the capture and create + display helpful analysis artifacts that are unique to the ecosystem in real time + Must be async aware and not interact with stdin """ raise NotImplementedError @abstractmethod - async def analyze_capture(self) -> None: + async def stop_capture(self) -> None: """ - Parse the capture and create + display helpful analysis artifacts that are unique to the ecosystem - Must be async aware and not interact with stdin + Stop the capture and pull any artifacts from remote devices + Write artifacts to artifact_dir passed on instantiation + Platform is already stopped """ raise NotImplementedError diff --git a/src/tools/interop/idt/capture/config.py b/src/tools/interop/idt/features/capture/config.py similarity index 97% rename from src/tools/interop/idt/capture/config.py rename to src/tools/interop/idt/features/capture/config.py index 12ce985eb4b3e4..818fbf4886b940 100644 --- a/src/tools/interop/idt/capture/config.py +++ b/src/tools/interop/idt/features/capture/config.py @@ -47,4 +47,4 @@ async def not_actually_async(): # Now it is_actually_async because we Result: The timeout error will be raised. """ -orchestrator_async_step_timeout_seconds = 240 +ORCHESTRATOR_ASYNC_STEP_TIMEOUT_SECONDS = 240 diff --git a/src/tools/interop/idt/capture/controller.py b/src/tools/interop/idt/features/capture/controller.py similarity index 67% rename from src/tools/interop/idt/capture/controller.py rename to src/tools/interop/idt/features/capture/controller.py index 624a6f292451d3..59ee5f23fcf3d9 100644 --- a/src/tools/interop/idt/capture/controller.py +++ b/src/tools/interop/idt/features/capture/controller.py @@ -19,38 +19,21 @@ import copy import os import sys -import traceback import typing -from dataclasses import dataclass +from asyncio import Task -import capture -from capture.base import EcosystemCapture, PlatformLogStreamer, UnsupportedCapturePlatformException -from utils.artifact import create_standard_log_name, log, safe_mkdir -from utils.log import add_border, border_print +from .. import capture +from utils.error import log_error +from features.capture.base import EcosystemCapture, PlatformLogStreamer, UnsupportedCapturePlatformException +from utils.artifact import safe_mkdir +from utils.log import border_print, get_logger from . import config - -@dataclass(repr=True) -class ErrorRecord: - ecosystem: str - help_message: str - stack_trace: str - - _PLATFORM_MAP: typing.Dict[str, PlatformLogStreamer] = {} -_ECOSYSTEM_MAP: typing.Dict[str, PlatformLogStreamer] = {} -_ERROR_REPORT: typing.Dict[str, ErrorRecord] = {} - -logger = log.get_logger(__file__) - +_ECOSYSTEM_MAP: typing.Dict[str, EcosystemCapture] = {} -def track_error(ecosystem: str, help_message: str) -> None: - if ecosystem not in _ERROR_REPORT: - _ERROR_REPORT[ecosystem] = [] - record = ErrorRecord(ecosystem, help_message, traceback.format_exc()) - logger.error(record) - _ERROR_REPORT[ecosystem].append(record) +logger = get_logger(__file__) def list_available_platforms() -> typing.List[str]: @@ -68,7 +51,7 @@ async def get_platform_impl( safe_mkdir(platform_artifact_dir) platform_inst = platform_class(platform_artifact_dir) _PLATFORM_MAP[platform] = platform_inst - async with asyncio.timeout(config.orchestrator_async_step_timeout_seconds): + async with asyncio.timeout(config.ORCHESTRATOR_ASYNC_STEP_TIMEOUT_SECONDS): await platform_inst.connect() return platform_inst @@ -103,11 +86,11 @@ async def init_ecosystems(platform, ecosystem, artifact_dir): except UnsupportedCapturePlatformException: help_message = f"Unsupported platform {ecosystem} {platform}" logger.error(help_message) - track_error(ecosystem, help_message) + log_error(ecosystem, help_message) except Exception: help_message = f"Unknown error instantiating ecosystem {ecosystem} {platform}" logger.error(help_message) - track_error(ecosystem, help_message) + log_error(ecosystem, help_message) async def handle_capture(attr): @@ -115,16 +98,16 @@ async def handle_capture(attr): for ecosystem in _ECOSYSTEM_MAP: try: border_print(f"{attr} for {ecosystem}") - async with asyncio.timeout(config.orchestrator_async_step_timeout_seconds): + async with asyncio.timeout(config.ORCHESTRATOR_ASYNC_STEP_TIMEOUT_SECONDS): await getattr(_ECOSYSTEM_MAP[ecosystem], attr)() except TimeoutError: - help_message = f"Timeout after {config.orchestrator_async_step_timeout_seconds} seconds {attr} {ecosystem}" + help_message = f"Timeout after {config.ORCHESTRATOR_ASYNC_STEP_TIMEOUT_SECONDS} seconds {attr} {ecosystem}" logger.error(help_message) - track_error(ecosystem, help_message) + log_error(ecosystem, help_message) except Exception: help_message = f"Unexpected error {attr} {ecosystem}" logger.error(help_message) - track_error(ecosystem, help_message) + log_error(ecosystem, help_message) async def start(): @@ -143,41 +126,44 @@ async def stop(): await handle_capture("stop") +def sub_task_error_tracker_done_callback(task: asyncio.Task) -> None: + try: + task.result() + except asyncio.CancelledError: + pass + except Exception: + platform_or_ecosystem = "UNKNOWN" + if hasattr(task, "error_tracking_name"): + platform_or_ecosystem = getattr(task, "error_tracking_name") + log_error(platform_or_ecosystem, "Unexpected error in sub task") + + async def run_analyzers(): border_print("Starting real time analysis, press enter to stop!", important=True) - analysis_tasks = [] - monitor_tasks = [] + analysis_tasks: typing.Dict[str, Task] = {} + monitor_tasks: typing.Dict[str, Task] = {} for platform_name, platform in _PLATFORM_MAP.items(): logger.info(f"Creating monitor task for {platform_name}") - monitor_tasks.append(asyncio.create_task(platform.run_observers())) + monitor_tasks[platform_name] = asyncio.create_task(platform.run_observers()) + setattr(monitor_tasks[platform_name], "error_tracking_name", platform_name) + monitor_tasks[platform_name].add_done_callback(sub_task_error_tracker_done_callback) for ecosystem_name, ecosystem in _ECOSYSTEM_MAP.items(): logger.info(f"Creating analysis task for {ecosystem_name}") - analysis_tasks.append(asyncio.create_task(ecosystem.analyze_capture())) + analysis_tasks[ecosystem_name] = asyncio.create_task(ecosystem.analyze_capture()) + setattr(analysis_tasks[ecosystem_name], "error_tracking_name", ecosystem_name) + analysis_tasks[ecosystem_name].add_done_callback(sub_task_error_tracker_done_callback) logger.info("Done creating analysis tasks") await asyncio.get_event_loop().run_in_executor( None, sys.stdin.readline) border_print("Cancelling monitor tasks") - for task in monitor_tasks: - task.cancel() + for platform_name, monitor_task in monitor_tasks.items(): + monitor_task.cancel() logger.info("Done cancelling monitor tasks") border_print("Cancelling analysis tasks") - for task in analysis_tasks: - task.cancel() + for ecosystem_name, analysis_task in analysis_tasks.items(): + analysis_task.cancel() logger.info("Done cancelling analysis tasks") async def probe(): await handle_capture("probe") - - -def write_error_report(artifact_dir: str): - if _ERROR_REPORT: - logger.critical("DETECTED ERRORS THIS RUN!") - error_report_file_name = create_standard_log_name("error_report", "txt", parent=artifact_dir) - with open(error_report_file_name, "a+") as error_report_file: - for ecosystem in _ERROR_REPORT: - log.print_and_write(add_border(f"Errors for {ecosystem}"), error_report_file) - for record in _ERROR_REPORT[ecosystem]: - log.print_and_write(str(record), error_report_file) - else: - logger.info("No errors seen this run!") diff --git a/src/tools/interop/idt/capture/ecosystem/__init__.py b/src/tools/interop/idt/features/capture/ecosystem/__init__.py similarity index 86% rename from src/tools/interop/idt/capture/ecosystem/__init__.py rename to src/tools/interop/idt/features/capture/ecosystem/__init__.py index 432af96d345a6b..e27844ea048cde 100644 --- a/src/tools/interop/idt/capture/ecosystem/__init__.py +++ b/src/tools/interop/idt/features/capture/ecosystem/__init__.py @@ -15,12 +15,12 @@ # limitations under the License. # -from capture.base import EcosystemCapture -from capture.loader import CaptureImplsLoader +from features.capture.base import EcosystemCapture +from utils.loader import CaptureImplsLoader impl_loader = CaptureImplsLoader( __path__[0], - "capture.ecosystem", + "features.capture.ecosystem", EcosystemCapture ) diff --git a/src/tools/interop/idt/capture/ecosystem/play_services/__init__.py b/src/tools/interop/idt/features/capture/ecosystem/play_services/__init__.py similarity index 100% rename from src/tools/interop/idt/capture/ecosystem/play_services/__init__.py rename to src/tools/interop/idt/features/capture/ecosystem/play_services/__init__.py diff --git a/src/tools/interop/idt/features/capture/ecosystem/play_services/analysis.py b/src/tools/interop/idt/features/capture/ecosystem/play_services/analysis.py new file mode 100644 index 00000000000000..7e3197c1fc4b08 --- /dev/null +++ b/src/tools/interop/idt/features/capture/ecosystem/play_services/analysis.py @@ -0,0 +1,119 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os + +from features.capture.platform.android import Android +from features.capture.platform.android.streams.logcat import LogcatStreamer +from utils.analysis import TextCause, PrescriptiveTextAnalysis, TextAnalysisObserver +from utils.artifact import create_standard_log_name, log +from utils.log import add_border, print_and_write + +logger = log.get_logger(__file__) + + +class PlayServicesAnalysis(PrescriptiveTextAnalysis, TextAnalysisObserver): + + def __init__(self, platform: Android, artifact_dir: str) -> None: + + self.logger = logger + self.output_file_name = os.path.join( + artifact_dir, create_standard_log_name( + 'commissioning_logcat', 'txt')) + + causes = [ + TextCause(["Failed to discover commissionable device."], + "failure_stack_trace", + "Play Services could not locate the device's initial advertisement. Use $ idt discover to check!", + []), + TextCause(["Failed to discover operational device"], + "failure_stack_trace", + "All steps of PASE completed as expected, but the secure session setup on IP network failed.", + [ + TextCause(["AddressResolve_DefaultImpl", "Timeout"], + "matter_commissioner_logs", + "Play Services failed to locate end device via DNS-SD. Inspect pcaps / $ idt " + "discover -t d", + []), + TextCause(["secure_channel/CASESession"], + "matter_commissioner_logs", + "End device discovered via DNS-SD, but session handshake failed! Try $ idt probe", + []), + ]) + ] + PrescriptiveTextAnalysis.__init__(self, causes, self.output_file_name, logger) + + logcat_stream: LogcatStreamer = platform.streams["LogcatStreamer"] + TextAnalysisObserver.__init__(self, logcat_stream.logcat_artifact, logger) + + self.matter_commissioner_logs = '' + self.failure_stack_trace = '' + self.pake_logs = '' + self.resolver_logs = '' + self.sigma_logs = '' + self.fail_trace_line_counter = -1 + + def _log_proc_matter_commissioner(self, line: str) -> None: + if 'MatterCommissioner' in line: + self.logger.info(line) + self.matter_commissioner_logs += line + + def _log_proc_commissioning_failed(self, line: str) -> None: + parsed_stack_trace_max_depth = 15 + if self.fail_trace_line_counter > parsed_stack_trace_max_depth: + self.fail_trace_line_counter = -1 + if self.fail_trace_line_counter > -1 and 'SetupDevice' in line: + self.failure_stack_trace += line + self.fail_trace_line_counter += 1 + if 'SetupDeviceView' and 'Commissioning failed' in line: + self.logger.info(line) + self.logger.warning("Failure detected, you may want to stop the capture now!") + self.fail_trace_line_counter = 0 + self.failure_stack_trace += line + + def _log_proc_pake(self, line: str) -> None: + if "Pake" in line and "chip_logging" in line: + self.logger.info(line) + self.pake_logs += line + + def _log_proc_mdns(self, line: str) -> None: + if "_matter" in line and "ServiceResolverAdapter" in line: + self.logger.info(line) + self.resolver_logs += line + + def _log_proc_sigma(self, line: str) -> None: + if "Sigma" in line and "chip_logging" in line: + self.logger.info(line) + self.sigma_logs += line + + # TODO: Option to analyze existing log + def show_analysis(self) -> None: + with open(self.output_file_name, mode="w+") as analysis_file: + print_and_write(add_border('Matter commissioner logs'), analysis_file) + print_and_write(self.matter_commissioner_logs, analysis_file) + print_and_write( + add_border('Commissioning failure stack trace'), + analysis_file) + print_and_write(self.failure_stack_trace, analysis_file) + print_and_write(add_border('PASE Handshake'), analysis_file) + print_and_write(self.pake_logs, analysis_file) + print_and_write(add_border('DNS-SD resolution'), analysis_file) + print_and_write(self.resolver_logs, analysis_file) + print_and_write(add_border('CASE handshake'), analysis_file) + print_and_write(self.sigma_logs, analysis_file) + print_and_write(add_border('Prescriptive analysis findings'), analysis_file) + self.check_causes() diff --git a/src/tools/interop/idt/capture/ecosystem/play_services/command_map.py b/src/tools/interop/idt/features/capture/ecosystem/play_services/command_map.py similarity index 69% rename from src/tools/interop/idt/capture/ecosystem/play_services/command_map.py rename to src/tools/interop/idt/features/capture/ecosystem/play_services/command_map.py index 4adf9b59bb5811..99b8e4cb9e8368 100644 --- a/src/tools/interop/idt/capture/ecosystem/play_services/command_map.py +++ b/src/tools/interop/idt/features/capture/ecosystem/play_services/command_map.py @@ -15,7 +15,7 @@ # limitations under the License. # -getprop = { +GET_PROP = { 'ro.product.model': 'android_model', 'ro.build.version.release': 'android_version', 'ro.build.version.sdk': 'android_api', @@ -25,15 +25,15 @@ 'ro.ecosystem.build.fingerprint': 'vendor_build_fingerprint', } -_ap = 'activity provider com.google.android.gms.chimera.container.GmsModuleProvider' -dumpsys = { +_MODULE_PROVIDER = 'activity provider com.google.android.gms.chimera.container.GmsModuleProvider' +DUMPSYS = { 'display_width': 'display | grep StableDisplayWidth | awk -F\'=\' \'{print $2}\'', 'display_height': 'display | grep StableDisplayHeight | awk -F\'=\' \'{print $2}\'', 'gha_info': ' package com.google.android.apps.chromecast.app | grep versionName', 'container_info': 'package com.google.android.gms | grep "versionName"', - 'home_module_info': f'{_ap} | grep "com.google.android.gms.home" | grep -v graph', - 'optional_home_module_info': f'{_ap} | grep "com.google.android.gms.optional_home" | grep -v graph', - 'policy_home_module_info': f'{_ap} | grep "com.google.android.gms.policy_home" | grep -v graph', - 'thread_info': f'{_ap} | grep "com.google.android.gms.threadnetwork"', - 'mdns_info': f'{_ap} | grep -i com.google.android.gms.mdns', + 'home_module_info': f'{_MODULE_PROVIDER} | grep "com.google.android.gms.home" | grep -v graph', + 'optional_home_module_info': f'{_MODULE_PROVIDER} | grep "com.google.android.gms.optional_home" | grep -v graph', + 'policy_home_module_info': f'{_MODULE_PROVIDER} | grep "com.google.android.gms.policy_home" | grep -v graph', + 'thread_info': f'{_MODULE_PROVIDER} | grep "com.google.android.gms.threadnetwork"', + 'mdns_info': f'{_MODULE_PROVIDER} | grep -i com.google.android.gms.mdns', } diff --git a/src/tools/interop/idt/features/capture/ecosystem/play_services/config.py b/src/tools/interop/idt/features/capture/ecosystem/play_services/config.py new file mode 100644 index 00000000000000..df80fa954a20c7 --- /dev/null +++ b/src/tools/interop/idt/features/capture/ecosystem/play_services/config.py @@ -0,0 +1,22 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os + +ENABLE_PROBERS = True +PROBER_TRACEROUTE_LIMIT = 32 +PROBER_URLS_FILE_NAME = os.path.join(os.path.dirname(__file__), "prober_urls.txt") diff --git a/src/tools/interop/idt/capture/ecosystem/play_services/play_services.py b/src/tools/interop/idt/features/capture/ecosystem/play_services/play_services.py similarity index 69% rename from src/tools/interop/idt/capture/ecosystem/play_services/play_services.py rename to src/tools/interop/idt/features/capture/ecosystem/play_services/play_services.py index e94a10f9260b94..8b48e66c459458 100644 --- a/src/tools/interop/idt/capture/ecosystem/play_services/play_services.py +++ b/src/tools/interop/idt/features/capture/ecosystem/play_services/play_services.py @@ -15,43 +15,42 @@ # limitations under the License. # -import asyncio import json import os from typing import IO, Dict -from capture.base import EcosystemCapture, UnsupportedCapturePlatformException -from capture.platform.android import Android -from capture.platform.android.streams.logcat import LogcatStreamer +from features.capture.base import EcosystemCapture, UnsupportedCapturePlatformException, PlatformLogStreamer +from features.capture.platform.android import Android +from features.capture.platform.android.streams.logcat import LogcatStreamer from utils.artifact import create_standard_log_name, log from . import config -from .command_map import dumpsys, getprop -from .play_services_analysis import PlayServicesAnalysis -from .prober import PlayServicesProber +from .command_map import DUMPSYS, GET_PROP +from .analysis import PlayServicesAnalysis +from .probe import PlayServicesProber logger = log.get_logger(__file__) -class PlayServices(EcosystemCapture): +class PlayServices(PlayServicesAnalysis, EcosystemCapture): """ Implementation of capture and analysis for Play Services """ - def __init__(self, platform: Android, artifact_dir: str) -> None: + def __init__(self, platform: PlatformLogStreamer, artifact_dir: str) -> None: self.artifact_dir = artifact_dir if not isinstance(platform, Android): raise UnsupportedCapturePlatformException( - 'only platform=android is supported for ecosystem=play_services') - self.platform = platform + 'only platform=Android is supported for ecosystem=PlayServices') + self.platform: Android = platform self.standard_info_file_path = os.path.join( self.artifact_dir, create_standard_log_name( 'phone_info', 'json')) self.standard_info_data: Dict[str, str] = {} - self.analysis = PlayServicesAnalysis(self.platform, self.artifact_dir) + PlayServicesAnalysis.__init__(self, platform, artifact_dir) self.service_ids = ['336', # Home '305', # Thread @@ -72,12 +71,12 @@ def _parse_get_prop(self) -> None: "shell getprop", capture_output=True).get_captured_output() for output in get_prop.split("\n"): - for prop in getprop: + for prop in GET_PROP: if prop in output: self.standard_info_data[prop] = output[output.rindex("["):] def _parse_dumpsys(self) -> None: - for attr_name, command in dumpsys.items(): + for attr_name, command in DUMPSYS.items(): command = f"shell dumpsys {command}" command_output = self.platform.run_adb_command( command, @@ -90,28 +89,17 @@ def _get_standard_info(self) -> None: self._write_standard_info_file() async def start_capture(self) -> None: + # TODO: Hub logs for service_id in self.service_ids: verbose_command = f"shell setprop log.tag.gms_svc_id:{service_id} VERBOSE" self.platform.run_adb_command(verbose_command) self._get_standard_info() - async def analyze_capture(self): - try: - self.logcat_file = open(self.logcat_stream.logcat_artifact, "r") - while True: - self.analysis.do_analysis(self.logcat_file.readlines()) - # Releasing async event loop for other analysis / monitor topics - await asyncio.sleep(0.5) - except asyncio.CancelledError: - logger.info("Closing logcat stream") - if self.logcat_file: - self.logcat_file.close() - async def stop_capture(self) -> None: - self.analysis.show_analysis() + self.show_analysis() async def probe_capture(self) -> None: - if config.enable_foyer_probers: + if config.ENABLE_PROBERS: await PlayServicesProber(self.platform, self.artifact_dir).probe_services() else: - logger.critical("Foyer probers disabled in config!") + logger.critical("Probers disabled in config!") diff --git a/src/tools/interop/idt/capture/ecosystem/play_services/prober.py b/src/tools/interop/idt/features/capture/ecosystem/play_services/probe.py similarity index 70% rename from src/tools/interop/idt/capture/ecosystem/play_services/prober.py rename to src/tools/interop/idt/features/capture/ecosystem/play_services/probe.py index 8dcda80ddf3133..08b7dddee408ce 100644 --- a/src/tools/interop/idt/capture/ecosystem/play_services/prober.py +++ b/src/tools/interop/idt/features/capture/ecosystem/play_services/probe.py @@ -21,7 +21,7 @@ from . import config -logger = log.get_logger(__file__) +_LOGGER = log.get_logger(__file__) class PlayServicesProber: @@ -30,42 +30,48 @@ def __init__(self, platform, artifact_dir): # TODO: Handle all resolved addresses self.platform = platform self.artifact_dir = artifact_dir - self.logger = logger + self.logger = _LOGGER self.probe_artifact = os.path.join(self.artifact_dir, "net_probes.txt") self.command_suffix = f" 2>&1 | tee -a {self.probe_artifact}" - self.target = "googlehomefoyer-pa.googleapis.com" - self.tracert_limit = config.foyer_prober_traceroute_limit + self.target = "" + self.tracert_limit = config.PROBER_TRACEROUTE_LIMIT def run_command(self, command): Bash(f"{command} {self.command_suffix}", sync=True).start_command() - async def _probe_tracert_icmp_foyer(self) -> None: + async def _probe_tracert_icmp(self) -> None: self.logger.info(f"icmp traceroute to {self.target}") self.run_command(f"traceroute -m {self.tracert_limit} {self.target}") - async def _probe_tracert_udp_foyer(self) -> None: + async def _probe_tracert_udp(self) -> None: # TODO: Per-host-platform impl self.logger.info(f"udp traceroute to {self.target}") self.run_command(f"traceroute -m {self.tracert_limit} -U -p 443 {self.target}") - async def _probe_tracert_tcp_foyer(self) -> None: + async def _probe_tracert_tcp(self) -> None: # TODO: Per-host-platform impl self.logger.info(f"tcp traceroute to {self.target}") self.run_command(f"traceroute -m {self.tracert_limit} -T -p 443 {self.target}") - async def _probe_ping_foyer(self) -> None: + async def _probe_ping(self) -> None: self.logger.info(f"ping {self.target}") self.run_command(f"ping -c 4 {self.target}") - async def _probe_dns_foyer(self) -> None: + async def _probe_dns(self) -> None: self.logger.info(f"dig {self.target}") self.run_command(f"dig {self.target}") - async def _probe_from_phone_ping_foyer(self) -> None: + async def _probe_from_phone_ping(self) -> None: self.logger.info(f"ping {self.target} from phone") self.platform.run_adb_command(f"shell ping -c 4 {self.target} {self.command_suffix}") async def probe_services(self) -> None: - self.logger.info(f"Probing {self.target}") - for probe_func in [s for s in dir(self) if s.startswith('_probe')]: - await getattr(self, probe_func)() + if not os.path.exists(config.PROBER_URLS_FILE_NAME): + self.logger.info("No probe targets configured") + return + with open(config.PROBER_URLS_FILE_NAME) as urls_file: + for line in urls_file: + self.target = line + self.logger.info(f"Probing {self.target}") + for probe_func in [s for s in dir(self) if s.startswith('_probe')]: + await getattr(self, probe_func)() diff --git a/src/tools/interop/idt/capture/ecosystem/play_services_user/__init__.py b/src/tools/interop/idt/features/capture/ecosystem/play_services_user/__init__.py similarity index 100% rename from src/tools/interop/idt/capture/ecosystem/play_services_user/__init__.py rename to src/tools/interop/idt/features/capture/ecosystem/play_services_user/__init__.py diff --git a/src/tools/interop/idt/capture/ecosystem/play_services_user/play_services_user.py b/src/tools/interop/idt/features/capture/ecosystem/play_services_user/play_services_user.py similarity index 70% rename from src/tools/interop/idt/capture/ecosystem/play_services_user/play_services_user.py rename to src/tools/interop/idt/features/capture/ecosystem/play_services_user/play_services_user.py index 2fdac62b8bbdb7..b0e40329eab8e0 100644 --- a/src/tools/interop/idt/capture/ecosystem/play_services_user/play_services_user.py +++ b/src/tools/interop/idt/features/capture/ecosystem/play_services_user/play_services_user.py @@ -15,24 +15,24 @@ # limitations under the License. # -import asyncio import os -from capture.base import EcosystemCapture, UnsupportedCapturePlatformException -from capture.platform.android.android import Android -from capture.platform.android.streams.logcat import LogcatStreamer +from features.capture.base import EcosystemCapture, UnsupportedCapturePlatformException, PlatformLogStreamer +from features.capture.platform.android.android import Android +from features.capture.platform.android.streams.logcat import LogcatStreamer +from utils.analysis import TextAnalysisObserver from utils.artifact import create_standard_log_name from utils.log import get_logger, print_and_write logger = get_logger(__file__) -class PlayServicesUser(EcosystemCapture): +class PlayServicesUser(TextAnalysisObserver, EcosystemCapture): """ Implementation of capture and analysis for Play Services 3P """ - def __init__(self, platform: Android, artifact_dir: str) -> None: + def __init__(self, platform: PlatformLogStreamer, artifact_dir: str) -> None: self.logger = logger self.artifact_dir = artifact_dir self.analysis_file = os.path.join( @@ -43,18 +43,16 @@ def __init__(self, platform: Android, artifact_dir: str) -> None: raise UnsupportedCapturePlatformException( 'only platform=android is supported for ' 'ecosystem=PlayServicesUser') - self.platform = platform + self.platform: Android = platform self.logcat_fd = None self.output = "" self.logcat_stream: LogcatStreamer = self.platform.streams["LogcatStreamer"] + TextAnalysisObserver.__init__(self, self.logcat_stream.logcat_artifact, logger) async def start_capture(self) -> None: pass - async def stop_capture(self) -> None: - self.show_analysis() - - def proc_line(self, line) -> None: + def _log_proc(self, line) -> None: if "CommissioningServiceBin: Binding to service" in line: s = f"3P commissioner initiated Play Services commissioning\n{line}" logger.info(s) @@ -68,23 +66,12 @@ def proc_line(self, line) -> None: logger.info(s) self.output += f"{s}\n" - async def analyze_capture(self) -> None: - """"Show the start and end times of commissioning boundaries""" - try: - self.logcat_fd = open(self.logcat_stream.logcat_artifact, "r") - while True: - for line in self.logcat_fd.readlines(): - self.proc_line(line) - # Releasing async event loop for other analysis / monitor tasks - await asyncio.sleep(0.5) - except asyncio.CancelledError: - self.logger.info("Closing logcat stream") - if self.logcat_fd is not None: - self.logcat_fd.close() - def show_analysis(self) -> None: with open(self.analysis_file, "w") as analysis_file: print_and_write(self.output, analysis_file) + async def stop_capture(self) -> None: + self.show_analysis() + async def probe_capture(self) -> None: pass diff --git a/src/tools/interop/idt/capture/pcap/__init__.py b/src/tools/interop/idt/features/capture/pcap/__init__.py similarity index 100% rename from src/tools/interop/idt/capture/pcap/__init__.py rename to src/tools/interop/idt/features/capture/pcap/__init__.py diff --git a/src/tools/interop/idt/capture/pcap/pcap.py b/src/tools/interop/idt/features/capture/pcap/base.py similarity index 77% rename from src/tools/interop/idt/capture/pcap/pcap.py rename to src/tools/interop/idt/features/capture/pcap/base.py index 2cb2285908b066..839ab12d24381e 100644 --- a/src/tools/interop/idt/capture/pcap/pcap.py +++ b/src/tools/interop/idt/features/capture/pcap/base.py @@ -18,15 +18,24 @@ import os import time +from abc import ABC +from typing import Optional + from utils.artifact import create_standard_log_name, log from utils.shell import Bash logger = log.get_logger(__file__) -class PacketCaptureRunner: +class PacketCaptureBase(ABC): - def __init__(self, artifact_dir: str, interface: str) -> None: + def __init__(self, + artifact_dir: str, + interface: str, + monitor_mode: bool, + channel: int, + band: str, + width: str) -> None: self.logger = logger self.artifact_dir = artifact_dir self.output_path = str( @@ -37,14 +46,18 @@ def __init__(self, artifact_dir: str, interface: str) -> None: "pcap"))) self.start_delay_seconds = 2 self.interface = interface - self.pcap_command = f"tcpdump -i {self.interface} -n -w {self.output_path}" - self.pcap_proc = Bash(self.pcap_command) + self.monitor_mode = monitor_mode + self.channel = channel + self.band = band + self.pcap_command: Optional[str] = None + self.pcap_proc: Optional[Bash] = None + self.width = width def start_pcap(self) -> None: self.pcap_proc.start_command() self.logger.info("Pausing to check if pcap started...") time.sleep(self.start_delay_seconds) - if not self.pcap_proc.command_is_running(): + if not self.pcap_proc.command_is_running() and "sudo" not in self.pcap_command: self.logger.error( "Pcap did not start, you might need root; please authorize if prompted.") Bash("sudo echo \"\"", sync=True).start_command() diff --git a/src/tools/interop/idt/features/capture/pcap/linux.py b/src/tools/interop/idt/features/capture/pcap/linux.py new file mode 100644 index 00000000000000..9cfc3d01aa6c12 --- /dev/null +++ b/src/tools/interop/idt/features/capture/pcap/linux.py @@ -0,0 +1,55 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os.path +from typing import Optional + +from features.capture.pcap.base import PacketCaptureBase +from utils.shell import Bash +from utils.host import current_platform + + +class LinuxPacketCaptureRunner(PacketCaptureBase): + + def __init__(self, + artifact_dir: str, + interface: str, + monitor_mode: bool, + channel: int, + band: str, + width: str) -> None: + super().__init__(artifact_dir, interface, monitor_mode, channel, band, width) + if self.monitor_mode: + self.interface = current_platform.get_link_local_interface() + setup_script_path = os.path.join(os.path.dirname(__file__), "setup_linux_interface.sh") + setup_proc = Bash(f"{setup_script_path} {self.interface} {self.channel}", sync=True) + setup_proc.start_command() + if not setup_proc.finished_success(): + self.logger.critical("Monitor mode setup failed!") + else: + self.interface = self.interface+"mon" + self.pcap_command: Optional[str] = f"tcpdump -i {self.interface} -n -w {self.output_path}" + self.pcap_proc = Bash(self.pcap_command) + + def start_pcap(self) -> None: + super().start_pcap() + + def stop_pcap(self) -> None: + super().stop_pcap() + if self.monitor_mode: + teardown_script_path = os.path.join(os.path.dirname(__file__), "teardown_linux_interface.sh") + teardown_proc = Bash(f"{teardown_script_path} {self.interface}", sync=True) + teardown_proc.start_command() diff --git a/src/tools/interop/idt/features/capture/pcap/mac.py b/src/tools/interop/idt/features/capture/pcap/mac.py new file mode 100644 index 00000000000000..702ae670b9458d --- /dev/null +++ b/src/tools/interop/idt/features/capture/pcap/mac.py @@ -0,0 +1,82 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import shutil +from typing import Optional + +from features.capture.pcap.base import PacketCaptureBase +from utils.shell import Bash + + +class MacPacketCaptureRunner(PacketCaptureBase): + + def __init__(self, + artifact_dir: str, + interface: str, + monitor_mode: bool, + channel: int, + band: str, + width: str) -> None: + super().__init__(artifact_dir, interface, monitor_mode, channel, band, width) + if self.monitor_mode: + self.pcap_command = f"sudo airport sniff {self.band}g{self.channel}" + if width: + self.pcap_command += "/"+width + self.logger.info(f"Mac pcap command {self.pcap_command}") + else: + self.pcap_command: Optional[str] = f"tcpdump -i {self.interface} -n -w {self.output_path}" + self.pcap_proc = Bash(self.pcap_command) + + def start_pcap(self) -> None: + if self.monitor_mode: + self.logger.warning("Disassociating from Wi-Fi, authorize if prompted!") + Bash("sudo airport -z", sync=True).start_command() # Do this to force a sudo auth + # We can remove this line and remove sudo from self.pcap_command above; + # simply call super() to add sudo if needed. + # However, airport does run without sudo and its unclear if that + # has any impact, so being explicit + self.logger.info("Starting 802.11 pcap using airport") + super().start_pcap() + + def find_latest_airport_pcap(self) -> str: + captures = [f for f in os.listdir("/tmp") if f.startswith("airportSniff")] + if captures: + path = os.path.join("/tmp", captures[0]) + oldest_mtime = os.path.getmtime(path) + oldest_path = path + for capture in captures: + path = os.path.join("/tmp", capture) + new_mtime = os.path.getmtime(path) + if new_mtime > oldest_mtime: + oldest_mtime = new_mtime + oldest_path = path + return oldest_path + else: + self.logger.error("Could not locate latest airport pcap!") + return "" + + def stop_pcap(self) -> None: + super().stop_pcap() + if self.monitor_mode: + airport_location = self.find_latest_airport_pcap() + if not airport_location: + self.logger.error("Could not locate latest airport pcap") + else: + self.logger.info(f"Found latest airport pcap at {airport_location}") + shutil.copy(airport_location, self.output_path) + self.logger.info(f"Copied airport pcap to artifacts {self.output_path}") diff --git a/src/tools/interop/idt/features/capture/pcap/pcap.py b/src/tools/interop/idt/features/capture/pcap/pcap.py new file mode 100644 index 00000000000000..6827c537a4875a --- /dev/null +++ b/src/tools/interop/idt/features/capture/pcap/pcap.py @@ -0,0 +1,45 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from features.capture.pcap.linux import LinuxPacketCaptureRunner +from features.capture.pcap.mac import MacPacketCaptureRunner +from utils.artifact import log +from utils.host import current_platform + +logger = log.get_logger(__file__) + + +class PacketCaptureRunner: + + def __init__(self, + artifact_dir: str, + interface: str, + monitor_mode: bool, + channel: int, + band: str, + width: int) -> None: + if current_platform.is_mac(): + self.runner = MacPacketCaptureRunner(artifact_dir, interface, monitor_mode, channel, band, width) + else: + self.runner = LinuxPacketCaptureRunner(artifact_dir, interface, monitor_mode, channel, band, width) + + def start_pcap(self) -> None: + self.runner.start_pcap() + + def stop_pcap(self) -> None: + # TODO: Display helpful starting wireshark queries to user here + self.runner.stop_pcap() diff --git a/src/tools/interop/idt/features/capture/pcap/setup_linux_interface.sh b/src/tools/interop/idt/features/capture/pcap/setup_linux_interface.sh new file mode 100755 index 00000000000000..acb01f88ccdd36 --- /dev/null +++ b/src/tools/interop/idt/features/capture/pcap/setup_linux_interface.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# +# Copyright (c) 2024 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e +echo "$0 setting interface $1 to monitor mode on channel $2" +sudo airmon-ng check +sudo airmon-ng check kill +sudo airmon-ng start $1 $2 diff --git a/src/tools/interop/idt/features/capture/pcap/teardown_linux_interface.sh b/src/tools/interop/idt/features/capture/pcap/teardown_linux_interface.sh new file mode 100755 index 00000000000000..d81317937a5f9f --- /dev/null +++ b/src/tools/interop/idt/features/capture/pcap/teardown_linux_interface.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# +# Copyright (c) 2024 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +echo "$0 setting interface $1 back to managed mode" +sudo airmon-ng stop $1 +sudo systemctl restart NetworkManager diff --git a/src/tools/interop/idt/capture/platform/__init__.py b/src/tools/interop/idt/features/capture/platform/__init__.py similarity index 86% rename from src/tools/interop/idt/capture/platform/__init__.py rename to src/tools/interop/idt/features/capture/platform/__init__.py index 92c1efcce79497..6f12d63e476d84 100644 --- a/src/tools/interop/idt/capture/platform/__init__.py +++ b/src/tools/interop/idt/features/capture/platform/__init__.py @@ -15,12 +15,12 @@ # limitations under the License. # -from capture.base import PlatformLogStreamer -from capture.loader import CaptureImplsLoader +from features.capture.base import PlatformLogStreamer +from utils.loader import CaptureImplsLoader impl_loader = CaptureImplsLoader( __path__[0], - "capture.platform", + "features.capture.platform", PlatformLogStreamer ) diff --git a/src/tools/interop/idt/capture/platform/android/__init__.py b/src/tools/interop/idt/features/capture/platform/android/__init__.py similarity index 100% rename from src/tools/interop/idt/capture/platform/android/__init__.py rename to src/tools/interop/idt/features/capture/platform/android/__init__.py diff --git a/src/tools/interop/idt/capture/platform/android/android.py b/src/tools/interop/idt/features/capture/platform/android/android.py similarity index 83% rename from src/tools/interop/idt/capture/platform/android/android.py rename to src/tools/interop/idt/features/capture/platform/android/android.py index 4ca8b26baeda6b..d1d3f5888bf86c 100644 --- a/src/tools/interop/idt/capture/platform/android/android.py +++ b/src/tools/interop/idt/features/capture/platform/android/android.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # + import asyncio import ipaddress import os @@ -21,7 +22,7 @@ import typing from asyncio import Task -from capture.base import PlatformLogStreamer +from features.capture.base import PlatformLogStreamer from utils.shell import Bash, log from . import config, streams @@ -36,10 +37,11 @@ def __init__(self, artifact_dir: str) -> None: self.logger = logger self.artifact_dir = artifact_dir self.device_id: str | None = None - self.adb_devices: typing.Dict[str, bool] = {} + self.adb_devices: typing.Dict[str, bool] = {} # Key: Device id, Value: Is authorized self.capabilities: None | Capabilities = None self.streams = {} self.connected = False + self.hci_out_path = os.path.join(self.artifact_dir, "btsnoop_hci.log") def run_adb_command( self, @@ -64,7 +66,7 @@ def get_adb_background_command( cwd=None) -> Bash: return Bash(f'adb -s {self.device_id} {command}', cwd=cwd) - def get_adb_devices(self) -> typing.Dict[str, bool]: + def _get_adb_devices(self) -> typing.Dict[str, bool]: """Returns a dict of device ids and whether they are authorized""" adb_devices = Bash('adb devices', sync=True, capture_output=True) adb_devices.start_command() @@ -132,7 +134,7 @@ def _check_connect_wireless_adb(self, temp_device_id: str) -> None: self.logger.warning( f"Detected connection string; attempting to connect: {connect_command}") Bash(connect_command, sync=True, capture_output=False).start_command() - self.get_adb_devices() + self._get_adb_devices() def _device_id_user_input(self) -> None: self.logger.error('Connect additional android devices via USB and press enter OR') @@ -141,7 +143,7 @@ def _device_id_user_input(self) -> None: self._log_adb_devices() temp_device_id = input('').strip() self._check_connect_wireless_adb(temp_device_id) - self.get_adb_devices() + self._get_adb_devices() if self._only_one_device_connected(): self._set_device_if_only_one_connected() elif temp_device_id not in self.adb_devices: @@ -155,17 +157,21 @@ def _choose_device_id(self) -> None: If only one device is ever connected, use it. """ self._set_device_if_only_one_connected() - while self.device_id not in self.get_adb_devices(): + while self.device_id not in self._get_adb_devices(): self._device_id_user_input() self.logger.info(f'Selected device {self.device_id}') + def _selected_device_connected_and_authd(self) -> bool: + current_devices = self._get_adb_devices() + return self.device_id in current_devices and current_devices[self.device_id] + def _authorize_adb(self) -> None: """ Prompts the user until a single device is selected and adb is auth'd """ - self.get_adb_devices() + self._get_adb_devices() self._choose_device_id() - while not self.get_adb_devices()[self.device_id]: + while not self._selected_device_connected_and_authd(): self.logger.info('Confirming authorization, press enter after auth') input('') self.logger.info(f'Target android device ID is authorized: {self.device_id}') @@ -179,7 +185,7 @@ async def connect(self) -> None: self.streams[stream] = getattr(streams, stream)(self) self.connected = True - async def handle_stream_action(self, action: str) -> None: + async def _handle_stream_action(self, action: str) -> None: had_error = False for stream_name, stream in self.streams.items(): self.logger.info(f"Doing {action} for {stream_name}!") @@ -192,7 +198,7 @@ async def handle_stream_action(self, action: str) -> None: raise Exception("Propagating to controller!") async def start_streaming(self) -> None: - await self.handle_stream_action("start") + await self._handle_stream_action("start") async def run_observers(self) -> None: try: @@ -210,17 +216,20 @@ async def run_observers(self) -> None: for observer_tasks in observer_tasks: observer_tasks.cancel() + def _pull_hci_logs(self) -> None: + for possible_hci_path in config.HCI_LOG_POSSIBLE_PATHS: + pull_command = self.run_adb_command(f"pull {possible_hci_path} {self.hci_out_path}") + if pull_command.finished_success(): + self.logger.info("Successfully pulled hci logs!") + return + self.logger.error("Could not locate hci logs!") + async def stop_streaming(self) -> None: - await self.handle_stream_action("stop") - if config.enable_bug_report: - found = False - for item in os.listdir(self.artifact_dir): - if "bugreport" in item and ".zip" in item: - found = True - if not found: - self.logger.info("Taking bugreport") - self.run_adb_command("bugreport", cwd=self.artifact_dir) - else: - self.logger.warning("bugreport already taken") + await self._handle_stream_action("stop") + if config.ENABLE_BUG_REPORT: + self.logger.info("Taking bugreport") + self.run_adb_command("bugreport", cwd=self.artifact_dir) else: - self.logger.critical("bugreport disabled in settings!") + self.logger.warning("bugreport disabled in settings!") + self.logger.info("Attempting to pull hci logs!") + self._pull_hci_logs() diff --git a/src/tools/interop/idt/capture/platform/android/capabilities.py b/src/tools/interop/idt/features/capture/platform/android/capabilities.py similarity index 75% rename from src/tools/interop/idt/capture/platform/android/capabilities.py rename to src/tools/interop/idt/features/capture/platform/android/capabilities.py index cd2f7f62797074..e2241bf467a22c 100644 --- a/src/tools/interop/idt/capture/platform/android/capabilities.py +++ b/src/tools/interop/idt/features/capture/platform/android/capabilities.py @@ -1,10 +1,27 @@ +# +# Copyright (c) 2024 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + from typing import TYPE_CHECKING from utils.artifact import create_standard_log_name, log from utils.shell import Bash if TYPE_CHECKING: - from capture.platform.android import Android + from features.capture.platform.android import Android from . import config @@ -29,7 +46,7 @@ def __repr__(self): return s def check_snoop_log(self) -> bool: - return config.hci_log_level in self.platform.run_adb_command("shell getprop persist.bluetooth.btsnooplogmode", + return config.HCI_LOG_LEVEL in self.platform.run_adb_command("shell getprop persist.bluetooth.btsnooplogmode", capture_output=True).get_captured_output() def check_capabilities(self): @@ -49,7 +66,7 @@ def check_capabilities(self): if not self.c_hci_snoop_enabled: self.logger.info("HCI not enabled, attempting to enable!") self.platform.run_adb_command( - f"shell setprop persist.bluetooth.btsnooplogmode {config.hci_log_level}") + f"shell setprop persist.bluetooth.btsnooplogmode {config.HCI_LOG_LEVEL}") self.platform.run_adb_command("shell svc bluetooth disable") self.platform.run_adb_command("shell svc bluetooth enable") self.c_hci_snoop_enabled = self.check_snoop_log() diff --git a/src/tools/interop/idt/features/capture/platform/android/config.py b/src/tools/interop/idt/features/capture/platform/android/config.py new file mode 100644 index 00000000000000..881c622caa50fd --- /dev/null +++ b/src/tools/interop/idt/features/capture/platform/android/config.py @@ -0,0 +1,25 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os + +ENABLE_BUILD_PUSH_TCPDUMP = True +ENABLE_BUG_REPORT = False +HCI_LOG_LEVEL = "full" +HCI_LOG_POSSIBLE_PATHS = [os.path.join(p, "btsnoop_hci.log") for p in + ["/sdcard/", + "/data/misc/bluetooth/logs/"]] diff --git a/src/tools/interop/idt/features/capture/platform/android/streams/__init__.py b/src/tools/interop/idt/features/capture/platform/android/streams/__init__.py new file mode 100644 index 00000000000000..d531da0f755cb9 --- /dev/null +++ b/src/tools/interop/idt/features/capture/platform/android/streams/__init__.py @@ -0,0 +1,31 @@ +# +# Copyright (c) 2024 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from utils.loader import CaptureImplsLoader + +from .base import AndroidStream + +impl_loader = CaptureImplsLoader( + __path__[0], + "features.capture.platform.android.streams", + AndroidStream +) + +for impl_name, impl in impl_loader.impls.items(): + globals()[impl_name] = impl + +__all__ = impl_loader.impl_names diff --git a/src/tools/interop/idt/capture/platform/android/streams/base.py b/src/tools/interop/idt/features/capture/platform/android/streams/base.py similarity index 100% rename from src/tools/interop/idt/capture/platform/android/streams/base.py rename to src/tools/interop/idt/features/capture/platform/android/streams/base.py diff --git a/src/tools/interop/idt/capture/platform/android/streams/logcat/__init__.py b/src/tools/interop/idt/features/capture/platform/android/streams/logcat/__init__.py similarity index 100% rename from src/tools/interop/idt/capture/platform/android/streams/logcat/__init__.py rename to src/tools/interop/idt/features/capture/platform/android/streams/logcat/__init__.py diff --git a/src/tools/interop/idt/capture/platform/android/streams/logcat/logcat.py b/src/tools/interop/idt/features/capture/platform/android/streams/logcat/logcat.py similarity index 95% rename from src/tools/interop/idt/capture/platform/android/streams/logcat/logcat.py rename to src/tools/interop/idt/features/capture/platform/android/streams/logcat/logcat.py index 78670bf6504df0..27bc26f76e642a 100644 --- a/src/tools/interop/idt/capture/platform/android/streams/logcat/logcat.py +++ b/src/tools/interop/idt/features/capture/platform/android/streams/logcat/logcat.py @@ -26,7 +26,7 @@ logger = log.get_logger(__file__) if TYPE_CHECKING: - from capture.platform.android import Android + from features.capture.platform.android import Android class LogcatStreamer(AndroidStream): @@ -58,8 +58,8 @@ async def run_observer(self) -> None: self.logcat_proc.start_command() await asyncio.sleep(4) - async def start(self): + async def start(self) -> None: self.logcat_proc.start_command() - async def stop(self): + async def stop(self) -> None: self.logcat_proc.stop_command() diff --git a/src/tools/interop/idt/capture/platform/android/streams/pcap/__init__.py b/src/tools/interop/idt/features/capture/platform/android/streams/pcap/__init__.py similarity index 100% rename from src/tools/interop/idt/capture/platform/android/streams/pcap/__init__.py rename to src/tools/interop/idt/features/capture/platform/android/streams/pcap/__init__.py diff --git a/src/tools/interop/idt/capture/platform/android/streams/pcap/linux_build_tcpdump_64.sh b/src/tools/interop/idt/features/capture/platform/android/streams/pcap/linux_build_tcpdump_64.sh similarity index 100% rename from src/tools/interop/idt/capture/platform/android/streams/pcap/linux_build_tcpdump_64.sh rename to src/tools/interop/idt/features/capture/platform/android/streams/pcap/linux_build_tcpdump_64.sh diff --git a/src/tools/interop/idt/capture/platform/android/streams/pcap/mac_build_tcpdump_64.sh b/src/tools/interop/idt/features/capture/platform/android/streams/pcap/mac_build_tcpdump_64.sh old mode 100644 new mode 100755 similarity index 75% rename from src/tools/interop/idt/capture/platform/android/streams/pcap/mac_build_tcpdump_64.sh rename to src/tools/interop/idt/features/capture/platform/android/streams/pcap/mac_build_tcpdump_64.sh index 15d911997b6d15..9ccf98775ac92b --- a/src/tools/interop/idt/capture/platform/android/streams/pcap/mac_build_tcpdump_64.sh +++ b/src/tools/interop/idt/features/capture/platform/android/streams/pcap/mac_build_tcpdump_64.sh @@ -1,4 +1,20 @@ #!/usr/bin/env bash +# +# Copyright (c) 2023 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# # This script cross-compiles libpcap and tcpdump for a specified architecture (default: ARM64) diff --git a/src/tools/interop/idt/features/capture/platform/android/streams/pcap/pcap.py b/src/tools/interop/idt/features/capture/platform/android/streams/pcap/pcap.py new file mode 100644 index 00000000000000..fe7ae47ab5397d --- /dev/null +++ b/src/tools/interop/idt/features/capture/platform/android/streams/pcap/pcap.py @@ -0,0 +1,131 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import asyncio +import os +import shutil +from typing import TYPE_CHECKING + +from utils.artifact import create_standard_log_name, log, safe_mkdir +from utils.host import current_platform +from utils.shell import Bash + +from ... import config +from ..base import AndroidStream + +if TYPE_CHECKING: + from features.capture.platform.android import Android + +logger = log.get_logger(__file__) + + +class AndroidPcap(AndroidStream): + + def __init__(self, platform: "Android"): + self.pcap_artifact = None + self.pcap_phone_out_path = None + self.pcap_command = None + self.pcap_proc = None + self.logger = logger + self.platform = platform + self.target_output_dir = "/sdcard/Download" + self.pcap_phone_bin_location = "tcpdump" if platform.capabilities.c_has_tcpdump \ + else f"{self.target_output_dir}/tcpdump" + self.pcap_pull = False + self.file_counter = 0 + self.build_dir = os.path.join(os.path.dirname(__file__), "BUILD") + self.pull_commands: [str] = [] + self.manifest_file = os.path.join(platform.artifact_dir, "packet_capture_manifest.txt") + + def start_pcap(self) -> None: + self.pcap_artifact = create_standard_log_name("android_tcpdump" + str(self.file_counter), + "pcap", + parent=self.platform.artifact_dir) + self.pcap_phone_out_path = f"{self.target_output_dir}/{os.path.basename(self.pcap_artifact)}" + pcap_pull_command = f"pull {self.pcap_phone_out_path} {self.pcap_artifact}" + self.pull_commands.append(pcap_pull_command) + with open(self.manifest_file, "a+") as manifest: + manifest.write(pcap_pull_command + "\n") + self.pcap_command = f"shell {self.pcap_phone_bin_location} -w {self.pcap_phone_out_path}" + self.pcap_proc = self.platform.get_adb_background_command(self.pcap_command) + self.pcap_proc.start_command() + self.file_counter += 1 + self.pcap_pull = True + self.logger.info(f"New pcap started {self.pcap_phone_out_path} {self.pcap_artifact}") + + async def start(self) -> None: + # TODO: Move build to setup function? + if not self.platform.capabilities.c_has_root: + self.logger.warning("Phone is not rooted, cannot take pcap!") + return + if self.platform.capabilities.c_has_tcpdump: + self.logger.info("tcpdump already available; using!") + self.start_pcap() + return + if not config.ENABLE_BUILD_PUSH_TCPDUMP: + self.logger.critical("Android TCP Dump build and push disabled in configs!") + return + if not os.path.exists(os.path.join(self.build_dir, "tcpdump")): + self.logger.warning("tcpdump bin not found, attempting to build, please wait a few moments!") + safe_mkdir(self.build_dir) + build_script = "mac_build_tcpdump_64.sh" if current_platform.is_mac() else "linux_build_tcpdump_64.sh" + build_script = os.path.join(os.path.dirname(__file__), build_script) + build_command = Bash(f"{build_script} 2>&1 >> BUILD_LOG.txt", sync=True, cwd=self.build_dir) + build_command.start_command() + if not build_command.finished_success(): + self.logger.error("Build failed, cleaning build dir!") + shutil.rmtree(self.build_dir) + return + else: + self.logger.warning("Reusing existing tcpdump build") + if not self.platform.run_adb_command(f"shell ls {self.target_output_dir}/tcpdump").finished_success(): + self.logger.warning("Pushing tcpdump to device") + push_command = self.platform.run_adb_command( + f"push {os.path.join(self.build_dir, 'tcpdump')} f{self.target_output_dir}") + chmod_command = self.platform.run_adb_command(f"chmod +x {self.target_output_dir}/tcpdump") + if not push_command.finished_success() or not chmod_command.finished_success(): + self.logger.error("Failed to push tcp dump!") + return + else: + self.logger.info("tcpdump already in the expected location, not pushing!") + self.logger.info("Starting Android pcap command") + self.start_pcap() + + async def run_observer(self) -> None: + while True: + if not self.pcap_pull: + await asyncio.sleep(120) + else: + if not self.pcap_proc.command_is_running(): + self.logger.warning( + f"Pcap proc needs restart (unexpected!) {self.platform.device_id}") + await self.start() + await asyncio.sleep(8) + + async def pull_packet_capture(self) -> None: + if self.pcap_pull: + self.logger.info("Attempting to pull android pcap") + await asyncio.sleep(3) + with open(self.manifest_file) as manifest: + for line in manifest: + self.platform.run_adb_command(line) + self.pcap_pull = False + + async def stop(self) -> None: + self.logger.info("Stopping android pcap proc") + self.pcap_proc.stop_command() + await self.pull_packet_capture() diff --git a/src/tools/interop/idt/capture/platform/android/streams/screen/__init__.py b/src/tools/interop/idt/features/capture/platform/android/streams/screen/__init__.py similarity index 100% rename from src/tools/interop/idt/capture/platform/android/streams/screen/__init__.py rename to src/tools/interop/idt/features/capture/platform/android/streams/screen/__init__.py diff --git a/src/tools/interop/idt/capture/platform/android/streams/screen/screen.py b/src/tools/interop/idt/features/capture/platform/android/streams/screen/screen.py similarity index 96% rename from src/tools/interop/idt/capture/platform/android/streams/screen/screen.py rename to src/tools/interop/idt/features/capture/platform/android/streams/screen/screen.py index e190e7cf212c27..045fdc336ae141 100644 --- a/src/tools/interop/idt/capture/platform/android/streams/screen/screen.py +++ b/src/tools/interop/idt/features/capture/platform/android/streams/screen/screen.py @@ -24,7 +24,7 @@ from ..base import AndroidStream if TYPE_CHECKING: - from capture.platform.android import Android + from features.capture.platform.android import Android logger = log.get_logger(__file__) @@ -69,7 +69,7 @@ def update_commands(self) -> None: manifest.write(screen_pull_command) self.file_counter += 1 - async def start(self): + async def start(self) -> None: await self.prepare_screen_recording() if self.check_screen(): self.screen_pull = True @@ -94,7 +94,7 @@ async def pull_screen_recording(self) -> None: self.platform.run_adb_command(line) self.screen_pull = False - async def stop(self): + async def stop(self) -> None: self.logger.info("Stopping screen proc") self.screen_proc.stop_command() await self.pull_screen_recording() diff --git a/src/tools/interop/idt/scripts/clean_child.sh b/src/tools/interop/idt/features/capture/thread/__init__.py similarity index 87% rename from src/tools/interop/idt/scripts/clean_child.sh rename to src/tools/interop/idt/features/capture/thread/__init__.py index fc4b2e21e438d0..11cc53bcdbe5b5 100644 --- a/src/tools/interop/idt/scripts/clean_child.sh +++ b/src/tools/interop/idt/features/capture/thread/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2023 Project CHIP Authors +# Copyright (c) 2024 Project CHIP Authors # All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,6 +14,3 @@ # See the License for the specific language governing permissions and # limitations under the License. # - -sudo killall tcpdump -sudo killall adb diff --git a/src/tools/interop/idt/features/capture/thread/base.py b/src/tools/interop/idt/features/capture/thread/base.py new file mode 100644 index 00000000000000..dc4ff5dd697ebc --- /dev/null +++ b/src/tools/interop/idt/features/capture/thread/base.py @@ -0,0 +1,33 @@ +# +# Copyright (c) 2024 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from abc import ABC, abstractmethod + + +class ThreadCapture(ABC): + + def __init__(self, artifact_dir: str) -> None: + pass + + @abstractmethod + def start(self) -> None: + raise NotImplementedError + + @abstractmethod + def stop(self) -> None: + raise NotImplementedError + diff --git a/src/tools/interop/idt/features/capture/thread/on_network.py b/src/tools/interop/idt/features/capture/thread/on_network.py new file mode 100644 index 00000000000000..9d9f90c6f70d5d --- /dev/null +++ b/src/tools/interop/idt/features/capture/thread/on_network.py @@ -0,0 +1,42 @@ +# +# Copyright (c) 2024 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from features.capture.thread.base import ThreadCapture + + +class MacThreadCaptureOnNetwork(ThreadCapture): + + def __init__(self, artifact_dir: str) -> None: + pass + + def start(self) -> None: + raise NotImplementedError + + def stop(self) -> None: + raise NotImplementedError + + +class LinuxThreadCaptureOnNetwork(ThreadCapture): + + def __init__(self, artifact_dir: str) -> None: + pass + + def start(self) -> None: + raise NotImplementedError + + def stop(self) -> None: + raise NotImplementedError diff --git a/src/tools/interop/idt/features/capture/thread/runner.py b/src/tools/interop/idt/features/capture/thread/runner.py new file mode 100644 index 00000000000000..759ad3f5cea04b --- /dev/null +++ b/src/tools/interop/idt/features/capture/thread/runner.py @@ -0,0 +1,52 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from abc import ABC + +from features.capture.thread.on_network import MacThreadCaptureOnNetwork, LinuxThreadCaptureOnNetwork +from features.capture.thread.sniff import MacThreadCaptureSniffer, LinuxThreadCaptureSniffer +from utils.artifact import log +from utils.host import current_platform + +logger = log.get_logger(__file__) + + +class ThreadCaptureRunner(ABC): + + def __init__(self, + artifact_dir: str, + mode: str, + ) -> None: + self.logger = logger + self.artifact_dir = artifact_dir + self.mode = mode + if current_platform.is_mac(): + if mode == "sniff": + self.runner = MacThreadCaptureSniffer(self.artifact_dir) + elif mode == "on_network": + self.runner = MacThreadCaptureOnNetwork(self.artifact_dir) + else: + if mode == "sniff": + self.runner = LinuxThreadCaptureSniffer(self.artifact_dir) + elif mode == "on_network": + self.runner = LinuxThreadCaptureOnNetwork(self.artifact_dir) + + def start(self) -> None: + self.runner.start() + + def stop(self) -> None: + self.runner.stop() diff --git a/src/tools/interop/idt/features/capture/thread/sniff.py b/src/tools/interop/idt/features/capture/thread/sniff.py new file mode 100644 index 00000000000000..7afc281764d801 --- /dev/null +++ b/src/tools/interop/idt/features/capture/thread/sniff.py @@ -0,0 +1,89 @@ +# +# Copyright (c) 2024 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os + +from features.capture.thread.base import ThreadCapture +from utils import log +from utils.artifact import create_standard_log_name +from utils.shell import Bash +import time + +_LOGGER = log.get_logger(__file__) + + +class MacThreadCaptureSniffer(ThreadCapture): + + def __init__(self, artifact_dir: str) -> None: + raise NotImplementedError + + def start(self) -> None: + raise NotImplementedError + + def stop(self) -> None: + pass + + +class LinuxThreadCaptureSniffer(ThreadCapture): + """ + PySpinel had to be patched like this + /thread/BUILD/sniff/pyspinel] + └─$ git diff + diff --git a/sniffer.py b/sniffer.py + index 8492431..c56a4b2 100755 + --- a/sniffer.py + +++ b/sniffer.py + @@ -325,6 +325,7 @@ def main(): + + # Some old version NCP doesn't contain timestamp information in metadata + else: + + metadata = "" + timestamp = datetime.utcnow() - epoch + timestamp_sec = timestamp.days * 24 * 60 * 60 + timestamp.seconds + timestamp_usec = timestamp.microseconds + """ + + def __init__(self, artifact_dir: str, logger=_LOGGER) -> None: + self.artifact_dir = artifact_dir + self.artifact = create_standard_log_name("THREAD_SNIFF", "cap", parent=self.artifact_dir) + self.logger = logger + self.script_dir = os.path.join(os.path.dirname(__file__), + "BUILD", + "sniff", + "pyspinel") + self.uart = "/dev/ttyACM0" + self.channel = 15 + self.run_sniffer_command: Bash | None = None + + def start(self) -> None: + # TODO: Need to verify board is connected here + # TODO: Allow port as arg + self.logger.info("Starting thread sniff") + # TODO: CRITICAL: Make sudo auth automatic in Bash! + Bash("sudo echo \"\"", sync=True).start_command() + self.run_sniffer_command = Bash( + f"sudo ./sniffer.py -c {self.channel} -n 1 --no-reset -u {self.uart} -b 115200 > {self.artifact}", + cwd=self.script_dir) + self.run_sniffer_command.start_command() + self.logger.info("Waiting to check if sniffer started") + time.sleep(5) + if not self.run_sniffer_command.command_is_running(): + self.logger.critical("Sniffer is down!") + self.logger.info("Done starting thread sniff") + + def stop(self) -> None: + self.run_sniffer_command.stop_command() diff --git a/src/tools/interop/idt/discovery/__init__.py b/src/tools/interop/idt/features/discovery/__init__.py similarity index 92% rename from src/tools/interop/idt/discovery/__init__.py rename to src/tools/interop/idt/features/discovery/__init__.py index b83e5dfe0e1f50..c275de7ca30654 100644 --- a/src/tools/interop/idt/discovery/__init__.py +++ b/src/tools/interop/idt/features/discovery/__init__.py @@ -14,8 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # + from .ble import MatterBleScanner -from .dnssd import MatterDnssdListener +from features.discovery.dnssd.dnssd import MatterDnssdListener __all__ = [ 'MatterBleScanner', diff --git a/src/tools/interop/idt/discovery/ble.py b/src/tools/interop/idt/features/discovery/ble.py similarity index 78% rename from src/tools/interop/idt/discovery/ble.py rename to src/tools/interop/idt/features/discovery/ble.py index 9549c74614de9a..5e07b12dfae0be 100644 --- a/src/tools/interop/idt/discovery/ble.py +++ b/src/tools/interop/idt/features/discovery/ble.py @@ -31,14 +31,17 @@ class MatterBleScanner: - def __init__(self, artifact_dir: str): + def __init__(self, artifact_dir: str, vendor_id: str | None, product_id: str | None): self.artifact_dir = artifact_dir self.logger = logger self.devices_seen_last_time: set[str] = set() self.devices_seen_this_time: set[str] = set() self.throttle_seconds = 1 self.error_seconds = 5 + self.vendor_id = None if not vendor_id else vendor_id.lstrip("0x") + self.product_id = None if not product_id else product_id.lstrip("0x") + # TODO: Parse discriminator def parse_vid_pid(self, loggable_data: str) -> str: try: vid = loggable_data[8:10] + loggable_data[6:8] @@ -48,6 +51,15 @@ def parse_vid_pid(self, loggable_data: str) -> str: return "" return f"VID: {vid} PID: {pid}" + def parse_vid_pid_for_filter(self, loggable_data: str) -> tuple[str, str]: + try: + vid = loggable_data[8:10] + loggable_data[6:8] + pid = loggable_data[12:14] + loggable_data[10:12] + except IndexError: + self.logger.warning("Error parsing vid / pid from BLE ad data") + return "", "" + return vid.lstrip("0"), pid.lstrip("0") + def write_device_log(self, device_name: str, to_write: str) -> None: log_file_name = os.path.join(self.artifact_dir, f"{device_name}.txt") with open(log_file_name, "a+") as log_file: @@ -68,6 +80,17 @@ def handle_device_states(self) -> None: self.devices_seen_last_time = self.devices_seen_this_time self.devices_seen_this_time = set() + def passing_filter(self, hex_service_data) -> bool: + vid, pid = self.parse_vid_pid_for_filter(hex_service_data) + if self.vendor_id: + if vid != self.vendor_id: + self.logger.debug(f"Filtered ad for vid {vid} not matching vid {self.vendor_id}") + return False + if self.product_id and pid != self.product_id: + self.logger.debug(f"Filtered ad for pid {pid} not matching pid {self.product_id}") + return False + return True + def log_ble_discovery( self, name: str, @@ -76,6 +99,8 @@ def log_ble_discovery( rssi: int) -> None: hex_service_data = bin_service_data.hex() if self.is_matter_device(name): + if not self.passing_filter(hex_service_data): + return device_id = f"{ble_device.name}_{ble_device.address}" self.devices_seen_this_time.add(device_id) if device_id not in self.devices_seen_last_time: diff --git a/src/tools/interop/idt/features/discovery/dnssd/__init__.py b/src/tools/interop/idt/features/discovery/dnssd/__init__.py new file mode 100644 index 00000000000000..11cc53bcdbe5b5 --- /dev/null +++ b/src/tools/interop/idt/features/discovery/dnssd/__init__.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2024 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/src/tools/interop/idt/features/discovery/dnssd/dnssd.py b/src/tools/interop/idt/features/discovery/dnssd/dnssd.py new file mode 100644 index 00000000000000..9ad93d315090b0 --- /dev/null +++ b/src/tools/interop/idt/features/discovery/dnssd/dnssd.py @@ -0,0 +1,161 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import asyncio +import os +from datetime import datetime + +import zeroconf + +from features.probe import ProbeTarget +from utils.net import get_addr_type +from utils.artifact import create_standard_log_name, log +from utils.log import add_border, border_print +from zeroconf import ServiceBrowser, ServiceInfo, ServiceListener, Zeroconf +from .parsers import ServiceLibrary, MatterTxtRecordParser + +logger = log.get_logger(__file__) + + +class MatterDnssdListener(ServiceListener): + + def __init__(self, + artifact_dir: str, + vendor_id: str | None, + product_id: str | None, + v4=True, + v6=True) -> None: + super().__init__() + self.artifact_dir = artifact_dir + self.logger = logger + self.discovered_matter_devices: [str, ServiceInfo] = {} + self.vendor_id = vendor_id + self.product_id = product_id + self.service_library = ServiceLibrary() + self.v4 = v4 + self.v6 = v6 + + def write_log(self, line: str, log_name: str) -> None: + with open(self.create_device_log_name(log_name), "a+") as log_file: + time_stamp = add_border(datetime.now().strftime("%Y-%m-%d\t%H %M\t%S.%f") + "\n") + log_file.write(time_stamp + line) + + def create_device_log_name(self, device_name) -> str: + return os.path.join( + self.artifact_dir, + create_standard_log_name(f"{device_name}_dnssd", "txt")) + + @staticmethod + def log_addr(info: ServiceInfo) -> str: + addrs = set() + ret = add_border("This device has the following IP addresses\n") + for addr in info.parsed_scoped_addresses(): + if addr not in addrs: # For some reason, there are duplicates in the list sometimes, so we dedup here + addrs.add(addr) + ret += f"{get_addr_type(addr)}: {addr}\n" + return ret + + def handle_service_info( + self, + zc: Zeroconf, + type_: str, + name: str, + delta_type: str) -> None: + service_type_info = self.service_library.get_service_type_info(type_) + to_log = f"{name}\n" \ + f"SERVICE {delta_type}\n" \ + f"BROADCAST ADDR IPv{zc.ipv}\n" \ + f"{service_type_info.type}\n" \ + f"{service_type_info.description}\n" + info = zc.get_service_info(type_, name) + if info is not None: + to_log += f"A/SRV TTL: {str(info.host_ttl)}\n" \ + f"PTR/TXT TTL: {str(info.other_ttl)}\n" + txt_parser = MatterTxtRecordParser() + to_log += txt_parser.parse_txt_records(info) + to_log += self.log_addr(info) + if self.vendor_id: + if txt_parser.vid != self.vendor_id: + return + if self.product_id and txt_parser.pid != self.product_id: + return + self.discovered_matter_devices[name + zc.ipv] = info + self.logger.info(to_log) + self.write_log(to_log, name + zc.ipv) + else: + self.logger.warning(f"No info found for {to_log}") + + def add_service(self, zc: Zeroconf, type_: str, name: str) -> None: + self.handle_service_info(zc, type_, name, "ADDED") + + def update_service(self, zc: Zeroconf, type_: str, name: str) -> None: + self.handle_service_info(zc, type_, name, "UPDATED") + + def remove_service(self, zc: Zeroconf, type_: str, name: str) -> None: + if name + zc.ipv in self.discovered_matter_devices: + del self.discovered_matter_devices[name + zc.ipv] + else: + return + service_type_info = self.service_library.get_service_type_info(type_) + to_log = f"{name}\n" \ + f"SERVICE REMOVED\n" \ + f"{service_type_info.type}\n" \ + f"{service_type_info.description}\n" + self.logger.warning(to_log) + self.write_log(to_log, name) + + def browse_interactive(self) -> None: + self.logger.warning("\nBrowsing Matter DNS-SD\n" + "DCL Lookup: https://webui.dcl.csa-iot.org/\n" + "See spec section 4.3 for details of Matter TXT records.\n") + border_print("Press enter to stop!", important=True) + if self.v4: + zc4 = Zeroconf(ip_version=zeroconf.IPVersion.V4Only) + zc4.ipv = "4" + ServiceBrowser(zc4, list(self.service_library.known_service_types()), self) + if self.v6: + zc6 = Zeroconf(ip_version=zeroconf.IPVersion.V6Only) + zc6.ipv = "6" + ServiceBrowser(zc6, list(self.service_library.known_service_types()), self) + try: + input("") + finally: + if self.v4: + zc4.close() + if self.v6: + zc6.close() + + async def browse_once(self, browse_time_seconds: int) -> [ProbeTarget]: + if self.v4: + zc4 = Zeroconf(ip_version=zeroconf.IPVersion.V4Only) + zc4.ipv = "4" + ServiceBrowser(zc4, list(self.service_library.known_service_types()), self) + if self.v6: + zc6 = Zeroconf(ip_version=zeroconf.IPVersion.V6Only) + zc6.ipv = "6" + ServiceBrowser(zc6, list(self.service_library.known_service_types()), self) + await asyncio.sleep(browse_time_seconds) + if self.v4: + zc4.close() + if self.v6: + zc6.close() + ret = [] + for name in self.discovered_matter_devices: + info: ServiceInfo = self.discovered_matter_devices[name] + for addr in info.parsed_scoped_addresses(): + ret.append(ProbeTarget(name, addr, info.port)) + return ret diff --git a/src/tools/interop/idt/features/discovery/dnssd/parsers.py b/src/tools/interop/idt/features/discovery/dnssd/parsers.py new file mode 100644 index 00000000000000..d408ad5f17bd55 --- /dev/null +++ b/src/tools/interop/idt/features/discovery/dnssd/parsers.py @@ -0,0 +1,204 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import traceback +from dataclasses import dataclass +from typing import Callable +from utils.artifact import log +from utils.log import add_border +from zeroconf import ServiceInfo +from utils.data import MATTER_APPLICATION_DEVICE_TYPES, MATTER_COMMISSIONING_MODE_DESCRIPTIONS, MATTER_PAIRING_HINTS + +logger = log.get_logger(__file__) + + +@dataclass() +class ServiceTypeInfo: + type: str + description: str + + +class ServiceLibrary: + + def __init__(self): + self._SERVICE_TYPE_INFO = { + "_matterd._udp.local.": ServiceTypeInfo( + "COMMISSIONER", + "A service for a Matter commissioner aka. controller" + ), + "_matterc._udp.local.": ServiceTypeInfo( + "COMMISSIONABLE / EXTENDED DISCOVERY", + "A service to be used in the commissioning process and provides more info about the device." + ), + "_matter._tcp.local.": ServiceTypeInfo( + "OPERATIONAL", + "A service for a commissioned Matter device. It exposes limited info about the device." + ), + "_meshcop._udp.local.": ServiceTypeInfo( + "THREAD BORDER ROUTER", + "A service for a thread border router; may be used for thread+Matter devices." + ), + "_trel._udp.local.": ServiceTypeInfo( + "THREAD RADIO ENCAPSULATION LINK", + "A service for Thread Radio Encapsulation Link which is a method for thread BRs to exchange data on " + "IP links." + ), + } + + def known_service_types(self) -> [str]: + return list(self._SERVICE_TYPE_INFO.keys()) + + def get_service_type_info(self, service_type: str) -> ServiceTypeInfo: + return self._SERVICE_TYPE_INFO[service_type] + + +@dataclass() +class TxtRecordParser: + readable_name: str + explanation: str + parse: Callable[[str], str] + + +# TODO: Meshcop parser + +class MatterTxtRecordParser: + + def __init__(self): + self.parsers = { + # Commissioning + "D": TxtRecordParser("Discriminator", + "Differentiates advertisements from this instance of the device from " + "advertisement from others devices w/ the same VID/PID.", + MatterTxtRecordParser.parse_d), # To hex + "VP": TxtRecordParser("VID/PID", + "The Vendor ID and Product ID (each two bytes of hex) that identify this product.", + self.parse_vp), # Split + to hex, also stores values for filtering + "CM": TxtRecordParser("Commissioning mode", + "Whether the device is in commissioning mode or not.", + MatterTxtRecordParser.parse_cm), # Decode + "DT": TxtRecordParser("Device type", + "Application type for this end device.", + MatterTxtRecordParser.parse_dt), # Decode map + "DN": TxtRecordParser("Device name", + "Manufacturer provided device name. MAY match NodeLabel in Basic info cluster.", + MatterTxtRecordParser.parse_pass_through), # None + "RI": TxtRecordParser("Rotating identifier", + "Vendor specific, non-trackable per-device ID.", + MatterTxtRecordParser.parse_pass_through), # None + "PH": TxtRecordParser("Pairing hint", + "Given the current device state, follow these instructions to make the " + "device commissionable.", + MatterTxtRecordParser.parse_ph), # Decode bitmap + "PI": TxtRecordParser("Pairing instructions", + "Used with the Pairing hint. If the Pairing hint mentions N, this " + "is the value of N.", + MatterTxtRecordParser.parse_pass_through), # None + # General + "SII": TxtRecordParser("Session idle interval", + "Message Reliability Protocol retry interval while the device is idle in " + "milliseconds.", + MatterTxtRecordParser.parse_pass_through), # None + "SAI": TxtRecordParser("Session active interval", + "Message Reliability Protocol retry interval while the device is " + "active in milliseconds.", + MatterTxtRecordParser.parse_pass_through), # None + "SAT": TxtRecordParser("Session active threshold", + "Duration of time this device stays active after last activity in milliseconds.", + MatterTxtRecordParser.parse_pass_through), # None + "T": TxtRecordParser("Supports TCP", + "Whether this device supports TCP.", + MatterTxtRecordParser.parse_t), # Decode + } + self.unparsed_records = "" + self.parsed_records = "" + self.vid = None + self.pid = None + + def parse_single_txt_record(self, key: str, value: str): + parser: TxtRecordParser = self.parsers[key] + self.parsed_records += add_border(parser.readable_name + "\n") + self.parsed_records += parser.explanation + "\n\n" + try: + self.parsed_records += "PARSED VALUE:\n" + parser.parse(value) + "\n" + except Exception: + logger.error("Exception parsing TXT record, appending raw value") + logger.error(traceback.format_exc()) + self.parsed_records += f"RAW VALUE: {value}\n" + + def get_output(self) -> str: + unparsed_exp = "\nThe following TXT records were not parsed or explained:\n" + parsed_exp = "\nThe following was discovered about this device via TXT records:\n" + ret = "" + if self.unparsed_records: + ret += unparsed_exp + self.unparsed_records + if self.parsed_records: + ret += parsed_exp + self.parsed_records + return ret + + def parse_txt_records(self, info: ServiceInfo) -> str: + if info.properties is not None: + for name, value in info.properties.items(): + try: + name = name.decode("utf-8") + except UnicodeDecodeError: + name = str(name) + try: + value = value.decode("utf-8") + except UnicodeDecodeError: + value = str(value) + if name not in self.parsers: + self.unparsed_records += f"KEY: {name} VALUE: {value}\n" + else: + self.parse_single_txt_record(name, value) + return self.get_output() + + @staticmethod + def parse_pass_through(txt_value: str) -> str: + return txt_value + + @staticmethod + def parse_d(txt_value: str) -> str: + return hex(int(txt_value)) + + def parse_vp(self, txt_value: str) -> str: + vid, pid = txt_value.split("+") + vid, pid = hex(int(vid)), hex(int(pid)) + self.vid = vid + self.pid = pid + return f"VID: {vid}, PID: {pid}" + + @staticmethod + def parse_cm(txt_value: str) -> str: + return MATTER_COMMISSIONING_MODE_DESCRIPTIONS[int(txt_value)] + + @staticmethod + def parse_dt(txt_value: str) -> str: + return MATTER_APPLICATION_DEVICE_TYPES[hex((int(txt_value))).upper().replace("0X", "0x")] + + @staticmethod + def parse_ph(txt_value: str) -> str: + ret = "\n" + b_arr = [int(b) for b in bin(int(txt_value))[2:]][::-1] + for i in range(0, len(b_arr)): + b = b_arr[i] + if b: + ret += MATTER_PAIRING_HINTS[i] + "\n" + return ret + + @staticmethod + def parse_t(txt_value: str) -> str: + return "TCP supported" if int(txt_value) else "TCP not supported" diff --git a/src/tools/interop/idt/probe/__init__.py b/src/tools/interop/idt/features/probe/__init__.py similarity index 100% rename from src/tools/interop/idt/probe/__init__.py rename to src/tools/interop/idt/features/probe/__init__.py diff --git a/src/tools/interop/idt/probe/config.py b/src/tools/interop/idt/features/probe/config.py similarity index 93% rename from src/tools/interop/idt/probe/config.py rename to src/tools/interop/idt/features/probe/config.py index 150e4079e23e31..f07cfd485780e1 100644 --- a/src/tools/interop/idt/probe/config.py +++ b/src/tools/interop/idt/features/probe/config.py @@ -15,5 +15,5 @@ # limitations under the License. # -ping_count = 4 -dnssd_browsing_time_seconds = 4 +PING_COUNT = 4 +DNSSD_BROWSING_TIME_SECONDS = 4 diff --git a/src/tools/interop/idt/probe/linux.py b/src/tools/interop/idt/features/probe/linux.py similarity index 73% rename from src/tools/interop/idt/probe/linux.py rename to src/tools/interop/idt/features/probe/linux.py index 05a8adb4410e28..342efbd13a3f9e 100644 --- a/src/tools/interop/idt/probe/linux.py +++ b/src/tools/interop/idt/features/probe/linux.py @@ -15,11 +15,10 @@ # limitations under the License. # -import probe.probe as p -from utils.host_platform import get_ll_interface +from utils.host import current_platform from utils.log import get_logger -from . import config +from . import config, probe as p logger = get_logger(__file__) @@ -27,22 +26,18 @@ class ProberLinuxHost(p.GenericMatterProber): def __init__(self, artifact_dir: str, dnssd_artifact_dir: str) -> None: - # TODO: Parity with macOS super().__init__(artifact_dir, dnssd_artifact_dir) self.logger = logger - self.ll_int = get_ll_interface() - - def discover_targets_by_neighbor(self) -> None: - pass + self.ll_int = current_platform.get_link_local_interface() def probe_v4(self, ipv4: str, port: str) -> None: - self.run_command(f"ping -c {config.ping_count} {ipv4}") + self.run_command(f"ping -c {config.PING_COUNT} {ipv4}") def probe_v6(self, ipv6: str, port: str) -> None: - self.run_command(f"ping -c {config.ping_count} -6 {ipv6}") + self.run_command(f"ping -c {config.PING_COUNT} -6 {ipv6}") def probe_v6_ll(self, ipv6_ll: str, port: str) -> None: - self.run_command(f"ping -c {config.ping_count} -6 {ipv6_ll}%{self.ll_int}") + self.run_command(f"ping -c {config.PING_COUNT} -6 {ipv6_ll}%{self.ll_int}") def get_general_details(self) -> None: pass diff --git a/src/tools/interop/idt/probe/mac.py b/src/tools/interop/idt/features/probe/mac.py similarity index 77% rename from src/tools/interop/idt/probe/mac.py rename to src/tools/interop/idt/features/probe/mac.py index b13e5d50fa46ce..b729335db92a1f 100644 --- a/src/tools/interop/idt/probe/mac.py +++ b/src/tools/interop/idt/features/probe/mac.py @@ -15,11 +15,10 @@ # limitations under the License. # -import probe.probe as p -from utils.host_platform import get_ll_interface +from utils.host import current_platform from utils.log import get_logger -from . import ProbeTarget, config +from . import ProbeTarget, config, probe as p logger = get_logger(__file__) @@ -27,25 +26,21 @@ class ProberMacHost(p.GenericMatterProber): def __init__(self, artifact_dir: str, dnssd_artifact_dir: str) -> None: - # TODO: Build out additional probes super().__init__(artifact_dir, dnssd_artifact_dir) self.logger = logger - self.ll_int = get_ll_interface() - - def discover_targets_by_neighbor(self) -> None: - pass + self.ll_int = current_platform.get_link_local_interface() def probe_v4(self, target: ProbeTarget) -> None: self.logger.info("Ping IPv4") - self.run_command(f"ping -c {config.ping_count} {target.ip}") + self.run_command(f"ping -c {config.PING_COUNT} {target.ip}") def probe_v6(self, target: ProbeTarget) -> None: self.logger.info("Ping IPv6") - self.run_command(f"ping6 -c {config.ping_count} {target.ip}") + self.run_command(f"ping6 -c {config.PING_COUNT} {target.ip}") def probe_v6_ll(self, target: ProbeTarget) -> None: self.logger.info("Ping IPv6 Link Local") - self.run_command(f"ping6 -c {config.ping_count} -I {self.ll_int} {target.ip}") + self.run_command(f"ping6 -c {config.PING_COUNT} -I {self.ll_int} {target.ip}") def get_general_details(self) -> None: self.logger.info("Host interfaces") diff --git a/src/tools/interop/idt/probe/probe.py b/src/tools/interop/idt/features/probe/probe.py similarity index 77% rename from src/tools/interop/idt/probe/probe.py rename to src/tools/interop/idt/features/probe/probe.py index 6fcc92ad0428fb..79d9545a0e7705 100644 --- a/src/tools/interop/idt/probe/probe.py +++ b/src/tools/interop/idt/features/probe/probe.py @@ -19,14 +19,13 @@ import os.path from abc import ABC, abstractmethod -from discovery import MatterDnssdListener -from discovery.dnssd import ServiceInfo +from features.discovery import MatterDnssdListener from utils.artifact import create_standard_log_name from utils.log import get_logger from utils.shell import Bash from . import ProbeTarget, config -from .ip_utils import is_ipv4, is_ipv6, is_ipv6_ll +from utils.net import is_ipv4, is_ipv6, is_ipv6_link_local logger = get_logger(__file__) @@ -40,10 +39,10 @@ def __init__(self, artifact_dir: str, dnssd_artifact_dir: str) -> None: self.targets: [GenericMatterProber.ProbeTarget] = [] self.output = os.path.join(self.artifact_dir, create_standard_log_name("generic_probes", "txt")) - self.suffix = f"2>&1 | tee -a {self.output}" def run_command(self, cmd: str, capture_output=False) -> Bash: - cmd = f"{cmd} {self.suffix}" + Bash(f"echo {cmd} >> {self.output}", sync=True, capture_output=True).start_command() + cmd = f" {cmd} 2>&1 | tee -a {self.output}" self.logger.debug(cmd) bash = Bash(cmd, sync=True, capture_output=capture_output) bash.start_command() @@ -61,27 +60,19 @@ def probe_v6(self, target: ProbeTarget) -> None: def probe_v6_ll(self, target: ProbeTarget) -> None: raise NotImplementedError - @abstractmethod - def discover_targets_by_neighbor(self) -> None: - raise NotImplementedError - @abstractmethod def get_general_details(self) -> None: raise NotImplementedError def discover_targets_by_browsing(self) -> None: - browser = MatterDnssdListener(self.dnssd_artifact_dir) - asyncio.run(browser.browse_once(config.dnssd_browsing_time_seconds)) - for name in browser.discovered_matter_devices: - info: ServiceInfo = browser.discovered_matter_devices[name] - for addr in info.parsed_scoped_addresses(): - self.targets.append(ProbeTarget(name, addr, info.port)) + browser = MatterDnssdListener(self.dnssd_artifact_dir, None, None) + self.targets = asyncio.run(browser.browse_once(config.DNSSD_BROWSING_TIME_SECONDS)) def probe_single_target(self, target: ProbeTarget) -> None: if is_ipv4(target.ip): self.logger.debug(f"Probing v4 {target.ip}") self.probe_v4(target) - elif is_ipv6_ll(target.ip): + elif is_ipv6_link_local(target.ip): self.logger.debug(f"Probing v6 ll {target.ip}") self.probe_v6_ll(target) elif is_ipv6(target.ip): @@ -95,6 +86,5 @@ def probe_targets(self) -> None: def probe(self) -> None: self.discover_targets_by_browsing() - self.discover_targets_by_neighbor() self.probe_targets() self.get_general_details() diff --git a/src/tools/interop/idt/probe/runner.py b/src/tools/interop/idt/features/probe/runner.py similarity index 92% rename from src/tools/interop/idt/probe/runner.py rename to src/tools/interop/idt/features/probe/runner.py index ac373178a0ef69..e027dbcdbae4bc 100644 --- a/src/tools/interop/idt/probe/runner.py +++ b/src/tools/interop/idt/features/probe/runner.py @@ -15,14 +15,14 @@ # limitations under the License. # -from utils.host_platform import is_mac +from utils.host import current_platform from .linux import ProberLinuxHost from .mac import ProberMacHost def run_probes(artifact_dir: str, dnssd_dir: str) -> None: - if is_mac(): + if current_platform.is_mac(): ProberMacHost(artifact_dir, dnssd_dir).probe() else: ProberLinuxHost(artifact_dir, dnssd_dir).probe() diff --git a/src/tools/interop/idt/features/setup/__init__.py b/src/tools/interop/idt/features/setup/__init__.py new file mode 100644 index 00000000000000..db19f9a0f55a95 --- /dev/null +++ b/src/tools/interop/idt/features/setup/__init__.py @@ -0,0 +1,20 @@ +# +# Copyright (c) 2024 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from .setup import do_setup, list_available_targets + +__all__ = ["do_setup", "list_available_targets"] diff --git a/src/tools/interop/idt/features/setup/setup.py b/src/tools/interop/idt/features/setup/setup.py new file mode 100644 index 00000000000000..63bbedc976db0c --- /dev/null +++ b/src/tools/interop/idt/features/setup/setup.py @@ -0,0 +1,30 @@ +# +# Copyright (c) 2024 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import copy + +from . import targets +from .targets.base import Setup + + +def list_available_targets() -> None: + return copy.deepcopy(targets.__all__) + + +def do_setup(target: str) -> None: + target_class: Setup = getattr(targets, target) + target_class().setup() diff --git a/src/tools/interop/idt/features/setup/targets/__init__.py b/src/tools/interop/idt/features/setup/targets/__init__.py new file mode 100644 index 00000000000000..3cbe8ce48a400d --- /dev/null +++ b/src/tools/interop/idt/features/setup/targets/__init__.py @@ -0,0 +1,31 @@ +# +# Copyright (c) 2024 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from utils.loader import CaptureImplsLoader + +from .base import Setup + +impl_loader = CaptureImplsLoader( + __path__[0], + "features.setup.targets", + Setup +) + +for impl_name, impl in impl_loader.impls.items(): + globals()[impl_name] = impl + +__all__ = impl_loader.impl_names diff --git a/src/tools/interop/idt/features/setup/targets/base.py b/src/tools/interop/idt/features/setup/targets/base.py new file mode 100644 index 00000000000000..6701f2b8420841 --- /dev/null +++ b/src/tools/interop/idt/features/setup/targets/base.py @@ -0,0 +1,25 @@ +# +# Copyright (c) 2024 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from abc import ABC, abstractmethod + + +class Setup(ABC): + + @abstractmethod + def setup(self) -> None: + raise NotImplementedError diff --git a/src/tools/interop/idt/features/setup/targets/nrf_52840_mdk_not_dongle/__init__.py b/src/tools/interop/idt/features/setup/targets/nrf_52840_mdk_not_dongle/__init__.py new file mode 100644 index 00000000000000..b73c429cd4ccd3 --- /dev/null +++ b/src/tools/interop/idt/features/setup/targets/nrf_52840_mdk_not_dongle/__init__.py @@ -0,0 +1,22 @@ +# +# Copyright (c) 2024 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from .nrf52840_mdk_not_dongle import Nrf52840MdkNotDongle + +__all__ = [ + "Nrf52840MdkNotDongle" +] diff --git a/src/tools/interop/idt/features/setup/targets/nrf_52840_mdk_not_dongle/linux_install_pyocd.sh b/src/tools/interop/idt/features/setup/targets/nrf_52840_mdk_not_dongle/linux_install_pyocd.sh new file mode 100755 index 00000000000000..2069b062a303db --- /dev/null +++ b/src/tools/interop/idt/features/setup/targets/nrf_52840_mdk_not_dongle/linux_install_pyocd.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# Copyright (c) 2023 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e +# Reference https://pyocd.io/docs/installing_on_non_x86.html +# The version is pinned here because we manually copy requirements from +# https://github.com/pyocd/pyOCD/blob/v0.36.0/setup.cfg +# as instructed by the reference above, to support RPi +# TODO: Script to auto update this file based on input version +pip install --no-deps pyocd==0.36.0 +requirements=$(cat <<-END +capstone>=4.0,<5.0 +colorama<1.0 +importlib_metadata>=3.6 +importlib_resources +intelhex>=2.0,<3.0 +intervaltree>=3.0.2,<4.0 +lark>=1.1.5,<2.0 +libusb-package>=1.0,<2.0 +natsort>=8.0.0,<9.0 +prettytable>=2.0,<4.0 +pyelftools<1.0 +pylink-square>=1.0,<2.0 +pyusb>=1.2.1,<2.0 +pyyaml>=6.0,<7.0 +six>=1.15.0,<2.0 +typing-extensions>=4.0,<5.0 +END + +) + +echo "$requirements" > linux_pyocd_requirements.txt +pip install -r linux_pyocd_requirements.txt + +# Reference https://pyocd.io/docs/installing#udev-rules-on-linux +set -e +git clone https://github.com/pyocd/pyOCD.git +cd pyOCD +git checkout v0.36.0 +cd udev +sudo cp *.rules /etc/udev/rules.d diff --git a/src/tools/interop/idt/features/setup/targets/nrf_52840_mdk_not_dongle/linux_install_wpantund.sh b/src/tools/interop/idt/features/setup/targets/nrf_52840_mdk_not_dongle/linux_install_wpantund.sh new file mode 100755 index 00000000000000..f5e96e7759161a --- /dev/null +++ b/src/tools/interop/idt/features/setup/targets/nrf_52840_mdk_not_dongle/linux_install_wpantund.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright (c) 2023 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e +sudo apt-get update +sudo apt-get install -y gcc g++ libdbus-1-dev libboost-dev libreadline-dev +git clone --recursive https://github.com/openthread/wpantund.git +cd wpantund +sudo apt-get install -y libtool autoconf autoconf-archive +./bootstrap.sh +./configure --sysconfdir=/etc +make +sudo make install diff --git a/src/tools/interop/idt/features/setup/targets/nrf_52840_mdk_not_dongle/nrf52840_mdk_not_dongle.py b/src/tools/interop/idt/features/setup/targets/nrf_52840_mdk_not_dongle/nrf52840_mdk_not_dongle.py new file mode 100644 index 00000000000000..356da30ffbe59a --- /dev/null +++ b/src/tools/interop/idt/features/setup/targets/nrf_52840_mdk_not_dongle/nrf52840_mdk_not_dongle.py @@ -0,0 +1,123 @@ +# +# Copyright (c) 2024 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import inspect +import os +import shutil + +from features.setup.targets.base import Setup +from utils import log +from utils.artifact import safe_mkdir +from utils.shell import Bash +from utils.error import log_error, write_error_report +from utils.host import current_platform +from features.capture.thread import sniff + +_LOGGER = log.get_logger(__file__) + + +class Nrf52840MdkNotDongle(Setup): + + def __init__(self, logger=_LOGGER) -> None: + self.build_dir = os.path.join(os.path.dirname(inspect.getfile(sniff)), "BUILD", "sniff") + self.build_log = os.path.join(self.build_dir, "BUILD_LOG.txt") + self.error_context = "setup_capture_thread_sniff_Nrf52840MdkNotDongle" + self.logger = logger + + def _create_command(self, command: str, join_path_for_script=False, cwd=None) -> Bash: + if not cwd: + cwd = self.build_dir + if join_path_for_script: + command = os.path.join(os.path.dirname(__file__), command) + return Bash(f"{command} 2>&1 | tee -a {self.build_log}", sync=True, cwd=cwd) + + def _log_error(self, msg: str) -> None: + log_error(self.error_context, msg, stack_trace=False) + + def _write_error_report(self) -> None: + write_error_report(self.build_dir) + + def _install_flash_utils(self) -> bool: + self.logger.info("Fetching NCP image!") + fetch_command = self._create_command("git clone https://github.com/makerdiary/nrf52840-mdk.git") + fetch_command.start_command() + if not fetch_command.finished_success(): + self._log_error("otncp image fetch failed!") + shutil.rmtree(self.build_dir) + return False + self.logger.info("Installing pyocd") + install_pyocd_command = self._create_command("linux_install_pyocd.sh", join_path_for_script=True) + install_pyocd_command.start_command() + if not install_pyocd_command.finished_success(): # TODO: Not all errors propagate from commands here + self._log_error("Failed to install pyocd") + shutil.rmtree(self.build_dir) + return False + return True + + def _install_control_utils(self) -> bool: + if current_platform.command_is_available("wpantund"): + self.logger.info("wpantund already installed; skipping!") + else: + self.logger.info("Installing wpantund") + wpantund_command = self._create_command("linux_install_wpantund.sh", join_path_for_script=True) + wpantund_command.start_command() + if not wpantund_command.finished_success(): + self._log_error("wpantund install failed!") + shutil.rmtree(self.build_dir) + return False + self.logger.info("Installing pyspinel") + install_pyspinel_command = self._create_command("pip install pyserial ipaddress && git clone " + "https://github.com/openthread/pyspinel && cd pyspinel && sudo " + "python3 setup.py install") + install_pyspinel_command.start_command() + if not install_pyspinel_command.finished_success(): + self._log_error("PySpinel install failed") + shutil.rmtree(self.build_dir) + return False + return True + + def _build(self) -> bool: + safe_mkdir(self.build_dir) + if not self._install_flash_utils(): + return False + if not self._install_control_utils(): + return False + return True + + def _flash(self) -> None: + self.logger.info("Connected devices:") + Bash("python3 -m pyocd list", sync=True).start_command() + self.logger.info("Trying to flash") + flash_dir = os.path.join(self.build_dir, "nrf52840-mdk/firmware/openthread/") + flash_command = Bash("python3 -m pyocd load --target nrf52 ot-ncp-ftd.hex", sync=True, cwd=flash_dir) + flash_command.start_command() + if not flash_command.finished_success(): + self._log_error("Failed to flash!") + return + + def setup(self) -> None: + # TODO: Alert user we might install deps and give the option to opt out + if current_platform.is_mac(): + raise NotImplementedError + if not os.path.exists(self.build_dir): + self.logger.warning(f"Existing build dir not found, building in {self.build_dir}!") + if self._build(): + self._flash() + else: + self.logger.info(f"Build already found in {self.build_dir}; skipping!") + self._flash() + diff --git a/src/tools/interop/idt/idt.py b/src/tools/interop/idt/idt.py index 9ddb1ddd90c37f..d46e6dd1a74a03 100644 --- a/src/tools/interop/idt/idt.py +++ b/src/tools/interop/idt/idt.py @@ -22,75 +22,84 @@ import sys from pathlib import Path -import probe.runner as probe_runner -from capture import PacketCaptureRunner, controller -from discovery import MatterBleScanner, MatterDnssdListener +import features.probe.runner as probe_runner +from features.advertise import FakeMatterAdBle, FakeMatterAdDnssd +from features.capture import PacketCaptureRunner, controller +from features.capture.thread.runner import ThreadCaptureRunner +from features.discovery import MatterBleScanner, MatterDnssdListener from utils.artifact import create_file_timestamp, safe_mkdir -from utils.host_platform import get_available_interfaces, verify_host_dependencies -from utils.log import border_print +from utils.host import current_platform +from utils.log import border_print, get_logger +from utils.error import write_error_report +from features.advertise import config as advertise_config +from utils.data import MATTER_APPLICATION_DEVICE_TYPES +from features.setup import list_available_targets, do_setup import config -splash = '''\x1b[0m -\x1b[32;1m┌────────┐\x1b[33;20m▪\x1b[32;1m \x1b[34;1m┌──────┐ \x1b[33;20m• \x1b[35;1m┌──────────┐ \x1b[33;20m● -\x1b[32;1m│░░░░░░░░│ \x1b[34;1m│░░░░░░└┐ \x1b[33;20mﾟ\x1b[35;1m│░░░░░░░░░░│ -\x1b[32;1m└──┐░░┌──┘\x1b[33;20m۰\x1b[32;1m \x1b[34;1m│░░┌┐░░░│ \x1b[35;1m└───┐░░┌───┘ -\x1b[32;1m │░░│ \x1b[34;1m│░░│└┐░░│\x1b[33;20m▫ \x1b[35;1m \x1b[33;20m۰\x1b[35;1m │░░│ \x1b[33;20m｡ -\x1b[32;1m \x1b[33;20m•\x1b[32;1m │░░│ \x1b[33;20m● \x1b[34;1m│░░│┌┘░░│ \x1b[35;1m │░░│ -\x1b[32;1m┌──┘░░└──┐ \x1b[34;1m│░░└┘░░░│ \x1b[35;1m │░░│ \x1b[33;20m• -\x1b[32;1m│░░░░░░░░│ \x1b[34;1m│░░░░░░┌┘\x1b[33;20m۰ \x1b[35;1m \x1b[33;20m▪\x1b[35;1m │░░│ -\x1b[32;1m└────────┘\x1b[33;20m•\x1b[32;1m \x1b[34;1m└──────┘\x1b[33;20m｡ \x1b[35;1m └──┘ \x1b[33;20m▫ -\x1b[32;1m✰ Interop\x1b[34;1m ✰ Debugging\x1b[35;1m ✰ Tool -\x1b[0m''' +logger = get_logger(__file__) class InteropDebuggingTool: def __init__(self) -> None: - if config.enable_color: - print(splash) + + if config.ENABLE_COLOR: + print(config.SPLASH) + + border_print(f"Version {config.IDT_VERSION}") + + current_platform.verify_host_dependencies() + self.artifact_dir = None create_artifact_dir = True if len(sys.argv) == 1: create_artifact_dir = False - elif sys.argv[1] != "capture" and sys.argv[1] != "discover": + elif sys.argv[1] != "capture" and sys.argv[1] != "discover" and sys.argv[1] != "probe": create_artifact_dir = False elif len(sys.argv) >= 3 and (sys.argv[2] == "-h" or sys.argv[2] == "--help"): create_artifact_dir = False - verify_host_dependencies(["adb", "tcpdump"]) - - if not os.environ['IDT_OUTPUT_DIR']: - print('Missing required env vars! Use /scripts!!!') + if not os.environ["IDT_OUTPUT_DIR"]: + print("Missing required env vars! Use /scripts!!!") sys.exit(1) self.artifact_dir_parent = os.path.join( Path(__file__).resolve().parent, - os.environ['IDT_OUTPUT_DIR']) + os.environ["IDT_OUTPUT_DIR"]) artifact_timestamp = create_file_timestamp() self.artifact_dir = os.path.join( self.artifact_dir_parent, - f'idt_{artifact_timestamp}') + f"idt_{artifact_timestamp}") if create_artifact_dir: safe_mkdir(self.artifact_dir) border_print(f"Using artifact dir {self.artifact_dir}") - self.available_platforms = controller.list_available_platforms() - self.available_platforms_default = 'Android' if 'Android' in self.available_platforms else None - self.platform_required = self.available_platforms_default is None + self.advertise_device_type_choices = [int(i, 16) for i in MATTER_APPLICATION_DEVICE_TYPES.keys()] - self.available_ecosystems = controller.list_available_ecosystems() - self.available_ecosystems_default = 'ALL' - self.available_ecosystems.append(self.available_ecosystems_default) + self.capture_platforms_choices = controller.list_available_platforms() + self.capture_platforms_default = "Android" if "Android" in self.capture_platforms_choices else None + self.capture_platform_required = self.capture_platforms_default is None + self.capture_ecosystems_choices = controller.list_available_ecosystems() + self.capture_ecosystems_default = "ALL" + self.capture_ecosystems_choices.append(self.capture_ecosystems_default) + self.capture_pcap_interfaces_choices = current_platform.get_interfaces_available_for_pcap() + # TODO: No None below + self.capture_pcap_interfaces_default = "any" if "any" in self.capture_pcap_interfaces_choices else None + self.capture_pcap_interface_required = self.capture_pcap_interfaces_default is None + self.capture_pcap_wlan_channel_default, self.capture_pcap_wlan_width_default = current_platform. \ + current_wifi_channel_width(display=True) # TODO: Remove display + self.capture_pcap_wifi_bands_choices = ["2", "5"] + self.capture_pcap_wifi_bands_default = "5" if current_platform.using_5g_band() else "2" + self.capture_pcap_artifact_dir = os.path.join(self.artifact_dir, "pcap") + self.capture_thread_modes_choices = ["none", "sniff", "on_network"] + self.capture_thread_mode_default = self.capture_thread_modes_choices[0] + self.capture_thread_artifact_dir = os.path.join(self.artifact_dir, "thread") - self.available_net_interfaces = get_available_interfaces() - self.available_net_interfaces_default = "any" if "any" in self.available_net_interfaces else None - self.pcap_artifact_dir = os.path.join(self.artifact_dir, "pcap") - self.net_interface_required = self.available_net_interfaces_default is None + self.discovery_ble_artifact_dir = os.path.join(self.artifact_dir, "ble") + self.discovery_dnssd_artifact_dir = os.path.join(self.artifact_dir, "dnssd") - self.ble_artifact_dir = os.path.join(self.artifact_dir, "ble") - self.dnssd_artifact_dir = os.path.join(self.artifact_dir, "dnssd") - self.prober_dir = os.path.join(self.artifact_dir, "probes") + self.probe_artifact_dir = os.path.join(self.artifact_dir, "features/probe") self.process_args() @@ -98,102 +107,254 @@ def process_args(self) -> None: parser = argparse.ArgumentParser( prog="idt", description="Interop Debugging Tool for Matter") - subparsers = parser.add_subparsers(title="subcommands") - discover_parser = subparsers.add_parser( - "discover", help="Discover all Matter devices") - discover_parser.set_defaults(func=self.command_discover) - discover_parser.add_argument( + advertise_parser = subparsers.add_parser("advertise", + help="Create a fake advertisement for a Matter device") + advertise_parser.add_argument( + "--vid", + "-v", + help=f"Vendor ID to use in the advertisement (int, default: {advertise_config.DEFAULT_VID})", + default=advertise_config.DEFAULT_VID + ) + advertise_parser.add_argument( + "--pid", + "-p", + help=f"Product ID to use in the advertisement (int, default: {advertise_config.DEFAULT_PID})", + default=advertise_config.DEFAULT_PID + ) + advertise_parser.add_argument( + "--discriminator", + "-i", + help=f"Discriminator to use in the advertisement (int, default: {advertise_config.DEFAULT_DISCRIMINATOR})", + default=advertise_config.DEFAULT_DISCRIMINATOR + ) + advertise_parser.add_argument( + "--device_name", + "-n", + help=f"Device name to be used in the advertisement (str, default: {advertise_config.DEFAULT_DEVICE_NAME})", + default=advertise_config.DEFAULT_DEVICE_NAME + ) + advertise_parser.add_argument( + "--device_type", + "-d", + help=f"Device type to be used in the advertisement (int, default: {advertise_config.DEFAULT_DEVICE_TYPE})", + choices=self.advertise_device_type_choices, + default=advertise_config.DEFAULT_DEVICE_TYPE + ) + advertise_parser.add_argument( + "--port", + "-o", + help=f"The port that this device is reachable on (int, default: {advertise_config.DEFAULT_PORT})", + default=advertise_config.DEFAULT_PORT + ) + advertise_parser.add_argument( + "--commissioning_open", + "-c", + choices=["t", "f"], + help=f"Whether commissioning window is open or not (default: {advertise_config.DEFAULT_COMMISSIONING_OPEN})", + default=advertise_config.DEFAULT_COMMISSIONING_OPEN + ) + advertise_parser.add_argument( + "--mac_address", + "-m", + help=f"MAC Address to use for the instance name in this advertisement " + f"(str, default: {advertise_config.DEFAULT_MAC_ADDR})", + default=advertise_config.DEFAULT_MAC_ADDR + ) + advertise_parser.add_argument( + "--allow_gua", + "-g", + help=f"Whether DNS-SD advertisements should include GUAs (if they're available on the host)" + f"(default: {advertise_config.DEFAULT_ALLOW_V6_GUA})", + choices=["t", "f"], + default=advertise_config.DEFAULT_ALLOW_V6_GUA + ) + advertise_parser.add_argument( "--type", "-t", - help="Specify the type of discovery to execute", + help="Specify the type of advertisement to create", required=True, choices=[ "ble", "b", "dnssd", "d"]) + advertise_parser.set_defaults(func=self.command_advertise) capture_parser = subparsers.add_parser( "capture", help="Capture all information of interest while running a manual test") - - platform_help = "Run capture for a particular platform" - if self.available_platforms_default: - platform_help += f" (default {self.available_platforms_default})" + capture_platform_help = "Run capture for a particular platform" + if self.capture_platforms_default: + capture_platform_help += f" (default {self.capture_platforms_default})" capture_parser.add_argument("--platform", "-p", - help=platform_help, - required=self.platform_required, - choices=self.available_platforms, - default=self.available_platforms_default) - + help=capture_platform_help, + required=self.capture_platform_required, + choices=self.capture_platforms_choices, + default=self.capture_platforms_default) capture_parser.add_argument( "--ecosystem", "-e", - help="Run capture for a particular ecosystem or ALL ecosystems (default ALL)", + help=f"Run capture for a particular ecosystem or ALL ecosystems (default {self.capture_ecosystems_default})", required=False, - choices=self.available_ecosystems, - default=self.available_ecosystems_default) - + choices=self.capture_ecosystems_choices, + default=self.capture_ecosystems_default) capture_parser.add_argument("--pcap", "-c", help="Run packet capture (default t)", required=False, - choices=['t', 'f'], - default='t') - - interface_help = "Specify packet capture interface" - if self.available_net_interfaces_default: - interface_help += f" (default {self.available_net_interfaces_default})" + choices=["t", "f"], + default="t") + capture_interface_help = "Specify packet capture interface" + if self.capture_pcap_interfaces_default: + capture_interface_help += f" (default {self.capture_pcap_interfaces_default})" capture_parser.add_argument( "--interface", "-i", - help=interface_help, - required=self.net_interface_required, - choices=self.available_net_interfaces, - default=self.available_net_interfaces_default) - + help=capture_interface_help, + required=self.capture_pcap_interface_required, + choices=self.capture_pcap_interfaces_choices, + default=self.capture_pcap_interfaces_default) + capture_parser.add_argument("--monitor", + "-m", + help="Run packet capture using a monitor mode interface (default f)", + required=False, + choices=["t", "f"], + default="f") + capture_channel_help = \ + f"Use this Wi-Fi channel if in monitor mode (default {self.capture_pcap_wlan_channel_default})" + capture_parser.add_argument("--channel", + "-n", + help=capture_channel_help, + required=False, + default=self.capture_pcap_wlan_channel_default) + capture_parser.add_argument("--band", + "-b", + help=f"Use 2 for 2.4GHz, 5 for 5GHz (default {self.capture_pcap_wifi_bands_default})", + required=False, + choices=self.capture_pcap_wifi_bands_choices, + default=self.capture_pcap_wifi_bands_default) + capture_parser.add_argument("--width", + "-w", + help=f"Optionally set the channel width of a monitor mode pcap " + f"(default {self.capture_pcap_wlan_width_default})", + required=False, + default=self.capture_pcap_wlan_width_default) + capture_parser.add_argument( + "--thread", + "-t", + help=f"Execute thread sniffer or join OTBR to network (Default {self.capture_thread_mode_default})", + choices=self.capture_thread_modes_choices, + default=self.capture_thread_mode_default + ) capture_parser.set_defaults(func=self.command_capture) - prober_parser = subparsers.add_parser("probe", - help="Probe the environment for Matter and general networking info") - prober_parser.set_defaults(func=self.command_probe) + discover_parser = subparsers.add_parser( + "discover", help="Discover all Matter devices") + discover_parser.set_defaults(func=self.command_discover) + discover_parser.add_argument( + "--type", + "-t", + help="Specify the type of discovery to execute", + required=True, + choices=[ + "ble", + "b", + "dnssd", + "d"]) + # TODO: Add filter for discriminator + # TODO: Verify that case is handled properly here + discover_parser.add_argument( + "--vid", + "-v", + help="Only display advertisements with this Vendor ID. Hex values (without 0x prefix) are expected. Eg " + "FFF1", + required=False) + discover_parser.add_argument( + "--pid", + "-p", + help="If vid argument is set, filter advertisements by this Product ID as well. Hex values (without 0x " + "prefix) are expected. Eg 8000", + required=False) + discover_parser.add_argument( + "--v4", + "-4", + help="Whether to browse on IPv4 or not (Default t)", + required=False, + choices=["t", "f"], + default="t") + discover_parser.add_argument( + "--v6", + "-6", + help="Whether to browse on IPv6 or not (Default t)", + required=False, + choices=["t", "f"], + default="t") + probe_parser = subparsers.add_parser("probe", + help="Probe the environment for Matter and general networking info") + probe_parser.set_defaults(func=self.command_probe) + + setup_parser = subparsers.add_parser("setup", + help="Build / flash devices to use with the tool, e.g. otncp") + setup_parser.add_argument("--target", + "-t", + help="The target to setup", + choices=list_available_targets(), + required=True) + setup_parser.set_defaults(func=self.command_setup) args, unknown = parser.parse_known_args() - if not hasattr(args, 'func'): + if not hasattr(args, "func"): parser.print_help() else: args.func(args) - def command_discover(self, args: argparse.Namespace) -> None: + def command_advertise(self, args: argparse.Namespace) -> None: + commissioning_open = args.commissioning_open == "t" + allow_gua = args.allow_gua == "t" if args.type[0] == "b": - safe_mkdir(self.ble_artifact_dir) - scanner = MatterBleScanner(self.ble_artifact_dir) - asyncio.run(scanner.browse_interactive()) - self.zip_artifacts() + asyncio.run(FakeMatterAdBle().advertise()) else: - safe_mkdir(self.dnssd_artifact_dir) - MatterDnssdListener(self.dnssd_artifact_dir).browse_interactive() - self.zip_artifacts() - - def zip_artifacts(self) -> None: - zip_basename = os.path.basename(self.artifact_dir) - archive_file = shutil.make_archive(zip_basename, - 'zip', - root_dir=self.artifact_dir) - output_zip = shutil.move(archive_file, self.artifact_dir_parent) - border_print(f'Output zip: {output_zip}') + asyncio.run(FakeMatterAdDnssd(args.vid, + args.pid, + args.discriminator, + args.device_name, + args.device_type, + args.port, + commissioning_open, + args.mac_address, + allow_gua).advertise()) def command_capture(self, args: argparse.Namespace) -> None: - pcap = args.pcap == 't' + pcap = args.pcap == "t" + monitor_mode = args.monitor == "t" + if monitor_mode: + logger.warning("You have selected monitor mode pcap. If you proceed, wireless connections from this " + "machine will not be available for the duration of the capture. On certain Linux systems, " + "you may need to restart your machine to return to normal Wi-Fi function.") + logger.critical("Enter y/Y to proceed with monitor mode, any other input to exit!") + answer = input() + if answer.lower().strip() != "y": + logger.info("y/Y not provided, exiting!") + sys.exit(1) + # TODO: Validate arguments pcap_runner = None if not pcap else PacketCaptureRunner( - self.pcap_artifact_dir, args.interface) + self.capture_pcap_artifact_dir, args.interface, monitor_mode, args.channel, args.band, args.width) if pcap: border_print("Starting pcap") - safe_mkdir(self.pcap_artifact_dir) + safe_mkdir(self.capture_pcap_artifact_dir) pcap_runner.start_pcap() + thread = args.thread != "none" + thread_runner = None if not thread else ThreadCaptureRunner( + self.capture_thread_artifact_dir, + args.thread + ) + if thread: + border_print(f"Starting thread capture in mode {args.thread}") + safe_mkdir(self.capture_thread_artifact_dir) + thread_runner.start() asyncio.run(controller.init_ecosystems(args.platform, args.ecosystem, self.artifact_dir)) @@ -202,16 +363,49 @@ def command_capture(self, args: argparse.Namespace) -> None: if pcap: border_print("Stopping pcap") pcap_runner.stop_pcap() + if thread: + border_print("Stopping thread") + thread_runner.stop() asyncio.run(controller.stop()) asyncio.run(controller.probe()) border_print("Checking error report") - controller.write_error_report(self.artifact_dir) + write_error_report(self.artifact_dir) border_print("Compressing artifacts...") self.zip_artifacts() + def command_discover(self, args: argparse.Namespace) -> None: + vendor_id = None if not args.vid else "0x" + args.vid.lstrip("0") + product_id = None if not args.pid else "0x" + args.pid.lstrip("0") + if args.type[0] == "b": + safe_mkdir(self.discovery_ble_artifact_dir) + scanner = MatterBleScanner(self.discovery_ble_artifact_dir, vendor_id, product_id) + asyncio.run(scanner.browse_interactive()) + self.zip_artifacts() + else: + safe_mkdir(self.discovery_dnssd_artifact_dir) + MatterDnssdListener(self.discovery_dnssd_artifact_dir, + vendor_id, + product_id, + v4=args.v4 == "t", + v6=args.v6 == "t").browse_interactive() + self.zip_artifacts() + def command_probe(self, args: argparse.Namespace) -> None: border_print("Starting generic Matter prober for local environment!") - safe_mkdir(self.dnssd_artifact_dir) - safe_mkdir(self.prober_dir) - probe_runner.run_probes(self.prober_dir, self.dnssd_artifact_dir) + safe_mkdir(self.discovery_dnssd_artifact_dir) + safe_mkdir(self.probe_artifact_dir) + probe_runner.run_probes(self.probe_artifact_dir, self.discovery_dnssd_artifact_dir) self.zip_artifacts() + + def command_setup(self, args: argparse.Namespace) -> None: + border_print("Executing setup!") + do_setup(args.target) + + def zip_artifacts(self) -> None: + # TODO: 1 Standardized machine readable report / platform & ecosystem + zip_basename = os.path.basename(self.artifact_dir) + archive_file = shutil.make_archive(zip_basename, + "zip", + root_dir=self.artifact_dir) + output_zip = shutil.move(archive_file, self.artifact_dir_parent) + border_print(f"Output zip: {output_zip}") diff --git a/src/tools/interop/idt/probe/ip_utils.py b/src/tools/interop/idt/probe/ip_utils.py deleted file mode 100644 index ca1e39ebf75f91..00000000000000 --- a/src/tools/interop/idt/probe/ip_utils.py +++ /dev/null @@ -1,50 +0,0 @@ -# -# Copyright (c) 2023 Project CHIP Authors -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import ipaddress - - -def is_ipv4(ip: str) -> bool: - try: - ipaddress.IPv4Address(ip) - return True - except ipaddress.AddressValueError: - return False - - -def is_ipv6_ll(ip: str) -> bool: - try: - return ipaddress.IPv6Address(ip).is_link_local - except ipaddress.AddressValueError: - return False - - -def is_ipv6(ip: str) -> bool: - try: - ipaddress.IPv6Address(ip) - return True - except ipaddress.AddressValueError: - return False - - -def get_addr_type(ip: str) -> str: - if is_ipv4(ip): - return "V4" - elif is_ipv6_ll(ip): - return "V6 Link Local" - elif is_ipv6(ip): - return "V6" diff --git a/src/tools/interop/idt/res/plugin_demo/ecosystem/demo_ext_ecosystem/demo_ext_ecosystem.py b/src/tools/interop/idt/res/plugin_demo/ecosystem/demo_ext_ecosystem/demo_ext_ecosystem.py index 11fe43f0204332..e4ceaa7c54bf58 100644 --- a/src/tools/interop/idt/res/plugin_demo/ecosystem/demo_ext_ecosystem/demo_ext_ecosystem.py +++ b/src/tools/interop/idt/res/plugin_demo/ecosystem/demo_ext_ecosystem/demo_ext_ecosystem.py @@ -15,12 +15,12 @@ # limitations under the License. # -from capture.base import EcosystemCapture +from features.capture.base import EcosystemCapture, PlatformLogStreamer class DemoExtEcosystem(EcosystemCapture): - def __init__(self, platform, artifact_dir: str) -> None: + def __init__(self, platform: PlatformLogStreamer, artifact_dir: str) -> None: self.artifact_dir = artifact_dir self.platform = platform self.message = "in the demo external ecosystem" @@ -32,4 +32,4 @@ async def stop_capture(self) -> None: print("Stop capture " + self.message) async def analyze_capture(self) -> None: - print("Analyze capture " + self.message) + print("Analyze capture real time" + self.message) diff --git a/src/tools/interop/idt/res/splash.py b/src/tools/interop/idt/res/splash.py new file mode 100644 index 00000000000000..a37f4185330878 --- /dev/null +++ b/src/tools/interop/idt/res/splash.py @@ -0,0 +1,28 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +splash = '''\x1b[0m +\x1b[32;1m┌────────┐\x1b[33;20m▪\x1b[32;1m \x1b[34;1m┌──────┐ \x1b[33;20m• \x1b[35;1m┌──────────┐ \x1b[33;20m● +\x1b[32;1m│░░░░░░░░│ \x1b[34;1m│░░░░░░└┐ \x1b[33;20mﾟ\x1b[35;1m│░░░░░░░░░░│ +\x1b[32;1m└──┐░░┌──┘\x1b[33;20m۰\x1b[32;1m \x1b[34;1m│░░┌┐░░░│ \x1b[35;1m└───┐░░┌───┘ +\x1b[32;1m │░░│ \x1b[34;1m│░░│└┐░░│\x1b[33;20m▫ \x1b[35;1m \x1b[33;20m۰\x1b[35;1m │░░│ \x1b[33;20m｡ +\x1b[32;1m \x1b[33;20m•\x1b[32;1m │░░│ \x1b[33;20m● \x1b[34;1m│░░│┌┘░░│ \x1b[35;1m │░░│ +\x1b[32;1m┌──┘░░└──┐ \x1b[34;1m│░░└┘░░░│ \x1b[35;1m │░░│ \x1b[33;20m• +\x1b[32;1m│░░░░░░░░│ \x1b[34;1m│░░░░░░┌┘\x1b[33;20m۰ \x1b[35;1m \x1b[33;20m▪\x1b[35;1m │░░│ +\x1b[32;1m└────────┘\x1b[33;20m•\x1b[32;1m \x1b[34;1m└──────┘\x1b[33;20m｡ \x1b[35;1m └──┘ \x1b[33;20m▫ +\x1b[32;1m✰ Interop\x1b[34;1m ✰ Debugging\x1b[35;1m ✰ Tool +\x1b[0m''' diff --git a/src/tools/interop/idt/scripts/alias.sh b/src/tools/interop/idt/scripts/alias.sh index e7b598bc3de5da..7f9e5663d233f0 100644 --- a/src/tools/interop/idt/scripts/alias.sh +++ b/src/tools/interop/idt/scripts/alias.sh @@ -15,6 +15,9 @@ # limitations under the License. # +# The IDT installation directory is located via relative path to this file. +# Different shells offer different facilities for checking the path of a sourced file at run time, +# so we have different configs here accordingly. if [[ $SHELL == "/bin/zsh" ]]; then echo "idt using zsh config" export IDT_SRC_PARENT="$(dirname "$0")/../.." @@ -23,30 +26,38 @@ else export IDT_SRC_PARENT="$(dirname "${BASH_SOURCE[0]:-$0}")/../.." fi +# Make sure these are setup before anything else runs export IDT_OUTPUT_DIR="IDT_ARTIFACTS" - alias idt_dir="echo \"idt dir $IDT_SRC_PARENT\"" -idt_dir alias idt_go="cd \"$IDT_SRC_PARENT\"" -alias idt_activate="idt_go && source idt/scripts/activate.sh" -alias idt_bootstrap="idt_go && source idt/scripts/bootstrap.sh" -alias idt_build="idt_go && source idt/scripts/build.sh" -alias idt_clean="idt_go && source idt/scripts/clean.sh" -alias idt_connect="idt_go && source idt/scripts/connect.sh" -alias idt_fetch_artifacts="idt_go && source idt/scripts/fetch_artifacts.sh" -alias idt_prune_docker="idt_go && source idt/scripts/prune_docker.sh" -alias idt_push="idt_go && source idt/scripts/push.sh" +# One time setup should not be aliases alias idt_vars="idt_go && source idt/scripts/vars.sh" -alias idt_clean_artifacts="idt_go && source idt/scripts/clean_artifacts.sh" -alias idt_clean_all="idt_go && source idt/scripts/clean_all.sh" -alias idt_create_vars="idt_go && source idt/scripts/create_vars.sh" -alias idt_check_child="idt_go && source idt/scripts/check_child.sh" -alias idt_clean_child="idt_go && source idt/scripts/clean_child.sh" +# Setting things up this way allows us to edit the source of the aliases without having to manually call +# source / relaunch shell for changes to take effect (unless changing the alias declaration itself) +# also, if the script is deleted, an old shell won't have access to run the alises anymore, which is desired +alias_src_path="idt_go && source idt/scripts/aliases" +alias idt_artifacts_fetch="$alias_src_path/artifacts_fetch.sh" +alias idt_child_check="$alias_src_path/child_check.sh" +alias idt_child_kill="$alias_src_path/child_kill.sh" +alias idt_clean_all="$alias_src_path/clean_all.sh" +alias idt_clean_artifacts="$alias_src_path/clean_artifacts.sh" +alias idt_connect="$alias_src_path/connect.sh" +alias idt_delete="$alias_src_path/delete.sh" +alias idt_docker_build="$alias_src_path/docker_build.sh" +alias idt_docker_prune="$alias_src_path/docker_prune.sh" +alias idt_docker_run="$alias_src_path/docker_run.sh" +alias idt_linux_bootstrap="$alias_src_path/linux_bootstrap.sh" +alias idt_push="$alias_src_path/push.sh" +alias idt_vars_create="$alias_src_path/vars_create.sh" +alias idt_linux_install_compilers_for_arm_tcpdump="$alias_src_path/linux_install_compilers_for_arm_tcpdump.sh" + +# Setup the venv if it doesn't exist; ensure byte code cache is compartmentalized alias idt="idt_go && \ if [ -z $PYTHONPYCACHEPREFIX ]; then export PYTHONPYCACHEPREFIX=$IDT_SRC_PARENT/idt/pycache; fi && \ -if [ -z $VIRTUAL_ENV]; then source idt/scripts/py_venv.sh; fi && \ +if [ -z $VIRTUAL_ENV ]; then source idt/scripts/py_venv.sh; fi && \ python3 idt " +idt_dir echo "idt commands available! type idt and press tab twice to see available commands." diff --git a/src/tools/interop/idt/scripts/fetch_artifacts.sh b/src/tools/interop/idt/scripts/aliases/artifacts_fetch.sh similarity index 100% rename from src/tools/interop/idt/scripts/fetch_artifacts.sh rename to src/tools/interop/idt/scripts/aliases/artifacts_fetch.sh diff --git a/src/tools/interop/idt/scripts/check_child.sh b/src/tools/interop/idt/scripts/aliases/child_check.sh similarity index 95% rename from src/tools/interop/idt/scripts/check_child.sh rename to src/tools/interop/idt/scripts/aliases/child_check.sh index 8a54cd2e65f6db..c57e10ed0602fc 100644 --- a/src/tools/interop/idt/scripts/check_child.sh +++ b/src/tools/interop/idt/scripts/aliases/child_check.sh @@ -15,4 +15,5 @@ # limitations under the License. # +# For use in development / crashes sudo ps -axf | grep -E "(tcpd|adb)" diff --git a/src/tools/interop/idt/scripts/aliases/child_kill.sh b/src/tools/interop/idt/scripts/aliases/child_kill.sh new file mode 100644 index 00000000000000..b53d98ddf5f565 --- /dev/null +++ b/src/tools/interop/idt/scripts/aliases/child_kill.sh @@ -0,0 +1,21 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# TODO: Script to populate child* scripts based off host dependencies config +# For use in development / crashes +sudo killall tcpdump +sudo killall adb diff --git a/src/tools/interop/idt/scripts/clean_all.sh b/src/tools/interop/idt/scripts/aliases/clean_all.sh similarity index 93% rename from src/tools/interop/idt/scripts/clean_all.sh rename to src/tools/interop/idt/scripts/aliases/clean_all.sh index d7703ab29d57cc..05f666f74df680 100644 --- a/src/tools/interop/idt/scripts/clean_all.sh +++ b/src/tools/interop/idt/scripts/aliases/clean_all.sh @@ -19,5 +19,5 @@ cd idt sudo rm -R venv/ sudo rm -R pycache/ sudo rm -R IDT_ARTIFACTS/ -sudo find . -type d -name "BUILD" -delete +sudo find . -type d -name "BUILD" -exec rm -rf {} \; cd .. diff --git a/src/tools/interop/idt/scripts/clean_artifacts.sh b/src/tools/interop/idt/scripts/aliases/clean_artifacts.sh similarity index 100% rename from src/tools/interop/idt/scripts/clean_artifacts.sh rename to src/tools/interop/idt/scripts/aliases/clean_artifacts.sh diff --git a/src/tools/interop/idt/scripts/connect.sh b/src/tools/interop/idt/scripts/aliases/connect.sh similarity index 100% rename from src/tools/interop/idt/scripts/connect.sh rename to src/tools/interop/idt/scripts/aliases/connect.sh diff --git a/src/tools/interop/idt/scripts/clean.sh b/src/tools/interop/idt/scripts/aliases/delete.sh similarity index 100% rename from src/tools/interop/idt/scripts/clean.sh rename to src/tools/interop/idt/scripts/aliases/delete.sh diff --git a/src/tools/interop/idt/scripts/build.sh b/src/tools/interop/idt/scripts/aliases/docker_build.sh similarity index 100% rename from src/tools/interop/idt/scripts/build.sh rename to src/tools/interop/idt/scripts/aliases/docker_build.sh diff --git a/src/tools/interop/idt/scripts/prune_docker.sh b/src/tools/interop/idt/scripts/aliases/docker_prune.sh similarity index 100% rename from src/tools/interop/idt/scripts/prune_docker.sh rename to src/tools/interop/idt/scripts/aliases/docker_prune.sh diff --git a/src/tools/interop/idt/scripts/activate.sh b/src/tools/interop/idt/scripts/aliases/docker_run.sh similarity index 100% rename from src/tools/interop/idt/scripts/activate.sh rename to src/tools/interop/idt/scripts/aliases/docker_run.sh diff --git a/src/tools/interop/idt/scripts/bootstrap.sh b/src/tools/interop/idt/scripts/aliases/linux_bootstrap.sh similarity index 100% rename from src/tools/interop/idt/scripts/bootstrap.sh rename to src/tools/interop/idt/scripts/aliases/linux_bootstrap.sh diff --git a/src/tools/interop/idt/capture/ecosystem/play_services/config.py b/src/tools/interop/idt/scripts/aliases/linux_install_compilers_for_arm_tcpdump.sh similarity index 86% rename from src/tools/interop/idt/capture/ecosystem/play_services/config.py rename to src/tools/interop/idt/scripts/aliases/linux_install_compilers_for_arm_tcpdump.sh index 2daacb38dcf909..7711d3355163b0 100644 --- a/src/tools/interop/idt/capture/ecosystem/play_services/config.py +++ b/src/tools/interop/idt/scripts/aliases/linux_install_compilers_for_arm_tcpdump.sh @@ -15,5 +15,4 @@ # limitations under the License. # -enable_foyer_probers = True -foyer_prober_traceroute_limit = 32 +sudo apt-get -y install gcc-arm-linux-gnueabi gcc-aarch64-linux-gnu byacc flex # gcc-arm-none-eabi diff --git a/src/tools/interop/idt/scripts/push.sh b/src/tools/interop/idt/scripts/aliases/push.sh similarity index 100% rename from src/tools/interop/idt/scripts/push.sh rename to src/tools/interop/idt/scripts/aliases/push.sh diff --git a/src/tools/interop/idt/scripts/create_vars.sh b/src/tools/interop/idt/scripts/aliases/vars_create.sh similarity index 76% rename from src/tools/interop/idt/scripts/create_vars.sh rename to src/tools/interop/idt/scripts/aliases/vars_create.sh index 44d7a4228150b6..33c3641e4c9895 100644 --- a/src/tools/interop/idt/scripts/create_vars.sh +++ b/src/tools/interop/idt/scripts/aliases/vars_create.sh @@ -15,8 +15,16 @@ # limitations under the License. # -read -p "Enter RPi host name " pihost -read -p "Enter RPi user name " piuser +host_prompt="Enter RPi host name " +user_prompt="Enter RPi user name " + +if [[ $SHELL == "/bin/zsh" ]]; then + read "pihost?$host_prompt" + read "piuser?$user_prompt" +else + read -p "$host_prompt" pihost + read -p "$user_prompt" piuser +fi echo "export PIHOST=\"$pihost\"" >"$IDT_SRC_PARENT"/idt/scripts/vars.sh echo "export PIUSER=\"$piuser\"" >>"$IDT_SRC_PARENT"/idt/scripts/vars.sh diff --git a/src/tools/interop/idt/scripts/compilers.sh b/src/tools/interop/idt/scripts/compilers.sh deleted file mode 100644 index ee50284fd6f489..00000000000000 --- a/src/tools/interop/idt/scripts/compilers.sh +++ /dev/null @@ -1,21 +0,0 @@ -# -# Copyright (c) 2023 Project CHIP Authors -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -sudo apt-get install gcc-arm-linux-gnueabi -sudo apt-get install gcc-aarch64-linux-gnu -sudo apt-get install byacc -sudo apt-get install flex diff --git a/src/tools/interop/idt/scripts/py_venv.sh b/src/tools/interop/idt/scripts/py_venv.sh index 3105cbdd391116..a037e2df537ba9 100644 --- a/src/tools/interop/idt/scripts/py_venv.sh +++ b/src/tools/interop/idt/scripts/py_venv.sh @@ -15,12 +15,15 @@ # limitations under the License. # -cd idt -if [ -d venv ]; then - source venv/bin/activate +if [ -f idt/venv/bin/activate ]; then + source idt/venv/bin/activate else - python3 -m venv venv - source venv/bin/activate - pip install -r requirements.txt + python3 -m venv idt/venv + if [ -f idt/venv/bin/activate ]; then + source idt/venv/bin/activate + pip install -r idt/requirements.txt + else + echo "Failed to created venv" + idt_clean_all + fi fi -cd .. diff --git a/src/tools/interop/idt/scripts/vars.sh b/src/tools/interop/idt/scripts/vars.sh index 5133e8d726c977..0923c40a74cb45 100644 --- a/src/tools/interop/idt/scripts/vars.sh +++ b/src/tools/interop/idt/scripts/vars.sh @@ -1,19 +1,2 @@ -# -# Copyright (c) 2023 Project CHIP Authors -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -export PIHOST="pi-host" -export PIUSER="pi-user" +export PIHOST="pihost" +export PIUSER="piuser" diff --git a/src/tools/interop/idt/utils/__init__.py b/src/tools/interop/idt/utils/__init__.py index 43ab9acb8a5558..c64f6cf12f7dbc 100644 --- a/src/tools/interop/idt/utils/__init__.py +++ b/src/tools/interop/idt/utils/__init__.py @@ -15,11 +15,11 @@ # limitations under the License. # -from . import artifact, host_platform, log, shell +from . import artifact, host, log, shell __all__ = [ 'artifact', - 'host_platform', + 'host', 'log', 'shell', ] diff --git a/src/tools/interop/idt/utils/analysis.py b/src/tools/interop/idt/utils/analysis.py new file mode 100644 index 00000000000000..f0c312ac3d2fc7 --- /dev/null +++ b/src/tools/interop/idt/utils/analysis.py @@ -0,0 +1,99 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import asyncio +from dataclasses import dataclass +from logging import Logger +from typing import IO + +from utils.log import print_and_write + + +@dataclass +class TextCause: + """ + A representation of a possible problem cause in a log + search_terms: terms to match in a log line (term 1 in line AND term 2 in line AND term N in line...) + search_attr: analyze this attribute of an analysis object for this cause + help_message: how to help the user if this cause is discovered + follow_up_causes: more checks to continue narrowing down the root cause (depth first) + """ + search_terms: [str] + search_attr: [str] + help_message: str + follow_up_causes: [] # : [Cause] (Recursive reference does not work for type hint here) + + +class PrescriptiveTextAnalysis: + + def __init__(self, causes: [TextCause], output_file_name: str, logger: Logger): + """ + An "analysis object" is an object which subclasses this class + and contains log lines collected in instance variables + """ + self.causes = causes + self.output_file_name = output_file_name + self.logger = logger + + def check_cause(self, cause: TextCause) -> None: + if not hasattr(self, cause.search_attr): + self.logger.error(f"{cause.search_attr} not found on this analysis object!") + return + to_search = getattr(self, cause.search_attr) + found = True + for search_term in cause.search_terms: + found = found and search_term in to_search + if found: + with open(self.output_file_name, mode="w+") as analysis_file: + print_and_write(cause.help_message, analysis_file, important=True) + if found and cause.follow_up_causes: + for follow_up in cause.follow_up_causes: + self.check_cause(follow_up) + + def check_causes(self) -> None: + for cause in self.causes: + self.check_cause(cause) + + +class TextAnalysisObserver: + """ + Subclasses should implement functions starting with _log to process lines into instance vars of any name + """ + + def __init__(self, log: str, logger: Logger): + self.log = log + self.logger = logger + self.log_file: IO | None = None + + def process_line(self, line: str) -> None: + for line_func in [s for s in dir(self) if s.startswith('_log')]: + getattr(self, line_func)(line) + + def do_analysis(self, batch: [str]) -> None: + for line in batch: + self.process_line(line) + + async def analyze_capture(self) -> None: + try: + self.log_file = open(self.log, "r") + while True: + self.do_analysis(self.log_file.readlines()) + await asyncio.sleep(0.5) # Releasing async event loop for other tasks + except asyncio.CancelledError: + self.logger.info(f"Closing log stream {self.log}") + if self.log_file: + self.log_file.close() diff --git a/src/tools/interop/idt/utils/data.py b/src/tools/interop/idt/utils/data.py new file mode 100644 index 00000000000000..b7b639a26e1b13 --- /dev/null +++ b/src/tools/interop/idt/utils/data.py @@ -0,0 +1,91 @@ +# +# Copyright (c) 2024 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +MATTER_APPLICATION_DEVICE_TYPES = { + # lighting + "0x100": "On/Off Light", + "0x101": "Dimmable Light", + "0x10C": "Color Temperature Light", + "0x10D": "Extended Color Light", + # smart plugs/outlets and other actuators + "0x10A": "On/Off Plug-in Unit", + "0x10B": "Dimmable Plug-In Unit", + "0x303": "Pump", + # switches and controls + "0x103": "On/Off Light Switch", + "0x104": "Dimmer Switch", + "0x105": "Color Dimmer Switch", + "0x840": "Control Bridge", + "0x304": "Pump Controller", + "0xF": "Generic Switch", + # sensors + "0x15": "Contact Sensor", + "0x106": "Light Sensor", + "0x107": "Occupancy Sensor", + "0x302": "Temperature Sensor", + "0x305": "Pressure Sensor", + "0x306": "Flow Sensor", + "0x307": "Humidity Sensor", + "0x850": "On/Off Sensor", + # closures + "0xA": "Door Lock", + "0xB": "Door Lock Controller", + "0x202": "Window Covering", + "0x203": "Window Covering Controller", + # HVAC + "0x300": "Heating/Cooling Unit", + "0x301": "Thermostat", + "0x2B": "Fan", + # media + "0x28": "Basic Video Player", + "0x23": "Casting Video Player", + "0x22": "Speaker", + "0x24": "Content App", + "0x29": "Casting Video Client", + "0x2A": "Video Remote Control", + # generic + "0x27": "Mode Select", +} + +MATTER_COMMISSIONING_MODE_DESCRIPTIONS = [ + "Not in commissioning mode", + "In passcode commissioning mode (standard mode)", + "In dynamic passcode commissioning mode", +] + +MATTER_PAIRING_HINTS = [ + "Power Cycle", + "Custom commissioning flow", + "Use existing administrator (already commissioned)", + "Use settings menu on device", + "Use the PI TXT record hint", + "Read the manual", + "Press the reset button", + "Press Reset Button with application of power", + "Press Reset Button for N seconds", + "Press Reset Button until light blinks", + "Press Reset Button for N seconds with application of power", + "Press Reset Button until light blinks with application of power", + "Press Reset Button N times", + "Press Setup Button", + "Press Setup Button with application of power", + "Press Setup Button for N seconds", + "Press Setup Button until light blinks", + "Press Setup Button for N seconds with application of power", + "Press Setup Button until light blinks with application of power", + "Press Setup Button N times", +] diff --git a/src/tools/interop/idt/utils/error.py b/src/tools/interop/idt/utils/error.py new file mode 100644 index 00000000000000..9c8c696be17e04 --- /dev/null +++ b/src/tools/interop/idt/utils/error.py @@ -0,0 +1,59 @@ +# +# Copyright (c) 2024 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import traceback +from typing import Dict, List +from dataclasses import dataclass + +from utils.artifact import create_standard_log_name +from utils.log import get_logger, print_and_write, add_border + +logger = get_logger(__file__) + + +@dataclass +class ErrorRecord: + help_message: str + stack_trace: str + + +_ERROR_RECORDS: Dict[str, List[ErrorRecord]] = {} + + +def log_error(context: str, help_message: str, stack_trace=True) -> None: + stack_trace = "" if not stack_trace else traceback.format_exc().replace("\\n", "\n") + logger.error(context) + logger.error(help_message) + if stack_trace: + logger.error(stack_trace) + if context not in _ERROR_RECORDS: + _ERROR_RECORDS[context] = [] + error_record = ErrorRecord(help_message, stack_trace) + _ERROR_RECORDS[context].append(error_record) + + +def write_error_report(artifact_dir: str): + if _ERROR_RECORDS: + logger.critical("DETECTED ERRORS THIS RUN!") + error_report_file_name = create_standard_log_name("error_report", "txt", parent=artifact_dir) + with open(error_report_file_name, "a+") as error_report_file: + for context in _ERROR_RECORDS: + print_and_write(add_border(f"Errors for {context}"), error_report_file, important=True) + for record in _ERROR_RECORDS[context]: + print_and_write(str(record), error_report_file, important=True) + else: + logger.info("No errors seen this run!") diff --git a/src/tools/interop/idt/utils/host/__init__.py b/src/tools/interop/idt/utils/host/__init__.py new file mode 100644 index 00000000000000..76b13dca2c0f0e --- /dev/null +++ b/src/tools/interop/idt/utils/host/__init__.py @@ -0,0 +1,20 @@ +# +# Copyright (c) 2024 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from .host import impl as current_platform + +__all__ = ['current_platform'] diff --git a/src/tools/interop/idt/utils/host/base.py b/src/tools/interop/idt/utils/host/base.py new file mode 100644 index 00000000000000..6470f3363e3573 --- /dev/null +++ b/src/tools/interop/idt/utils/host/base.py @@ -0,0 +1,209 @@ +# +# Copyright (c) 2024 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import re +import sys +import uuid +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Callable + +from config import HOST_DEPENDENCIES +from utils import log +from utils.log import border_print +from utils.shell import Bash +from utils.net import is_ipv4, is_ipv6_global_unicast, is_ipv6_unique_local, is_ipv6_link_local, WIFI_CHANNELS_2G + +import config + +logger = log.get_logger(__file__) + + +@dataclass +class HostIps: + """ + Object that holds three types of unicast addresses the host may have + """ + v4: [str] + v6_global: [str] + v6_unique_local: [str] + v6_link_local: [str] + + +class HostPlatform(ABC): + + def __init__(self) -> None: + self.interface_config = self.get_interface_config() + + @abstractmethod + def get_interfaces_available_for_pcap(self) -> str: + """ + Returns a list of available network interfaces for pcaps + """ + raise NotImplementedError + + @abstractmethod + def get_link_local_interface(self) -> str: + """ + Returns an interface appropriate for link local connections + """ + raise NotImplementedError + + def get_mac_addr(self) -> str: + return ':'.join(re.findall('..', '%012x' % uuid.getnode())).replace(":", "").upper() + + def get_interface_config(self) -> [str]: + """ + Returns the chunk of ifconfig relevant to the link local interface + """ + all_ifconfig_cmd = Bash("ifconfig", sync=True, capture_output=True) + all_ifconfig_cmd.start_command() + all_ifconfig_output = all_ifconfig_cmd.get_captured_output().split("\n") + start_search = self.get_link_local_interface() + ":" + stop_search = ": flags" + in_chunk, accum = False, [] + for line in all_ifconfig_output: + if in_chunk and stop_search in line: + in_chunk = False + if start_search in line: + in_chunk = True + if in_chunk: + accum.append(line) + logger.debug("Found ifconfig chunk:") + for line in accum: + logger.debug(line) + return accum + + def get_addresses_of_types(self, + address_type: str, + search_prefix: str, + match_function: Callable[[str], bool]) -> [str]: + ret = [] + for line in self.interface_config: + if line.strip().startswith(f"{search_prefix} "): + potential_match = line.strip().split(" ")[1] + if match_function(potential_match): + ret.append(potential_match) + logger.debug(f"{address_type} addresses {ret}") + return ret + + def get_ipv4_addresses(self) -> [str]: + return self.get_addresses_of_types("v4", + "inet", + is_ipv4) + + def get_ipv6_global_addresses(self) -> [str]: + return self.get_addresses_of_types("v6 globally unique", + "inet6", + is_ipv6_global_unicast) + + def get_ipv6_unique_local_addresses(self) -> [str]: + return self.get_addresses_of_types("v6 unique local", + "inet6", + is_ipv6_unique_local) + + def get_ipv6_link_local_addresses(self) -> str | None: + return self.get_addresses_of_types("v6 link local", + "inet6", + is_ipv6_link_local) + + def ips(self) -> HostIps: + """ + Returns addresses this host has + """ + return HostIps(self.get_ipv4_addresses(), + self.get_ipv6_global_addresses(), + self.get_ipv6_unique_local_addresses(), + self.get_ipv6_link_local_addresses()) + + @abstractmethod + def current_wifi_channel_width(self, display=False) -> tuple[int, int | None]: + """ + Returns the currently used Wi-Fi channel and width if available + """ + raise NotImplementedError + + def using_5g_band(self) -> bool: + """ + Returns true if the host is using 5g band + """ + using_5g = str(self.current_wifi_channel_width()[0]) not in WIFI_CHANNELS_2G + if using_5g: + logger.info(f"Using 5g Wi-Fi") + else: + logger.info(f"Using 2g Wi-Fi") + return using_5g + + def verify_py_version(self) -> None: + """ + Verify the python version used on the host + Exit the entire program is there is a version mismatch + """ + py_version_major = sys.version_info[0] + py_version_minor = sys.version_info[1] + have = f"{py_version_major}.{py_version_minor}" + need = f"{config.PY_MAJOR_VERSION}.{config.PY_MINOR_VERSION}" + if not (py_version_major == config.PY_MAJOR_VERSION + and py_version_minor >= config.PY_MINOR_VERSION): + # TODO: Autoclean venv + logger.critical( + f"IDT requires python >= {need} but you have {have} for the command python3") + logger.critical("Please install / configure the correct python version, delete idt/venv, and re-run!") + sys.exit(1) + + def command_is_available(self, command: str) -> bool: + """ + Returns True if the command is available on the host system + """ + cmd = Bash(f"which {command}", sync=True, capture_output=True) + cmd.start_command() + return cmd.finished_success() + + def check_dependencies(self, deps: [str]) -> None: + """ + Checks if a dependency from the provided list is missing and logs it + Exit the entire program is there is any missing dependency + """ + if not self.command_is_available("which"): + logger.critical("which is required to verify host dependencies, exiting as its not available!") + sys.exit(1) + missing_deps = [] + for dep in deps: + logger.info(f"Verifying host dependency {dep}") + if not self.command_is_available(dep): + missing_deps.append(dep) + if missing_deps: + for missing_dep in missing_deps: + border_print(f"Missing dependency, please install {missing_dep}!", important=True) + print(f"Help: {deps[missing_dep]}") + sys.exit(1) + + def verify_host_dependencies(self) -> None: + self.check_dependencies(HOST_DEPENDENCIES["ALL"]) + if self.is_mac(): + logger.info("Verifying Mac specific dependencies") + self.check_dependencies(HOST_DEPENDENCIES["MAC"]) + else: + logger.info("Verifying Linux specific dependencies") + self.check_dependencies(HOST_DEPENDENCIES["LINUX"]) + + def is_mac(self) -> bool: + """ + Returns True if the current host is a Mac + """ + from .mac import MacPlatform # Avoid circular import + return isinstance(self, MacPlatform) diff --git a/src/tools/interop/idt/utils/host/host.py b/src/tools/interop/idt/utils/host/host.py new file mode 100644 index 00000000000000..8e316ba79a7fc1 --- /dev/null +++ b/src/tools/interop/idt/utils/host/host.py @@ -0,0 +1,32 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import platform +import sys + +from utils.host.linux import LinuxPlatform +from utils.host.mac import MacPlatform +from utils.log import border_print + +p = platform.platform().lower() +if "darwin" in p or "mac" in p: + impl = MacPlatform() +elif "linux" in p: + impl = LinuxPlatform() +else: + border_print("Could not identify if host is Linux or MacOS, exiting!", important=True) + sys.exit(1) diff --git a/src/tools/interop/idt/utils/host/linux.py b/src/tools/interop/idt/utils/host/linux.py new file mode 100644 index 00000000000000..a7a6fc9326483a --- /dev/null +++ b/src/tools/interop/idt/utils/host/linux.py @@ -0,0 +1,75 @@ +# +# Copyright (c) 2024 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os + +from utils.host.base import HostPlatform +from utils.shell import Bash +from utils.log import get_logger + +logger = get_logger(__file__) + + +class LinuxPlatform(HostPlatform): + + def __init__(self): + HostPlatform.__init__(self) + + def get_interfaces_available_for_pcap(self) -> str: + """ + Returns a list of available network interfaces + """ + net_interface_path = "/sys/class/net/" + available_net_interfaces = os.listdir(net_interface_path) \ + if os.path.exists(net_interface_path) \ + else [] + available_net_interfaces.append("any") + return available_net_interfaces + + def get_link_local_interface(self) -> str: + """ + Returns an interface appropriate for link local connections + """ + net_interface_path = "/sys/class/net/" + available_net_interfaces = os.listdir(net_interface_path) \ + if os.path.exists(net_interface_path) \ + else [] + for interface in available_net_interfaces: + if "wl" in interface: + return interface + + def current_wifi_channel_width(self, display=False) -> tuple[int, int | None]: + """ + Returns the currently used Wi-Fi channel and width if available + """ + interface = self.get_link_local_interface() + channel_cmd = Bash(f"iw {interface} info", sync=True, capture_output=True) + channel_cmd.start_command() + try: + lines = channel_cmd.get_captured_output().split("\n") + for i in range(0, len(lines)): + lines[i] = lines[i].strip() + for line in lines: + if line.startswith("channel"): + channel = int(line.split(" ")[1]) + if display: + logger.info(f"Current Wi-Fi channel is {channel}") + return channel, None + except Exception: + logger.critical(f"Error parsing channel for interface {interface}") + logger.warning("Could not find default Wi-Fi channel, using 6") + return 6, None diff --git a/src/tools/interop/idt/utils/host/mac.py b/src/tools/interop/idt/utils/host/mac.py new file mode 100644 index 00000000000000..bc11d18a64a5b1 --- /dev/null +++ b/src/tools/interop/idt/utils/host/mac.py @@ -0,0 +1,68 @@ +# +# Copyright (c) 2024 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from utils import log +from utils.host.base import HostPlatform +from utils.shell import Bash + +logger = log.get_logger(__file__) + + +class MacPlatform(HostPlatform): + + def __init__(self): + HostPlatform.__init__(self) + + def get_interfaces_available_for_pcap(self) -> [str]: + """ + Returns a list of available network interfaces + """ + return ["any"] + + def get_link_local_interface(self) -> str: + """ + Returns an interface appropriate for link local connections + """ + return "en0" + + def current_wifi_channel_width(self, display=False) -> tuple[str, str | None]: + """ + Returns the currently used Wi-Fi channel and width if available + """ + get_config = Bash("airport -I", sync=True, capture_output=True) + get_config.start_command() + config_output = get_config.get_captured_output() + for line in config_output.split("\n"): + search_term = "channel:" + if search_term in line: + try: + value = line[line.index(search_term) + len(search_term) + 1:] + if "," in value: + channel = str(int(value[:value.index(",")])) + width = str(int(value[value.index(",")+1:])) + if display: + logger.info(f"Current Wi-Fi channel is {channel} width {width}") + return channel, width + else: + channel = str(int(value)) + if display: + logger.info(f"Current Wi-Fi channel is {channel} width not detected") + return channel, None + except Exception: + logger.critical(f"Error parsing channel/width from {line}") + logger.warning("Could not find default Wi-Fi channel, using 6 with no width") + return "6", None diff --git a/src/tools/interop/idt/utils/host_platform.py b/src/tools/interop/idt/utils/host_platform.py deleted file mode 100644 index fd6425666104d9..00000000000000 --- a/src/tools/interop/idt/utils/host_platform.py +++ /dev/null @@ -1,90 +0,0 @@ -# -# Copyright (c) 2023 Project CHIP Authors -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import os -import platform as host_platform -import sys - -from utils import log -from utils.log import border_print -from utils.shell import Bash - -import config - -logger = log.get_logger(__file__) - - -def is_mac(): - p = host_platform.platform().lower() - return "darwin" in p or "mac" in p - - -def get_ll_interface(): - # TODO: Makes too many assumptions - if is_mac(): - return "en0" - net_interface_path = "/sys/class/net/" - available_net_interfaces = os.listdir(net_interface_path) \ - if os.path.exists(net_interface_path) \ - else [] - for interface in available_net_interfaces: - if "wl" in interface: - return interface - - -def get_available_interfaces(): - net_interface_path = "/sys/class/net/" - available_net_interfaces = os.listdir(net_interface_path) \ - if os.path.exists(net_interface_path) \ - else [] - available_net_interfaces.append("any") - return available_net_interfaces - - -def command_is_available(cmd_name) -> bool: - cmd = Bash(f"which {cmd_name}", sync=True, capture_output=True) - cmd.start_command() - return cmd.finished_success() - - -def verify_host_dependencies(deps: [str]) -> None: - if not command_is_available("which"): - # TODO: Check $PATH explicitly as well - logger.critical("which is required to verify host dependencies, exiting as its not available!") - sys.exit(1) - missing_deps = [] - for dep in deps: - logger.info(f"Verifying host dependency {dep}") - if not command_is_available(dep): - missing_deps.append(dep) - if missing_deps: - for missing_dep in missing_deps: - border_print(f"Missing dependency, please install {missing_dep}!", important=True) - sys.exit(1) - - -def verify_py_version() -> None: - py_version_major = sys.version_info[0] - py_version_minor = sys.version_info[1] - have = f"{py_version_major}.{py_version_minor}" - need = f"{config.py_major_version}.{config.py_minor_version}" - if not (py_version_major == config.py_major_version - and py_version_minor >= config.py_minor_version): - logger.critical( - f"IDT requires python >= {need} but you have {have}") - logger.critical("Please install the correct version, delete idt/venv, and re-run!") - sys.exit(1) diff --git a/src/tools/interop/idt/capture/loader.py b/src/tools/interop/idt/utils/loader.py similarity index 65% rename from src/tools/interop/idt/capture/loader.py rename to src/tools/interop/idt/utils/loader.py index 8e02e9d492c2b5..c92d5efbd02dfd 100644 --- a/src/tools/interop/idt/capture/loader.py +++ b/src/tools/interop/idt/utils/loader.py @@ -23,12 +23,12 @@ from utils import log -logger = log.get_logger(__file__) +_LOGGER = log.get_logger(__file__) class CaptureImplsLoader: - def __init__(self, root_dir: str, root_package: str, search_type: type): + def __init__(self, root_dir: str, root_package: str, search_type: type, logger=_LOGGER): self.logger = logger self.root_dir = root_dir self.root_package = root_package @@ -39,35 +39,55 @@ def __init__(self, root_dir: str, root_package: str, search_type: type): @staticmethod def is_package(potential_package: str) -> bool: - init_path = os.path.join(potential_package, - "__init__.py") + init_path = os.path.join(potential_package, "__init__.py") return os.path.exists(init_path) def verify_coroutines(self, subclass) -> bool: - # ABC does not verify coroutines on subclass instantiation, it merely checks the presence of methods + """ + ABC does not verify coroutines on subclass instantiation, it merely checks the presence of methods + """ for item in dir(self.search_type): item_attr = getattr(self.search_type, item) if inspect.iscoroutinefunction(item_attr): if not hasattr(subclass, item): - self.logger.warning(f"Missing coroutine in {subclass}") + self.logger.error(f"Missing coroutine attr {subclass}.{item}") return False if not inspect.iscoroutinefunction(getattr(subclass, item)): - self.logger.warning(f"Missing coroutine in {subclass}") + self.logger.error(f"{subclass}.{item} is not a coroutine, but it should be") return False for item in dir(subclass): item_attr = getattr(subclass, item) if inspect.iscoroutinefunction(item_attr) and hasattr(self.search_type, item): if not inspect.iscoroutinefunction(getattr(self.search_type, item)): - self.logger.warning(f"Unexpected coroutine in {subclass}") + self.logger.error(f"{subclass}.{item} is a coroutine, when it should not be") return False return True + def verify_arguments(self, subclass) -> bool: + arguments_match = True + for item in dir(self.search_type): + item_attr = getattr(self.search_type, item) + if inspect.isfunction(item_attr): + if not hasattr(subclass, item): + self.logger.error(f"Missing attr {subclass}.{item}()") + arguments_match = False + if not inspect.isfunction(getattr(subclass, item)): + self.logger.error(f"Missing function {subclass}.{item}()") + expected_signature = inspect.signature(item_attr) + actual_signature = inspect.signature(getattr(subclass, item)) + if expected_signature != actual_signature: + self.logger.error(f"Invalid signature in {subclass}.{item}().\n" + f"Expected {expected_signature}\n" + f"Actual {actual_signature}\n") + arguments_match = False + return arguments_match + def is_type_match(self, potential_class_match: Any) -> bool: if inspect.isclass(potential_class_match): self.logger.debug(f"Checking {self.search_type} match against {potential_class_match}") if issubclass(potential_class_match, self.search_type): self.logger.debug(f"Found type match search: {self.search_type} match: {potential_class_match}") - if self.verify_coroutines(potential_class_match): + if self.verify_coroutines(potential_class_match) and self.verify_arguments(potential_class_match): return True return False @@ -89,7 +109,9 @@ def load_module(self, to_load): self.impl_names.append(found_class) self.impls[found_class] = found_impl elif saw_more_than_one_impl: - self.logger.warning(f"more than one impl in {module_item}") + self.logger.error(f"more than one impl in {module_item}") + if not saw_one_impl: + self.logger.error(f"No impl found in {module_item}") def fetch_impls(self): self.logger.debug(f"Searching for implementations in {self.root_dir}") @@ -101,4 +123,4 @@ def fetch_impls(self): module = importlib.import_module("." + item, self.root_package) self.load_module(module) except ModuleNotFoundError: - self.logger.warning(f"No module matching package name for {item}\n{traceback.format_exc()}") + self.logger.error(f"No module matching package name for {item}\n{traceback.format_exc()}") diff --git a/src/tools/interop/idt/utils/log.py b/src/tools/interop/idt/utils/log.py index 73dc4e0d876f65..226f0629c7c2a0 100644 --- a/src/tools/interop/idt/utils/log.py +++ b/src/tools/interop/idt/utils/log.py @@ -22,15 +22,15 @@ import config -_CONFIG_LEVEL = config.log_level +_CONFIG_LEVEL = config.LOG_LEVEL -_FORMAT_PRE_FSTRING = "%(asctime)s %(levelname)s {%(module)s} [%(funcName)s] " -_FORMAT_PRE = colored(_FORMAT_PRE_FSTRING, "blue") if config.enable_color else _FORMAT_PRE_FSTRING +_FORMAT_PRE_FSTRING = "%(asctime)s %(levelname)s {%(module)s} [%(funcName)s] " if config.DEBUG else "%(asctime)s " +_FORMAT_PRE = colored(_FORMAT_PRE_FSTRING, "blue") if config.ENABLE_COLOR else _FORMAT_PRE_FSTRING _FORMAT_POST = "%(message)s" _FORMAT_NO_COLOR = _FORMAT_PRE_FSTRING+_FORMAT_POST FORMATS = { - logging.DEBUG: _FORMAT_PRE + colored(_FORMAT_POST, "blue"), + logging.DEBUG: _FORMAT_PRE + colored(_FORMAT_POST, "cyan"), logging.INFO: _FORMAT_PRE + colored(_FORMAT_POST, "green"), logging.WARNING: _FORMAT_PRE + colored(_FORMAT_POST, "yellow"), logging.ERROR: _FORMAT_PRE + colored(_FORMAT_POST, "red", attrs=["bold"]), @@ -41,7 +41,7 @@ class LoggingFormatter(logging.Formatter): def format(self, record): - log_fmt = FORMATS.get(record.levelno) if config.enable_color else _FORMAT_NO_COLOR + log_fmt = FORMATS.get(record.levelno) if config.ENABLE_COLOR else _FORMAT_NO_COLOR formatter = logging.Formatter(log_fmt) return formatter.format(record) @@ -62,13 +62,15 @@ def border_print(to_print: str, important: bool = False) -> None: border = f"\n{'_' * len_borders}\n" i_border = f"\n{'!' * len_borders}\n" if important else "" to_print = f"{border}{i_border}{to_print}{i_border}{border}" - if config.enable_color: + if config.ENABLE_COLOR: to_print = colored(to_print, "magenta") print(to_print) -def print_and_write(to_print: str, file: TextIO) -> None: - if config.enable_color: +def print_and_write(to_print: str, file: TextIO, important: bool = False) -> None: + if config.ENABLE_COLOR and important: + print(colored(to_print, "red", attrs=["bold"])) + elif config.ENABLE_COLOR: print(colored(to_print, "green")) else: print(to_print) diff --git a/src/tools/interop/idt/utils/net.py b/src/tools/interop/idt/utils/net.py new file mode 100644 index 00000000000000..515c996e481146 --- /dev/null +++ b/src/tools/interop/idt/utils/net.py @@ -0,0 +1,72 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import ipaddress +from typing import Callable +from .log import get_logger + +_LOGGER = get_logger(__file__) +WIFI_CHANNELS_2G = [str(j) for j in [i for i in range(1, 15)]] + + +def safe_ip_type_check(ip_type_filter: Callable[[str], bool]) -> Callable[[str], bool]: + def wrapper(ip_to_check: str) -> bool: + try: + return ip_type_filter(ip_to_check) + except ipaddress.AddressValueError: + return False + return wrapper + + +@safe_ip_type_check +def is_ipv4(ip: str) -> bool: + return ipaddress.IPv4Address(ip) is not None + + +@safe_ip_type_check +def is_ipv6_global_unicast(ip: str) -> bool: + return ipaddress.IPv6Address(ip).is_global + + +@safe_ip_type_check +def is_ipv6_link_local(ip: str) -> bool: + return ipaddress.IPv6Address(ip).is_link_local + + +@safe_ip_type_check +def is_ipv6_unique_local(ip: str) -> bool: + return ipaddress.IPv6Address(ip).is_private and not is_ipv6_link_local(ip) + + +@safe_ip_type_check +def is_ipv6(ip: str) -> bool: + return ipaddress.IPv6Address(ip) is not None + + +def get_addr_type(ip: str) -> str: + if is_ipv4(ip): + return "V4" + elif is_ipv6_global_unicast(ip): + return "V6 Global unicast" + elif is_ipv6_link_local(ip): + return "V6 Link local" + elif is_ipv6_unique_local(ip): + return "V6 Unique local" + elif is_ipv6(ip): + return "V6" + _LOGGER.debug(f"Failed to parse IP {ip}") + return "ERROR: UNKNOWN ADDR TYPE" diff --git a/src/tools/interop/idt/utils/shell.py b/src/tools/interop/idt/utils/shell.py index e2b0d27a58d800..bb0aac7e44c318 100644 --- a/src/tools/interop/idt/utils/shell.py +++ b/src/tools/interop/idt/utils/shell.py @@ -16,10 +16,12 @@ # import multiprocessing +import os import shlex import subprocess import psutil +from utils.error import log_error from . import log @@ -47,6 +49,7 @@ def __init__(self, command: str, sync: bool = False, self.args: list[str] = [] self._init_args() self.proc: None | subprocess.CompletedProcess | subprocess.Popen = None + self.logger.debug(f"New Bash sync={sync} instantiated: {command}") def _init_args(self) -> None: command_escaped = self.command.replace('"', '\"') @@ -59,8 +62,22 @@ def get_captured_output(self) -> str: return "" if not self.capture_output or not self.sync \ else self.proc.stdout.decode().strip() + def get_available_output(self) -> str: + stdout = "" + if self.proc.stdout: + stdout = self.proc.stdout.decode().strip() + stderr = "" + if self.proc.stderr: + stderr = self.proc.stderr.decode().strip() + return stdout, stderr + + # TODO: Add facility for awaitable background process def start_command(self) -> None: + # TODO: Make sudo auth automatic here if needed if self.proc is None: + # TODO: This does guard against a crash for a bad dir, but need to ensure no side effects before adding + # if self.cwd and not os.path.exists(self.cwd): + # self.logger.error(f"Bash instantiated in a directory that doesn't exist: {self.cwd}") if self.sync: self.proc = subprocess.run(self.args, capture_output=self.capture_output, cwd=self.cwd) else: @@ -88,6 +105,27 @@ def kill(self, proc: multiprocessing.Process) -> None: else: proc.kill() + @staticmethod + def get_current_command_for_pid(pid: int) -> str: + get_command_for_pid_command = Bash(f"ps -p {pid} -o comm=", sync=True, capture_output=True) + get_command_for_pid_command.start_command() + return get_command_for_pid_command.get_captured_output() + + @staticmethod + def get_current_command_for_pid_full(pid: int) -> str: + get_command_for_pid_command = Bash(f"ps {pid}", sync=True, capture_output=True) + get_command_for_pid_command.start_command() + return get_command_for_pid_command.get_captured_output() + + def verify_pid_not_recycled(self) -> (bool, str): + currently_running_command = self.get_current_command_for_pid(self.proc.pid) + # subshell if redirect is present; otherwise just compare the first arg of self.command + command_recycled = \ + currently_running_command == "/bin/bash" or currently_running_command == "bash" if ">" in self.command else \ + currently_running_command == self.command.split(" ")[0] + self.logger.debug(f"Found {currently_running_command} Expected {self.command}") + return command_recycled, currently_running_command + def stop_single_proc(self, proc: multiprocessing.Process) -> None: self.logger.debug(f"Killing process {proc.pid}") try: @@ -105,6 +143,13 @@ def stop_single_proc(self, proc: multiprocessing.Process) -> None: def stop_command(self) -> None: if self.command_is_running(): + pid_not_recycled, currently_running_command = self.verify_pid_not_recycled() + if not pid_not_recycled: + help_message = f"pid {self.proc.pid} appears to be recycled, expected {self.command} but found " \ + f"{currently_running_command}!" + self.logger.critical(help_message) + log_error("utils.shell", help_message, stack_trace=False) + return psutil_proc = psutil.Process(self.proc.pid) suffix = f"{psutil_proc.pid} for command {self.command}" self.logger.debug(f"Stopping children of {suffix}")