From a49a59cd8b94cee1e2c7cf93b1b04b94a4ed3e25 Mon Sep 17 00:00:00 2001 From: JarbasAl Date: Tue, 17 Oct 2023 22:10:50 +0000 Subject: [PATCH 01/29] Increment Version --- CHANGELOG.md | 4 ++++ hivemind_core/version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fbec06..f5ec400 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## [V0.13.1a5](https://github.com/JarbasHiveMind/HiveMind-core/tree/V0.13.1a5) (2023-10-17) + +[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.1a4...V0.13.1a5) + ## [V0.13.1a4](https://github.com/JarbasHiveMind/HiveMind-core/tree/V0.13.1a4) (2023-10-13) [Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.1a3...V0.13.1a4) diff --git a/hivemind_core/version.py b/hivemind_core/version.py index ff6e0ac..154637d 100644 --- a/hivemind_core/version.py +++ b/hivemind_core/version.py @@ -3,5 +3,5 @@ VERSION_MAJOR = 0 VERSION_MINOR = 13 VERSION_BUILD = 1 -VERSION_ALPHA = 5 +VERSION_ALPHA = 6 # END_VERSION_BLOCK From 7388a311699b0c5045b4c90615c59f350b222313 Mon Sep 17 00:00:00 2001 From: JarbasAi Date: Thu, 26 Oct 2023 05:55:24 +0100 Subject: [PATCH 02/29] permissions --- hivemind_core/database.py | 11 +++++++-- hivemind_core/protocol.py | 49 ++++++++++++++++++++++++++++++++------- hivemind_core/service.py | 3 +++ 3 files changed, 52 insertions(+), 11 deletions(-) diff --git a/hivemind_core/database.py b/hivemind_core/database.py index 571fb7f..368f549 100644 --- a/hivemind_core/database.py +++ b/hivemind_core/database.py @@ -10,7 +10,8 @@ def cast_to_client_obj(): valid_kwargs: Iterable[str] = ("client_id", "api_key", "name", "description", "is_admin", "last_seen", "blacklist", "allowed_types", "crypto_key", - "password") + "password", "can_broadcast", "can_escalate", + "can_propagate") def _handler(func): @@ -49,7 +50,10 @@ def __init__(self, blacklist: Optional[Dict[str, List[str]]] = None, allowed_types: Optional[List[str]] = None, crypto_key: Optional[str] = None, - password: Optional[str] = None): + password: Optional[str] = None, + can_broadcast: bool = True, + can_escalate: bool = True, + can_propagate: bool = True): self.client_id = client_id self.description = description @@ -67,6 +71,9 @@ def __init__(self, self.allowed_types = allowed_types or ["recognizer_loop:utterance"] if "recognizer_loop:utterance" not in self.allowed_types: self.allowed_types.append("recognizer_loop:utterance") + self.can_broadcast = can_broadcast + self.can_escalate = can_escalate + self.can_propagate = can_propagate def __getitem__(self, item: str) -> Any: return self.__dict__.get(item) diff --git a/hivemind_core/protocol.py b/hivemind_core/protocol.py index 6422e48..57cf131 100644 --- a/hivemind_core/protocol.py +++ b/hivemind_core/protocol.py @@ -58,6 +58,9 @@ class HiveMindClientConnection: allowed_types: List[str] = field(default_factory=list) # list of ovos message_type to allow to be sent from this client binarize: bool = False site_id: str = "unknown" + can_broadcast: bool = True + can_escalate: bool = True + can_propagate: bool = True @property def peer(self) -> str: @@ -227,6 +230,7 @@ class HiveMindListenerProtocol: escalate_callback = None # slave asked to escalate payload illegal_callback = None # slave asked to broadcast payload (illegal action) propagate_callback = None # slave asked to propagate payload + broadcast_callback = None # slave asked to broadcast payload mycroft_bus_callback = None # slave asked to inject payload into mycroft bus shared_bus_callback = None # passive sharing of slave device bus (info) @@ -348,6 +352,7 @@ def handle_unknown_message(self, message: HiveMessage, client: HiveMindClientCon def handle_binary_message(self, message: HiveMessage, client: HiveMindClientConnection): assert message.msg_type == HiveMessageType.BINARY + # TODO def handle_handshake_message(self, message: HiveMessage, client: HiveMindClientConnection): @@ -413,12 +418,24 @@ def handle_broadcast_message(self, message: HiveMessage, client: HiveMindClientC """ message (HiveMessage): HiveMind message object """ - # Slaves are not allowed to broadcast, by definition broadcast goes - # downstream only, use propagate instead - LOG.warning("Received broadcast message from downstream, illegal action") - if self.illegal_callback: - self.illegal_callback(message.payload) - # TODO kick client for misbehaviour so it stops doing that? + payload = self._unpack_message(message, client) + + if not client.can_broadcast: + LOG.warning("Received broadcast message from downstream, illegal action") + if self.illegal_callback: + self.illegal_callback(payload) + # TODO kick client for misbehaviour so it stops doing that? + return + + if self.broadcast_callback: + self.broadcast_callback(payload) + + # broadcast message to other peers + payload = self._unpack_message(message, client) + for peer in self.clients: + if peer == client.peer: + continue + self.clients[peer].send(payload) def _unpack_message(self, message: HiveMessage, client: HiveMindClientConnection): # propagate message to other peers @@ -440,13 +457,20 @@ def handle_propagate_message(self, message: HiveMessage, payload = self._unpack_message(message, client) + if not client.can_propagate: + LOG.warning("Received propagate message from downstream, illegal action") + if self.illegal_callback: + self.illegal_callback(payload) + # TODO kick client for misbehaviour so it stops doing that? + return + if self.propagate_callback: self.propagate_callback(payload) # propagate message to other peers - for peer in payload.target_peers: - if peer in self.clients: - self.clients[peer].send(payload) + for peer in self.clients: + if peer == client.peer: + continue # send to other masters message = Message("hive.send.upstream", payload, @@ -469,6 +493,13 @@ def handle_escalate_message(self, message: HiveMessage, # unpack message payload = self._unpack_message(message, client) + if not client.can_escalate: + LOG.warning("Received escalate message from downstream, illegal action") + if self.illegal_callback: + self.illegal_callback(payload) + # TODO kick client for misbehaviour so it stops doing that? + return + if self.escalate_callback: self.escalate_callback(payload) diff --git a/hivemind_core/service.py b/hivemind_core/service.py index cb54f5b..a873889 100644 --- a/hivemind_core/service.py +++ b/hivemind_core/service.py @@ -130,6 +130,9 @@ def open(self): self.client.crypto_key = user.crypto_key self.client.blacklist = user.blacklist.get("messages", []) self.client.allowed_types = user.allowed_types + self.client.can_broadcast = user.can_broadcast + self.client.can_propagate = user.can_propagate + self.client.can_escalate = user.can_escalate if user.password: # pre-shared password to derive aes_key self.client.pswd_handshake = PasswordHandShake(user.password) From b07baa44a0a03057118601f70e74bace73efcc09 Mon Sep 17 00:00:00 2001 From: JarbasAi Date: Thu, 26 Oct 2023 05:56:03 +0100 Subject: [PATCH 03/29] permissions --- hivemind_core/protocol.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hivemind_core/protocol.py b/hivemind_core/protocol.py index 57cf131..db7a7e3 100644 --- a/hivemind_core/protocol.py +++ b/hivemind_core/protocol.py @@ -471,6 +471,7 @@ def handle_propagate_message(self, message: HiveMessage, for peer in self.clients: if peer == client.peer: continue + self.clients[peer].send(payload) # send to other masters message = Message("hive.send.upstream", payload, From 8021450a66b7130e9993047e0704ff3b58925cb4 Mon Sep 17 00:00:00 2001 From: JarbasAi Date: Tue, 21 Nov 2023 22:18:38 +0000 Subject: [PATCH 04/29] add session_id to HELLO message allows master to inform the client what session_id it was assigned internally while this info can be ignored by clients and they should still work, it indicates the session_id that gets assigned inside ovos-core --- hivemind_core/protocol.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hivemind_core/protocol.py b/hivemind_core/protocol.py index db7a7e3..36ce398 100644 --- a/hivemind_core/protocol.py +++ b/hivemind_core/protocol.py @@ -261,7 +261,8 @@ def handle_new_client(self, client: HiveMindClientConnection): payload={"pubkey": client.handshake.pubkey, # allows any node to verify messages are signed with this "peer": client.peer, # this identifies the connected client in ovos message.context - "node_id": self.peer}) + "node_id": self.peer, + "session_id": client.sess.session_id}) LOG.debug(f"saying HELLO to: {client.peer}") client.send(msg) From a8ba0c004a37934d16d1a4ba64d4ab576f048b1a Mon Sep 17 00:00:00 2001 From: JarbasAl Date: Tue, 21 Nov 2023 22:33:40 +0000 Subject: [PATCH 05/29] Increment Version --- CHANGELOG.md | 4 ++++ hivemind_core/version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5ec400..da18c97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## [V0.13.1a6](https://github.com/JarbasHiveMind/HiveMind-core/tree/V0.13.1a6) (2023-10-17) + +[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.1a5...V0.13.1a6) + ## [V0.13.1a5](https://github.com/JarbasHiveMind/HiveMind-core/tree/V0.13.1a5) (2023-10-17) [Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.1a4...V0.13.1a5) diff --git a/hivemind_core/version.py b/hivemind_core/version.py index 154637d..0f6d359 100644 --- a/hivemind_core/version.py +++ b/hivemind_core/version.py @@ -3,5 +3,5 @@ VERSION_MAJOR = 0 VERSION_MINOR = 13 VERSION_BUILD = 1 -VERSION_ALPHA = 6 +VERSION_ALPHA = 7 # END_VERSION_BLOCK From a1221b03a956c78bf1dbcc1bb9e8efe9ebb4ca5e Mon Sep 17 00:00:00 2001 From: JarbasAi Date: Fri, 24 Nov 2023 14:21:55 +0000 Subject: [PATCH 06/29] update logging --- hivemind_core/service.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/hivemind_core/service.py b/hivemind_core/service.py index a873889..9c2b77e 100644 --- a/hivemind_core/service.py +++ b/hivemind_core/service.py @@ -6,14 +6,14 @@ from os import makedirs from os.path import exists, join from socket import gethostname -from threading import Thread from typing import Callable, Dict, Any, Optional, Tuple from OpenSSL import crypto +from ovos_bus_client import MessageBusClient +from ovos_bus_client.session import Session from ovos_config import Configuration from ovos_utils.log import LOG from ovos_utils.process_utils import ProcessStatus, StatusCallbackMap -from ovos_bus_client.session import Session from ovos_utils.xdg_utils import xdg_data_home from poorman_handshake import HandShake, PasswordHandShake from pyee import EventEmitter @@ -22,11 +22,11 @@ from tornado.websocket import WebSocketHandler from hivemind_bus_client.identity import NodeIdentity +from hivemind_bus_client.message import HiveMessageType from hivemind_core.database import ClientDatabase from hivemind_core.protocol import HiveMindListenerProtocol, HiveMindClientConnection, HiveMindNodeType -from hivemind_presence import LocalPresence from hivemind_ggwave import GGWaveMaster -from ovos_bus_client import MessageBusClient +from hivemind_presence import LocalPresence def create_self_signed_cert(cert_dir=f"{xdg_data_home()}/hivemind", @@ -105,7 +105,10 @@ def decode_auth(auth) -> Tuple[str, str]: def on_message(self, message): message = self.client.decode(message) - LOG.info(f"received {self.client.peer} message: {message}") + if message.msg_type == HiveMessageType.BUS and message.payload.msg_type == "recognizer_loop:b64_audio": + LOG.info(f"received {self.client.peer} sent base64 audio for STT") + else: + LOG.info(f"received {self.client.peer} message: {message}") self.protocol.handle_message(message, self.client) def open(self): @@ -163,7 +166,7 @@ def check_origin(self, origin) -> bool: class HiveMindService: identity = NodeIdentity() - def __init__(self, + def __init__(self, alive_hook: Callable = on_alive, started_hook: Callable = on_started, ready_hook: Callable = on_ready, @@ -175,7 +178,7 @@ def __init__(self, ws_handler=MessageBusEventHandler): websocket_config = websocket_config or \ - Configuration().get('hivemind_websocket', {}) + Configuration().get('hivemind_websocket', {}) callbacks = StatusCallbackMap(on_started=started_hook, on_alive=alive_hook, on_ready=ready_hook, From 5b174a88808b8cb47abdbba3384a1b8478badf6b Mon Sep 17 00:00:00 2001 From: JarbasAl Date: Fri, 24 Nov 2023 14:23:31 +0000 Subject: [PATCH 07/29] Increment Version --- CHANGELOG.md | 14 +++++++++----- hivemind_core/version.py | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da18c97..dfbdddc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## [V0.13.1a7](https://github.com/JarbasHiveMind/HiveMind-core/tree/V0.13.1a7) (2023-11-21) + +[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.1a6...V0.13.1a7) + ## [V0.13.1a6](https://github.com/JarbasHiveMind/HiveMind-core/tree/V0.13.1a6) (2023-10-17) [Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.1a5...V0.13.1a6) @@ -62,15 +66,15 @@ ## [V0.13.0a4](https://github.com/JarbasHiveMind/HiveMind-core/tree/V0.13.0a4) (2023-08-03) -[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.0a2...V0.13.0a4) +[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.0a3...V0.13.0a4) -## [V0.13.0a2](https://github.com/JarbasHiveMind/HiveMind-core/tree/V0.13.0a2) (2023-08-03) +## [V0.13.0a3](https://github.com/JarbasHiveMind/HiveMind-core/tree/V0.13.0a3) (2023-08-03) -[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.0a3...V0.13.0a2) +[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.0a2...V0.13.0a3) -## [V0.13.0a3](https://github.com/JarbasHiveMind/HiveMind-core/tree/V0.13.0a3) (2023-08-03) +## [V0.13.0a2](https://github.com/JarbasHiveMind/HiveMind-core/tree/V0.13.0a2) (2023-08-03) -[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.11.0a3...V0.13.0a3) +[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.11.0a3...V0.13.0a2) **Breaking changes:** diff --git a/hivemind_core/version.py b/hivemind_core/version.py index 0f6d359..11cfeca 100644 --- a/hivemind_core/version.py +++ b/hivemind_core/version.py @@ -3,5 +3,5 @@ VERSION_MAJOR = 0 VERSION_MINOR = 13 VERSION_BUILD = 1 -VERSION_ALPHA = 7 +VERSION_ALPHA = 8 # END_VERSION_BLOCK From 7558c69d79152b62175cedac7d9e91809e3ea424 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Trellu?= Date: Tue, 2 Jan 2024 14:25:40 -0500 Subject: [PATCH 08/29] Add support to remote bus address and port (#79) * Fixing some linting * Fixing some linting part 2 * Add support for remote bus address and port * Changes requested from review --- .gitignore | 2 +- CHANGELOG.md | 12 +-- examples/fakecroft.py | 39 +++++--- hivemind_core/database.py | 106 +++++++++++--------- hivemind_core/protocol.py | 196 ++++++++++++++++++++++++------------- hivemind_core/scripts.py | 84 ++++++++++++---- hivemind_core/service.py | 148 +++++++++++++++++----------- readme.md | 20 ++-- scripts/bump_alpha.py | 2 +- scripts/bump_build.py | 2 +- scripts/bump_major.py | 2 +- scripts/bump_minor.py | 2 +- scripts/remove_alpha.py | 2 +- setup.py | 60 ++++++------ test/unittests/test_bus.py | 121 ++++++++++++++--------- test/unittests/test_db.py | 7 +- 16 files changed, 497 insertions(+), 308 deletions(-) diff --git a/.gitignore b/.gitignore index 6769e21..68bc17f 100644 --- a/.gitignore +++ b/.gitignore @@ -157,4 +157,4 @@ cython_debug/ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ \ No newline at end of file +#.idea/ diff --git a/CHANGELOG.md b/CHANGELOG.md index dfbdddc..6c5556b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,7 +38,7 @@ **Implemented enhancements:** -- add site\_id [\#74](https://github.com/JarbasHiveMind/HiveMind-core/pull/74) ([JarbasAl](https://github.com/JarbasAl)) +- add site_id [\#74](https://github.com/JarbasHiveMind/HiveMind-core/pull/74) ([JarbasAl](https://github.com/JarbasAl)) ## [V0.13.0a0](https://github.com/JarbasHiveMind/HiveMind-core/tree/V0.13.0a0) (2023-09-08) @@ -100,7 +100,7 @@ **Merged pull requests:** -- filter allowed\_types [\#71](https://github.com/JarbasHiveMind/HiveMind-core/pull/71) ([emphasize](https://github.com/emphasize)) +- filter allowed_types [\#71](https://github.com/JarbasHiveMind/HiveMind-core/pull/71) ([emphasize](https://github.com/emphasize)) - add typehints [\#70](https://github.com/JarbasHiveMind/HiveMind-core/pull/70) ([emphasize](https://github.com/emphasize)) - \[requirements\] Add missing pyOpenSSL [\#69](https://github.com/JarbasHiveMind/HiveMind-core/pull/69) ([goldyfruit](https://github.com/goldyfruit)) @@ -130,7 +130,7 @@ - Xdg [\#51](https://github.com/JarbasHiveMind/HiveMind-core/pull/51) ([JarbasAl](https://github.com/JarbasAl)) - V2 [\#50](https://github.com/JarbasHiveMind/HiveMind-core/pull/50) ([JarbasAl](https://github.com/JarbasAl)) - Refactor/hivemind presence [\#49](https://github.com/JarbasHiveMind/HiveMind-core/pull/49) ([JarbasAl](https://github.com/JarbasAl)) -- refactor/deprecate\_sql [\#48](https://github.com/JarbasHiveMind/HiveMind-core/pull/48) ([JarbasAl](https://github.com/JarbasAl)) +- refactor/deprecate_sql [\#48](https://github.com/JarbasHiveMind/HiveMind-core/pull/48) ([JarbasAl](https://github.com/JarbasAl)) - launcher scripts / deprecate mail param / increase RSA key size [\#46](https://github.com/JarbasHiveMind/HiveMind-core/pull/46) ([Joanguitar](https://github.com/Joanguitar)) - move to asyncio [\#41](https://github.com/JarbasHiveMind/HiveMind-core/pull/41) ([JarbasAl](https://github.com/JarbasAl)) @@ -162,7 +162,7 @@ - migrate to pycryptodomex [\#30](https://github.com/JarbasHiveMind/HiveMind-core/pull/30) ([JarbasAl](https://github.com/JarbasAl)) - Add instructions to README [\#27](https://github.com/JarbasHiveMind/HiveMind-core/pull/27) ([ChanceNCounter](https://github.com/ChanceNCounter)) -- migrate from jarbas\_utils to ovos\_utils [\#24](https://github.com/JarbasHiveMind/HiveMind-core/pull/24) ([JarbasAl](https://github.com/JarbasAl)) +- migrate from jarbas_utils to ovos_utils [\#24](https://github.com/JarbasHiveMind/HiveMind-core/pull/24) ([JarbasAl](https://github.com/JarbasAl)) - revert cryptodomex requirement change [\#21](https://github.com/JarbasHiveMind/HiveMind-core/pull/21) ([JarbasAl](https://github.com/JarbasAl)) - Switch from pycryptodome to pycryptodomex [\#12](https://github.com/JarbasHiveMind/HiveMind-core/pull/12) ([j1nx](https://github.com/j1nx)) - escalate [\#8](https://github.com/JarbasHiveMind/HiveMind-core/pull/8) ([JarbasAl](https://github.com/JarbasAl)) @@ -171,6 +171,4 @@ - Feat/emulation [\#5](https://github.com/JarbasHiveMind/HiveMind-core/pull/5) ([JarbasAl](https://github.com/JarbasAl)) - refactor + http support [\#4](https://github.com/JarbasHiveMind/HiveMind-core/pull/4) ([JarbasAl](https://github.com/JarbasAl)) - - -\* *This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)* +\* _This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)_ diff --git a/examples/fakecroft.py b/examples/fakecroft.py index 57d9a53..73e1b91 100644 --- a/examples/fakecroft.py +++ b/examples/fakecroft.py @@ -3,12 +3,16 @@ from ovos_utils.log import LOG from ovos_utils.messagebus import FakeBus from hivemind_listener.service import MessageBusEventHandler -from hivemind_listener.protocol import HiveMindListenerProtocol, \ - HiveMindListenerInternalProtocol, HiveMindClientConnection +from hivemind_listener.protocol import ( + HiveMindListenerProtocol, + HiveMindListenerInternalProtocol, + HiveMindClientConnection, +) class HiveMindFakeCroftProtocol(HiveMindListenerProtocol): - """ Fake ovos-core instance, not actually connected to a messagebus""" + """Fake ovos-core instance, not actually connected to a messagebus""" + peer: str = "fakecroft:0.0.0.0" def bind(self, websocket, bus=None): @@ -17,28 +21,31 @@ def bind(self, websocket, bus=None): bus = FakeBus() self.internal_protocol = HiveMindListenerInternalProtocol(bus) - def handle_incoming_mycroft(self, message: Message, client: HiveMindClientConnection): + def handle_incoming_mycroft( + self, message: Message, client: HiveMindClientConnection + ): """ message (Message): mycroft bus message object """ super().handle_inject_mycroft_msg(message, client) answer = "mycroft is dead! long live mycroft!" - payload = HiveMessage(HiveMessageType.BUS, - message.reply("speak", {"utterance": answer})) + payload = HiveMessage( + HiveMessageType.BUS, message.reply("speak", {"utterance": answer}) + ) client.send(payload) def on_ready(): - LOG.info('FakeCroft started!') + LOG.info("FakeCroft started!") -def on_error(e='Unknown'): - LOG.info('FakeCroft failed to start ({})'.format(repr(e))) +def on_error(e="Unknown"): + LOG.info("FakeCroft failed to start ({})".format(repr(e))) def on_stopping(): - LOG.info('FakeCroft is shutting down...') + LOG.info("FakeCroft is shutting down...") def main(ready_hook=on_ready, error_hook=on_error, stopping_hook=on_stopping): @@ -46,17 +53,17 @@ def main(ready_hook=on_ready, error_hook=on_error, stopping_hook=on_stopping): from tornado import web, ioloop from ovos_config import Configuration - LOG.info('Starting FakeCroft...') + LOG.info("Starting FakeCroft...") try: - websocket_configs = Configuration()['websocket'] + websocket_configs = Configuration()["websocket"] except KeyError as ke: - LOG.error('No websocket configs found ({})'.format(repr(ke))) + LOG.error("No websocket configs found ({})".format(repr(ke))) raise - host = websocket_configs.get('host') - port = websocket_configs.get('port') - route = websocket_configs.get('route') + host = websocket_configs.get("host") + port = websocket_configs.get("port") + route = websocket_configs.get("route") port = 5678 route = "/" diff --git a/hivemind_core/database.py b/hivemind_core/database.py index 368f549..17b1d90 100644 --- a/hivemind_core/database.py +++ b/hivemind_core/database.py @@ -7,27 +7,36 @@ def cast_to_client_obj(): - valid_kwargs: Iterable[str] = ("client_id", "api_key", "name", - "description", "is_admin", "last_seen", - "blacklist", "allowed_types", "crypto_key", - "password", "can_broadcast", "can_escalate", - "can_propagate") + valid_kwargs: Iterable[str] = ( + "client_id", + "api_key", + "name", + "description", + "is_admin", + "last_seen", + "blacklist", + "allowed_types", + "crypto_key", + "password", + "can_broadcast", + "can_escalate", + "can_propagate", + ) def _handler(func): - def _cast(ret): if ret is None or isinstance(ret, Client): return ret if isinstance(ret, list): return [_cast(r) for r in ret] if isinstance(ret, dict): - if not all((k in valid_kwargs - for k in ret.keys())): + if not all((k in valid_kwargs for k in ret.keys())): raise RuntimeError(f"{func} returned a dict with unknown keys") return Client(**ret) raise TypeError( - "cast_to_client_obj decorator can only be used in functions that return None, dict, Client or a list of those types") + "cast_to_client_obj decorator can only be used in functions that return None, dict, Client or a list of those types" + ) @wraps(func) def call_function(*args, **kwargs): @@ -40,21 +49,22 @@ def call_function(*args, **kwargs): class Client: - def __init__(self, - client_id: int, - api_key: str, - name: str = "", - description: str = "", - is_admin: bool = False, - last_seen: float = -1, - blacklist: Optional[Dict[str, List[str]]] = None, - allowed_types: Optional[List[str]] = None, - crypto_key: Optional[str] = None, - password: Optional[str] = None, - can_broadcast: bool = True, - can_escalate: bool = True, - can_propagate: bool = True): - + def __init__( + self, + client_id: int, + api_key: str, + name: str = "", + description: str = "", + is_admin: bool = False, + last_seen: float = -1, + blacklist: Optional[Dict[str, List[str]]] = None, + allowed_types: Optional[List[str]] = None, + crypto_key: Optional[str] = None, + password: Optional[str] = None, + can_broadcast: bool = True, + can_escalate: bool = True, + can_propagate: bool = True, + ): self.client_id = client_id self.description = description self.api_key = api_key @@ -63,11 +73,7 @@ def __init__(self, self.is_admin = is_admin self.crypto_key = crypto_key self.password = password - self.blacklist = blacklist or { - "messages": [], - "skills": [], - "intents": [] - } + self.blacklist = blacklist or {"messages": [], "skills": [], "intents": []} self.allowed_types = allowed_types or ["recognizer_loop:utterance"] if "recognizer_loop:utterance" not in self.allowed_types: self.allowed_types.append("recognizer_loop:utterance") @@ -155,9 +161,7 @@ def change_name(self, new_name: str, key: str) -> bool: self.update_item(item_id, user) return True - def change_blacklist(self, - blacklist: Union[str, Dict[str, Any]], - key: str) -> bool: + def change_blacklist(self, blacklist: Union[str, Dict[str, Any]], key: str) -> bool: if isinstance(blacklist, dict): blacklist = json.dumps(blacklist) user = self.get_client_by_api_key(key) @@ -186,15 +190,16 @@ def get_clients_by_name(self, name: str) -> List[Client]: return self.search_by_value("name", name) @cast_to_client_obj() - def add_client(self, - name: str, - key: str = "", - admin: bool = False, - blacklist: Optional[Dict[str, Any]] = None, - allowed_types: Optional[List[str]] = None, - crypto_key: Optional[str] = None, - password: Optional[str] = None) -> Client: - + def add_client( + self, + name: str, + key: str = "", + admin: bool = False, + blacklist: Optional[Dict[str, Any]] = None, + allowed_types: Optional[List[str]] = None, + crypto_key: Optional[str] = None, + password: Optional[str] = None, + ) -> Client: user = self.get_client_by_api_key(key) item_id = self.get_item_id(user) if crypto_key is not None: @@ -214,11 +219,16 @@ def add_client(self, user["password"] = password self.update_item(item_id, user) else: - user = Client(api_key=key, name=name, - blacklist=blacklist, crypto_key=crypto_key, - client_id=self.total_clients() + 1, - is_admin=admin, password=password, - allowed_types=allowed_types) + user = Client( + api_key=key, + name=name, + blacklist=blacklist, + crypto_key=crypto_key, + client_id=self.total_clients() + 1, + is_admin=admin, + password=password, + allowed_types=allowed_types, + ) self.add_item(user) return user @@ -226,11 +236,11 @@ def total_clients(self) -> int: return len(self) def __enter__(self): - """ Context handler """ + """Context handler""" return self def __exit__(self, _type, value, traceback): - """ Commits changes and Closes the session """ + """Commits changes and Closes the session""" try: self.commit() except Exception as e: diff --git a/hivemind_core/protocol.py b/hivemind_core/protocol.py index 36ce398..aa56b92 100644 --- a/hivemind_core/protocol.py +++ b/hivemind_core/protocol.py @@ -13,7 +13,12 @@ from hivemind_bus_client.message import HiveMessage, HiveMessageType from hivemind_bus_client.serialization import decode_bitstring, get_bitstring -from hivemind_bus_client.util import decrypt_bin, encrypt_bin, decrypt_from_json, encrypt_as_json +from hivemind_bus_client.util import ( + decrypt_bin, + encrypt_bin, + decrypt_from_json, + encrypt_as_json, +) class ProtocolVersion(IntEnum): @@ -43,7 +48,8 @@ class HiveMindNodeType(str, Enum): @dataclass class HiveMindClientConnection: - """ represents a connection to the hivemind listener """ + """represents a connection to the hivemind listener""" + key: str ip: str loop: ioloop.IOLoop @@ -54,8 +60,12 @@ class HiveMindClientConnection: pswd_handshake: Optional[PasswordHandShake] = None socket: Optional[WebSocketHandler] = None crypto_key: Optional[str] = None - blacklist: List[str] = field(default_factory=list) # list of ovos message_type to never be sent to this client - allowed_types: List[str] = field(default_factory=list) # list of ovos message_type to allow to be sent from this client + blacklist: List[str] = field( + default_factory=list + ) # list of ovos message_type to never be sent to this client + allowed_types: List[str] = field( + default_factory=list + ) # list of ovos message_type to allow to be sent from this client binarize: bool = False site_id: str = "unknown" can_broadcast: bool = True @@ -76,22 +86,27 @@ def send(self, message: HiveMessage): _msg_type = message.payload.msg_type if _msg_type in self.blacklist: - return LOG.debug(f"message type {_msg_type} " - f"is blacklisted for {self.peer}") + return LOG.debug( + f"message type {_msg_type} " f"is blacklisted for {self.peer}" + ) LOG.debug(f"sending to {self.peer}: {message.msg_type}") if message.msg_type == HiveMessageType.BUS: LOG.debug(f"mycroft_type {_msg_type}") payload = message.serialize() # json string is_bin = False - if self.crypto_key and message.msg_type not in [HiveMessageType.HANDSHAKE, - HiveMessageType.HELLO]: + if self.crypto_key and message.msg_type not in [ + HiveMessageType.HANDSHAKE, + HiveMessageType.HELLO, + ]: if self.binarize: payload = get_bitstring(message.msg_type, message.payload).bytes payload = encrypt_bin(self.crypto_key, payload) is_bin = True else: - payload = encrypt_as_json(self.crypto_key, payload) # still a json string + payload = encrypt_as_json( + self.crypto_key, payload + ) # still a json string LOG.debug(f"encrypted payload: {len(payload)}") else: LOG.debug(f"sent unencrypted!") @@ -120,7 +135,7 @@ def decode(self, payload: str) -> HiveMessage: return HiveMessage(**payload) def authorize(self, message: Message) -> bool: - """ parse the message being injected into ovos-core bus + """parse the message being injected into ovos-core bus if this client is not authorized to inject it return False""" if message.msg_type not in self.allowed_types: return False @@ -132,7 +147,8 @@ def authorize(self, message: Message) -> bool: @dataclass() class HiveMindListenerInternalProtocol: - """ this class handles all interactions between a hivemind listener and a ovos-core messagebus""" + """this class handles all interactions between a hivemind listener and a ovos-core messagebus""" + bus: MessageBusClient def register_bus_handlers(self): @@ -146,7 +162,7 @@ def clients(self) -> Dict[str, HiveMindClientConnection]: # mycroft handlers - from master -> slave def handle_send(self, message: Message): - """ ovos wants to send a HiveMessage + """ovos wants to send a HiveMessage a device can be both a master and a slave, downstream messages are handled here @@ -157,9 +173,7 @@ def handle_send(self, message: Message): peer = message.data.get("peer") msg_type = message.data["msg_type"] - hmessage = HiveMessage(msg_type, - payload=payload, - target_peers=[peer]) + hmessage = HiveMessage(msg_type, payload=payload, target_peers=[peer]) if msg_type in [HiveMessageType.PROPAGATE, HiveMessageType.BROADCAST]: # this message is meant to be sent to all slave nodes @@ -180,12 +194,15 @@ def handle_send(self, message: Message): client.send(hmessage) else: LOG.error("That client is not connected") - self.bus.emit(message.forward( - "hive.client.send.error", - {"error": "That client is not connected", "peer": peer})) + self.bus.emit( + message.forward( + "hive.client.send.error", + {"error": "That client is not connected", "peer": peer}, + ) + ) def handle_internal_mycroft(self, message: str): - """ forward internal messages to clients if they are the target + """forward internal messages to clients if they are the target here is where the client isolation happens, clients only get responses to their own messages""" @@ -208,10 +225,12 @@ def handle_internal_mycroft(self, message: str): # forward internal messages to clients if they are the target LOG.debug(f"{message.msg_type} - destination: {peer}") message.context["source"] = "hive" - msg = HiveMessage(HiveMessageType.BUS, - source_peer=peer, - target_peers=target_peers, - payload=message) + msg = HiveMessage( + HiveMessageType.BUS, + source_peer=peer, + target_peers=target_peers, + payload=message, + ) client.send(msg) @@ -246,23 +265,32 @@ def get_bus(self, client: HiveMindClientConnection): def handle_new_client(self, client: HiveMindClientConnection): LOG.debug(f"new client: {client.peer}") self.clients[client.peer] = client - message = Message("hive.client.connect", - {"ip": client.ip, "session_id": client.sess.session_id}, - {"source": client.peer}) + message = Message( + "hive.client.connect", + {"ip": client.ip, "session_id": client.sess.session_id}, + {"source": client.peer}, + ) bus = self.get_bus(client) bus.emit(message) - min_version = ProtocolVersion.ONE if client.crypto_key is None and self.require_crypto \ + min_version = ( + ProtocolVersion.ONE + if client.crypto_key is None and self.require_crypto else ProtocolVersion.ZERO + ) max_version = ProtocolVersion.ONE - msg = HiveMessage(HiveMessageType.HELLO, - payload={"pubkey": client.handshake.pubkey, - # allows any node to verify messages are signed with this - "peer": client.peer, # this identifies the connected client in ovos message.context - "node_id": self.peer, - "session_id": client.sess.session_id}) + msg = HiveMessage( + HiveMessageType.HELLO, + payload={ + "pubkey": client.handshake.pubkey, + # allows any node to verify messages are signed with this + "peer": client.peer, # this identifies the connected client in ovos message.context + "node_id": self.peer, + "session_id": client.sess.session_id, + }, + ) LOG.debug(f"saying HELLO to: {client.peer}") client.send(msg) @@ -274,9 +302,11 @@ def handle_new_client(self, client: HiveMindClientConnection): "min_protocol_version": min_version, "max_protocol_version": max_version, "binarize": True, # report we support the binarization scheme - "preshared_key": client.crypto_key is not None, # do we have a pre-shared key (V0 proto) - "password": client.pswd_handshake is not None, # is password available (V1 proto, replaces pre-shared key) - "crypto_required": self.require_crypto # do we allow unencrypted payloads + "preshared_key": client.crypto_key + is not None, # do we have a pre-shared key (V0 proto) + "password": client.pswd_handshake + is not None, # is password available (V1 proto, replaces pre-shared key) + "crypto_required": self.require_crypto, # do we allow unencrypted payloads } msg = HiveMessage(HiveMessageType.HANDSHAKE, payload) LOG.debug(f"starting {client.peer} HANDSHAKE: {payload}") @@ -288,25 +318,31 @@ def handle_client_disconnected(self, client: HiveMindClientConnection): if client.peer in self.clients: self.clients.pop(client.peer) client.socket.close() - message = Message("hive.client.disconnect", - {"ip": client.ip}, - {"source": client.peer, "session": client.sess.serialize()}) + message = Message( + "hive.client.disconnect", + {"ip": client.ip}, + {"source": client.peer, "session": client.sess.serialize()}, + ) bus = self.get_bus(client) bus.emit(message) def handle_invalid_key_connected(self, client: HiveMindClientConnection): LOG.error("Client provided an invalid api key") - message = Message("hive.client.connection.error", - {"error": "invalid api key", "peer": client.peer}, - {"source": client.peer}) + message = Message( + "hive.client.connection.error", + {"error": "invalid api key", "peer": client.peer}, + {"source": client.peer}, + ) bus = self.get_bus(client) bus.emit(message) def handle_invalid_protocol_version(self, client: HiveMindClientConnection): LOG.error("Client does not satisfy protocol requirements") - message = Message("hive.client.connection.error", - {"error": "protocol error", "peer": client.peer}, - {"source": client.peer}) + message = Message( + "hive.client.connection.error", + {"error": "protocol error", "peer": client.peer}, + {"source": client.peer}, + ) bus = self.get_bus(client) bus.emit(message) @@ -344,19 +380,24 @@ def handle_message(self, message: HiveMessage, client: HiveMindClientConnection) self.handle_unknown_message(message, client) # HiveMind protocol messages - from slave -> master - def handle_unknown_message(self, message: HiveMessage, client: HiveMindClientConnection): - """ message handler for non default message types, subclasses can + def handle_unknown_message( + self, message: HiveMessage, client: HiveMindClientConnection + ): + """message handler for non default message types, subclasses can handle their own types here message (HiveMessage): HiveMind message object """ - def handle_binary_message(self, message: HiveMessage, client: HiveMindClientConnection): + def handle_binary_message( + self, message: HiveMessage, client: HiveMindClientConnection + ): assert message.msg_type == HiveMessageType.BINARY # TODO - def handle_handshake_message(self, message: HiveMessage, - client: HiveMindClientConnection): + def handle_handshake_message( + self, message: HiveMessage, client: HiveMindClientConnection + ): LOG.debug("handshake received, generating session key") payload = message.payload if "site_id" in payload: @@ -409,13 +450,16 @@ def handle_handshake_message(self, message: HiveMessage, msg = HiveMessage(HiveMessageType.HANDSHAKE, payload) client.send(msg) # client can recreate crypto_key on his side now - def handle_bus_message(self, message: HiveMessage, - client: HiveMindClientConnection): + def handle_bus_message( + self, message: HiveMessage, client: HiveMindClientConnection + ): self.handle_inject_mycroft_msg(message.payload, client) if self.mycroft_bus_callback: self.mycroft_bus_callback(message.payload) - def handle_broadcast_message(self, message: HiveMessage, client: HiveMindClientConnection): + def handle_broadcast_message( + self, message: HiveMessage, client: HiveMindClientConnection + ): """ message (HiveMessage): HiveMind message object """ @@ -447,8 +491,9 @@ def _unpack_message(self, message: HiveMessage, client: HiveMindClientConnection pload.remove_target_peer(client.peer) return pload - def handle_propagate_message(self, message: HiveMessage, - client: HiveMindClientConnection): + def handle_propagate_message( + self, message: HiveMessage, client: HiveMindClientConnection + ): """ message (HiveMessage): HiveMind message object """ @@ -475,15 +520,21 @@ def handle_propagate_message(self, message: HiveMessage, self.clients[peer].send(payload) # send to other masters - message = Message("hive.send.upstream", payload, - {"destination": "hive", - "source": self.peer, - "session": client.sess.serialize()}) + message = Message( + "hive.send.upstream", + payload, + { + "destination": "hive", + "source": self.peer, + "session": client.sess.serialize(), + }, + ) bus = self.get_bus(client) bus.emit(message) - def handle_escalate_message(self, message: HiveMessage, - client: HiveMindClientConnection): + def handle_escalate_message( + self, message: HiveMessage, client: HiveMindClientConnection + ): """ message (HiveMessage): HiveMind message object """ @@ -506,22 +557,29 @@ def handle_escalate_message(self, message: HiveMessage, self.escalate_callback(payload) # send to other masters - message = Message("hive.send.upstream", payload, - {"destination": "hive", - "source": self.peer, - "session": client.sess.serialize()}) + message = Message( + "hive.send.upstream", + payload, + { + "destination": "hive", + "source": self.peer, + "session": client.sess.serialize(), + }, + ) bus = self.get_bus(client) bus.emit(message) # HiveMind mycroft bus messages - from slave -> master def update_slave_session(self, message: Message, client: HiveMindClientConnection): - """ slave injected a message, master decides what the session is unconditionally (active skills etc) + """slave injected a message, master decides what the session is unconditionally (active skills etc) handle special message that influence session per client and update HM session as needed here """ message.context["session"] = client.sess.serialize() return message - def handle_inject_mycroft_msg(self, message: Message, client: HiveMindClientConnection): + def handle_inject_mycroft_msg( + self, message: Message, client: HiveMindClientConnection + ): """ message (Message): mycroft bus message object """ @@ -538,7 +596,9 @@ def handle_inject_mycroft_msg(self, message: Message, client: HiveMindClientConn if message.msg_type == "speak": message.context["destination"] = ["audio"] elif message.context.get("destination") is None: - message.context["destination"] = "skills" # ensure not treated as a broadcast + message.context[ + "destination" + ] = "skills" # ensure not treated as a broadcast # send client message to internal mycroft bus LOG.info(f"Forwarding message to mycroft bus from client: {client.peer}") diff --git a/hivemind_core/scripts.py b/hivemind_core/scripts.py index 93ef320..404327f 100644 --- a/hivemind_core/scripts.py +++ b/hivemind_core/scripts.py @@ -22,9 +22,13 @@ def hmcore_cmds(): def add_client(name, access_key, password, crypto_key): key = crypto_key if key: - print("WARNING: crypto key is deprecated, use password instead if your client supports it") - print("WARNING: for security the encryption key should be randomly generated\n" - "Defining your own key is discouraged") + print( + "WARNING: crypto key is deprecated, use password instead if your client supports it" + ) + print( + "WARNING: for security the encryption key should be randomly generated\n" + "Defining your own key is discouraged" + ) if len(key) != 16: print("Encryption key needs to be exactly 16 characters!") raise ValueError @@ -49,7 +53,9 @@ def add_client(name, access_key, password, crypto_key): print("Password:", password) print("Encryption Key:", key) - print("WARNING: Encryption Key is deprecated, only use if your client does not support password") + print( + "WARNING: Encryption Key is deprecated, only use if your client does not support password" + ) @hmcore_cmds.command(help="allow message types sent from a client", name="allow-msg") @@ -65,9 +71,11 @@ def allow_msg(msg_type, node_id): _choices = [] for client in ClientDatabase(): if client["client_id"] != -1: - table.add_row(str(client["client_id"]), - client["name"], - str(client.get("allowed_types", []))) + table.add_row( + str(client["client_id"]), + client["name"], + str(client.get("allowed_types", [])), + ) _choices.append(str(client["client_id"])) if not _choices: @@ -77,8 +85,10 @@ def allow_msg(msg_type, node_id): console = Console() console.print(table) _exit = str(max(int(i) for i in _choices) + 1) - node_id = Prompt.ask(f"To which client you want to add '{msg_type}'? ({_exit}='Exit')", - choices=_choices + [_exit]) + node_id = Prompt.ask( + f"To which client you want to add '{msg_type}'? ({_exit}='Exit')", + choices=_choices + [_exit], + ) if node_id == _exit: console.print("User exit", style="red") exit() @@ -101,8 +111,9 @@ def allow_msg(msg_type, node_id): break -@hmcore_cmds.command(help="remove credentials for a client (numeric unique ID)", - name="delete-client") +@hmcore_cmds.command( + help="remove credentials for a client (numeric unique ID)", name="delete-client" +) @click.argument("node_id", required=True, type=int) def delete_client(node_id): with ClientDatabase() as db: @@ -134,28 +145,67 @@ def list_clients(): with ClientDatabase() as db: for x in db: if x["client_id"] != -1: - table.add_row(str(x["client_id"]), x["name"], x["api_key"], x["password"], x["crypto_key"]) + table.add_row( + str(x["client_id"]), + x["name"], + x["api_key"], + x["password"], + x["crypto_key"], + ) console.print(table) @hmcore_cmds.command(help="start listening for HiveMind connections", name="listen") +@click.option( + "--ovos_bus_address", + help="Open Voice OS bus address", + type=str, + default="127.0.0.1", +) +@click.option( + "--ovos_bus_port", help="Open Voice OS bus port number", type=int, default=8181 +) @click.option("--port", help="HiveMind port number", type=int, default=5678) @click.option("--ssl", help="use wss://", type=bool, default=False) -@click.option("--cert_dir", help="HiveMind SSL certificate directory", type=str, default=f"{xdg_data_home()}/hivemind") -@click.option("--cert_name", help="HiveMind SSL certificate file name", type=str, default="hivemind") -def listen(port: int, ssl: bool, cert_dir: str, cert_name: str): +@click.option( + "--cert_dir", + help="HiveMind SSL certificate directory", + type=str, + default=f"{xdg_data_home()}/hivemind", +) +@click.option( + "--cert_name", + help="HiveMind SSL certificate file name", + type=str, + default="hivemind", +) +def listen( + ovos_bus_address: str, + ovos_bus_port: int, + port: int, + ssl: bool, + cert_dir: str, + cert_name: str, +): from hivemind_core.service import HiveMindService + ovos_bus_config = { + "address": ovos_bus_address, + "port": ovos_bus_port, + } + websocket_config = { "host": "0.0.0.0", "port": port, "ssl": ssl, "cert_dir": cert_dir, - "cert_name": cert_name + "cert_name": cert_name, } - service = HiveMindService(websocket_config=websocket_config) + service = HiveMindService( + ovos_bus_config=ovos_bus_config, websocket_config=websocket_config + ) service.run() diff --git a/hivemind_core/service.py b/hivemind_core/service.py index 9c2b77e..50206f9 100644 --- a/hivemind_core/service.py +++ b/hivemind_core/service.py @@ -24,13 +24,18 @@ from hivemind_bus_client.identity import NodeIdentity from hivemind_bus_client.message import HiveMessageType from hivemind_core.database import ClientDatabase -from hivemind_core.protocol import HiveMindListenerProtocol, HiveMindClientConnection, HiveMindNodeType +from hivemind_core.protocol import ( + HiveMindListenerProtocol, + HiveMindClientConnection, + HiveMindNodeType, +) from hivemind_ggwave import GGWaveMaster from hivemind_presence import LocalPresence -def create_self_signed_cert(cert_dir=f"{xdg_data_home()}/hivemind", - name="hivemind") -> Tuple[str, str]: +def create_self_signed_cert( + cert_dir=f"{xdg_data_home()}/hivemind", name="hivemind" +) -> Tuple[str, str]: """ If name.crt and name.key don't exist in cert_dir, create a new self-signed cert and key pair and write them into that directory. @@ -41,8 +46,7 @@ def create_self_signed_cert(cert_dir=f"{xdg_data_home()}/hivemind", key_path = join(cert_dir, KEY_FILE) makedirs(cert_dir, exist_ok=True) - if not exists(join(cert_dir, CERT_FILE)) \ - or not exists(join(cert_dir, KEY_FILE)): + if not exists(join(cert_dir, CERT_FILE)) or not exists(join(cert_dir, KEY_FILE)): # create a key pair k = crypto.PKey() k.generate_key(crypto.TYPE_RSA, 2048) @@ -61,36 +65,36 @@ def create_self_signed_cert(cert_dir=f"{xdg_data_home()}/hivemind", cert.set_issuer(cert.get_subject()) cert.set_pubkey(k) # TODO don't use sha1 - cert.sign(k, 'sha1') + cert.sign(k, "sha1") if not exists(cert_dir): makedirs(cert_dir) - open(cert_path, "wb").write( - crypto.dump_certificate(crypto.FILETYPE_PEM, cert)) + open(cert_path, "wb").write(crypto.dump_certificate(crypto.FILETYPE_PEM, cert)) open(join(cert_dir, KEY_FILE), "wb").write( - crypto.dump_privatekey(crypto.FILETYPE_PEM, k)) + crypto.dump_privatekey(crypto.FILETYPE_PEM, k) + ) return cert_path, key_path def on_ready(): - LOG.info('HiveMind bus service ready!') + LOG.info("HiveMind bus service ready!") def on_alive(): - LOG.info('HiveMind bus service alive') + LOG.info("HiveMind bus service alive") def on_started(): - LOG.info('HiveMind bus service started!') + LOG.info("HiveMind bus service started!") -def on_error(e='Unknown'): - LOG.info('HiveMind bus failed to start ({})'.format(repr(e))) +def on_error(e="Unknown"): + LOG.info("HiveMind bus failed to start ({})".format(repr(e))) def on_stopping(): - LOG.info('HiveMind bus is shutting down...') + LOG.info("HiveMind bus is shutting down...") class MessageBusEventHandler(WebSocketHandler): @@ -105,7 +109,10 @@ def decode_auth(auth) -> Tuple[str, str]: def on_message(self, message): message = self.client.decode(message) - if message.msg_type == HiveMessageType.BUS and message.payload.msg_type == "recognizer_loop:b64_audio": + if ( + message.msg_type == HiveMessageType.BUS + and message.payload.msg_type == "recognizer_loop:b64_audio" + ): LOG.info(f"received {self.client.peer} sent base64 audio for STT") else: LOG.info(f"received {self.client.peer} message: {message}") @@ -118,9 +125,15 @@ def open(self): # in regular handshake an asymmetric key pair is used handshake = HandShake(HiveMindService.identity.private_key) - self.client = HiveMindClientConnection(key=key, name=name, - ip=self.request.remote_ip, socket=self, sess=Session(), - handshake=handshake, loop=self.protocol.loop) + self.client = HiveMindClientConnection( + key=key, + name=name, + ip=self.request.remote_ip, + socket=self, + sess=Session(), + handshake=handshake, + loop=self.protocol.loop, + ) with ClientDatabase() as users: user = users.get_client_by_api_key(key) @@ -142,11 +155,15 @@ def open(self): self.client.node_type = HiveMindNodeType.NODE # TODO . placeholder - if not self.client.crypto_key and \ - not self.protocol.handshake_enabled \ - and self.protocol.require_crypto: - LOG.error("No pre-shared crypto key for client and handshake disabled, " - "but configured to require crypto!") + if ( + not self.client.crypto_key + and not self.protocol.handshake_enabled + and self.protocol.require_crypto + ): + LOG.error( + "No pre-shared crypto key for client and handshake disabled, " + "but configured to require crypto!" + ) # clients requiring handshake support might fail here self.protocol.handle_invalid_protocol_version(self.client) self.close() @@ -166,45 +183,62 @@ def check_origin(self, origin) -> bool: class HiveMindService: identity = NodeIdentity() - def __init__(self, - alive_hook: Callable = on_alive, - started_hook: Callable = on_started, - ready_hook: Callable = on_ready, - error_hook: Callable = on_error, - stopping_hook: Callable = on_stopping, - websocket_config: Optional[Dict[str, Any]] = None, - protocol=HiveMindListenerProtocol, - bus=None, - ws_handler=MessageBusEventHandler): - - websocket_config = websocket_config or \ - Configuration().get('hivemind_websocket', {}) - callbacks = StatusCallbackMap(on_started=started_hook, - on_alive=alive_hook, - on_ready=ready_hook, - on_error=error_hook, - on_stopping=stopping_hook) + def __init__( + self, + alive_hook: Callable = on_alive, + started_hook: Callable = on_started, + ready_hook: Callable = on_ready, + error_hook: Callable = on_error, + stopping_hook: Callable = on_stopping, + websocket_config: Optional[Dict[str, Any]] = None, + ovos_bus_config: Optional[Dict[str, Any]] = None, + protocol=HiveMindListenerProtocol, + bus=None, + ws_handler=MessageBusEventHandler, + ): + websocket_config = websocket_config or Configuration().get( + "hivemind_websocket", {} + ) + callbacks = StatusCallbackMap( + on_started=started_hook, + on_alive=alive_hook, + on_ready=ready_hook, + on_error=error_hook, + on_stopping=stopping_hook, + ) self._proto = protocol self._ws_handler = ws_handler if bus: self.bus = bus else: - self.bus = MessageBusClient(emitter=EventEmitter()) + self.ovos_bus_address = ovos_bus_config.get("address") or "127.0.0.1" + self.ovos_bus_port = ovos_bus_config.get("port") or 8181 + self.bus = MessageBusClient( + host=self.ovos_bus_address, + port=self.ovos_bus_port, + emitter=EventEmitter(), + ) self.bus.run_in_thread() self.bus.connected_event.wait() - self.status = ProcessStatus('HiveMind', callback_map=callbacks) - self.host = websocket_config.get('host') or "0.0.0.0" - self.port = websocket_config.get('port') or 5678 - self.ssl = websocket_config.get('ssl', False) - self.cert_dir = websocket_config.get('cert_dir') or f"{xdg_data_home()}/hivemind" - self.cert_name = websocket_config.get('cert_name') or "hivemind" # name + ".crt"/".key" - - self.presence = LocalPresence(name=self.identity.name, - service_type=HiveMindNodeType.MIND, - upnp=websocket_config.get('upnp', False), - port=self.port, - zeroconf=websocket_config.get('zeroconf', False)) + self.status = ProcessStatus("HiveMind", callback_map=callbacks) + self.host = websocket_config.get("host") or "0.0.0.0" + self.port = websocket_config.get("port") or 5678 + self.ssl = websocket_config.get("ssl", False) + self.cert_dir = ( + websocket_config.get("cert_dir") or f"{xdg_data_home()}/hivemind" + ) + self.cert_name = ( + websocket_config.get("cert_name") or "hivemind" + ) # name + ".crt"/".key" + + self.presence = LocalPresence( + name=self.identity.name, + service_type=HiveMindNodeType.MIND, + upnp=websocket_config.get("upnp", False), + port=self.port, + zeroconf=websocket_config.get("zeroconf", False), + ) try: # TODO - silent_mode should be controlled via external events # to start enrolling new devices on demand @@ -231,7 +265,9 @@ def run(self): KEY_FILE = f"{self.cert_dir}/{self.cert_name}.key" if not os.path.isfile(KEY_FILE): LOG.info(f"generating self-signed SSL certificate") - CERT_FILE, KEY_FILE = create_self_signed_cert(self.cert_dir, self.cert_name) + CERT_FILE, KEY_FILE = create_self_signed_cert( + self.cert_dir, self.cert_name + ) LOG.debug("using ssl key at " + KEY_FILE) LOG.debug("using ssl certificate at " + CERT_FILE) ssl_options = {"certfile": CERT_FILE, "keyfile": KEY_FILE} diff --git a/readme.md b/readme.md index c8026c3..0b18dfc 100644 --- a/readme.md +++ b/readme.md @@ -45,12 +45,14 @@ Usage: hivemind-core listen [OPTIONS] start listening for HiveMind connections Options: - --host TEXT HiveMind host - --port INTEGER HiveMind port number - --ssl BOOLEAN use wss:// - --cert_dir TEXT HiveMind SSL certificate directory - --cert_name TEXT HiveMind SSL certificate file name - --help Show this message and exit. + --host TEXT HiveMind host + --port INTEGER HiveMind port number + --ovos_bus_address TEXT Open Voice OS bus address + --ovos_bus_port INTEGER Open Voice OS bus port + --ssl BOOLEAN use wss:// + --cert_dir TEXT HiveMind SSL certificate directory + --cert_name TEXT HiveMind SSL certificate file name + --help Show this message and exit. $ hivemind-core delete-client --help @@ -72,10 +74,12 @@ Options: ``` +By default HiveMind listens for the Open Voice OS bus on `127.0.0.1` which should not be changed when running as the same place. In some cases such as Kubernetes when the HiveMind Listener and Open Voice OS bus are in different pods, the HiveMind Listener should be able to connect to the pod address by using the `ovos_bus_address` and `ovos_bus_port` options. + # Protocol | Protocol Version | 0 | 1 | -|----------------------|-----|-----| +| -------------------- | --- | --- | | json serialization | yes | yes | | binary serialization | no | yes | | pre-shared AES key | yes | yes | @@ -83,7 +87,6 @@ Options: | PGP handshake | no | yes | | zlib compression | no | yes | - some clients such as HiveMind-Js do not yet support protocol V1 # HiveMind components @@ -112,4 +115,3 @@ some clients such as HiveMind-Js do not yet support protocol V1 ## Minds - [NodeRed](https://github.com/OpenJarbas/HiveMind-NodeRed) - diff --git a/scripts/bump_alpha.py b/scripts/bump_alpha.py index bc18619..b91c437 100644 --- a/scripts/bump_alpha.py +++ b/scripts/bump_alpha.py @@ -15,4 +15,4 @@ if line.startswith(version_var_name): print(f"{version_var_name} = {new_version}") else: - print(line.rstrip('\n')) + print(line.rstrip("\n")) diff --git a/scripts/bump_build.py b/scripts/bump_build.py index 3307ebc..0010f59 100644 --- a/scripts/bump_build.py +++ b/scripts/bump_build.py @@ -18,4 +18,4 @@ elif line.startswith(alpha_var_name): print(f"{alpha_var_name} = 0") else: - print(line.rstrip('\n')) + print(line.rstrip("\n")) diff --git a/scripts/bump_major.py b/scripts/bump_major.py index 175f81e..47e7cee 100644 --- a/scripts/bump_major.py +++ b/scripts/bump_major.py @@ -24,4 +24,4 @@ elif line.startswith(alpha_var_name): print(f"{alpha_var_name} = 0") else: - print(line.rstrip('\n')) + print(line.rstrip("\n")) diff --git a/scripts/bump_minor.py b/scripts/bump_minor.py index 9f3590d..bf26bbe 100644 --- a/scripts/bump_minor.py +++ b/scripts/bump_minor.py @@ -21,4 +21,4 @@ elif line.startswith(alpha_var_name): print(f"{alpha_var_name} = 0") else: - print(line.rstrip('\n')) + print(line.rstrip("\n")) diff --git a/scripts/remove_alpha.py b/scripts/remove_alpha.py index c52a4f0..64a43ef 100644 --- a/scripts/remove_alpha.py +++ b/scripts/remove_alpha.py @@ -10,4 +10,4 @@ if line.startswith(alpha_var_name): print(f"{alpha_var_name} = 0") else: - print(line.rstrip('\n')) + print(line.rstrip("\n")) diff --git a/setup.py b/setup.py index e8deba5..3e23544 100644 --- a/setup.py +++ b/setup.py @@ -5,23 +5,22 @@ def get_version(): - """ Find the version of the package""" + """Find the version of the package""" version = None - version_file = os.path.join(BASEDIR, 'hivemind_core', 'version.py') + version_file = os.path.join(BASEDIR, "hivemind_core", "version.py") major, minor, build, alpha = (None, None, None, None) with open(version_file) as f: for line in f: - if 'VERSION_MAJOR' in line: - major = line.split('=')[1].strip() - elif 'VERSION_MINOR' in line: - minor = line.split('=')[1].strip() - elif 'VERSION_BUILD' in line: - build = line.split('=')[1].strip() - elif 'VERSION_ALPHA' in line: - alpha = line.split('=')[1].strip() - - if ((major and minor and build and alpha) or - '# END_VERSION_BLOCK' in line): + if "VERSION_MAJOR" in line: + major = line.split("=")[1].strip() + elif "VERSION_MINOR" in line: + minor = line.split("=")[1].strip() + elif "VERSION_BUILD" in line: + build = line.split("=")[1].strip() + elif "VERSION_ALPHA" in line: + alpha = line.split("=")[1].strip() + + if (major and minor and build and alpha) or "# END_VERSION_BLOCK" in line: break version = f"{major}.{minor}.{build}" if int(alpha) > 0: @@ -30,30 +29,29 @@ def get_version(): def required(requirements_file): - """ Read requirements file and remove comments and empty lines. """ - with open(os.path.join(BASEDIR, requirements_file), 'r') as f: + """Read requirements file and remove comments and empty lines.""" + with open(os.path.join(BASEDIR, requirements_file), "r") as f: requirements = f.read().splitlines() - if 'MYCROFT_LOOSE_REQUIREMENTS' in os.environ: - print('USING LOOSE REQUIREMENTS!') - requirements = [r.replace('==', '>=').replace('~=', '>=') for r in requirements] - return [pkg for pkg in requirements - if pkg.strip() and not pkg.startswith("#")] + if "MYCROFT_LOOSE_REQUIREMENTS" in os.environ: + print("USING LOOSE REQUIREMENTS!") + requirements = [ + r.replace("==", ">=").replace("~=", ">=") for r in requirements + ] + return [pkg for pkg in requirements if pkg.strip() and not pkg.startswith("#")] setup( - name='jarbas_hive_mind', + name="jarbas_hive_mind", version=get_version(), - packages=['hivemind_core'], + packages=["hivemind_core"], include_package_data=True, install_requires=required("requirements.txt"), - url='https://github.com/JarbasHiveMind/HiveMind-core', - license='MIT', - author='jarbasAI', - author_email='jarbasai@mailfence.com', - description='Mesh Networking utilities for OpenVoiceOS', + url="https://github.com/JarbasHiveMind/HiveMind-core", + license="MIT", + author="jarbasAI", + author_email="jarbasai@mailfence.com", + description="Mesh Networking utilities for OpenVoiceOS", entry_points={ - 'console_scripts': [ - 'hivemind-core=hivemind_core.scripts:hmcore_cmds' - ] - } + "console_scripts": ["hivemind-core=hivemind_core.scripts:hmcore_cmds"] + }, ) diff --git a/test/unittests/test_bus.py b/test/unittests/test_bus.py index 9a97580..ec4ac7f 100644 --- a/test/unittests/test_bus.py +++ b/test/unittests/test_bus.py @@ -9,6 +9,7 @@ # TODO - rewrite tests + def get_hive(): # TODO add/mock db for the test key = "dummy_key" @@ -43,24 +44,44 @@ def get_hive(): sleep(1) - mid = FakeMycroft(MID_PORT, connection=HiveNodeClient( - key=key, crypto_key=crypto_key, port=MASTER_PORT, ssl=False)) + mid = FakeMycroft( + MID_PORT, + connection=HiveNodeClient( + key=key, crypto_key=crypto_key, port=MASTER_PORT, ssl=False + ), + ) mid.start() - mid2 = FakeMycroft(MID2_PORT, connection=HiveNodeClient( - key=key, crypto_key=crypto_key, port=MASTER_PORT, ssl=False)) + mid2 = FakeMycroft( + MID2_PORT, + connection=HiveNodeClient( + key=key, crypto_key=crypto_key, port=MASTER_PORT, ssl=False + ), + ) mid2.start() sleep(1) - end = FakeMycroft(END_PORT, connection=HiveNodeClient( - key=key, crypto_key=crypto_key, port=MID_PORT, ssl=False)) + end = FakeMycroft( + END_PORT, + connection=HiveNodeClient( + key=key, crypto_key=crypto_key, port=MID_PORT, ssl=False + ), + ) end.start() - end2 = FakeMycroft(END2_PORT, connection=HiveNodeClient( - key=key, crypto_key=crypto_key, port=MID_PORT, ssl=False)) + end2 = FakeMycroft( + END2_PORT, + connection=HiveNodeClient( + key=key, crypto_key=crypto_key, port=MID_PORT, ssl=False + ), + ) end2.start() - end3 = FakeMycroft(END3_PORT, connection=HiveNodeClient( - key=key, crypto_key=crypto_key, port=MID2_PORT, ssl=False)) + end3 = FakeMycroft( + END3_PORT, + connection=HiveNodeClient( + key=key, crypto_key=crypto_key, port=MID2_PORT, ssl=False + ), + ) end3.start() sleep(10) # allow hive to fully connect @@ -90,9 +111,7 @@ def test_connections(self): class TestHiveBus(TestCase): - def test_midtomaster(self): - # # Master # * @@ -122,8 +141,9 @@ def handle_mid2_bus(message): mid.register_downstream_handlers() mid2.register_downstream_handlers() - pload = HiveMessage(msg_type=HiveMessageType.BUS, - payload=Message("test", {"ping": "pong"})) + pload = HiveMessage( + msg_type=HiveMessageType.BUS, payload=Message("test", {"ping": "pong"}) + ) mid.connection.emit(pload) sleep(2) @@ -167,8 +187,9 @@ def handle_mid2_bus(message): mid.register_downstream_handlers() mid2.register_downstream_handlers() - pload = HiveMessage(msg_type=HiveMessageType.BUS, - payload=Message("test", {"ping": "pong"})) + pload = HiveMessage( + msg_type=HiveMessageType.BUS, payload=Message("test", {"ping": "pong"}) + ) end3.connection.emit(pload) sleep(2) @@ -225,8 +246,9 @@ def handle_end3_bus(message): end2.register_upstream_handlers() end2.register_upstream_handlers() - pload = HiveMessage(msg_type=HiveMessageType.BUS, - payload=Message("test", {"ping": "pong"})) + pload = HiveMessage( + msg_type=HiveMessageType.BUS, payload=Message("test", {"ping": "pong"}) + ) master.interface.send(pload, mid.connection.peer) sleep(0.5) @@ -283,8 +305,9 @@ def handle_end3_bus(message): end2.register_upstream_handlers() end2.register_upstream_handlers() - pload = HiveMessage(msg_type=HiveMessageType.BUS, - payload=Message("test", {"ping": "pong"})) + pload = HiveMessage( + msg_type=HiveMessageType.BUS, payload=Message("test", {"ping": "pong"}) + ) mid.interface.send(pload, end.connection.peer) sleep(0.5) @@ -301,9 +324,7 @@ def handle_end3_bus(message): class TestEscalate(TestCase): - def test_midtomaster(self): - # # Master # * @@ -333,9 +354,11 @@ def handle_mid2_escalate(message): mid.register_downstream_handlers() mid2.register_downstream_handlers() - pload = HiveMessage(msg_type=HiveMessageType.THIRDPRTY, - payload=Message("test", {"ping": "pong"})) - pload = HiveMessage(msg_type=HiveMessageType.ESCALATE, payload=pload) + pload = HiveMessage( + msg_type=HiveMessageType.THIRDPRTY, + payload=Message("test", {"ping": "pong"}), + ) + pload = HiveMessage(msg_type=HiveMessageType.ESCALATE, payload=pload) mid.connection.emit(pload) sleep(2) @@ -381,8 +404,10 @@ def handle_mid2_escalate(message): mid.register_downstream_handlers() mid2.register_downstream_handlers() - pload = HiveMessage(msg_type=HiveMessageType.THIRDPRTY, - payload=Message("test", {"ping": "pong"})) + pload = HiveMessage( + msg_type=HiveMessageType.THIRDPRTY, + payload=Message("test", {"ping": "pong"}), + ) pload = HiveMessage(msg_type=HiveMessageType.ESCALATE, payload=pload) end.connection.emit(pload) sleep(2) @@ -428,8 +453,10 @@ def handle_mid2_escalate(message): mid.register_downstream_handlers() mid2.register_downstream_handlers() - pload = HiveMessage(msg_type=HiveMessageType.THIRDPRTY, - payload=Message("test", {"ping": "pong"})) + pload = HiveMessage( + msg_type=HiveMessageType.THIRDPRTY, + payload=Message("test", {"ping": "pong"}), + ) pload = HiveMessage(msg_type=HiveMessageType.ESCALATE, payload=pload) end3.connection.emit(pload) sleep(2) @@ -447,7 +474,6 @@ def handle_mid2_escalate(message): class TestHiveBroadcast(TestCase): - def test_master(self): # Master # / \ @@ -490,8 +516,10 @@ def handle_end3_broadcast(message): end2.register_upstream_handlers() end3.register_upstream_handlers() - pload = HiveMessage(msg_type=HiveMessageType.THIRDPRTY, - payload=Message("test", {"ping": "pong"})) + pload = HiveMessage( + msg_type=HiveMessageType.THIRDPRTY, + payload=Message("test", {"ping": "pong"}), + ) master.interface.broadcast(pload) sleep(1) @@ -550,8 +578,10 @@ def handle_end3_broadcast(message): end2.register_upstream_handlers() end3.register_upstream_handlers() - pload = HiveMessage(msg_type=HiveMessageType.THIRDPRTY, - payload=Message("test", {"ping": "pong"})) + pload = HiveMessage( + msg_type=HiveMessageType.THIRDPRTY, + payload=Message("test", {"ping": "pong"}), + ) mid.interface.broadcast(pload) sleep(1) @@ -572,9 +602,7 @@ def handle_end3_broadcast(message): @skip("TODO Fix me") class TestPropagate(TestCase): - def test_mid(self): - # # Master # * \ @@ -628,9 +656,11 @@ def handle_end3_propagate(message): end2.register_upstream_handlers() end3.register_upstream_handlers() - pload = HiveMessage(msg_type=HiveMessageType.THIRDPRTY, - payload=Message("test", {"ping": "pong"})) - pload = HiveMessage(msg_type=HiveMessageType.PROPAGATE, payload=pload) + pload = HiveMessage( + msg_type=HiveMessageType.THIRDPRTY, + payload=Message("test", {"ping": "pong"}), + ) + pload = HiveMessage(msg_type=HiveMessageType.PROPAGATE, payload=pload) mid.connection.emit(pload) sleep(2) @@ -702,8 +732,10 @@ def handle_end3_propagate(message): end2.register_upstream_handlers() end3.register_upstream_handlers() - pload = HiveMessage(msg_type=HiveMessageType.THIRDPRTY, - payload=Message("test", {"ping": "pong"})) + pload = HiveMessage( + msg_type=HiveMessageType.THIRDPRTY, + payload=Message("test", {"ping": "pong"}), + ) pload = HiveMessage(msg_type=HiveMessageType.PROPAGATE, payload=pload) end.connection.emit(pload) sleep(2) @@ -778,8 +810,10 @@ def handle_end3_propagate(message): end2.register_upstream_handlers() end3.register_upstream_handlers() - pload = HiveMessage(msg_type=HiveMessageType.THIRDPRTY, - payload=Message("test", {"ping": "pong"})) + pload = HiveMessage( + msg_type=HiveMessageType.THIRDPRTY, + payload=Message("test", {"ping": "pong"}), + ) pload = HiveMessage(msg_type=HiveMessageType.PROPAGATE, payload=pload) end3.connection.emit(pload) sleep(2) @@ -797,4 +831,3 @@ def handle_end3_propagate(message): continue self.assertEqual(message.msg_type, HiveMessageType.BUS) self.assertTrue(isinstance(message.payload, Message)) - diff --git a/test/unittests/test_db.py b/test/unittests/test_db.py index 5fbce6e..fd8c1d5 100644 --- a/test/unittests/test_db.py +++ b/test/unittests/test_db.py @@ -5,9 +5,7 @@ class TestDB(TestCase): - def test_add_entry(self): - key = os.urandom(8).hex() access_key = os.urandom(16).hex() password = None @@ -15,9 +13,7 @@ def test_add_entry(self): with ClientDatabase() as db: n = db.total_clients() name = f"HiveMind-Node-{n}" - user = db.add_client(name, access_key, - crypto_key=key, - password=password) + user = db.add_client(name, access_key, crypto_key=key, password=password) # verify data self.assertTrue(isinstance(user, Client)) self.assertEqual(user.name, name) @@ -39,4 +35,3 @@ def test_add_entry(self): self.assertEqual(node_id, -1) user = db.get_client_by_api_key(access_key) self.assertIsNone(user) - From 0a412eb5fa1f59816defd3d53e05afd81f2bcb6c Mon Sep 17 00:00:00 2001 From: JarbasAl Date: Tue, 2 Jan 2024 19:26:26 +0000 Subject: [PATCH 09/29] Increment Version --- CHANGELOG.md | 28 +++++++++++++++++++++++----- hivemind_core/version.py | 2 +- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c5556b..5bdff0b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog +## [Unreleased](https://github.com/JarbasHiveMind/HiveMind-core/tree/HEAD) + +[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.1a8...HEAD) + +**Closed issues:** + +- Handle client deletion by name [\#68](https://github.com/JarbasHiveMind/HiveMind-core/issues/68) + +**Merged pull requests:** + +- Add support to remote bus address and port [\#79](https://github.com/JarbasHiveMind/HiveMind-core/pull/79) ([goldyfruit](https://github.com/goldyfruit)) + +## [V0.13.1a8](https://github.com/JarbasHiveMind/HiveMind-core/tree/V0.13.1a8) (2023-11-24) + +[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.1a7...V0.13.1a8) + ## [V0.13.1a7](https://github.com/JarbasHiveMind/HiveMind-core/tree/V0.13.1a7) (2023-11-21) [Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.1a6...V0.13.1a7) @@ -38,7 +54,7 @@ **Implemented enhancements:** -- add site_id [\#74](https://github.com/JarbasHiveMind/HiveMind-core/pull/74) ([JarbasAl](https://github.com/JarbasAl)) +- add site\_id [\#74](https://github.com/JarbasHiveMind/HiveMind-core/pull/74) ([JarbasAl](https://github.com/JarbasAl)) ## [V0.13.0a0](https://github.com/JarbasHiveMind/HiveMind-core/tree/V0.13.0a0) (2023-09-08) @@ -100,7 +116,7 @@ **Merged pull requests:** -- filter allowed_types [\#71](https://github.com/JarbasHiveMind/HiveMind-core/pull/71) ([emphasize](https://github.com/emphasize)) +- filter allowed\_types [\#71](https://github.com/JarbasHiveMind/HiveMind-core/pull/71) ([emphasize](https://github.com/emphasize)) - add typehints [\#70](https://github.com/JarbasHiveMind/HiveMind-core/pull/70) ([emphasize](https://github.com/emphasize)) - \[requirements\] Add missing pyOpenSSL [\#69](https://github.com/JarbasHiveMind/HiveMind-core/pull/69) ([goldyfruit](https://github.com/goldyfruit)) @@ -130,7 +146,7 @@ - Xdg [\#51](https://github.com/JarbasHiveMind/HiveMind-core/pull/51) ([JarbasAl](https://github.com/JarbasAl)) - V2 [\#50](https://github.com/JarbasHiveMind/HiveMind-core/pull/50) ([JarbasAl](https://github.com/JarbasAl)) - Refactor/hivemind presence [\#49](https://github.com/JarbasHiveMind/HiveMind-core/pull/49) ([JarbasAl](https://github.com/JarbasAl)) -- refactor/deprecate_sql [\#48](https://github.com/JarbasHiveMind/HiveMind-core/pull/48) ([JarbasAl](https://github.com/JarbasAl)) +- refactor/deprecate\_sql [\#48](https://github.com/JarbasHiveMind/HiveMind-core/pull/48) ([JarbasAl](https://github.com/JarbasAl)) - launcher scripts / deprecate mail param / increase RSA key size [\#46](https://github.com/JarbasHiveMind/HiveMind-core/pull/46) ([Joanguitar](https://github.com/Joanguitar)) - move to asyncio [\#41](https://github.com/JarbasHiveMind/HiveMind-core/pull/41) ([JarbasAl](https://github.com/JarbasAl)) @@ -162,7 +178,7 @@ - migrate to pycryptodomex [\#30](https://github.com/JarbasHiveMind/HiveMind-core/pull/30) ([JarbasAl](https://github.com/JarbasAl)) - Add instructions to README [\#27](https://github.com/JarbasHiveMind/HiveMind-core/pull/27) ([ChanceNCounter](https://github.com/ChanceNCounter)) -- migrate from jarbas_utils to ovos_utils [\#24](https://github.com/JarbasHiveMind/HiveMind-core/pull/24) ([JarbasAl](https://github.com/JarbasAl)) +- migrate from jarbas\_utils to ovos\_utils [\#24](https://github.com/JarbasHiveMind/HiveMind-core/pull/24) ([JarbasAl](https://github.com/JarbasAl)) - revert cryptodomex requirement change [\#21](https://github.com/JarbasHiveMind/HiveMind-core/pull/21) ([JarbasAl](https://github.com/JarbasAl)) - Switch from pycryptodome to pycryptodomex [\#12](https://github.com/JarbasHiveMind/HiveMind-core/pull/12) ([j1nx](https://github.com/j1nx)) - escalate [\#8](https://github.com/JarbasHiveMind/HiveMind-core/pull/8) ([JarbasAl](https://github.com/JarbasAl)) @@ -171,4 +187,6 @@ - Feat/emulation [\#5](https://github.com/JarbasHiveMind/HiveMind-core/pull/5) ([JarbasAl](https://github.com/JarbasAl)) - refactor + http support [\#4](https://github.com/JarbasHiveMind/HiveMind-core/pull/4) ([JarbasAl](https://github.com/JarbasAl)) -\* _This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)_ + + +\* *This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)* diff --git a/hivemind_core/version.py b/hivemind_core/version.py index 11cfeca..2f5b9ab 100644 --- a/hivemind_core/version.py +++ b/hivemind_core/version.py @@ -3,5 +3,5 @@ VERSION_MAJOR = 0 VERSION_MINOR = 13 VERSION_BUILD = 1 -VERSION_ALPHA = 8 +VERSION_ALPHA = 9 # END_VERSION_BLOCK From 26cba2ac68969dcab60558ae140c89f3ecaba1d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Trellu?= Date: Tue, 2 Jan 2024 17:57:26 -0500 Subject: [PATCH 10/29] Fix missing host argument (#80) --- hivemind_core/scripts.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/hivemind_core/scripts.py b/hivemind_core/scripts.py index 404327f..592c897 100644 --- a/hivemind_core/scripts.py +++ b/hivemind_core/scripts.py @@ -166,6 +166,12 @@ def list_clients(): @click.option( "--ovos_bus_port", help="Open Voice OS bus port number", type=int, default=8181 ) +@click.option( + "--host", + help="HiveMind host", + type=str, + default="0.0.0.0", +) @click.option("--port", help="HiveMind port number", type=int, default=5678) @click.option("--ssl", help="use wss://", type=bool, default=False) @click.option( @@ -183,6 +189,7 @@ def list_clients(): def listen( ovos_bus_address: str, ovos_bus_port: int, + host: str, port: int, ssl: bool, cert_dir: str, @@ -196,7 +203,7 @@ def listen( } websocket_config = { - "host": "0.0.0.0", + "host": host, "port": port, "ssl": ssl, "cert_dir": cert_dir, From 0eff0303c6bb995683ba4eea15d941f3eeaa1b52 Mon Sep 17 00:00:00 2001 From: JarbasAl Date: Tue, 2 Jan 2024 22:58:03 +0000 Subject: [PATCH 11/29] Increment Version --- CHANGELOG.md | 10 +++++++++- hivemind_core/version.py | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bdff0b..5e5b87c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,15 @@ ## [Unreleased](https://github.com/JarbasHiveMind/HiveMind-core/tree/HEAD) -[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.1a8...HEAD) +[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.1a9...HEAD) + +**Merged pull requests:** + +- Fix missing host argument [\#80](https://github.com/JarbasHiveMind/HiveMind-core/pull/80) ([goldyfruit](https://github.com/goldyfruit)) + +## [V0.13.1a9](https://github.com/JarbasHiveMind/HiveMind-core/tree/V0.13.1a9) (2024-01-02) + +[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.1a8...V0.13.1a9) **Closed issues:** diff --git a/hivemind_core/version.py b/hivemind_core/version.py index 2f5b9ab..d99e96f 100644 --- a/hivemind_core/version.py +++ b/hivemind_core/version.py @@ -3,5 +3,5 @@ VERSION_MAJOR = 0 VERSION_MINOR = 13 VERSION_BUILD = 1 -VERSION_ALPHA = 9 +VERSION_ALPHA = 10 # END_VERSION_BLOCK From 5facb2c5a3f735466da5cca4b439480a5dcb8806 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Sun, 21 Apr 2024 22:25:23 +0100 Subject: [PATCH 12/29] fix/clients_manage_their_own_sessions (#84) * fix/clients_manage_their_own_sessions * fix/clients_manage_their_own_sessions --- hivemind_core/protocol.py | 35 ++++++++++------------------------- 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/hivemind_core/protocol.py b/hivemind_core/protocol.py index aa56b92..de535ef 100644 --- a/hivemind_core/protocol.py +++ b/hivemind_core/protocol.py @@ -212,15 +212,7 @@ def handle_internal_mycroft(self, message: str): if not isinstance(target_peers, list): target_peers = [target_peers] - new_sess = Session.from_message(message) for peer, client in self.clients.items(): - # ovos-core decides the runtime contents of the Session, - # let's sync any internal changes - if new_sess.session_id == client.sess.session_id: - LOG.debug(f"syncing session from ovos with {peer}") - client.sess = Session.from_message(message) - client.sess.site_id = client.site_id - if peer in target_peers: # forward internal messages to clients if they are the target LOG.debug(f"{message.msg_type} - destination: {peer}") @@ -303,9 +295,9 @@ def handle_new_client(self, client: HiveMindClientConnection): "max_protocol_version": max_version, "binarize": True, # report we support the binarization scheme "preshared_key": client.crypto_key - is not None, # do we have a pre-shared key (V0 proto) + is not None, # do we have a pre-shared key (V0 proto) "password": client.pswd_handshake - is not None, # is password available (V1 proto, replaces pre-shared key) + is not None, # is password available (V1 proto, replaces pre-shared key) "crypto_required": self.require_crypto, # do we allow unencrypted payloads } msg = HiveMessage(HiveMessageType.HANDSHAKE, payload) @@ -453,6 +445,10 @@ def handle_handshake_message( def handle_bus_message( self, message: HiveMessage, client: HiveMindClientConnection ): + # update the session as received by the client + client.sess = Session.from_message(message.payload) + LOG.debug(f"Client session updated: {client.sess.serialize()}") + self.handle_inject_mycroft_msg(message.payload, client) if self.mycroft_bus_callback: self.mycroft_bus_callback(message.payload) @@ -570,13 +566,6 @@ def handle_escalate_message( bus.emit(message) # HiveMind mycroft bus messages - from slave -> master - def update_slave_session(self, message: Message, client: HiveMindClientConnection): - """slave injected a message, master decides what the session is unconditionally (active skills etc) - handle special message that influence session per client and update HM session as needed here - """ - message.context["session"] = client.sess.serialize() - return message - def handle_inject_mycroft_msg( self, message: Message, client: HiveMindClientConnection ): @@ -592,22 +581,18 @@ def handle_inject_mycroft_msg( return # ensure client specific session data is injected in query to ovos - message.context["session"] = client.sess.serialize() + if "session" not in message.context: + message.context["session"] = client.sess.serialize() if message.msg_type == "speak": - message.context["destination"] = ["audio"] + message.context["destination"] = ["audio"] # make audible, this is injected "speak" command elif message.context.get("destination") is None: - message.context[ - "destination" - ] = "skills" # ensure not treated as a broadcast + message.context["destination"] = "skills" # ensure not treated as a broadcast # send client message to internal mycroft bus LOG.info(f"Forwarding message to mycroft bus from client: {client.peer}") message.context["peer"] = message.context["source"] = client.peer message.context["source"] = client.peer - # validate slave session - message = self.update_slave_session(message, client) - bus = self.get_bus(client) bus.emit(message) From 8e97e04fc55c89c2a3fc3b6be089857c57558d23 Mon Sep 17 00:00:00 2001 From: JarbasAl Date: Sun, 21 Apr 2024 21:25:58 +0000 Subject: [PATCH 13/29] Increment Version --- CHANGELOG.md | 14 +++++++++++++- hivemind_core/version.py | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e5b87c..d077d62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,19 @@ ## [Unreleased](https://github.com/JarbasHiveMind/HiveMind-core/tree/HEAD) -[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.1a9...HEAD) +[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.1a10...HEAD) + +**Fixed bugs:** + +- fix/clients\_manage\_their\_own\_sessions [\#84](https://github.com/JarbasHiveMind/HiveMind-core/pull/84) ([JarbasAl](https://github.com/JarbasAl)) + +**Closed issues:** + +- Proposal/s: compressing messages, URIs for nodes, and how they relate [\#14](https://github.com/JarbasHiveMind/HiveMind-core/issues/14) + +## [V0.13.1a10](https://github.com/JarbasHiveMind/HiveMind-core/tree/V0.13.1a10) (2024-01-02) + +[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.1a9...V0.13.1a10) **Merged pull requests:** diff --git a/hivemind_core/version.py b/hivemind_core/version.py index d99e96f..ce587e7 100644 --- a/hivemind_core/version.py +++ b/hivemind_core/version.py @@ -3,5 +3,5 @@ VERSION_MAJOR = 0 VERSION_MINOR = 13 VERSION_BUILD = 1 -VERSION_ALPHA = 10 +VERSION_ALPHA = 11 # END_VERSION_BLOCK From 893938c44fbd56731e8d8a5eddc0afcaf5f4771e Mon Sep 17 00:00:00 2001 From: miro Date: Mon, 22 Apr 2024 07:59:04 +0100 Subject: [PATCH 14/29] fix/client_session_id_sync --- hivemind_core/protocol.py | 29 +++++++++++++++++------------ hivemind_core/service.py | 5 +++-- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/hivemind_core/protocol.py b/hivemind_core/protocol.py index de535ef..a4e6823 100644 --- a/hivemind_core/protocol.py +++ b/hivemind_core/protocol.py @@ -212,18 +212,19 @@ def handle_internal_mycroft(self, message: str): if not isinstance(target_peers, list): target_peers = [target_peers] - for peer, client in self.clients.items(): - if peer in target_peers: - # forward internal messages to clients if they are the target - LOG.debug(f"{message.msg_type} - destination: {peer}") - message.context["source"] = "hive" - msg = HiveMessage( - HiveMessageType.BUS, - source_peer=peer, - target_peers=target_peers, - payload=message, - ) - client.send(msg) + if target_peers: + for peer, client in self.clients.items(): + if peer in target_peers: + # forward internal messages to clients if they are the target + LOG.debug(f"{message.msg_type} - destination: {peer}") + message.context["source"] = "hive" + msg = HiveMessage( + HiveMessageType.BUS, + source_peer=peer, + target_peers=target_peers, + payload=message, + ) + client.send(msg) @dataclass() @@ -446,8 +447,12 @@ def handle_bus_message( self, message: HiveMessage, client: HiveMindClientConnection ): # update the session as received by the client + old = client.peer client.sess = Session.from_message(message.payload) LOG.debug(f"Client session updated: {client.sess.serialize()}") + if old != client.peer: + LOG.debug(f"Client session_id changed! new peer_id: {client.peer}") + self.clients[client.peer] = self.clients.pop(old) self.handle_inject_mycroft_msg(message.payload, client) if self.mycroft_bus_callback: diff --git a/hivemind_core/service.py b/hivemind_core/service.py index 50206f9..bd50f97 100644 --- a/hivemind_core/service.py +++ b/hivemind_core/service.py @@ -130,7 +130,7 @@ def open(self): name=name, ip=self.request.remote_ip, socket=self, - sess=Session(), + sess=Session(session_id="default"), # will be re-assigned once client sends it's own handshake=handshake, loop=self.protocol.loop, ) @@ -211,7 +211,8 @@ def __init__( if bus: self.bus = bus else: - self.ovos_bus_address = ovos_bus_config.get("address") or "127.0.0.1" + ovos_bus_config = ovos_bus_config or Configuration().get("websocket", {}) + self.ovos_bus_address = ovos_bus_config.get("host") or "127.0.0.1" self.ovos_bus_port = ovos_bus_config.get("port") or 8181 self.bus = MessageBusClient( host=self.ovos_bus_address, From ccb8ffb721c647c274f78c2cd2dafbd3af8e270c Mon Sep 17 00:00:00 2001 From: JarbasAl Date: Mon, 22 Apr 2024 07:01:51 +0000 Subject: [PATCH 15/29] Increment Version --- CHANGELOG.md | 4 ++-- hivemind_core/version.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d077d62..fd532a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ # Changelog -## [Unreleased](https://github.com/JarbasHiveMind/HiveMind-core/tree/HEAD) +## [V0.13.1a11](https://github.com/JarbasHiveMind/HiveMind-core/tree/V0.13.1a11) (2024-04-21) -[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.1a10...HEAD) +[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.1a10...V0.13.1a11) **Fixed bugs:** diff --git a/hivemind_core/version.py b/hivemind_core/version.py index ce587e7..65a67ee 100644 --- a/hivemind_core/version.py +++ b/hivemind_core/version.py @@ -3,5 +3,5 @@ VERSION_MAJOR = 0 VERSION_MINOR = 13 VERSION_BUILD = 1 -VERSION_ALPHA = 11 +VERSION_ALPHA = 12 # END_VERSION_BLOCK From 5e6c4aa53dbfc0b8fe340265cc5d57466e48c152 Mon Sep 17 00:00:00 2001 From: miro Date: Mon, 22 Apr 2024 18:34:58 +0100 Subject: [PATCH 16/29] fix/typo_host --- hivemind_core/scripts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hivemind_core/scripts.py b/hivemind_core/scripts.py index 592c897..bd44ace 100644 --- a/hivemind_core/scripts.py +++ b/hivemind_core/scripts.py @@ -198,7 +198,7 @@ def listen( from hivemind_core.service import HiveMindService ovos_bus_config = { - "address": ovos_bus_address, + "host": ovos_bus_address, "port": ovos_bus_port, } From b6d4844cd7ab5ae7ab930fd5c5add72eb084ae41 Mon Sep 17 00:00:00 2001 From: JarbasAl Date: Mon, 22 Apr 2024 17:35:50 +0000 Subject: [PATCH 17/29] Increment Version --- CHANGELOG.md | 4 ++++ hivemind_core/version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd532a4..3024267 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## [V0.13.1a12](https://github.com/JarbasHiveMind/HiveMind-core/tree/V0.13.1a12) (2024-04-22) + +[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.1a11...V0.13.1a12) + ## [V0.13.1a11](https://github.com/JarbasHiveMind/HiveMind-core/tree/V0.13.1a11) (2024-04-21) [Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.1a10...V0.13.1a11) diff --git a/hivemind_core/version.py b/hivemind_core/version.py index 65a67ee..2638239 100644 --- a/hivemind_core/version.py +++ b/hivemind_core/version.py @@ -3,5 +3,5 @@ VERSION_MAJOR = 0 VERSION_MINOR = 13 VERSION_BUILD = 1 -VERSION_ALPHA = 12 +VERSION_ALPHA = 13 # END_VERSION_BLOCK From 772c38b38549636789facc229ea0496b9e252567 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Thu, 2 May 2024 17:46:09 +0100 Subject: [PATCH 18/29] fix/allow_audio_state_messages_for_session (#85) companion to https://github.com/OpenVoiceOS/ovos-bus-client/pull/93 by default allows voice sats to report if they are recording/playing audio, exposing that info to Session avoids the need of configuring devices individually by default ``` $hivemind-core allow-msg "recognizer_loop:record_begin" $hivemind-core allow-msg "recognizer_loop:record_end" $hivemind-core allow-msg "recognizer_loop:audio_output_start" $hivemind-core allow-msg "recognizer_loop:audio_output_end" ``` --- hivemind_core/database.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/hivemind_core/database.py b/hivemind_core/database.py index 17b1d90..9b37ee5 100644 --- a/hivemind_core/database.py +++ b/hivemind_core/database.py @@ -74,7 +74,11 @@ def __init__( self.crypto_key = crypto_key self.password = password self.blacklist = blacklist or {"messages": [], "skills": [], "intents": []} - self.allowed_types = allowed_types or ["recognizer_loop:utterance"] + self.allowed_types = allowed_types or ["recognizer_loop:utterance", + "recognizer_loop:record_begin", + "recognizer_loop:record_end", + "recognizer_loop:audio_output_start", + "recognizer_loop:audio_output_end"] if "recognizer_loop:utterance" not in self.allowed_types: self.allowed_types.append("recognizer_loop:utterance") self.can_broadcast = can_broadcast From e2ccd8cd18abae1b0669fcc11a22cfb2abdf2460 Mon Sep 17 00:00:00 2001 From: JarbasAl Date: Thu, 2 May 2024 16:46:54 +0000 Subject: [PATCH 19/29] Increment Version --- CHANGELOG.md | 22 +++++++++++++++++----- hivemind_core/version.py | 2 +- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3024267..334ef4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## [Unreleased](https://github.com/JarbasHiveMind/HiveMind-core/tree/HEAD) + +[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.1a13...HEAD) + +**Implemented enhancements:** + +- fix/allow\_audio\_state\_messages\_for\_session [\#85](https://github.com/JarbasHiveMind/HiveMind-core/pull/85) ([JarbasAl](https://github.com/JarbasAl)) + +## [V0.13.1a13](https://github.com/JarbasHiveMind/HiveMind-core/tree/V0.13.1a13) (2024-04-22) + +[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.1a12...V0.13.1a13) + ## [V0.13.1a12](https://github.com/JarbasHiveMind/HiveMind-core/tree/V0.13.1a12) (2024-04-22) [Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.1a11...V0.13.1a12) @@ -106,15 +118,15 @@ ## [V0.13.0a4](https://github.com/JarbasHiveMind/HiveMind-core/tree/V0.13.0a4) (2023-08-03) -[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.0a3...V0.13.0a4) +[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.0a2...V0.13.0a4) -## [V0.13.0a3](https://github.com/JarbasHiveMind/HiveMind-core/tree/V0.13.0a3) (2023-08-03) +## [V0.13.0a2](https://github.com/JarbasHiveMind/HiveMind-core/tree/V0.13.0a2) (2023-08-03) -[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.0a2...V0.13.0a3) +[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.0a3...V0.13.0a2) -## [V0.13.0a2](https://github.com/JarbasHiveMind/HiveMind-core/tree/V0.13.0a2) (2023-08-03) +## [V0.13.0a3](https://github.com/JarbasHiveMind/HiveMind-core/tree/V0.13.0a3) (2023-08-03) -[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.11.0a3...V0.13.0a2) +[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.11.0a3...V0.13.0a3) **Breaking changes:** diff --git a/hivemind_core/version.py b/hivemind_core/version.py index 2638239..d4172e7 100644 --- a/hivemind_core/version.py +++ b/hivemind_core/version.py @@ -3,5 +3,5 @@ VERSION_MAJOR = 0 VERSION_MINOR = 13 VERSION_BUILD = 1 -VERSION_ALPHA = 13 +VERSION_ALPHA = 14 # END_VERSION_BLOCK From e287e057f879c5c9cfaf695f60da9b90a12fb0c7 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Wed, 5 Jun 2024 01:08:44 +0100 Subject: [PATCH 20/29] allow to report SEI to OCP (#87) companion to https://github.com/OpenVoiceOS/ovos-core/pull/495 makes voice sats able to report what stream extractor plugins they have installed for usage with OCP without needing to explicit whitelist the bus message --- hivemind_core/database.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hivemind_core/database.py b/hivemind_core/database.py index 9b37ee5..b4bd987 100644 --- a/hivemind_core/database.py +++ b/hivemind_core/database.py @@ -78,7 +78,8 @@ def __init__( "recognizer_loop:record_begin", "recognizer_loop:record_end", "recognizer_loop:audio_output_start", - "recognizer_loop:audio_output_end"] + "recognizer_loop:audio_output_end", + "ovos.common_play.SEI.get.response"] if "recognizer_loop:utterance" not in self.allowed_types: self.allowed_types.append("recognizer_loop:utterance") self.can_broadcast = can_broadcast From bd8bab879f95155e136320c8ce0158b0dea41fd6 Mon Sep 17 00:00:00 2001 From: JarbasAl Date: Wed, 5 Jun 2024 00:09:32 +0000 Subject: [PATCH 21/29] Increment Version --- CHANGELOG.md | 10 +++++++++- hivemind_core/version.py | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 334ef4e..0b0f957 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,15 @@ ## [Unreleased](https://github.com/JarbasHiveMind/HiveMind-core/tree/HEAD) -[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.1a13...HEAD) +[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.1a14...HEAD) + +**Implemented enhancements:** + +- allow to report SEI to OCP [\#87](https://github.com/JarbasHiveMind/HiveMind-core/pull/87) ([JarbasAl](https://github.com/JarbasAl)) + +## [V0.13.1a14](https://github.com/JarbasHiveMind/HiveMind-core/tree/V0.13.1a14) (2024-05-02) + +[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.1a13...V0.13.1a14) **Implemented enhancements:** diff --git a/hivemind_core/version.py b/hivemind_core/version.py index d4172e7..4eeefb8 100644 --- a/hivemind_core/version.py +++ b/hivemind_core/version.py @@ -3,5 +3,5 @@ VERSION_MAJOR = 0 VERSION_MINOR = 13 VERSION_BUILD = 1 -VERSION_ALPHA = 14 +VERSION_ALPHA = 15 # END_VERSION_BLOCK From 5ef3563c9d57e7c5bec21869c0689bdbab150662 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Wed, 5 Jun 2024 02:49:33 +0100 Subject: [PATCH 22/29] feat/intercom (#86) * feat/intercom companion to https://github.com/JarbasHiveMind/hivemind-websocket-client/pull/27 * feat/intercom * requirements.txt --- .github/workflows/publish_alpha.yml | 2 +- hivemind_core/protocol.py | 87 ++++++++++++++++++++++++++++- hivemind_core/service.py | 2 +- requirements.txt | 13 +++-- 4 files changed, 94 insertions(+), 10 deletions(-) diff --git a/.github/workflows/publish_alpha.yml b/.github/workflows/publish_alpha.yml index a59006b..a60cbf8 100644 --- a/.github/workflows/publish_alpha.yml +++ b/.github/workflows/publish_alpha.yml @@ -14,7 +14,7 @@ on: - 'LICENSE' - 'CHANGELOG.md' - 'MANIFEST.in' - - 'readme.md' + - 'README.md' - 'scripts/**' workflow_dispatch: diff --git a/hivemind_core/protocol.py b/hivemind_core/protocol.py index a4e6823..834177e 100644 --- a/hivemind_core/protocol.py +++ b/hivemind_core/protocol.py @@ -2,6 +2,7 @@ from dataclasses import dataclass, field from enum import Enum, IntEnum from typing import List, Dict, Optional +import pgpy from ovos_bus_client import MessageBusClient from ovos_bus_client.message import Message @@ -13,6 +14,7 @@ from hivemind_bus_client.message import HiveMessage, HiveMessageType from hivemind_bus_client.serialization import decode_bitstring, get_bitstring +from hivemind_bus_client.identity import NodeIdentity from hivemind_bus_client.util import ( decrypt_bin, encrypt_bin, @@ -236,7 +238,7 @@ class HiveMindListenerProtocol: require_crypto: bool = True # throw error if crypto key not available handshake_enabled: bool = True # generate a key per session if not pre-shared - + identity: Optional[NodeIdentity] = None # below are optional callbacks to handle payloads # receives the payload + HiveMindClient that sent it escalate_callback = None # slave asked to escalate payload @@ -246,7 +248,8 @@ class HiveMindListenerProtocol: mycroft_bus_callback = None # slave asked to inject payload into mycroft bus shared_bus_callback = None # passive sharing of slave device bus (info) - def bind(self, websocket, bus): + def bind(self, websocket, bus, identity): + self.identity = identity websocket.protocol = self self.internal_protocol = HiveMindListenerInternalProtocol(bus) self.internal_protocol.register_bus_handlers() @@ -367,6 +370,8 @@ def handle_message(self, message: HiveMessage, client: HiveMindClientConnection) self.handle_broadcast_message(message, client) elif message.msg_type == HiveMessageType.ESCALATE: self.handle_escalate_message(message, client) + elif message.msg_type == HiveMessageType.INTERCOM: + self.handle_intercom_message(message, client) elif message.msg_type == HiveMessageType.BINARY: self.handle_binary_message(message, client) else: @@ -476,6 +481,16 @@ def handle_broadcast_message( if self.broadcast_callback: self.broadcast_callback(payload) + if message.payload.msg_type == HiveMessageType.INTERCOM: + if self.handle_intercom_message(message.payload, client): + return + + if message.payload.msg_type == HiveMessageType.BUS: + # if the message targets our site_id, send it to internal bus + site = message.target_site_id + if site and site == self.identity.site_id: + self.handle_bus_message(message.payload, client) + # broadcast message to other peers payload = self._unpack_message(message, client) for peer in self.clients: @@ -514,6 +529,16 @@ def handle_propagate_message( if self.propagate_callback: self.propagate_callback(payload) + if message.payload.msg_type == HiveMessageType.INTERCOM: + if self.handle_intercom_message(message.payload, client): + return + + if message.payload.msg_type == HiveMessageType.BUS: + # if the message targets our site_id, send it to internal bus + site = message.target_site_id + if site and site == self.identity.site_id: + self.handle_bus_message(message.payload, client) + # propagate message to other peers for peer in self.clients: if peer == client.peer: @@ -557,6 +582,16 @@ def handle_escalate_message( if self.escalate_callback: self.escalate_callback(payload) + if message.payload.msg_type == HiveMessageType.INTERCOM: + if self.handle_intercom_message(message.payload, client): + return + + if message.payload.msg_type == HiveMessageType.BUS: + # if the message targets our site_id, send it to internal bus + site = message.target_site_id + if site and site == self.identity.site_id: + self.handle_bus_message(message.payload, client) + # send to other masters message = Message( "hive.send.upstream", @@ -570,6 +605,54 @@ def handle_escalate_message( bus = self.get_bus(client) bus.emit(message) + def handle_intercom_message( + self, message: HiveMessage, client: HiveMindClientConnection + ) -> bool: + + # if the message targets us, send it to internal bus + k = message.target_public_key + if k and k != self.identity.public_key: + # not for us + return False + + pload = message.payload + if isinstance(pload, dict) and "ciphertext" in pload: + try: + message_from_blob = pgpy.PGPMessage.from_blob(pload["ciphertext"]) + + with open(self.identity.private_key, "r") as f: + private_key = pgpy.PGPKey.from_blob(f.read()) + + decrypted: str = private_key.decrypt(message_from_blob) + message._payload = HiveMessage.deserialize(decrypted) + except: + if k: + LOG.error("failed to decrypt message!") + else: + LOG.debug("failed to decrypt message, not for us") + return False + + if message.msg_type == HiveMessageType.BUS: + self.handle_bus_message(message, client) + return True + elif message.msg_type == HiveMessageType.PROPAGATE: + self.handle_propagate_message(message, client) + return True + elif message.msg_type == HiveMessageType.BROADCAST: + self.handle_broadcast_message(message, client) + return True + elif message.msg_type == HiveMessageType.ESCALATE: + self.handle_escalate_message(message, client) + return True + elif message.msg_type == HiveMessageType.BINARY: + self.handle_binary_message(message, client) + return True + elif message.msg_type == HiveMessageType.SHARED_BUS: + self.handle_client_bus(message.payload, client) + return True + + return False + # HiveMind mycroft bus messages - from slave -> master def handle_inject_mycroft_msg( self, message: Message, client: HiveMindClientConnection diff --git a/hivemind_core/service.py b/hivemind_core/service.py index bd50f97..0f857d6 100644 --- a/hivemind_core/service.py +++ b/hivemind_core/service.py @@ -254,7 +254,7 @@ def run(self): loop = ioloop.IOLoop.current() self.protocol = self._proto(loop=loop) - self.protocol.bind(self._ws_handler, self.bus) + self.protocol.bind(self._ws_handler, self.bus, self.identity) self.status.bind(self.bus) self.status.set_started() diff --git a/requirements.txt b/requirements.txt index fb116e4..2014088 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,12 @@ tornado -ovos_utils>=0.0.33 -pycryptodomex -HiveMind_presence>=0.0.2a3 -ovos-bus-client>=0.0.6a5 -poorman_handshake>=0.1.0 click click_default_group rich pyOpenSSL -hivemind-ggwave \ No newline at end of file +pycryptodomex +poorman_handshake>=0.1.0 +hivemind-ggwave +hivemind_bus_client>=0.0.4a25 +HiveMind_presence>=0.0.2a3 +ovos_utils>=0.0.33 +ovos-bus-client>=0.0.6a5 \ No newline at end of file From 0a1b72321984c32e05ede6f0fa3dfb048783cd17 Mon Sep 17 00:00:00 2001 From: JarbasAl Date: Wed, 5 Jun 2024 01:50:17 +0000 Subject: [PATCH 23/29] Increment Version --- CHANGELOG.md | 10 +++++++++- hivemind_core/version.py | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b0f957..a2d9be1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,15 @@ ## [Unreleased](https://github.com/JarbasHiveMind/HiveMind-core/tree/HEAD) -[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.1a14...HEAD) +[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.1a15...HEAD) + +**Implemented enhancements:** + +- feat/intercom [\#86](https://github.com/JarbasHiveMind/HiveMind-core/pull/86) ([JarbasAl](https://github.com/JarbasAl)) + +## [V0.13.1a15](https://github.com/JarbasHiveMind/HiveMind-core/tree/V0.13.1a15) (2024-06-05) + +[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.1a14...V0.13.1a15) **Implemented enhancements:** diff --git a/hivemind_core/version.py b/hivemind_core/version.py index 4eeefb8..83ab182 100644 --- a/hivemind_core/version.py +++ b/hivemind_core/version.py @@ -3,5 +3,5 @@ VERSION_MAJOR = 0 VERSION_MINOR = 13 VERSION_BUILD = 1 -VERSION_ALPHA = 15 +VERSION_ALPHA = 16 # END_VERSION_BLOCK From eef5b14e27ca754b573ac056bba476c612675dd9 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Mon, 1 Jul 2024 14:50:52 +0100 Subject: [PATCH 24/29] fix/91 (#92) closes https://github.com/JarbasHiveMind/HiveMind-core/issues/91 --- hivemind_core/protocol.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hivemind_core/protocol.py b/hivemind_core/protocol.py index 834177e..f4bad8d 100644 --- a/hivemind_core/protocol.py +++ b/hivemind_core/protocol.py @@ -457,7 +457,10 @@ def handle_bus_message( LOG.debug(f"Client session updated: {client.sess.serialize()}") if old != client.peer: LOG.debug(f"Client session_id changed! new peer_id: {client.peer}") - self.clients[client.peer] = self.clients.pop(old) + if old in self.clients: + self.clients[client.peer] = self.clients.pop(old) + else: + self.clients[client.peer] = client self.handle_inject_mycroft_msg(message.payload, client) if self.mycroft_bus_callback: From 3c5540c8f478ec866b74dd457abfac3491715ee3 Mon Sep 17 00:00:00 2001 From: JarbasAl Date: Mon, 1 Jul 2024 13:51:29 +0000 Subject: [PATCH 25/29] Increment Version --- CHANGELOG.md | 14 +++++++++++++- hivemind_core/version.py | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a2d9be1..d039862 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,19 @@ ## [Unreleased](https://github.com/JarbasHiveMind/HiveMind-core/tree/HEAD) -[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.1a15...HEAD) +[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.1a16...HEAD) + +**Fixed bugs:** + +- fix/91 [\#92](https://github.com/JarbasHiveMind/HiveMind-core/pull/92) ([JarbasAl](https://github.com/JarbasAl)) + +**Closed issues:** + +- KeyError: 'VoiceSatelliteV0.3.0:10.233.100.128::default' [\#91](https://github.com/JarbasHiveMind/HiveMind-core/issues/91) + +## [V0.13.1a16](https://github.com/JarbasHiveMind/HiveMind-core/tree/V0.13.1a16) (2024-06-05) + +[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.1a15...V0.13.1a16) **Implemented enhancements:** diff --git a/hivemind_core/version.py b/hivemind_core/version.py index 83ab182..cd60a46 100644 --- a/hivemind_core/version.py +++ b/hivemind_core/version.py @@ -3,5 +3,5 @@ VERSION_MAJOR = 0 VERSION_MINOR = 13 VERSION_BUILD = 1 -VERSION_ALPHA = 16 +VERSION_ALPHA = 17 # END_VERSION_BLOCK From 3ad42de4c928228ca8a0b582dda3af37e08b3a51 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Mon, 1 Jul 2024 17:59:14 +0100 Subject: [PATCH 26/29] fix/session_mapping (#94) * fix/session_mapping ensure session is tracked properly per client, allow it to be sent in initial handshake * fix/session_mapping ensure session is tracked properly per client, allow it to be sent in initial handshake * fix/session_mapping ensure session is tracked properly per client, allow it to be sent in initial handshake * fix/session_mapping ensure session is tracked properly per client, allow it to be sent in initial handshake --- hivemind_core/protocol.py | 40 +++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/hivemind_core/protocol.py b/hivemind_core/protocol.py index f4bad8d..3a604a8 100644 --- a/hivemind_core/protocol.py +++ b/hivemind_core/protocol.py @@ -1,4 +1,5 @@ import json +import uuid from dataclasses import dataclass, field from enum import Enum, IntEnum from typing import List, Dict, Optional @@ -260,7 +261,6 @@ def get_bus(self, client: HiveMindClientConnection): def handle_new_client(self, client: HiveMindClientConnection): LOG.debug(f"new client: {client.peer}") - self.clients[client.peer] = client message = Message( "hive.client.connect", {"ip": client.ip, "session_id": client.sess.session_id}, @@ -398,6 +398,8 @@ def handle_handshake_message( ): LOG.debug("handshake received, generating session key") payload = message.payload + if "session" in payload: + client.sess = Session.deserialize(payload["session"]) if "site_id" in payload: client.sess.site_id = client.site_id = payload["site_id"] if "pubkey" in payload and client.handshake is not None: @@ -445,22 +447,36 @@ def handle_handshake_message( client.socket.close() return + LOG.debug(f"client site_id: {client.sess.site_id}") + if client.sess.session_id != "default": + LOG.debug(f"client session_id: {client.sess.session_id}") + self.clients[client.peer] = client + else: + LOG.warning("client did not send a session in it's handshake") + msg = HiveMessage(HiveMessageType.HANDSHAKE, payload) client.send(msg) # client can recreate crypto_key on his side now def handle_bus_message( self, message: HiveMessage, client: HiveMindClientConnection ): - # update the session as received by the client - old = client.peer - client.sess = Session.from_message(message.payload) - LOG.debug(f"Client session updated: {client.sess.serialize()}") - if old != client.peer: - LOG.debug(f"Client session_id changed! new peer_id: {client.peer}") - if old in self.clients: - self.clients[client.peer] = self.clients.pop(old) + # track any Session updates from client side + sess = Session.from_message(message.payload) + if client.sess.session_id == "default": + LOG.warning(f"{client.peer} did not send a Session via handshake") + if sess.session_id == "default": + client.sess.session_id = str(uuid.uuid4()) + LOG.debug(f"Client session_id randomly generated: {client.sess.session_id}") else: - self.clients[client.peer] = client + client.sess.session_id = sess.session_id + LOG.debug(f"Client session_id assigned via client first message: {client.sess.session_id}") + self.clients[client.peer] = client + + if sess.session_id == "default": + sess.session_id = client.sess.session_id + if client.sess.session_id == sess.session_id: + client.sess = sess + LOG.debug(f"Client session updated from payload: {sess.serialize()}") self.handle_inject_mycroft_msg(message.payload, client) if self.mycroft_bus_callback: @@ -672,8 +688,8 @@ def handle_inject_mycroft_msg( return # ensure client specific session data is injected in query to ovos - if "session" not in message.context: - message.context["session"] = client.sess.serialize() + LOG.debug("replacing message metadata with hivemind client session") + message.context["session"] = client.sess.serialize() if message.msg_type == "speak": message.context["destination"] = ["audio"] # make audible, this is injected "speak" command elif message.context.get("destination") is None: From 204bd399f802fe49929b3f0f7d1914990f6c26a6 Mon Sep 17 00:00:00 2001 From: JarbasAl Date: Mon, 1 Jul 2024 16:59:54 +0000 Subject: [PATCH 27/29] Increment Version --- CHANGELOG.md | 20 ++++++++++++++------ hivemind_core/version.py | 2 +- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d039862..4627f95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,15 @@ ## [Unreleased](https://github.com/JarbasHiveMind/HiveMind-core/tree/HEAD) -[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.1a16...HEAD) +[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.1a17...HEAD) + +**Fixed bugs:** + +- fix/session\_mapping [\#94](https://github.com/JarbasHiveMind/HiveMind-core/pull/94) ([JarbasAl](https://github.com/JarbasAl)) + +## [V0.13.1a17](https://github.com/JarbasHiveMind/HiveMind-core/tree/V0.13.1a17) (2024-07-01) + +[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.1a16...V0.13.1a17) **Fixed bugs:** @@ -146,15 +154,15 @@ ## [V0.13.0a4](https://github.com/JarbasHiveMind/HiveMind-core/tree/V0.13.0a4) (2023-08-03) -[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.0a2...V0.13.0a4) +[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.0a3...V0.13.0a4) -## [V0.13.0a2](https://github.com/JarbasHiveMind/HiveMind-core/tree/V0.13.0a2) (2023-08-03) +## [V0.13.0a3](https://github.com/JarbasHiveMind/HiveMind-core/tree/V0.13.0a3) (2023-08-03) -[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.0a3...V0.13.0a2) +[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.0a2...V0.13.0a3) -## [V0.13.0a3](https://github.com/JarbasHiveMind/HiveMind-core/tree/V0.13.0a3) (2023-08-03) +## [V0.13.0a2](https://github.com/JarbasHiveMind/HiveMind-core/tree/V0.13.0a2) (2023-08-03) -[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.11.0a3...V0.13.0a3) +[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.11.0a3...V0.13.0a2) **Breaking changes:** diff --git a/hivemind_core/version.py b/hivemind_core/version.py index cd60a46..feb0ee5 100644 --- a/hivemind_core/version.py +++ b/hivemind_core/version.py @@ -3,5 +3,5 @@ VERSION_MAJOR = 0 VERSION_MINOR = 13 VERSION_BUILD = 1 -VERSION_ALPHA = 17 +VERSION_ALPHA = 18 # END_VERSION_BLOCK From 16eee1136878632030369962bb37362cfe2c12e0 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Fri, 5 Jul 2024 20:24:48 +0100 Subject: [PATCH 28/29] feat/skill_intent_blacklist_per_client (#89) * feat/skill_intent_blacklist_per_client * add script blacklist-intent blacklist intents from being triggered by a client blacklist-skill blacklist skills from being triggered by a client * unblacklist commands * dont require restart --- hivemind_core/protocol.py | 42 +++++-- hivemind_core/scripts.py | 226 ++++++++++++++++++++++++++++++++++++-- hivemind_core/service.py | 6 +- 3 files changed, 258 insertions(+), 16 deletions(-) diff --git a/hivemind_core/protocol.py b/hivemind_core/protocol.py index 3a604a8..e58cc49 100644 --- a/hivemind_core/protocol.py +++ b/hivemind_core/protocol.py @@ -3,8 +3,8 @@ from dataclasses import dataclass, field from enum import Enum, IntEnum from typing import List, Dict, Optional -import pgpy +import pgpy from ovos_bus_client import MessageBusClient from ovos_bus_client.message import Message from ovos_bus_client.session import Session @@ -13,15 +13,16 @@ from tornado import ioloop from tornado.websocket import WebSocketHandler +from hivemind_bus_client.identity import NodeIdentity from hivemind_bus_client.message import HiveMessage, HiveMessageType from hivemind_bus_client.serialization import decode_bitstring, get_bitstring -from hivemind_bus_client.identity import NodeIdentity from hivemind_bus_client.util import ( decrypt_bin, encrypt_bin, decrypt_from_json, encrypt_as_json, ) +from hivemind_core.database import ClientDatabase class ProtocolVersion(IntEnum): @@ -63,9 +64,15 @@ class HiveMindClientConnection: pswd_handshake: Optional[PasswordHandShake] = None socket: Optional[WebSocketHandler] = None crypto_key: Optional[str] = None - blacklist: List[str] = field( + msg_blacklist: List[str] = field( default_factory=list ) # list of ovos message_type to never be sent to this client + skill_blacklist: List[str] = field( + default_factory=list + ) # list of skill_id that can't match for this client + intent_blacklist: List[str] = field( + default_factory=list + ) # list of skill_id:intent_name that can't match for this client allowed_types: List[str] = field( default_factory=list ) # list of ovos message_type to allow to be sent from this client @@ -88,7 +95,7 @@ def send(self, message: HiveMessage): else: _msg_type = message.payload.msg_type - if _msg_type in self.blacklist: + if _msg_type in self.msg_blacklist: return LOG.debug( f"message type {_msg_type} " f"is blacklisted for {self.peer}" ) @@ -673,8 +680,30 @@ def handle_intercom_message( return False # HiveMind mycroft bus messages - from slave -> master + def _update_blacklist(self, message: Message, client: HiveMindClientConnection): + LOG.debug("replacing message metadata with hivemind client session") + message.context["session"] = client.sess.serialize() + + # update blacklist from db, to account for changes without requiring a restart + with ClientDatabase() as users: + user = users.get_client_by_api_key(client.key) + client.skill_blacklist = user.blacklist.get("skills", []) + client.intent_blacklist = user.blacklist.get("intents", []) + + # inject client specific blacklist into session + if "blacklisted_skills" not in message.context["session"]: + message.context["session"]["blacklisted_skills"] = [] + if "blacklisted_intents" not in message.context["session"]: + message.context["session"]["blacklisted_intents"] = [] + + message.context["session"]["blacklisted_skills"] += [s for s in client.skill_blacklist + if s not in message.context["session"]["blacklisted_skills"]] + message.context["session"]["blacklisted_intents"] += [s for s in client.intent_blacklist + if s not in message.context["session"]["blacklisted_intents"]] + return message + def handle_inject_mycroft_msg( - self, message: Message, client: HiveMindClientConnection + self, message: Message, client: HiveMindClientConnection ): """ message (Message): mycroft bus message object @@ -688,8 +717,7 @@ def handle_inject_mycroft_msg( return # ensure client specific session data is injected in query to ovos - LOG.debug("replacing message metadata with hivemind client session") - message.context["session"] = client.sess.serialize() + message = self._update_blacklist(message, client) if message.msg_type == "speak": message.context["destination"] = ["audio"] # make audible, this is injected "speak" command elif message.context.get("destination") is None: diff --git a/hivemind_core/scripts.py b/hivemind_core/scripts.py index bd44ace..d3f1f5b 100644 --- a/hivemind_core/scripts.py +++ b/hivemind_core/scripts.py @@ -187,13 +187,13 @@ def list_clients(): default="hivemind", ) def listen( - ovos_bus_address: str, - ovos_bus_port: int, - host: str, - port: int, - ssl: bool, - cert_dir: str, - cert_name: str, + ovos_bus_address: str, + ovos_bus_port: int, + host: str, + port: int, + ssl: bool, + cert_dir: str, + cert_name: str, ): from hivemind_core.service import HiveMindService @@ -216,5 +216,217 @@ def listen( service.run() +@hmcore_cmds.command(help="blacklist skills from being triggered by a client", name="blacklist-skill") +@click.argument("skill_id", required=True, type=str) +@click.argument("node_id", required=False, type=int) +def blacklist_skill(skill_id, node_id): + if not node_id: + # list clients and prompt for id using rich + table = Table(title="HiveMind Clients") + table.add_column("ID", justify="right", style="cyan", no_wrap=True) + table.add_column("Name", style="magenta") + table.add_column("Allowed Msg Types", style="yellow") + _choices = [] + for client in ClientDatabase(): + if client["client_id"] != -1: + table.add_row( + str(client["client_id"]), + client["name"], + str(client.get("allowed_types", [])), + ) + _choices.append(str(client["client_id"])) + + if not _choices: + print("No clients found!") + exit() + elif len(_choices) > 1: + console = Console() + console.print(table) + _exit = str(max(int(i) for i in _choices) + 1) + node_id = Prompt.ask( + f"To which client you want to blacklist '{skill_id}'? ({_exit}='Exit')", + choices=_choices + [_exit], + ) + if node_id == _exit: + console.print("User exit", style="red") + exit() + else: + node_id = _choices[0] + + with ClientDatabase() as db: + for client in db: + if client["client_id"] == int(node_id): + blacklist = client.get("blacklist", {"messages": [], "skills": [], "intents": []}) + if skill_id in blacklist["skills"]: + print(f"Client {client['name']} already blacklisted '{skill_id}'") + exit() + + blacklist["skills"].append(skill_id) + client["blacklist"] = blacklist + item_id = db.get_item_id(client) + db.update_item(item_id, client) + print(f"Blacklisted '{skill_id}' for {client['name']}") + break + + +@hmcore_cmds.command(help="remove skills from a client blacklist", name="unblacklist-skill") +@click.argument("skill_id", required=True, type=str) +@click.argument("node_id", required=False, type=int) +def unblacklist_skill(skill_id, node_id): + if not node_id: + # list clients and prompt for id using rich + table = Table(title="HiveMind Clients") + table.add_column("ID", justify="right", style="cyan", no_wrap=True) + table.add_column("Name", style="magenta") + table.add_column("Allowed Msg Types", style="yellow") + _choices = [] + for client in ClientDatabase(): + if client["client_id"] != -1: + table.add_row( + str(client["client_id"]), + client["name"], + str(client.get("allowed_types", [])), + ) + _choices.append(str(client["client_id"])) + + if not _choices: + print("No clients found!") + exit() + elif len(_choices) > 1: + console = Console() + console.print(table) + _exit = str(max(int(i) for i in _choices) + 1) + node_id = Prompt.ask( + f"To which client you want to blacklist '{skill_id}'? ({_exit}='Exit')", + choices=_choices + [_exit], + ) + if node_id == _exit: + console.print("User exit", style="red") + exit() + else: + node_id = _choices[0] + + with ClientDatabase() as db: + for client in db: + if client["client_id"] == int(node_id): + blacklist = client.get("blacklist", {"messages": [], "skills": [], "intents": []}) + if skill_id not in blacklist["skills"]: + print(f"'{skill_id}' is not blacklisted for client {client['name']}") + exit() + + blacklist["skills"].pop(skill_id) + client["blacklist"] = blacklist + item_id = db.get_item_id(client) + db.update_item(item_id, client) + print(f"Blacklisted '{skill_id}' for {client['name']}") + break + + +@hmcore_cmds.command(help="blacklist intents from being triggered by a client", name="blacklist-intent") +@click.argument("intent_id", required=True, type=str) +@click.argument("node_id", required=False, type=int) +def blacklist_intent(intent_id, node_id): + if not node_id: + # list clients and prompt for id using rich + table = Table(title="HiveMind Clients") + table.add_column("ID", justify="right", style="cyan", no_wrap=True) + table.add_column("Name", style="magenta") + table.add_column("Allowed Msg Types", style="yellow") + _choices = [] + for client in ClientDatabase(): + if client["client_id"] != -1: + table.add_row( + str(client["client_id"]), + client["name"], + str(client.get("allowed_types", [])), + ) + _choices.append(str(client["client_id"])) + + if not _choices: + print("No clients found!") + exit() + elif len(_choices) > 1: + console = Console() + console.print(table) + _exit = str(max(int(i) for i in _choices) + 1) + node_id = Prompt.ask( + f"To which client you want to blacklist '{intent_id}'? ({_exit}='Exit')", + choices=_choices + [_exit], + ) + if node_id == _exit: + console.print("User exit", style="red") + exit() + else: + node_id = _choices[0] + + with ClientDatabase() as db: + for client in db: + if client["client_id"] == int(node_id): + blacklist = client.get("blacklist", {"messages": [], "skills": [], "intents": []}) + if intent_id in blacklist["intents"]: + print(f"Client {client['name']} already blacklisted '{intent_id}'") + exit() + + blacklist["intents"].append(intent_id) + client["blacklist"] = blacklist + item_id = db.get_item_id(client) + db.update_item(item_id, client) + print(f"Blacklisted '{intent_id}' for {client['name']}") + break + + +@hmcore_cmds.command(help="remove intents from a client blacklist", name="unblacklist-intent") +@click.argument("intent_id", required=True, type=str) +@click.argument("node_id", required=False, type=int) +def unblacklist_intent(intent_id, node_id): + if not node_id: + # list clients and prompt for id using rich + table = Table(title="HiveMind Clients") + table.add_column("ID", justify="right", style="cyan", no_wrap=True) + table.add_column("Name", style="magenta") + table.add_column("Allowed Msg Types", style="yellow") + _choices = [] + for client in ClientDatabase(): + if client["client_id"] != -1: + table.add_row( + str(client["client_id"]), + client["name"], + str(client.get("allowed_types", [])), + ) + _choices.append(str(client["client_id"])) + + if not _choices: + print("No clients found!") + exit() + elif len(_choices) > 1: + console = Console() + console.print(table) + _exit = str(max(int(i) for i in _choices) + 1) + node_id = Prompt.ask( + f"To which client you want to blacklist '{intent_id}'? ({_exit}='Exit')", + choices=_choices + [_exit], + ) + if node_id == _exit: + console.print("User exit", style="red") + exit() + else: + node_id = _choices[0] + + with ClientDatabase() as db: + for client in db: + if client["client_id"] == int(node_id): + blacklist = client.get("blacklist", {"messages": [], "skills": [], "intents": []}) + if intent_id not in blacklist["intents"]: + print(f" '{intent_id}' not blacklisted for Client {client['name']} ") + exit() + + blacklist["intents"].pop(intent_id) + client["blacklist"] = blacklist + item_id = db.get_item_id(client) + db.update_item(item_id, client) + print(f"Blacklisted '{intent_id}' for {client['name']}") + break + + if __name__ == "__main__": hmcore_cmds() diff --git a/hivemind_core/service.py b/hivemind_core/service.py index 0f857d6..b070952 100644 --- a/hivemind_core/service.py +++ b/hivemind_core/service.py @@ -130,7 +130,7 @@ def open(self): name=name, ip=self.request.remote_ip, socket=self, - sess=Session(session_id="default"), # will be re-assigned once client sends it's own + sess=Session(session_id="default"), # will be re-assigned once client sends handshake handshake=handshake, loop=self.protocol.loop, ) @@ -144,7 +144,9 @@ def open(self): return self.client.crypto_key = user.crypto_key - self.client.blacklist = user.blacklist.get("messages", []) + self.client.msg_blacklist = user.blacklist.get("messages", []) + self.client.skill_blacklist = user.blacklist.get("skills", []) + self.client.intent_blacklist = user.blacklist.get("intents", []) self.client.allowed_types = user.allowed_types self.client.can_broadcast = user.can_broadcast self.client.can_propagate = user.can_propagate From 7b483b5c58d86113e1cc79d0d9493dc3b5fab382 Mon Sep 17 00:00:00 2001 From: JarbasAl Date: Fri, 5 Jul 2024 19:25:25 +0000 Subject: [PATCH 29/29] Increment Version --- CHANGELOG.md | 10 +++++++++- hivemind_core/version.py | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4627f95..7401997 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,15 @@ ## [Unreleased](https://github.com/JarbasHiveMind/HiveMind-core/tree/HEAD) -[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.1a17...HEAD) +[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.1a18...HEAD) + +**Implemented enhancements:** + +- feat/skill\_intent\_blacklist\_per\_client [\#89](https://github.com/JarbasHiveMind/HiveMind-core/pull/89) ([JarbasAl](https://github.com/JarbasAl)) + +## [V0.13.1a18](https://github.com/JarbasHiveMind/HiveMind-core/tree/V0.13.1a18) (2024-07-01) + +[Full Changelog](https://github.com/JarbasHiveMind/HiveMind-core/compare/V0.13.1a17...V0.13.1a18) **Fixed bugs:** diff --git a/hivemind_core/version.py b/hivemind_core/version.py index feb0ee5..c62325b 100644 --- a/hivemind_core/version.py +++ b/hivemind_core/version.py @@ -3,5 +3,5 @@ VERSION_MAJOR = 0 VERSION_MINOR = 13 VERSION_BUILD = 1 -VERSION_ALPHA = 18 +VERSION_ALPHA = 19 # END_VERSION_BLOCK