From fd550ee5646b8d244d20b7ad028bc01640386310 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Tue, 24 Aug 2021 15:40:53 +0100 Subject: [PATCH 1/3] Onion-based message channels with directory nodes Joinmarket bots run their own onion services allowing inbound connections. Both takers and makers connect to other makers at the mentioned onion services, over Tor. Directory nodes run persistent onion services allowing peers to find other (maker) peers to connect to, and also forwarding messages where necessary. This is implemented as an alternative to IRC, i.e. a new implementation of the abstract class MessageChannel, in onionmc.py. Note that using both this *and* IRC servers is supported; Joinmarket supports multiple, redundant different communication methods, simultaneously. Messaging is done with a derived class of twisted's LineReceiver, and there is an additional layer of syntax, similar to but not the same as the IRC syntax for ensuring that messages are passed with the same J5.. nick as is used on IRC. This allows us to keep the message signing logic the same as before. As well as Joinmarket line messages, we use additional control messages to communicate peer lists, and to manage connections. Peers which send messages not conforming to the syntax are dropped. See https://github.com/JoinMarket-Org/JoinMarket-Docs/pull/12 for documentation of the syntax. Connections to directory nodes are robust as for IRC servers, in that we use a ReconnectingClientFactory to keep trying to re-establish broken connections with exponential backoff. Connections to maker peers do not require this feature, as they will often disconnect in normal operation. Multiple directory nodes can and should be configured by bots. --- docs/onion-message-channels.md | 173 ++++ jmbase/jmbase/twisted_utils.py | 77 +- jmclient/jmclient/__init__.py | 2 +- jmclient/jmclient/client_protocol.py | 14 +- jmclient/jmclient/configure.py | 132 ++- jmclient/jmclient/wallet_rpc.py | 10 +- jmdaemon/jmdaemon/__init__.py | 1 + jmdaemon/jmdaemon/daemon_protocol.py | 18 +- jmdaemon/jmdaemon/message_channel.py | 6 +- jmdaemon/jmdaemon/onionmc.py | 1179 +++++++++++++++++++++++++ jmdaemon/test/test_daemon_protocol.py | 4 +- jmdaemon/test/test_irc_messaging.py | 6 +- jmdaemon/test/test_orderbookwatch.py | 4 +- scripts/obwatch/ob-watcher.py | 4 +- test/e2e-coinjoin-test.py | 364 ++++++++ test/regtest_joinmarket.cfg | 42 +- test/ygrunner.py | 3 +- 17 files changed, 1945 insertions(+), 94 deletions(-) create mode 100644 docs/onion-message-channels.md create mode 100644 jmdaemon/jmdaemon/onionmc.py create mode 100644 test/e2e-coinjoin-test.py diff --git a/docs/onion-message-channels.md b/docs/onion-message-channels.md new file mode 100644 index 000000000..deb5af6cb --- /dev/null +++ b/docs/onion-message-channels.md @@ -0,0 +1,173 @@ +# HOW TO SETUP ONION MESSAGE CHANNELS IN JOINMARKET + +### Contents + +1. [Overview](#overview) + +2. [Testing, configuring for signet](#testing) + +4. [Directory nodes](#directory) + + + +## Overview + +This is a new way for Joinmarket bots to communicate, namely by serving and connecting to Tor onion services. This does not +introduce any new requirements to your Joinmarket installation, technically, because the use of Payjoin already required the need +to service such onion services, and connecting to IRC used a SOCKS5 proxy (by default, and used by almost all users) over Tor to +a remote onion service. + +The purpose of this new type of message channel is as follows: + +* less reliance on any service external to Joinmarket +* most of the transaction negotiation will be happening directly peer to peer, not passed over a central server ( +albeit it was and remains E2E encrypted data, in either case) +* the above can lead to better scalability at large numbers +* a substantial increase in the speed of transaction negotiation; this is mostly related to the throttling of high bursts of traffic on IRC + +The configuration for a user is simple; in their `joinmarket.cfg` they will add a messaging section like this: + +``` +[MESSAGING:onion1] +type = onion +onion_serving_port = 8082 +# This is a comma separated list (comma can be omitted if only one item). +# Each item has format host:port +directory_nodes = rr6f6qtleiiwic45bby4zwmiwjrj3jsbmcvutwpqxjziaydjydkk5iad.onion:80 +``` + +Here, I have deliberately omitted the several other settings in this section which will almost always be fine as default; +see `jmclient/jmclient/configure.py` for what those defaults are, and the extensive comments explaining. + +The main point is the list of **directory nodes** (the one shown here is one being run on signet, right now), which will +be comma separated if multiple directory nodes are configured (we expect there will be 2 or 3 as a normal situation). +The `onion_serving_port` is on which port on the local machine the onion service is served. +The `type` field must always be `onion` in this case, and distinguishes it from IRC message channels and others. + +### Can/should I still run IRC message channels? + +In short, yes. + +### Do I need to configure Tor, and if so, how? + +These message channels use both outbound and inbound connections to onion services (or "hidden services"). + +As previously mentioned, both of these features were already in use in Joinmarket. If you never served an +onion service before, it should work fine as long as you have the Tor service running in the background, +and the default control port 9051 (if not, change that value in the `joinmarket.cfg`, see above. + +#### Why not use Lightning based onions? + +(*Feel free to skip this section if you don't know what "Lightning based onions" refers to!*). The reason this architecture is +proposed as an alternative to the previously suggested Lightning-node-based network (see +[this PR](https://github.com/JoinMarket-Org/joinmarket-clientserver/pull/1000)), is mostly that: + +* the latter has a bunch of extra installation and maintenance dependencies (just one example: pyln-client requires coincurve, which we just +removed) +* the latter requires establishing a new node "identity" which can be refreshed, but that creates more concern +* longer term ideas to integrate Lightning payments to the coinjoin workflow (and vice versa!) are not realizable yet +* using multi-hop onion messaging in the LN network itself is also a way off, and a bit problematic + +So the short version is: the Lightning based alternative is certainly feasible, but has a lot more baggage that can't really be justified +unless we're actually using it for something. + + + + +## Testing, and configuring for signet. + +This testing section focuses on signet since that will be the less troublesome way of getting involved in tests for +the non-hardcore JM developer :) + +(For the latter, please use the regtest setup by running `test/e2e-coinjoin-test.py` under `pytest`, +and pay attention to the settings in `regtest_joinmarket.cfg`.) + +There is no separate/special configuration for signet other than the configuration that is already needed for running +Joinmarket against a signet backend (so e.g. RPC port of 38332). + +Add the `[MESSAGING:onion1]` message channel section to your `joinmarket.cfg`, as listed above, including the +signet directory node listed above (rr6f6qtleiiwic45bby4zwmiwjrj3jsbmcvutwpqxjziaydjydkk5iad.onion:80), and, +for the simplest test, remove the other `[MESSAGING:*]` sections that you have. + +Then just make sure your bot has some signet coins and try running as maker or taker or both. + + + +## Directory nodes + +**This last section is for people with a lot of technical knowledge in this area, +who would like to help by running a directory node. You can ignore it if that does not apply.**. + +This requires a long running bot. It should be on a server you can keep running permanently, so perhaps a VPS, +but in any case, very high uptime. For reliability it also makes sense to configure to run as a systemd service. + +A note: in this early stage, the usage of Lightning is only really network-layer stuff, and the usage of bitcoin, is none; feel free to add elements that remove any need for a backend bitcoin blockchain, but beware: future upgrades *could* mean that the directory node really does need the bitcoin backend. + +#### Joinmarket-specific configuration + +Add `hidden_service_dir` to your `[MESSAGING:onion1]` with a directory accessible to your user. You may want to lock this down +a bit! +The point to understand is: Joinmarket's `jmbase.JMHiddenService` will, if configured with a non-empty `hidden_service_dir` +field, actually start an *independent* instance of Tor specifically for serving this, under the current user. +(our tor interface library `txtorcon` needs read access to the Tor HS dir, so it's troublesome to do this another way). + +##### Question: How to configure the `directory-nodes` list in our `joinmarket.cfg` for this directory node bot? + +Answer: **you must only enter your own node in this list!** (otherwise you may find your bot infinitely rebroadcasting messages). + + +#### Suggested setup of a service: + +You will need two components: bitcoind, and Joinmarket itself, which you can run as a yg. +Since this task is going to be attempted by someone with significant technical knowledge, +only an outline is provided here; several details will need to be filled in. +Here is a sketch of how the systemd service files can be set up for signet: + +If someone wants to put together a docker setup of this for a more "one-click install", that would be great. + +1. bitcoin-signet.service + +``` +[Unit] +Description=bitcoind signet +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +ExecStart=/usr/local/bin/bitcoind -signet +User=user + +[Install] +WantedBy=multi-user.target +``` + +This is deliberately a super-basic setup (see above). Don't forget to setup your `bitcoin.conf` as usual, +for the bitcoin user, and make it match (specifically in terms of RPC) what you set up for Lightning below. + + +2. + +``` +[Unit] +Description=joinmarket directory node on signet +Requires=bitcoin-signet.service +After=bitcoin-signet.service + +[Service] +Type=simple +ExecStart=/bin/bash -c 'cd /path/to/joinmarket-clientserver && source jmvenv/bin/activate && cd scripts && echo -n "password" | python yg-privacyenhanced.py --wallet-password-stdin --datadir=/custom/joinmarket-datadir some-signet-wallet.jmdat' +User=user + +[Install] +WantedBy=multi-user.target +``` + +To state the obvious, the idea here is that this second service will run the JM directory node and have a dependency on the previous one, +to ensure they start up in the correct order. + +Re: password echo, obviously this kind of password entry is bad; +for now we needn't worry as these nodes don't need to carry any real coins (and it's better they don't!). +Later we may need to change that (though of course you can use standard measures to protect the box). + +TODO: add some material on network hardening/firewalls here, I guess. diff --git a/jmbase/jmbase/twisted_utils.py b/jmbase/jmbase/twisted_utils.py index f7e2f287b..b7594d181 100644 --- a/jmbase/jmbase/twisted_utils.py +++ b/jmbase/jmbase/twisted_utils.py @@ -128,16 +128,23 @@ def config_to_hs_ports(virtual_port, host, port): class JMHiddenService(object): """ Wrapper class around the actions needed to create and serve on a hidden service; an object of - type Resource must be provided in the constructor, - which does the HTTP serving actions (GET, POST serving). + type either Resource or server.ProtocolFactory must + be provided in the constructor, which does the HTTP + (GET, POST) or other protocol serving actions. """ - def __init__(self, resource, info_callback, error_callback, - onion_hostname_callback, tor_control_host, + def __init__(self, proto_factory_or_resource, info_callback, + error_callback, onion_hostname_callback, tor_control_host, tor_control_port, serving_host, serving_port, - virtual_port = None, - shutdown_callback = None): - self.site = Site(resource) - self.site.displayTracebacks = False + virtual_port=None, + shutdown_callback=None, + hidden_service_dir=""): + if isinstance(proto_factory_or_resource, Resource): + # TODO bad naming, in this case it doesn't start + # out as a protocol factory; a Site is one, a Resource isn't. + self.proto_factory = Site(proto_factory_or_resource) + self.proto_factory.displayTracebacks = False + else: + self.proto_factory = proto_factory_or_resource self.info_callback = info_callback self.error_callback = error_callback # this has a separate callback for convenience, it should @@ -155,6 +162,13 @@ def __init__(self, resource, info_callback, error_callback, # config object, so no default here: self.serving_host = serving_host self.serving_port = serving_port + # this is used to serve an onion from the filesystem, + # NB: Because of how txtorcon is set up, this option + # uses a *separate tor instance* owned by the owner of + # this script (because txtorcon needs to read the + # HS dir), whereas if this option is "", we set up + # an ephemeral HS on the global or pre-existing tor. + self.hidden_service_dir = hidden_service_dir def start_tor(self): """ This function executes the workflow @@ -162,19 +176,31 @@ def start_tor(self): """ self.info_callback("Attempting to start onion service on port: {} " "...".format(self.virtual_port)) - if str(self.tor_control_host).startswith('unix:'): - control_endpoint = UNIXClientEndpoint(reactor, - self.tor_control_host[5:]) + if self.hidden_service_dir == "": + if str(self.tor_control_host).startswith('unix:'): + control_endpoint = UNIXClientEndpoint(reactor, + self.tor_control_host[5:]) + else: + control_endpoint = TCP4ClientEndpoint(reactor, + self.tor_control_host, self.tor_control_port) + d = txtorcon.connect(reactor, control_endpoint) + d.addCallback(self.create_onion_ep) + d.addErrback(self.setup_failed) + # TODO: add errbacks to the next two calls in + # the chain: + d.addCallback(self.onion_listen) + d.addCallback(self.print_host) else: - control_endpoint = TCP4ClientEndpoint(reactor, - self.tor_control_host, self.tor_control_port) - d = txtorcon.connect(reactor, control_endpoint) - d.addCallback(self.create_onion_ep) - d.addErrback(self.setup_failed) - # TODO: add errbacks to the next two calls in - # the chain: - d.addCallback(self.onion_listen) - d.addCallback(self.print_host) + ep = "onion:" + str(self.virtual_port) + ":localPort=" + ep += str(self.serving_port) + # endpoints.TCPHiddenServiceEndpoint creates version 2 by + # default for backwards compat (err, txtorcon needs to update that ...) + ep += ":version=3" + ep += ":hiddenServiceDir="+self.hidden_service_dir + onion_endpoint = serverFromString(reactor, ep) + d = onion_endpoint.listen(self.proto_factory) + d.addCallback(self.print_host_filesystem) + def setup_failed(self, arg): # Note that actions based on this failure are deferred to callers: @@ -195,7 +221,8 @@ def onion_listen(self, onion): serverstring = "tcp:{}:interface={}".format(self.serving_port, self.serving_host) onion_endpoint = serverFromString(reactor, serverstring) - return onion_endpoint.listen(self.site) + print("created the onion endpoint, now calling listen") + return onion_endpoint.listen(self.proto_factory) def print_host(self, ep): """ Callback fired once the HS is available @@ -204,6 +231,14 @@ def print_host(self, ep): """ self.onion_hostname_callback(self.onion.hostname) + def print_host_filesystem(self, port): + """ As above but needed to respect slightly different + callback chain for this case (where we start our own tor + instance for the filesystem-based onion). + """ + self.onion = port.onion_service + self.onion_hostname_callback(self.onion.hostname) + def shutdown(self): self.tor_connection.protocol.transport.loseConnection() self.info_callback("Hidden service shutdown complete") diff --git a/jmclient/jmclient/__init__.py b/jmclient/jmclient/__init__.py index e410638b6..ca91ace64 100644 --- a/jmclient/jmclient/__init__.py +++ b/jmclient/jmclient/__init__.py @@ -24,7 +24,7 @@ TYPE_P2PKH, TYPE_P2SH_P2WPKH, TYPE_P2WPKH, detect_script_type) from .configure import (load_test_config, process_shutdown, load_program_config, jm_single, get_network, update_persist_config, - validate_address, is_burn_destination, get_irc_mchannels, + validate_address, is_burn_destination, get_mchannels, get_blockchain_interface_instance, set_config, is_segwit_mode, is_native_segwit_mode, JMPluginService, get_interest_rate, get_bondless_makers_allowance) from .blockchaininterface import (BlockchainInterface, diff --git a/jmclient/jmclient/client_protocol.py b/jmclient/jmclient/client_protocol.py index 68ea865ac..01b00b8e7 100644 --- a/jmclient/jmclient/client_protocol.py +++ b/jmclient/jmclient/client_protocol.py @@ -15,7 +15,7 @@ import sys from jmbase import (get_log, EXIT_FAILURE, hextobin, bintohex, utxo_to_utxostr, bdict_sdict_convert) -from jmclient import (jm_single, get_irc_mchannels, +from jmclient import (jm_single, get_mchannels, RegtestBitcoinCoreInterface, SNICKERReceiver, process_shutdown) import jmbitcoin as btc @@ -434,7 +434,7 @@ def clientStart(self): "blockchain_source") #needed only for channel naming convention network = jm_single().config.get("BLOCKCHAIN", "network") - irc_configs = get_irc_mchannels() + irc_configs = self.factory.get_mchannels() #only here because Init message uses this field; not used by makers TODO minmakers = jm_single().config.getint("POLICY", "minimum_makers") maker_timeout_sec = jm_single().maker_timeout_sec @@ -601,7 +601,7 @@ def clientStart(self): "blockchain_source") #needed only for channel naming convention network = jm_single().config.get("BLOCKCHAIN", "network") - irc_configs = get_irc_mchannels() + irc_configs = self.factory.get_mchannels() minmakers = jm_single().config.getint("POLICY", "minimum_makers") maker_timeout_sec = jm_single().maker_timeout_sec @@ -795,6 +795,14 @@ def getClient(self): def buildProtocol(self, addr): return self.protocol(self, self.client) + def get_mchannels(self): + """ A transparent wrapper that allows override, + so that a script can return a customised set of + message channel configs; currently used for testing + multiple bots on regtest. + """ + return get_mchannels() + def start_reactor(host, port, factory=None, snickerfactory=None, bip78=False, jm_coinjoin=True, ish=True, daemon=False, rs=True, gui=False): #pragma: no cover diff --git a/jmclient/jmclient/configure.py b/jmclient/jmclient/configure.py index b12e194bf..0c3ff6b10 100644 --- a/jmclient/jmclient/configure.py +++ b/jmclient/jmclient/configure.py @@ -140,6 +140,9 @@ def jm_single(): ## SERVER 1/3) Darkscience IRC (Tor, IP) ################################################################################ [MESSAGING:server1] +# by default the legacy format without a `type` field is +# understood to be IRC, but you can, optionally, add it: +# type = irc channel = joinmarket-pit port = 6697 usessl = true @@ -154,24 +157,47 @@ def jm_single(): #socks5_host = localhost #socks5_port = 9050 -## SERVER 2/3) hackint IRC (Tor, IP) -################################################################################ -[MESSAGING:server2] -channel = joinmarket-pit +[MESSAGING:onion1] +# onion based message channels must have the exact type 'onion' +# (while the section name above can be MESSAGING:whatever), and there must +# be only ONE such message channel configured (note the directory servers +# can be multiple, below): +type = onion -# For traditional IP (default): -host = irc.hackint.org -port = 6697 -usessl = true -socks5 = false +socks5_host = localhost +socks5_port = 9050 -# For Tor (recommended as clearnet alternative): -#host = ncwkrwxpq2ikcngxq3dy2xctuheniggtqeibvgofixpzvrwpa77tozqd.onion -#port = 6667 -#usessl = false -#socks5 = true -#socks5_host = localhost -#socks5_port = 9050 +# the tor control configuration. +# for most people running the tor daemon +# on Linux, no changes are required here: +tor_control_host = localhost +# or, to use a UNIX socket +# tor_control_host = unix:/var/run/tor/control +tor_control_port = 9051 + +# the host/port actually serving the hidden service +# (note the *virtual port*, that the client uses, +# is hardcoded to 80): +onion_serving_host = 127.0.0.1 +onion_serving_port = 8080 + +# directory node configuration +# +# This is mandatory for directory nodes (who must also set their +# own *.onion:port as the only directory in directory_nodes, below), +# but NOT TO BE USED by non-directory nodes (which is you, unless +# you know otherwise!), as it will greatly degrade your privacy. +# (note the default is no value, don't replace it with ""). +hidden_service_dir = +# +# This is a comma separated list (comma can be omitted if only one item). +# Each item has format host:port ; both are required, though port will +# be 80 if created in this code. +directory_nodes = rr6f6qtleiiwic45bby4zwmiwjrj3jsbmcvutwpqxjziaydjydkk5iad.onion:80 + +# This setting is ONLY for developer regtest setups, +# running multiple bots at once. Don't alter it otherwise +regtest_count = 0,0 ## SERVER 3/3) ILITA IRC (Tor - disabled by default) ################################################################################ @@ -484,7 +510,7 @@ def set_config(cfg, bcint=None): global_singleton.bc_interface = bcint -def get_irc_mchannels(): +def get_mchannels(): SECTION_NAME = 'MESSAGING' # FIXME: remove in future release if jm_single().config.has_section(SECTION_NAME): @@ -495,16 +521,30 @@ def get_irc_mchannels(): return _get_irc_mchannels_old() SECTION_NAME += ':' - irc_sections = [] + sections = [] for s in jm_single().config.sections(): if s.startswith(SECTION_NAME): - irc_sections.append(s) - assert irc_sections + sections.append(s) + assert sections - req_fields = [("host", str), ("port", int), ("channel", str), ("usessl", str)] + irc_fields = [("host", str), ("port", int), ("channel", str), ("usessl", str), + ("socks5", str), ("socks5_host", str), ("socks5_port", str)] + onion_fields = [("type", str), ("directory_nodes", str), ("regtest_count", str), + ("socks5_host", str), ("socks5_port", int), + ("tor_control_host", str), ("tor_control_port", int), + ("onion_serving_host", str), ("onion_serving_port", int), + ("hidden_service_dir", str)] configs = [] - for section in irc_sections: + + # processing the IRC sections: + for section in sections: + if jm_single().config.has_option(section, "type"): + # legacy IRC configs do not have "type" but just + # in case, we'll allow the "irc" type: + if not jm_single().config.get(section, "type").lower( + ) == "irc": + break server_data = {} # check if socks5 is enabled for tor and load relevant config if so @@ -516,13 +556,30 @@ def get_irc_mchannels(): server_data["socks5_host"] = jm_single().config.get(section, "socks5_host") server_data["socks5_port"] = jm_single().config.get(section, "socks5_port") - for option, otype in req_fields: + for option, otype in irc_fields: val = jm_single().config.get(section, option) server_data[option] = otype(val) server_data['btcnet'] = get_network() configs.append(server_data) - return configs + # processing the onion sections: + for section in sections: + if not jm_single().config.has_option(section, "type") or \ + not jm_single().config.get(section, "type").lower() == "onion": + continue + onion_data = {} + for option, otype in onion_fields: + try: + val = jm_single().config.get(section, option) + except NoOptionError: + continue + onion_data[option] = otype(val) + onion_data['btcnet'] = get_network() + # Just to allow a dynamic set of var: + onion_data["section-name"] = section + configs.append(onion_data) + + return configs def _get_irc_mchannels_old(): fields = [("host", str), ("port", int), ("channel", str), ("usessl", str), @@ -651,28 +708,6 @@ def load_program_config(config_path="", bs=None, plugin_services=[]): "settings and restart joinmarket.", "info") sys.exit(EXIT_FAILURE) - #These are left as sanity checks but currently impossible - #since any edits are overlays to the default, these sections/options will - #always exist. - # FIXME: This check is a best-effort attempt. Certain incorrect section - # names can pass and so can non-first invalid sections. - for s in required_options: #pragma: no cover - # check for sections - avail = None - if not global_singleton.config.has_section(s): - for avail in global_singleton.config.sections(): - if avail.startswith(s): - break - else: - raise Exception( - "Config file does not contain the required section: " + s) - # then check for specific options - k = avail or s - for o in required_options[s]: - if not global_singleton.config.has_option(k, o): - raise Exception("Config file does not contain the required " - "option '{}' in section '{}'.".format(o, k)) - loglevel = global_singleton.config.get("LOGGING", "console_log_level") try: set_logging_level(loglevel) @@ -742,6 +777,11 @@ def load_program_config(config_path="", bs=None, plugin_services=[]): if not os.path.exists(plogsdir): os.makedirs(plogsdir) p.set_log_dir(plogsdir) + # Check if a onion message channel was configured, and if so, + # check there is only 1; multiple directory nodes will be inside the config. + chans = get_mchannels() + onion_chans = [x for x in chans if "type" in x and x["type"] == "onion"] + assert len(onion_chans) < 2 def load_test_config(**kwargs): if "config_path" not in kwargs: diff --git a/jmclient/jmclient/wallet_rpc.py b/jmclient/jmclient/wallet_rpc.py index 4597cf475..68c8f895d 100644 --- a/jmclient/jmclient/wallet_rpc.py +++ b/jmclient/jmclient/wallet_rpc.py @@ -159,6 +159,9 @@ def __init__(self, port, wss_port, tls=True): # can be shut down cleanly: self.coinjoin_connection = None + def get_client_factory(self): + return JMClientProtocolFactory(self.taker) + def activate_coinjoin_state(self, state): """ To be set when a maker or taker operation is initialized; they cannot @@ -420,7 +423,8 @@ def dummy_restart_callback(msg): walletname=self.wallet_name, token=self.cookie) - def taker_finished(self, res, fromtx=False, waittime=0.0, txdetails=None): + def taker_finished(self, res, fromtx=False, + waittime=0.0, txdetails=None): # This is a slimmed down version compared with what is seen in # the CLI code, since that code encompasses schedules with multiple # entries; for now, the RPC only supports single joins. @@ -1003,13 +1007,13 @@ def dummy_user_callback(rel, abs): self.taker = Taker(self.services["wallet"], schedule, max_cj_fee = max_cj_fee, callbacks=(self.filter_orders_callback, - None, self.taker_finished)) + None, self.taker_finished)) # TODO ; this makes use of a pre-existing hack to allow # selectively disabling the stallMonitor function that checks # if transactions went through or not; here we want to cleanly # destroy the Taker after an attempt is made, successful or not. self.taker.testflag = True - self.clientfactory = JMClientProtocolFactory(self.taker) + self.clientfactory = self.get_client_factory() dhost, dport = self.check_daemon_ready() diff --git a/jmdaemon/jmdaemon/__init__.py b/jmdaemon/jmdaemon/__init__.py index 384b5f720..fc1c4070b 100644 --- a/jmdaemon/jmdaemon/__init__.py +++ b/jmdaemon/jmdaemon/__init__.py @@ -4,6 +4,7 @@ from .enc_wrapper import as_init_encryption, decode_decrypt, \ encrypt_encode, init_keypair, init_pubkey, get_pubkey, NaclError from .irc import IRCMessageChannel +from .onionmc import OnionMessageChannel from jmbase.support import get_log from .message_channel import MessageChannel, MessageChannelCollection from .orderbookwatch import OrderbookWatch diff --git a/jmdaemon/jmdaemon/daemon_protocol.py b/jmdaemon/jmdaemon/daemon_protocol.py index b20a55107..d84bbb514 100644 --- a/jmdaemon/jmdaemon/daemon_protocol.py +++ b/jmdaemon/jmdaemon/daemon_protocol.py @@ -7,8 +7,9 @@ from .protocol import (COMMAND_PREFIX, ORDER_KEYS, NICK_HASH_LENGTH, NICK_MAX_ENCODED, JM_VERSION, JOINMARKET_NICK_HEADER, COMMITMENT_PREFIXES) -from .irc import IRCMessageChannel +from .irc import IRCMessageChannel +from .onionmc import OnionMessageChannel from jmbase import (is_hs_uri, get_tor_agent, JMHiddenService, get_nontor_agent, BytesProducer, wrapped_urlparse, bdict_sdict_convert, JMHTTPResource) @@ -527,10 +528,15 @@ def on_JM_INIT(self, bcsource, network, irc_configs, minmakers, self.mc_shutdown() self.irc_configs = irc_configs self.restart_mc_required = True - mcs = [IRCMessageChannel(c, - daemon=self, - realname='btcint=' + bcsource) - for c in self.irc_configs] + mcs = [] + for c in self.irc_configs: + if "type" in c and c["type"] == "onion": + mcs.append(OnionMessageChannel(c, daemon=self)) + else: + # default is IRC; TODO allow others + mcs.append(IRCMessageChannel(c, + daemon=self, + realname='btcint=' + bcsource)) self.mcc = MessageChannelCollection(mcs) OrderbookWatch.set_msgchan(self, self.mcc) #register taker-specific msgchan callbacks here @@ -947,6 +953,7 @@ def init_connections(self, nick): incomplete transaction is wiped. """ self.jm_state = 0 #uninited + self.mcc.set_nick(nick) if self.restart_mc_required: self.mcc.run() self.restart_mc_required = False @@ -954,7 +961,6 @@ def init_connections(self, nick): #if we are not restarting the MC, #we must simulate the on_welcome message: self.on_welcome() - self.mcc.set_nick(nick) def transfer_commitment(self, commit): """Send this commitment via privmsg to one (random) diff --git a/jmdaemon/jmdaemon/message_channel.py b/jmdaemon/jmdaemon/message_channel.py index 96be37ec6..9549f193d 100644 --- a/jmdaemon/jmdaemon/message_channel.py +++ b/jmdaemon/jmdaemon/message_channel.py @@ -263,9 +263,9 @@ def privmsg(self, nick, cmd, message, mc=None): #is supposed to be sent. There used to be an exception raise. #to prevent a crash (especially in makers), we just inform #the user about it for now - log.error("Tried to communicate on this IRC server but " + log.error("Tried to communicate on this message channel but " "failed: " + str(mc)) - log.error("You might have to comment out this IRC server " + log.error("You might have to comment out this message channel" "in joinmarket.cfg and restart.") log.error("No action needed for makers / yield generators!") # todo: add logic to continue on other available mc @@ -444,7 +444,7 @@ def on_welcome_trigger(self, mc): if (not self.on_welcome_announce_id) and self.on_welcome: self.on_welcome_announce_id = reactor.callLater(60, self.on_welcome_setup_finished,) else: - log.info("All IRC servers connected, starting execution.") + log.info("All message channels connected, starting execution.") if self.on_welcome_announce_id: self.on_welcome_announce_id.cancel() self.on_welcome_setup_finished() diff --git a/jmdaemon/jmdaemon/onionmc.py b/jmdaemon/jmdaemon/onionmc.py new file mode 100644 index 000000000..a426674bb --- /dev/null +++ b/jmdaemon/jmdaemon/onionmc.py @@ -0,0 +1,1179 @@ +from jmdaemon.message_channel import MessageChannel +from jmdaemon.protocol import COMMAND_PREFIX, JM_VERSION +from jmbase import get_log, JM_APP_NAME, JMHiddenService +import json +import copy +from typing import Callable, Union +from twisted.internet import reactor, task, protocol +from twisted.protocols import basic +from twisted.internet.endpoints import TCP4ClientEndpoint +from twisted.internet.address import IPv4Address, IPv6Address +from txtorcon.socks import TorSocksEndpoint + +log = get_log() + +def network_addr_to_string(location: Union[IPv4Address, IPv4Address]) -> str: + if isinstance(location, (IPv4Address, IPv6Address)): + host = location.host + port = location.port + else: + # TODO handle other addr types + assert False + return host + ":" + str(port) + +# module-level var to control whether we use Tor or not +# (specifically for tests): +testing_mode = False +def set_testing_mode(configdata: dict) -> None: + """ Toggles testing mode which enables non-Tor + network setup: + """ + global testing_mode + if not "regtest_count" in configdata: + log.debug("Onion message channel is not using regtest mode.") + testing_mode = False + return + try: + s, e = [int(x) for x in configdata["regtest_count"].split(",")] + except Exception as e: + log.info("Failed to get regtest count settings, error: {}".format(repr(e))) + testing_mode = False + return + if s == 0 and e == 0: + testing_mode = False + return + testing_mode = True + +""" +Messaging protocol (which wraps the underlying Joinmarket +messaging protocol) used here is documented in: +Joinmarket-Docs/onion-messaging.md +""" + +LOCAL_CONTROL_MESSAGE_TYPES = {"connect": 785, "disconnect": 787, "connect-in": 797} +CONTROL_MESSAGE_TYPES = {"peerlist": 789, "getpeerlist": 791, + "handshake": 793, "dn-handshake": 795, + "ping": 797, "pong": 799, "disconnect": 801} +JM_MESSAGE_TYPES = {"privmsg": 685, "pubmsg": 687} + +# Used for some control message construction, as detailed below. +NICK_PEERLOCATOR_SEPARATOR = ";" + +# location_string and nick must be set before sending, +# otherwise invalid: +client_handshake_json = {"app-name": JM_APP_NAME, + "directory": False, + "location-string": "", + "proto-ver": JM_VERSION, + "features": {}, + "nick": "" +} + +# default acceptance false; code must switch it on: +server_handshake_json = {"app-name": JM_APP_NAME, + "directory": True, + "proto-ver-min": JM_VERSION, + "proto-ver-max": JM_VERSION, + "features": {}, + "accepted": False, + "nick": "", + "motd": "Default MOTD, replace with information for the directory." + } + +# states that keep track of relationship to a peer +PEER_STATUS_UNCONNECTED, PEER_STATUS_CONNECTED, PEER_STATUS_HANDSHAKED, \ + PEER_STATUS_DISCONNECTED = range(4) + + +class OnionPeerError(Exception): + pass + +class OnionPeerDirectoryWithoutHostError(OnionPeerError): + pass + +class OnionPeerConnectionError(OnionPeerError): + pass + +class OnionCustomMessageDecodingError(Exception): + pass + +class OnionCustomMessage(object): + """ Encapsulates the messages passed over the wire + to and from other onion peers + """ + def __init__(self, text: str, msgtype: int): + self.text = text + self.msgtype = msgtype + + def encode(self) -> str: + self.encoded = json.dumps({"type": self.msgtype, + "line": self.text}).encode("utf-8") + return self.encoded + + @classmethod + def from_string_decode(cls, msg: str) -> 'OnionCustomMessage': + """ Build a custom message from a json-ified string. + """ + try: + msg_obj = json.loads(msg) + text = msg_obj["line"] + msgtype = msg_obj["type"] + except: + raise OnionCustomMessageDecodingError + return cls(text, msgtype) + +class OnionLineProtocol(basic.LineReceiver): + def connectionMade(self): + self.factory.register_connection(self) + + def connectionLost(self, reason): + self.factory.register_disconnection(self) + + def lineReceived(self, line: str) -> None: + #print("received", repr(line)) + try: + msg = OnionCustomMessage.from_string_decode(line) + except OnionCustomMessageDecodingError: + log.debug("Received invalid message, dropping connection.") + self.transport.loseConnection() + return + self.factory.receive_message(msg, self) + + def message(self, message: OnionCustomMessage) -> None: + #log.info("in OnionLineProtocol, about to send message: {} to peer {}".format(message.encode(), self.transport.getPeer())) + self.transport.write(message.encode() + self.delimiter) + +class OnionLineProtocolFactory(protocol.ServerFactory): + """ This factory allows us to start up instances + of the LineReceiver protocol that are instantiated + towards us. + As such, it is responsible for keeping track + """ + protocol = OnionLineProtocol + + def __init__(self, client: 'OnionMessageChannel'): + self.client = client + self.peers = {} + + def register_connection(self, p: OnionLineProtocol) -> None: + # make a local control message registering + # the new connection + peer_location = network_addr_to_string(p.transport.getPeer()) + self.client.register_connection(peer_location, direction=0) + self.peers[peer_location] = p + + def register_disconnection(self, p: OnionLineProtocol) -> None: + # make a local control message registering + # the new connection + peer_location = network_addr_to_string(p.transport.getPeer()) + self.client.register_disconnection(peer_location) + if not peer_location in self.peers: + log.warn("Disconnection event registered for non-existent peer.") + return + del self.peers[peer_location] + + def receive_message(self, message: OnionCustomMessage, + p: OnionLineProtocol) -> None: + self.client.receive_msg(message, network_addr_to_string( + p.transport.getPeer())) + + def send(self, message: OnionCustomMessage, destination: str) -> bool: + #print("trying to send in OnionLineProtocolFactory.") + #print("message: {}, destination: {}".format(message.encode(), destination)) + if not (destination in self.peers): + print("sending message {}, destination {} was not in peers {}".format(message.encode(), destination, self.peers)) + return False + proto = self.peers[destination] + proto.message(message) + return True + +class OnionClientFactory(protocol.ServerFactory): + """ We define a distinct protocol factory for outbound connections. + Notably, this factory supports only *one* protocol instance at a time. + """ + protocol = OnionLineProtocol + + def __init__(self, message_receive_callback: Callable, + connection_callback: Callable, + disconnection_callback: Callable): + self.proto_client = None + # callback takes OnionCustomMessage as arg and returns None + self.message_receive_callback = message_receive_callback + # connection callback, no args, returns None + self.connection_callback = connection_callback + # disconnection the same + self.disconnection_callback = disconnection_callback + + def register_connection(self, p: OnionLineProtocol) -> None: + #print("in OnionClientFactory, registered a connection, proto instance: ", p) + self.proto_client = p + self.connection_callback() + + def register_disconnection(self, p: OnionLineProtocol) -> None: + self.proto_client = None + self.disconnection_callback() + + def send(self, msg: OnionCustomMessage) -> bool: + self.proto_client.message(msg) + + def receive_message(self, message: OnionCustomMessage, + p: OnionLineProtocol) -> None: + self.message_receive_callback(message) + + """ + def clientConnectionLost(self, connector, reason): + log.debug('Connection to peer lost: {}, reason: {}'.format(connector, reason)) + if reactor.running: + log.info('Attempting to reconnect...') + protocol.ReconnectingClientFactory.clientConnectionLost( + self, connector, reason) + + def clientConnectionFailed(self, connector, reason): + log.debug('Connection to peer failed: {}, reason: {}'.format( + connector, reason)) + if reactor.running: + log.info('Attempting to reconnect...') + protocol.ReconnectingClientFactory.clientConnectionFailed( + self, connector, reason) + """ + +class OnionPeer(object): + + def __init__(self, messagechannel: 'OnionMessageChannel', + socks5_host: str, socks5_port: int, + hostname: str=None, port: int=-1, + directory: bool=False, nick: str="", + handshake_callback: Callable=None): + # reference to the managing OnionMessageChannel instance is + # needed so that we know where to send the messages received + # from this peer: + self.messagechannel = messagechannel + self.nick = nick + # client side net config: + self.socks5_host = socks5_host + self.socks5_port = socks5_port + # remote net config: + self.hostname = hostname + self.port = port + if directory and not (self.hostname): + raise OnionPeerDirectoryWithoutHostError() + self.directory = directory + self._status = PEER_STATUS_UNCONNECTED + #A function to be called to initiate a handshake; + # it should take a single argument, an OnionPeer object, + #and return None. + self.handshake_callback = handshake_callback + # Keep track of the protocol factory used to connect + # to the remote peer. Note that this won't always be used, + # if we have an inbound connection from this peer: + self.factory = None + # alternate location strings are used for inbound + # connections for this peer (these will be used first + # and foremost by directories, sending messages backwards + # on a connection created towards them). + self.alternate_location = "" + + def set_alternate_location(self, location_string: str): + self.alternate_location = location_string + + def update_status(self, destn_status: int) -> None: + """ Wrapping state updates to enforce: + (a) that the handshake is triggered by connection + outwards, and (b) to ensure no illegal state transitions. + """ + assert destn_status in range(4) + ignored_updates = [] + if self._status == PEER_STATUS_UNCONNECTED: + allowed_updates = [PEER_STATUS_CONNECTED, + PEER_STATUS_DISCONNECTED] + elif self._status == PEER_STATUS_CONNECTED: + # updates from connected->connected are harmless + allowed_updates = [PEER_STATUS_CONNECTED, + PEER_STATUS_DISCONNECTED, + PEER_STATUS_HANDSHAKED] + elif self._status == PEER_STATUS_HANDSHAKED: + allowed_updates = [PEER_STATUS_DISCONNECTED] + ignored_updates = [PEER_STATUS_CONNECTED] + elif self._status == PEER_STATUS_DISCONNECTED: + allowed_updates = [PEER_STATUS_CONNECTED] + ignored_updates = [PEER_STATUS_DISCONNECTED] + if destn_status in ignored_updates: + # TODO: this happens sometimes from 2->1; why? + log.debug("Attempt to update status of peer from {} " + "to {} ignored.".format(self._status, destn_status)) + return + assert destn_status in allowed_updates, ("couldn't update state " + "from {} to {}".format(self._status, destn_status)) + self._status = destn_status + # the handshakes are always initiated by a client: + if destn_status == PEER_STATUS_CONNECTED: + log.info("We, {}, are calling the handshake callback as client.".format(self.messagechannel.self_as_peer.peer_location())) + self.handshake_callback(self) + + def status(self) -> int: + """ Simple getter function for the wrapped _status: + """ + return self._status + + def set_nick(self, nick: str) -> None: + self.nick = nick + + def get_nick_peerlocation_ser(self) -> str: + if not self.nick: + raise OnionPeerError("Cannot serialize " + "identifier string without nick.") + return self.nick + NICK_PEERLOCATOR_SEPARATOR + \ + self.peer_location() + + @classmethod + def from_location_string(cls, mc: 'OnionMessageChannel', + location: str, + socks5_host: str, + socks5_port: int, + directory: bool=False, + handshake_callback: Callable=None) -> 'OnionPeer': + """ Allows construction of an OnionPeer from the + connection information given by the network interface. + TODO: special handling for inbound is needed. + """ + host, port = location.split(":") + return cls(mc, socks5_host, socks5_port, hostname=host, + port=int(port), directory=directory, + handshake_callback=handshake_callback) + + def set_host_port(self, hostname: str, port: int) -> None: + """ If the connection info is discovered + after this peer was already added to our list, + we can set it with this method. + """ + self.hostname = hostname + self.port = port + + def set_location(self, location_string: str) -> bool: + """ Allows setting location from an unchecked + input string argument; if the string does not have + the required format, + will return False, otherwise self.hostname, self.port are + updated for future `peer_location` calls, and True is returned. + """ + try: + host, port = location_string.split(":") + portint = int(port) + assert portint > 0 + except Exception as e: + log.debug("Failed to update host and port of this peer, " + "error: {}".format(repr(e))) + return False + self.hostname = host + self.port = portint + return True + + def peer_location(self) -> str: + assert (self.hostname and self.port > 0) + return self.hostname + ":" + str(self.port) + + def send(self, message: OnionCustomMessage) -> bool: + """ If the message can be sent on either an inbound or + outbound connection, True is returned, else False. + """ + if not self.factory: + #print("We are: {}. peer, wich was directory {}, did not have factory, so we send via mc".format( + # self.messagechannel.self_as_peer.peer_location(), self.directory)) + # we try to send via the overall message channel serving + # protocol, i.e. we assume the connection was made inbound: + #print("and to this location: ", self.peer_location()) + return self.messagechannel.proto_factory.send(message, self.alternate_location) + #print("peer which was directory {} did have factory {}, we send via that".format(self.directory, self.factory)) + return self.factory.send(message) + + def receive_message(self, message: OnionCustomMessage) -> None: + self.messagechannel.receive_msg(message, self.peer_location()) + + def connect(self) -> None: + """ This method is called to connect, over Tor, to the remote + peer at the given onion host/port. + The connection is 'persistent' in the sense that we use a + ReconnectingClientFactory. + """ + if self._status in [PEER_STATUS_HANDSHAKED, PEER_STATUS_CONNECTED]: + return + if not (self.hostname and self.port > 0): + raise OnionPeerConnectionError( + "Cannot connect without host, port info") + + self.factory = OnionClientFactory(self.receive_message, + self.register_connection, self.register_disconnection) + if testing_mode: + print("{} is making a tcp connection to {}, {}, {},".format( + self.messagechannel.self_as_peer.peer_location(), self.hostname, self.port, self.factory)) + self.tcp_connector = reactor.connectTCP(self.hostname, self.port, self.factory) + else: + torEndpoint = TCP4ClientEndpoint(reactor, self.socks5_host, self.socks5_port) + onionEndpoint = TorSocksEndpoint(torEndpoint, self.hostname, self.port) + onionEndpoint.connect(self.factory) + + def register_connection(self) -> None: + self.messagechannel.register_connection(self.peer_location(), direction=1) + + def register_disconnection(self) -> None: + self.messagechannel.register_disconnection(self.peer_location()) + + def try_to_connect(self) -> None: + """ This method wraps OnionPeer.connect and accepts + any error if that fails. + """ + try: + self.connect() + except OnionPeerConnectionError as e: + log.debug("Tried to connect but failed: {}".format(repr(e))) + except Exception as e: + log.warn("Got unexpected exception in connect attempt: {}".format( + repr(e))) + + def disconnect(self) -> None: + if self._status in [PEER_STATUS_UNCONNECTED, PEER_STATUS_DISCONNECTED]: + return + if not (self.hostname and self.port > 0): + raise OnionPeerConnectionError( + "Cannot disconnect without host, port info") + d = self.reconnecting_service.stopService() + d.addCallback(self.complete_disconnection) + d.addErrback(log.warn, "Failed to disconnect from peer {}.".format( + self.peer_location())) + + def complete_disconnection(self): + log.debug("Disconnected from peer: {}".format(self.peer_location())) + self.update_status(PEER_STATUS_DISCONNECTED) + self.factory = None + +class OnionDirectoryPeer(OnionPeer): + delay = 4.0 + def try_to_connect(self) -> None: + # Delay deliberately expands out to very + # long times as yg-s tend to be very long + # running bots: + self.delay *= 1.5 + if self.delay > 10000: + log.warn("Cannot connect to directory node peer: {} " + "after 20 attempts, giving up.".format(self.peer_location())) + return + try: + self.connect() + except OnionPeerConnectionError: + reactor.callLater(self.delay, self.try_to_connect) + +class OnionMessageChannel(MessageChannel): + """ Receives messages via a Torv3 hidden/onion service. + Sends messages to other nodes of the same type over Tor + via SOCKS5. + Uses one or more configured "directory nodes" + to access a list of current active nodes, and updates + dynamically from messages seen. + """ + + def __init__(self, + configdata, + daemon=None): + MessageChannel.__init__(self, daemon=daemon) + # hostid is a feature to avoid replay attacks across message channels; + # TODO investigate, but for now, treat onion-based as one "server". + self.hostid = "onion-network" + self.tor_control_host = configdata["tor_control_host"] + self.tor_control_port = int(configdata["tor_control_port"]) + self.onion_serving_host=configdata["onion_serving_host"] + self.onion_serving_port=int(configdata["onion_serving_port"]) + self.hidden_service_dir = configdata["hidden_service_dir"] + # client side config: + self.socks5_host = "127.0.0.1" + self.socks5_port = 9050 + # we use the setting in the config sent over from + # the client, to decide whether to set up our connections + # over localhost (if testing), without Tor: + set_testing_mode(configdata) + log.info("after call to testing_mode, it is: {}".format(testing_mode)) + # keep track of peers. the list will be instances + # of OnionPeer: + self.peers = set() + for dn in configdata["directory_nodes"].split(","): + # note we don't use a nick for directories: + self.peers.add(OnionDirectoryPeer.from_location_string( + self, dn, self.socks5_host, self.socks5_port, + directory=True, handshake_callback=self.handshake_as_client)) + # we can direct messages via the protocol factory, which + # will index protocol connections by peer location: + self.proto_factory = OnionLineProtocolFactory(self) + if testing_mode: + # we serve over TCP: + self.testing_serverconn = reactor.listenTCP(self.onion_serving_port, + self.proto_factory, interface="localhost") + self.onion_hostname = "127.0.0.1" + else: + self.hs = JMHiddenService(self.proto_factory, + self.info_callback, + self.setup_error_callback, + self.onion_hostname_callback, + self.tor_control_host, + self.tor_control_port, + self.onion_serving_host, + self.onion_serving_port, + shutdown_callback=self.shutdown_callback, + hidden_service_dir=self.hidden_service_dir) + # this call will start bringing up the HS; when it's finished, + # it will fire the `onion_hostname_callback`, or if it fails, + # it'll fire the `setup_error_callback`. + self.hs.start_tor() + + # This will serve as our unique identifier, indicating + # that we are ready to communicate (in both directions) over Tor. + self.onion_hostname = None + + # intended to represent the special case of 'we are the + # only directory node known', however for now dns don't interact + # so this has no role. TODO probably remove it. + self.genesis_node = False + + # waiting loop for all directories to have + # connected (note we could use a deferred but + # the rpc connection calls are not using twisted) + self.wait_for_directories_loop = None + + def info_callback(self, msg): + log.info(msg) + + def setup_error_callback(self, msg): + log.error(msg) + + def shutdown_callback(self, msg): + log.info("in shutdown callback: {}".format(msg)) + + def onion_hostname_callback(self, hostname): + """ This entrypoint marks the start of the OnionMessageChannel + running, since we need this unique identifier as our name + before we can start working (we need to compare it with the + configured directory nodes). + """ + print("hostname: ", hostname) + print("type: ", type(hostname)) + log.info("setting onion hostname to : {}".format(hostname)) + self.onion_hostname = hostname + +# ABC implementation section + def run(self) -> None: + self.hs_up_loop = task.LoopingCall(self.check_onion_hostname) + self.hs_up_loop.start(0.5) + + def get_pubmsg(self, msg:str, source_nick:str ="") -> str: + """ Converts a message into the known format for + pubmsgs; if we are not sending this (because we + are a directory, forwarding it), `source_nick` must be set. + Note that pubmsg does NOT prefix the *message* with COMMAND_PREFIX. + """ + nick = source_nick if source_nick else self.nick + return nick + COMMAND_PREFIX + "PUBLIC" + msg + + def get_privmsg(self, nick: str, cmd: str, message: str, + source_nick=None) -> None: + """ See `get_pubmsg` for comment on `source_nick`. + """ + from_nick = source_nick if source_nick else self.nick + return from_nick + COMMAND_PREFIX + nick + COMMAND_PREFIX + \ + cmd + " " + message + + def _pubmsg(self, msg:str) -> None: + """ Best effort broadcast of message `msg`: + send the message to every known directory node, + with the PUBLIC message type and nick. + """ + peerids = self.get_directory_peers() + msg = OnionCustomMessage(self.get_pubmsg(msg), + JM_MESSAGE_TYPES["pubmsg"]) + for peerid in peerids: + # currently a directory node can send its own + # pubmsgs (act as maker or taker); this will + # probably be removed but is useful in testing: + if peerid == self.self_as_peer.peer_location(): + self.receive_msg(msg, "00") + else: + self._send(self.get_peer_by_id(peerid), msg) + + def _privmsg(self, nick: str, cmd: str, msg:str) -> None: + log.debug("Privmsging to: {}, {}, {}".format(nick, cmd, msg)) + encoded_privmsg = OnionCustomMessage(self.get_privmsg(nick, cmd, msg), + JM_MESSAGE_TYPES["privmsg"]) + peerid = self.get_peerid_by_nick(nick) + if peerid: + peer = self.get_peer_by_id(peerid) + # notice the order matters here!: + if not peerid or not peer or not peer.status() == PEER_STATUS_HANDSHAKED: + # If we are trying to message a peer via their nick, we + # may not yet have a connection; then we just + # forward via directory nodes. + log.debug("Privmsg peer: {} but don't have peerid; " + "sending via directory.".format(nick)) + try: + # TODO change this to redundant or switching? + peer = self.get_connected_directory_peers()[0] + except Exception as e: + log.warn("Failed to send privmsg because no " + "directory peer is connected. Error: {}".format(repr(e))) + return + self._send(peer, encoded_privmsg) + + def _announce_orders(self, offerlist: list) -> None: + for offer in offerlist: + self._pubmsg(offer) + +# End ABC implementation section + + def check_onion_hostname(self): + if not self.onion_hostname: + return + self.hs_up_loop.stop() + # now our hidden service is up, we must check our peer status + # then set up directories. + self.get_our_peer_info() + # at this point the only peers added are directory + # nodes from config; we try to connect to all. + # We will get other peers to add to our list once they + # start sending us messages. + reactor.callLater(0.0, self.connect_to_directories) + + def get_our_peer_info(self) -> None: + """ Create a special OnionPeer object, + outside of our peerlist, to refer to ourselves. + """ + dp = self.get_directory_peers() + self_dir = False + # only for publically exposed onion does the 'virtual port' exist; + # for local tests we always connect to an actual machine port: + port_to_check = 80 if not testing_mode else self.onion_serving_port + my_location_str = self.onion_hostname + ":" + str(port_to_check) + log.info("To check if we are genesis, we compare {} with {}".format(my_location_str, dp)) + if [my_location_str] == dp: + log.info("This is the genesis node: {}".format(self.onion_hostname)) + self.genesis_node = True + self_dir = True + elif my_location_str in dp: + # Here we are just one of many directory nodes, + # which should be fine, we should just be careful + # to not query ourselves. + self_dir = True + self.self_as_peer = OnionPeer(self, self.socks5_host, self.socks5_port, + self.onion_hostname, self.onion_serving_port, + self_dir, nick=self.nick, + handshake_callback=None) + + def connect_to_directories(self) -> None: + if self.genesis_node: + # we are a directory and we have no directory peers; + # just start. + self.on_welcome(self) + return + # the remaining code is only executed by non-directories: + for p in self.peers: + log.info("Trying to connect to node: {}".format(p.peer_location())) + try: + p.connect() + except OnionPeerConnectionError: + pass + # do not trigger on_welcome event until all directories + # configured are ready: + self.on_welcome_sent = False + self.wait_for_directories_loop = task.LoopingCall( + self.wait_for_directories) + self.wait_for_directories_loop.start(10.0) + + def handshake_as_client(self, peer: OnionPeer) -> None: + assert peer.status() == PEER_STATUS_CONNECTED + if self.self_as_peer.directory: + log.debug("Not sending client handshake to {} because we are directory.".format(peer.peer_location())) + return + our_hs = copy.deepcopy(client_handshake_json) + our_hs["location-string"] = self.self_as_peer.peer_location() + our_hs["nick"] = self.nick + # We fire and forget the handshake; successful setting + # of the `is_handshaked` var in the Peer object will depend + # on a valid/success return via the custommsg hook in the plugin. + log.info("Sending this handshake: {} to peer {}".format(json.dumps(our_hs), peer.peer_location())) + self._send(peer, OnionCustomMessage(json.dumps(our_hs), + CONTROL_MESSAGE_TYPES["handshake"])) + + def handshake_as_directory(self, peer: OnionPeer, our_hs: dict) -> None: + assert peer.status() == PEER_STATUS_CONNECTED + log.info("Sending this handshake as directory: {}".format(json.dumps(our_hs))) + self._send(peer, OnionCustomMessage(json.dumps(our_hs), + CONTROL_MESSAGE_TYPES["dn-handshake"])) + + def get_directory_peers(self) -> list: + return [ p.peer_location() for p in self.peers if p.directory is True] + + def get_peerid_by_nick(self, nick:str) -> Union[OnionPeer, None]: + for p in self.get_all_connected_peers(): + if p.nick == nick: + return p.peer_location() + return None + + def _send(self, peer: OnionPeer, message: OnionCustomMessage) -> bool: + try: + return peer.send(message) + except Exception as e: + # This can happen when a peer disconnects, depending + # on the timing: + log.warn("Failed to send message to: {}, error: {}".format( + peer.peer_location(), repr(e))) + return False + + def shutdown(self): + """ TODO + """ + + def receive_msg(self, message: OnionCustomMessage, peer_location: str) -> None: + """ Messages from peers and also connection related control + messages. These messages either come via OnionPeer or via + the main OnionLineProtocolFactory instance that handles all + inbound connections. + """ + if self.self_as_peer.directory: + print("received message as directory: ", message.encode()) + peer = self.get_peer_by_id(peer_location) + if not peer: + log.warn("Received message but could not find peer: {}".format(peer_location)) + return + msgtype = message.msgtype + msgval = message.text + if msgtype in LOCAL_CONTROL_MESSAGE_TYPES.values(): + self.process_control_message(peer_location, msgtype, msgval) + # local control messages are processed first. + # TODO this is a historical artifact, we can simplify. + return + + if self.process_control_message(peer_location, msgtype, msgval): + # will return True if it is, elsewise, a control message. + return + + # ignore non-JM messages: + if not msgtype in JM_MESSAGE_TYPES.values(): + log.debug("Invalid message type, ignoring: {}".format(msgtype)) + return + + # real JM message; should be: from_nick, to_nick, cmd, message + try: + nicks_msgs = msgval.split(COMMAND_PREFIX) + from_nick, to_nick = nicks_msgs[:2] + msg = COMMAND_PREFIX + COMMAND_PREFIX.join(nicks_msgs[2:]) + if to_nick == "PUBLIC": + #log.debug("A pubmsg is being processed by {} from {}; it " + # "is {}".format(self.self_as_peer.nick, from_nick, msg)) + self.on_pubmsg(from_nick, msg) + if self.self_as_peer.directory: + self.forward_pubmsg_to_peers(msg, from_nick) + elif to_nick != self.nick: + if not self.self_as_peer.directory: + log.debug("Ignoring message, not for us: {}".format(msg)) + else: + self.forward_privmsg_to_peer(to_nick, msg, from_nick) + else: + self.on_privmsg(from_nick, msg) + except Exception as e: + log.debug("Invalid joinmarket message: {}, error was: {}".format( + msgval, repr(e))) + return + + def forward_pubmsg_to_peers(self, msg: str, from_nick: str) -> None: + """ Used by directory nodes currently. Takes a received + message that was PUBLIC and broadcasts it to the non-directory + peers. + """ + assert self.self_as_peer.directory + pubmsg = self.get_pubmsg(msg, source_nick=from_nick) + msgtype = JM_MESSAGE_TYPES["pubmsg"] + # NOTE!: Specifically with forwarding/broadcasting, + # we introduce the danger of infinite re-broadcast, + # if there is more than one party forwarding. + # For now we are having directory nodes not talk to + # each other (i.e. they are *required* to only configure + # themselves, not other dns). But this could happen by + # accident. + encoded_msg = OnionCustomMessage(pubmsg, msgtype) + for peer in self.get_connected_nondirectory_peers(): + # don't loop back to the sender: + if peer.nick == from_nick: + continue + log.debug("Sending {}:{} to nondir peer {}".format( + msgtype, pubmsg, peer.peer_location())) + self._send(peer, encoded_msg) + + def forward_privmsg_to_peer(self, nick: str, message: str, + from_nick: str) -> None: + assert self.self_as_peer.directory + peerid = self.get_peerid_by_nick(nick) + if not peerid: + log.debug("We were asked to send a message from {} to {}, " + "but {} is not connected.".format(from_nick, nick, nick)) + return + # The `message` passed in has format COMMAND_PREFIX||command||" "||msg + # we need to parse out cmd, message for sending. + _, cmdmsg = message.split(COMMAND_PREFIX) + cmdmsglist = cmdmsg.split(" ") + cmd = cmdmsglist[0] + msg = " ".join(cmdmsglist[1:]) + privmsg = self.get_privmsg(nick, cmd, msg, source_nick=from_nick) + #log.debug("Sending out privmsg: {} to peer: {}".format(privmsg, peerid)) + encoded_msg = OnionCustomMessage(privmsg, + JM_MESSAGE_TYPES["privmsg"]) + self._send(self.get_peer_by_id(peerid), encoded_msg) + # If possible, we forward the from-nick's network location + # to the to-nick peer, so they can just talk directly next time. + peerid_from = self.get_peerid_by_nick(from_nick) + if not peerid_from: + return + peer_to = self.get_peer_by_id(peerid) + self.send_peers(peer_to, peerid_filter=[peerid_from]) + + def process_control_message(self, peerid: str, msgtype: int, + msgval: str) -> bool: + """ Triggered by a directory node feeding us + peers, or by a connect/disconnect hook; this is our housekeeping + to try to create, and keep track of, useful connections. + """ + all_ctrl = list(LOCAL_CONTROL_MESSAGE_TYPES.values( + )) + list(CONTROL_MESSAGE_TYPES.values()) + if msgtype not in all_ctrl: + return False + # this is too noisy, but TODO, investigate allowing + # some kind of control message monitoring e.g. default-off + # log-to-file (we don't currently have a 'TRACE' level debug). + #log.debug("received control message: {},{}".format(msgtype, msgval)) + if msgtype == CONTROL_MESSAGE_TYPES["peerlist"]: + # This is the base method of seeding connections; + # a directory node can send this any time. We may well + # need to control this; for now it just gets processed, + # whereever it came from: + try: + peerlist = msgval.split(",") + for peer in peerlist: + # defaults mean we just add the peer, not + # add or alter its connection status: + self.add_peer(peer, with_nick=True) + except Exception as e: + log.debug("Incorrectly formatted peer list: {}, " + "ignoring, {}".format(msgval, e)) + # returning True either way, because although it was an + # invalid message, it *was* a control message, and should + # not be processed as something else. + return True + elif msgtype == CONTROL_MESSAGE_TYPES["getpeerlist"]: + # getpeerlist must be accompanied by a full node + # locator, and nick; + # add that peer before returning our peer list. + p = self.add_peer(msgval, connection=True, + overwrite_connection=True, with_nick=True) + try: + self.send_peers(p) + except OnionPeerConnectionError: + pass + # comment much as above; if we can't connect, it's none + # of our business. + return True + elif msgtype == CONTROL_MESSAGE_TYPES["handshake"]: + # sent by non-directory peers on startup + self.process_handshake(peerid, msgval) + return True + elif msgtype == CONTROL_MESSAGE_TYPES["dn-handshake"]: + self.process_handshake(peerid, msgval, dn=True) + return True + elif msgtype == LOCAL_CONTROL_MESSAGE_TYPES["connect"]: + self.add_peer(msgval, connection=True, + overwrite_connection=True) + elif msgtype == LOCAL_CONTROL_MESSAGE_TYPES["connect-in"]: + self.add_peer(msgval, connection=True, + overwrite_connection=True) + elif msgtype == LOCAL_CONTROL_MESSAGE_TYPES["disconnect"]: + log.debug("We got a disconnect event: {}".format(msgval)) + if msgval in [x.peer_location() for x in self.get_connected_directory_peers()]: + # we need to use the full peer locator string, so that + # add_peer knows it can try to reconnect: + msgval = self.get_peer_by_id(msgval).peer_location() + self.add_peer(msgval, connection=False, + overwrite_connection=True) + else: + assert False + # If we got here it is *not* a non-local control message; + # so we must process it as a Joinmarket message. + return False + + + def process_handshake(self, peerid: str, message: str, + dn: bool=False) -> None: + peer = self.get_peer_by_id(peerid) + if not peer: + # rando sent us a handshake? + log.warn("Unexpected handshake from unknown peer: {}, " + "ignoring.".format(peerid)) + return + assert isinstance(peer, OnionPeer) + if not peer.status() == PEER_STATUS_CONNECTED: + # we were not waiting for it: + log.warn("Unexpected handshake from peer: {}, " + "ignoring. Peer's current status is: {}".format( + peerid, peer.status())) + return + if dn: + print("We, {}, are processing a handshake with dn {} from peer {}".format(self.self_as_peer.peer_location(), dn, peerid)) + # it means, we are a non-dn and we are expecting + # a returned `dn-handshake` message: + # (currently dns don't talk to other dns): + assert not self.self_as_peer.directory + if not peer.directory: + # got dn-handshake from non-dn: + log.warn("Unexpected dn-handshake from non-dn " + "node: {}, ignoring.".format(peerid)) + return + # we got the right message from the right peer; + # check it is formatted correctly and represents + # acceptance of the connection + try: + handshake_json = json.loads(message) + app_name = handshake_json["app-name"] + is_directory = handshake_json["directory"] + proto_min = handshake_json["proto-ver-min"] + proto_max = handshake_json["proto-ver-max"] + features = handshake_json["features"] + accepted = handshake_json["accepted"] + nick = handshake_json["nick"] + assert isinstance(proto_max, int) + assert isinstance(proto_min, int) + assert isinstance(features, dict) + assert isinstance(nick, str) + except Exception as e: + log.warn("Invalid handshake message from: {}, exception: {}, message: {}," + "ignoring".format(peerid, repr(e), message)) + return + # currently we are not using any features, but the intention + # is forwards compatibility, so we don't check its contents + # at all. + if not accepted: + log.warn("Directory: {} rejected our handshake.".format(peerid)) + return + if not (app_name == JM_APP_NAME and is_directory and JM_VERSION \ + <= proto_max and JM_VERSION >= proto_min and accepted): + log.warn("Handshake from directory is incompatible or " + "rejected: {}".format(handshake_json)) + return + # We received a valid, accepting dn-handshake. Update the peer. + peer.update_status(PEER_STATUS_HANDSHAKED) + peer.set_nick(nick) + else: + print("We, {}, are processing a handshake with dn {} from peer {}".format(self.self_as_peer.peer_location(), dn, peerid)) + # it means, we are receiving an initial handshake + # message from a 'client' (non-dn) peer. + # dns don't talk to each other: + assert not peer.directory + accepted = True + try: + handshake_json = json.loads(message) + app_name = handshake_json["app-name"] + is_directory = handshake_json["directory"] + proto_ver = handshake_json["proto-ver"] + features = handshake_json["features"] + full_location_string = handshake_json["location-string"] + nick = handshake_json["nick"] + assert isinstance(proto_ver, int) + assert isinstance(features, dict) + assert isinstance(nick, str) + except Exception as e: + log.warn("(not dn) Invalid handshake message from: {}, exception: {}, message: {}," + "ignoring".format(peerid, repr(e), message)) + accepted = False + if not (app_name == JM_APP_NAME and proto_ver == JM_VERSION \ + and not is_directory): + log.warn("Invalid handshake name/version data: {}, from peer: " + "{}, rejecting.".format(message, peerid)) + accepted = False + # If accepted, we should update the peer to have the full + # location which in general will not yet be present, so as to + # allow publishing their location via `getpeerlist`: + if not peer.set_location(full_location_string): + accepted = False + if not peerid == full_location_string: + print("we are reading a handshake from location {} but they sent" + "us full location string {}, setting an alternate".format( + peerid, full_location_string)) + peer.set_alternate_location(peerid) + peer.set_nick(nick) + # client peer's handshake message was valid; send ours, and + # then mark this peer as successfully handshaked: + our_hs = copy.deepcopy(server_handshake_json) + our_hs["nick"] = self.nick + our_hs["accepted"] = accepted + if self.self_as_peer.directory: + self.handshake_as_directory(peer, our_hs) + if accepted: + peer.update_status(PEER_STATUS_HANDSHAKED) + + def get_peer_by_id(self, p: str) -> Union[OnionPeer, bool]: + """ Returns the OnionPeer with peer location p, + if it is in self.peers, otherwise returns False. + """ + if p == "00": + return self.self_as_peer + for x in self.peers: + if x.peer_location() == p: + return x + if x.alternate_location == p: + return x + return False + + def register_connection(self, peer_location: str, direction: int) -> None: + """ We send ourselves a local control message indicating + the new connection. + If the connection is inbound, direction == 0, else 1. + """ + assert direction in range(2) + if direction == 1: + msgtype = LOCAL_CONTROL_MESSAGE_TYPES["connect"] + else: + msgtype = LOCAL_CONTROL_MESSAGE_TYPES["connect-in"] + msg = OnionCustomMessage(peer_location, msgtype) + self.receive_msg(msg, "00") + + def register_disconnection(self, peer_location: str) -> None: + """ We send ourselves a local control message indicating + the disconnection. + """ + msg = OnionCustomMessage(peer_location, + LOCAL_CONTROL_MESSAGE_TYPES["disconnect"]) + self.receive_msg(msg, "00") + + def add_peer(self, peerdata: str, connection: bool=False, + overwrite_connection: bool=False, with_nick=False) -> None: + """ add non-directory peer from (nick, peer) serialization `peerdata`, + where "peer" is host:port; + return the created OnionPeer object. Or, with_nick=False means + that `peerdata` has only the peer location. + If the peer is already in our peerlist it can be updated in + one of these ways: + * the nick can be added + * it can be marked as 'connected' if it was previously unconnected, + with this conditional on whether the flag `overwrite_connection` is + set. Note that this peer removal, unlike the peer addition above, + can also occur for directory nodes, if we lose connection (and then + we persistently try to reconnect; see OnionDirectoryPeer). + """ + if with_nick: + try: + nick, peer = peerdata.split(NICK_PEERLOCATOR_SEPARATOR) + except Exception as e: + # TODO: as of now, this is not an error, but expected. + # Don't log? Do something else? + log.debug("Received invalid peer identifier string: {}, {}".format( + peerdata, e)) + return + else: + peer = peerdata + + # assumed that it's passing a full string + try: + temp_p = OnionPeer.from_location_string(self, peer, + self.socks5_host, self.socks5_port, + handshake_callback=self.handshake_as_client) + except Exception as e: + # There are currently a few ways the location + # parsing and Peer object construction can fail; + # TODO specify exception types. + log.warn("Failed to add peer: {}, exception: {}".format(peer, repr(e))) + return + if not self.get_peer_by_id(temp_p.peer_location()): + if connection: + log.info("Updating status of peer: {} to connected.".format(temp_p.peer_location())) + temp_p.update_status(PEER_STATUS_CONNECTED) + else: + temp_p.update_status(PEER_STATUS_DISCONNECTED) + if with_nick: + temp_p.set_nick(nick) + self.peers.add(temp_p) + if not connection: + # Here, we are not currently connected. We + # try to connect asynchronously. We don't pay attention + # to any return. This attempt is one-shot and opportunistic, + # for non-dns, but will retry with exp-backoff for dns. + # Notice this is only possible for non-dns to other non-dns, + # since dns will never reach this point without an active + # connection. + reactor.callLater(0.0, temp_p.try_to_connect) + return temp_p + else: + p = self.get_peer_by_id(temp_p.peer_location()) + if overwrite_connection: + if connection: + log.info("Updating status to connected for peer {}.".format(temp_p.peer_location())) + p.update_status(PEER_STATUS_CONNECTED) + else: + p.update_status(PEER_STATUS_DISCONNECTED) + if with_nick: + p.set_nick(nick) + return p + + def get_all_connected_peers(self) -> list: + return self.get_connected_directory_peers() + \ + self.get_connected_nondirectory_peers() + + def get_connected_directory_peers(self) -> list: + return [p for p in self.peers if p.directory and p.status() == \ + PEER_STATUS_HANDSHAKED] + + def get_connected_nondirectory_peers(self) -> list: + return [p for p in self.peers if (not p.directory) and p.status() == \ + PEER_STATUS_HANDSHAKED] + + def wait_for_directories(self) -> None: + # Notice this is checking for *handshaked* dps; + # the handshake will have been initiated once a + # connection was seen: + log.warn("in the wait for directories loop, this is the connected dps: {}".format(self.get_connected_directory_peers())) + if len(self.get_connected_directory_peers()) == 0: + return + # This is what triggers the start of taker/maker workflows. + if not self.on_welcome_sent: + self.on_welcome(self) + self.on_welcome_sent = True + self.wait_for_directories_loop.stop() + + """ CONTROL MESSAGES SENT BY US + """ + def send_peers(self, requesting_peer: OnionPeer, + peerid_filter: list=[]) -> None: + """ This message is sent by directory peers on request + by non-directory peers. + If peerid_filter is specified, only peers whose peerid is in + this list will be sent. (TODO this is inefficient). + The peerlist message should have this format: + (1) entries comma separated + (2) each entry is serialized nick then the NICK_PEERLOCATOR_SEPARATOR + then *either* 66 char hex peerid, *or* peerid@host:port + (3) However this message might be long enough to exceed a 1300 byte limit, + if we don't use a filter, so we may need to split it into multiple + messages (TODO). + """ + if not requesting_peer.status() == PEER_STATUS_HANDSHAKED: + raise OnionPeerConnectionError( + "Cannot send peer list to unhandshaked peer") + peerlist = set() + for p in self.get_connected_nondirectory_peers(): + # don't send a peer to itself + if p.peer_location() == requesting_peer.peer_location(): + continue + if len(peerid_filter) > 0 and p.peer_location() not in peerid_filter: + continue + if not p.status() == PEER_STATUS_HANDSHAKED: + # don't advertise what is not online. + continue + # peers that haven't sent their nick yet are not + # privmsg-reachable; don't send them + if p.nick == "": + continue + peerlist.add(p.get_nick_peerlocation_ser()) + # For testing: dns won't usually participate: + peerlist.add(self.self_as_peer.get_nick_peerlocation_ser()) + self._send(requesting_peer, OnionCustomMessage(",".join( + peerlist), CONTROL_MESSAGE_TYPES["peerlist"])) diff --git a/jmdaemon/test/test_daemon_protocol.py b/jmdaemon/test/test_daemon_protocol.py index 71beba734..f9dbf390e 100644 --- a/jmdaemon/test/test_daemon_protocol.py +++ b/jmdaemon/test/test_daemon_protocol.py @@ -7,7 +7,7 @@ from jmdaemon.protocol import NICK_HASH_LENGTH, NICK_MAX_ENCODED, JM_VERSION,\ JOINMARKET_NICK_HEADER from jmbase import get_log -from jmclient import (load_test_config, jm_single, get_irc_mchannels) +from jmclient import (load_test_config, jm_single, get_mchannels) from twisted.python.log import msg as tmsg from twisted.python.log import startLogging from twisted.internet import protocol, reactor, task @@ -59,7 +59,7 @@ def connectionMade(self): def clientStart(self): self.sigs_received = 0 - irc = get_irc_mchannels() + irc = [get_mchannels()[0]] d = self.callRemote(JMInit, bcsource="dummyblockchain", network="dummynetwork", diff --git a/jmdaemon/test/test_irc_messaging.py b/jmdaemon/test/test_irc_messaging.py index 755a20c69..0e9812fd7 100644 --- a/jmdaemon/test/test_irc_messaging.py +++ b/jmdaemon/test/test_irc_messaging.py @@ -6,7 +6,7 @@ from twisted.internet import reactor, task from jmdaemon import IRCMessageChannel, MessageChannelCollection #needed for test framework -from jmclient import (load_test_config, get_irc_mchannels, jm_single) +from jmclient import (load_test_config, get_mchannels, jm_single) si = 1 class DummyDaemon(object): @@ -95,7 +95,7 @@ def junk_fill(mc): def getmc(nick): dm = DummyDaemon() - mc = DummyMC(get_irc_mchannels()[0], nick, dm) + mc = DummyMC(get_mchannels()[0], nick, dm) mc.register_orderbookwatch_callbacks(on_order_seen=on_order_seen) mc.register_taker_callbacks(on_pubkey=on_pubkey) mc.on_connect = on_connect @@ -108,7 +108,7 @@ class TrialIRC(unittest.TestCase): def setUp(self): load_test_config() - print(get_irc_mchannels()[0]) + print(get_mchannels()[0]) jm_single().maker_timeout_sec = 1 dm, mc, mcc = getmc("irc_publisher") dm2, mc2, mcc2 = getmc("irc_receiver") diff --git a/jmdaemon/test/test_orderbookwatch.py b/jmdaemon/test/test_orderbookwatch.py index 39d4de791..17797a635 100644 --- a/jmdaemon/test/test_orderbookwatch.py +++ b/jmdaemon/test/test_orderbookwatch.py @@ -2,7 +2,7 @@ from jmdaemon.orderbookwatch import OrderbookWatch from jmdaemon import IRCMessageChannel, fidelity_bond_cmd_list -from jmclient import get_irc_mchannels, load_test_config +from jmclient import get_mchannels, load_test_config from jmdaemon.protocol import JM_VERSION, ORDER_KEYS from jmbase.support import hextobin from jmclient.fidelity_bond import FidelityBondProof @@ -24,7 +24,7 @@ def on_welcome(x): def get_ob(): load_test_config() dm = DummyDaemon() - mc = DummyMC(get_irc_mchannels()[0], "test", dm) + mc = DummyMC(get_mchannels()[0], "test", dm) ob = OrderbookWatch() ob.on_welcome = on_welcome ob.set_msgchan(mc) diff --git a/scripts/obwatch/ob-watcher.py b/scripts/obwatch/ob-watcher.py index 324f92c98..b6a2950ed 100755 --- a/scripts/obwatch/ob-watcher.py +++ b/scripts/obwatch/ob-watcher.py @@ -44,7 +44,7 @@ import matplotlib.pyplot as plt from jmclient import jm_single, load_program_config, calc_cj_fee, \ - get_irc_mchannels, add_base_options + get_mchannels, add_base_options from jmdaemon import OrderbookWatch, MessageChannelCollection, IRCMessageChannel #TODO this is only for base58, find a solution for a client without jmbitcoin import jmbitcoin as btc @@ -804,7 +804,7 @@ def main(): (options, args) = parser.parse_args() load_program_config(config_path=options.datadir) hostport = (options.host, options.port) - mcs = [ObIRCMessageChannel(c) for c in get_irc_mchannels()] + mcs = [ObIRCMessageChannel(c) for c in get_mchannels()] mcc = MessageChannelCollection(mcs) mcc.set_nick(get_dummy_nick()) taker = ObBasic(mcc, hostport) diff --git a/test/e2e-coinjoin-test.py b/test/e2e-coinjoin-test.py new file mode 100644 index 000000000..600d6ecd5 --- /dev/null +++ b/test/e2e-coinjoin-test.py @@ -0,0 +1,364 @@ +#! /usr/bin/env python +'''Creates wallets and yield generators in regtest, + then runs both them and a JMWalletDaemon instance + for the taker, injecting the newly created taker + wallet into it and running sendpayment once. + Number of ygs is configured in the joinmarket.cfg + with `regtest-count` in the `ln-onion` type MESSAGING + section. + See notes below for more detail on config. + Run it like: + pytest \ + --btcroot=/path/to/bitcoin/bin/ \ + --btcpwd=123456abcdef --btcconf=/blah/bitcoin.conf \ + -s test/ln-ygrunner.py + ''' +from twisted.internet import reactor, defer +from twisted.web.client import readBody, Headers +from common import make_wallets +import pytest +import random +import json +from datetime import datetime +from jmbase import (get_nontor_agent, BytesProducer, jmprint, + get_log, stop_reactor, hextobin, bintohex) +from jmclient import (YieldGeneratorBasic, load_test_config, jm_single, + JMClientProtocolFactory, start_reactor, SegwitWallet, get_mchannels, + SegwitLegacyWallet, JMWalletDaemon) +from jmclient.wallet_utils import wallet_gettimelockaddress +from jmclient.wallet_rpc import api_version_string + +log = get_log() + +# For quicker testing, restrict the range of timelock +# addresses to avoid slow load of multiple bots. +# Note: no need to revert this change as ygrunner runs +# in isolation. +from jmclient import FidelityBondMixin +FidelityBondMixin.TIMELOCK_ERA_YEARS = 2 +FidelityBondMixin.TIMELOCK_EPOCH_YEAR = datetime.now().year +FidelityBondMixin.TIMENUMBERS_PER_PUBKEY = 12 + +wallet_name = "test-onion-yg-runner.jmdat" + +mean_amt = 2.0 + +directory_node_indices = [1] + +# +def get_onion_messaging_config_regtest(run_num: int, dns=[1], hsd=""): + """ Sets a onion messaging channel section for a regtest instance + indexed by `run_num`. The indices to be used as directory nodes + should be passed as `dns`, as a list of ints. + """ + def location_string(directory_node_run_num): + return "127.0.0.1:" + str( + 8080 + directory_node_run_num) + if run_num in dns: + # means *we* are a dn, and dns currently + # do not use other dns: + dns_to_use = [location_string(run_num)] + else: + dns_to_use = [location_string(a) for a in dns] + dn_nodes_list = ",".join(dns_to_use) + log.info("For node: {}, set dn list to: {}".format(run_num, dn_nodes_list)) + cf = {"type": "onion", + "socks5_host": "127.0.0.1", + "socks5_port": 9050, + "tor_control_host": "127.0.0.1", + "tor_control_port": 9051, + "onion_serving_host": "127.0.0.1", + "onion_serving_port": 8080 + run_num, + "hidden_service_dir": "", + "directory_nodes": dn_nodes_list, + "regtest_count": "1, 1"} + if run_num in dns: + # only directories need to use fixed hidden service directories: + cf["hidden_service_dir"] = hsd + return cf + + +class RegtestJMClientProtocolFactory(JMClientProtocolFactory): + i = 1 + def set_directory_nodes(self, dns): + # a list of integers representing the directory nodes + # for this test: + self.dns = dns + + def get_mchannels(self): + # swaps out any existing lightning configs + # in the config settings on startup, for one + # that's indexed to the regtest counter var: + default_chans = get_mchannels() + new_chans = [] + onion_found = False + hsd = "" + for c in default_chans: + if "type" in c and c["type"] == "onion": + onion_found = True + if c["hidden_service_dir"] != "": + hsd = c["hidden_service_dir"] + continue + else: + new_chans.append(c) + if onion_found: + new_chans.append(get_onion_messaging_config_regtest( + self.i, self.dns, hsd)) + return new_chans + +class JMWalletDaemonT(JMWalletDaemon): + def check_cookie(self, request): + if self.auth_disabled: + return True + return super().check_cookie(request) + +class TWalletRPCManager(object): + """ Base class for set up of tests of the + Wallet RPC calls using the wallet_rpc.JMWalletDaemon service. + """ + # the port for the jmwallet daemon + dport = 28183 + # the port for the ws + wss_port = 28283 + + def __init__(self): + # a client connnection object which is often but not always + # instantiated: + self.client_connector = None + self.daemon = JMWalletDaemonT(self.dport, self.wss_port, tls=False) + self.daemon.auth_disabled = True + # because we sync and start the wallet service manually here + # (and don't use wallet files yet), we won't have set a wallet name, + # so we set it here: + self.daemon.wallet_name = wallet_name + + def start(self): + r, s = self.daemon.startService() + self.listener_rpc = r + self.listener_ws = s + + def get_route_root(self): + addr = "http://127.0.0.1:" + str(self.dport) + addr += api_version_string + return addr + + def stop(self): + for dc in reactor.getDelayedCalls(): + dc.cancel() + d1 = defer.maybeDeferred(self.listener_ws.stopListening) + d2 = defer.maybeDeferred(self.listener_rpc.stopListening) + if self.client_connector: + self.client_connector.disconnect() + # only fire if everything is finished: + return defer.gatherResults([d1, d2]) + + @defer.inlineCallbacks + def do_request(self, agent, method, addr, body, handler, token=None): + if token: + headers = Headers({"Authorization": ["Bearer " + self.jwt_token]}) + else: + headers = None + response = yield agent.request(method, addr, headers, bodyProducer=body) + yield self.response_handler(response, handler) + + @defer.inlineCallbacks + def response_handler(self, response, handler): + body = yield readBody(response) + # these responses should always be 200 OK. + #assert response.code == 200 + # handlers check the body is as expected; no return. + yield handler(body) + return True + +def test_start_yg_and_taker_setup(setup_onion_ygrunner): + """Set up some wallets, for the ygs and 1 taker. + Then start LN and the ygs in the background, then fire + a startup of a wallet daemon for the taker who then + makes a coinjoin payment. + """ + if jm_single().config.get("POLICY", "native") == "true": + walletclass = SegwitWallet + else: + # TODO add Legacy + walletclass = SegwitLegacyWallet + + start_bot_num, end_bot_num = [int(x) for x in jm_single().config.get( + "MESSAGING:onion1", "regtest_count").split(",")] + num_ygs = end_bot_num - start_bot_num + # specify the number of wallets and bots of each type: + wallet_services = make_wallets(num_ygs + 1, + wallet_structures=[[1, 3, 0, 0, 0]] * (num_ygs + 1), + mean_amt=2.0, + walletclass=walletclass) + #the sendpayment bot uses the last wallet in the list + wallet_service = wallet_services[end_bot_num - 1]['wallet'] + jmprint("\n\nTaker wallet seed : " + wallet_services[end_bot_num - 1]['seed']) + # for manual audit if necessary, show the maker's wallet seeds + # also (note this audit should be automated in future, see + # test_full_coinjoin.py in this directory) + jmprint("\n\nMaker wallet seeds: ") + for i in range(start_bot_num, end_bot_num): + jmprint("Maker seed: " + wallet_services[i - 1]['seed']) + jmprint("\n") + wallet_service.sync_wallet(fast=True) + ygclass = YieldGeneratorBasic + + # As per previous note, override non-default command line settings: + options = {} + for x in ["ordertype", "txfee_contribution", "txfee_contribution_factor", + "cjfee_a", "cjfee_r", "cjfee_factor", "minsize", "size_factor"]: + options[x] = jm_single().config.get("YIELDGENERATOR", x) + ordertype = options["ordertype"] + txfee_contribution = int(options["txfee_contribution"]) + txfee_contribution_factor = float(options["txfee_contribution_factor"]) + cjfee_factor = float(options["cjfee_factor"]) + size_factor = float(options["size_factor"]) + if ordertype == 'reloffer': + cjfee_r = options["cjfee_r"] + # minimum size is such that you always net profit at least 20% + #of the miner fee + minsize = max(int(1.2 * txfee_contribution / float(cjfee_r)), + int(options["minsize"])) + cjfee_a = None + elif ordertype == 'absoffer': + cjfee_a = int(options["cjfee_a"]) + minsize = int(options["minsize"]) + cjfee_r = None + else: + assert False, "incorrect offertype config for yieldgenerator." + + txtype = wallet_service.get_txtype() + if txtype == "p2wpkh": + prefix = "sw0" + elif txtype == "p2sh-p2wpkh": + prefix = "sw" + elif txtype == "p2pkh": + prefix = "" + else: + assert False, "Unsupported wallet type for yieldgenerator: " + txtype + + ordertype = prefix + ordertype + + for i in range(start_bot_num, end_bot_num): + cfg = [txfee_contribution, cjfee_a, cjfee_r, ordertype, minsize, + txfee_contribution_factor, cjfee_factor, size_factor] + wallet_service_yg = wallet_services[i - 1]["wallet"] + + wallet_service_yg.startService() + + yg = ygclass(wallet_service_yg, cfg) + clientfactory = RegtestJMClientProtocolFactory(yg, proto_type="MAKER") + # This ensures that the right rpc/port config is passed into the daemon, + # for this specific bot: + clientfactory.i = i + # This ensures that this bot knows which other bots are directory nodes: + clientfactory.set_directory_nodes(directory_node_indices) + nodaemon = jm_single().config.getint("DAEMON", "no_daemon") + daemon = True if nodaemon == 1 else False + #rs = True if i == num_ygs - 1 else False + start_reactor(jm_single().config.get("DAEMON", "daemon_host"), + jm_single().config.getint("DAEMON", "daemon_port"), + clientfactory, daemon=daemon, rs=False) + reactor.callLater(1.0, start_test_taker, wallet_services[end_bot_num - 1]['wallet'], end_bot_num) + reactor.run() + +@defer.inlineCallbacks +def start_test_taker(wallet_service, i): + # this rpc manager has auth disabled, + # and the wallet_service is set manually, + # so no unlock etc. + mgr = TWalletRPCManager() + mgr.daemon.wallet_service = wallet_service + # because we are manually setting the wallet_service + # of the JMWalletDaemon instance, we do not follow the + # usual flow of `initialize_wallet_service`, we do not set + # the auth token or start the websocket; so we must manually + # sync the wallet, including bypassing any restart callback: + def dummy_restart_callback(msg): + log.warn("Ignoring rescan request from backend wallet service: " + msg) + mgr.daemon.wallet_service.add_restart_callback(dummy_restart_callback) + mgr.daemon.wallet_name = wallet_name + while not mgr.daemon.wallet_service.synced: + mgr.daemon.wallet_service.sync_wallet(fast=True) + mgr.daemon.wallet_service.startService() + def get_client_factory(): + clientfactory = RegtestJMClientProtocolFactory(mgr.daemon.taker, + proto_type="TAKER") + clientfactory.i = i + clientfactory.set_directory_nodes(directory_node_indices) + return clientfactory + + mgr.daemon.get_client_factory = get_client_factory + # before preparing the RPC call to the wallet daemon, + # we decide a coinjoin destination and amount. Choosing + # a destination in the wallet is a bit easier because + # we can query the mixdepth balance at the end. + coinjoin_destination = mgr.daemon.wallet_service.get_internal_addr(4) + cj_amount = 22000000 + # once the taker is finished we sanity check before + # shutting down: + def dummy_taker_finished(res, fromtx=False, + waittime=0.0, txdetails=None): + jmprint("Taker is finished") + # check that the funds have arrived. + mbal = mgr.daemon.wallet_service.get_balance_by_mixdepth()[4] + assert mbal == cj_amount + jmprint("Funds: {} sats successfully arrived into mixdepth 4.".format(cj_amount)) + stop_reactor() + mgr.daemon.taker_finished = dummy_taker_finished + mgr.start() + agent = get_nontor_agent() + addr = mgr.get_route_root() + addr += "/wallet/" + addr += mgr.daemon.wallet_name + addr += "/taker/coinjoin" + addr = addr.encode() + body = BytesProducer(json.dumps({"mixdepth": "1", + "amount_sats": cj_amount, + "counterparties": "2", + "destination": coinjoin_destination}).encode()) + yield mgr.do_request(agent, b"POST", addr, body, + process_coinjoin_response) + +def process_coinjoin_response(response): + json_body = json.loads(response.decode("utf-8")) + print("coinjoin response: {}".format(json_body)) + +def get_addr_and_fund(yg): + """ This function allows us to create + and publish a fidelity bond for a particular + yield generator object after the wallet has reached + a synced state and is therefore ready to serve up + timelock addresses. We create the TL address, fund it, + refresh the wallet and then republish our offers, which + will also publish the new FB. + """ + if not yg.wallet_service.synced: + return + if yg.wallet_service.timelock_funded: + return + addr = wallet_gettimelockaddress(yg.wallet_service.wallet, "2021-11") + print("Got timelockaddress: {}".format(addr)) + + # pay into it; amount is randomized for now. + # Note that grab_coins already mines 1 block. + fb_amt = random.randint(1, 5) + jm_single().bc_interface.grab_coins(addr, fb_amt) + + # we no longer have to run this loop (TODO kill with nonlocal) + yg.wallet_service.timelock_funded = True + + # force wallet to check for the new coins so the new + # yg offers will include them: + yg.wallet_service.transaction_monitor() + + # publish a new offer: + yg.offerlist = yg.create_my_orders() + yg.fidelity_bond = yg.get_fidelity_bond_template() + jmprint('updated offerlist={}'.format(yg.offerlist)) + +@pytest.fixture(scope="module") +def setup_onion_ygrunner(): + load_test_config() + jm_single().bc_interface.tick_forward_chain_interval = 10 + jm_single().bc_interface.simulate_blocks() diff --git a/test/regtest_joinmarket.cfg b/test/regtest_joinmarket.cfg index 4d3c211cf..3345e29ff 100644 --- a/test/regtest_joinmarket.cfg +++ b/test/regtest_joinmarket.cfg @@ -16,6 +16,7 @@ network = testnet rpc_wallet_file = jm-test-wallet [MESSAGING:server1] +type = irc host = localhost hostid = localhost1 channel = joinmarket-pit @@ -26,6 +27,7 @@ socks5_host = localhost socks5_port = 9150 [MESSAGING:server2] +type = irc host = localhost hostid = localhost2 channel = joinmarket-pit @@ -35,8 +37,46 @@ socks5 = false socks5_host = localhost socks5_port = 9150 +[MESSAGING:onion1] +# onion based message channels must have the exact type 'onion' +# (while the section name above can be MESSAGING:whatever), and there must +# be only ONE such message channel configured (note the directory servers +# can be multiple, below): +type = onion +socks5_host = localhost +socks5_port = 9050 +# the tor control configuration: +tor_control_host = localhost +# or, to use a UNIX socket +# control_host = unix:/var/run/tor/control +tor_control_port = 9051 +# the host/port actually serving the hidden service +# (note the *virtual port*, that the client uses, +# is hardcoded to 80): +onion_serving_host = 127.0.0.1 +onion_serving_port = 8080 +# This is mandatory for directory nodes (who must also set their +# own .onion:port as the only directory in directory_nodes, below), +# but NOT TO BE USED by non-directory nodes (which is you, unless +# you know otherwise!), as it will greatly degrade your privacy. +# +# Special handling on regtest, so just ignore and let the code handle it: +hidden_service_dir = "" +# This is a comma separated list (comma can be omitted if only one item). +# Each item has format host:port +# On regtest we are going to increment the port numbers served from, with +# the value used here as the starting value: +directory_nodes = localhost:8081 +# this is not present in default real config +# and is specifically used to flag tests: +# means we use indices 1,2,3,4,5: +regtest_count=1,5 + [TIMEOUT] -maker_timeout_sec = 15 +maker_timeout_sec = 10 + +[LOGGING] +console_log_level = DEBUG [POLICY] # for dust sweeping, try merge_algorithm = gradual diff --git a/test/ygrunner.py b/test/ygrunner.py index 88ef65b97..d657179d5 100644 --- a/test/ygrunner.py +++ b/test/ygrunner.py @@ -96,7 +96,7 @@ def on_tx_received(self, nick, tx, offerinfo): "num_ygs, wallet_structures, fb_indices, mean_amt, malicious, deterministic", [ # 1sp 3yg, honest makers, one maker has FB: - (3, [[1, 3, 0, 0, 0]] * 4, [1, 2], 2, 0, False), + (3, [[1, 3, 0, 0, 0]] * 4, [], 2, 0, False), # 1sp 3yg, malicious makers reject on auth and on tx 30% of time #(3, [[1, 3, 0, 0, 0]] * 4, 2, 30, False), # 1 sp 9 ygs, deterministically malicious 50% of time @@ -173,6 +173,7 @@ def test_start_ygs(setup_ygrunner, num_ygs, wallet_structures, fb_indices, ygclass = DeterministicMaliciousYieldGenerator else: ygclass = MaliciousYieldGenerator + for i in range(num_ygs): cfg = [txfee_contribution, cjfee_a, cjfee_r, ordertype, minsize, txfee_contribution_factor, cjfee_factor, size_factor] From 830ac229345ebb865d2ea2ba5c3bcf6921307ee1 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Thu, 24 Feb 2022 15:01:50 +0000 Subject: [PATCH 2/3] Allow taker peers to not serve onions + bugfixes. In the previous commit, all peers served an onion. After this commit, taker client instances will automatically send a config var to the jmdaemon backend that instructs the OnionMessageChannel instance to not start an onion service, and the handshake messages sent by these peers replace the onion location with a placeholder string NOT-SERVING-ONION. Directories and maker peers will not therefore to connect outbound to them, but privmsging still happens p2p with connections from takers to makers after the directory has communicated their reachable .onion addresses. This change reduces the configuration requirement for takers and is better for their privacy and security (without sacrificing the gain we get from having p2p connections). The above comments re: takers also apply to ob-watcher bots. This commit also fixes a large number of minor bugs and errors in documentation, as well as many Python cleanups after review from @PulpCattel. A few concrete items are: It fixes the ob-watcher functionality to work with the new subclass of MessageChannel (OnionMessageChannel). It corrects the on_nick_leave trigger to make dynamic nick switching between MessageChannels (as implemented in MessageChannelCollection) work correctly. It corrects the order of events in the add_peer workflow to ensure that a handshake can always be sent so that the activation of the connection always works. It sets a default messaging config with onion, 2 active IRC servers and one inactive IRC server. The onion config has 2 signet directory nodes, so this will change to mainnet after the PR is merged to master. --- docs/onion-message-channels.md | 88 +++- jmbase/jmbase/commands.py | 4 +- jmbase/test/test_commands.py | 6 +- jmclient/jmclient/client_protocol.py | 13 +- jmclient/jmclient/configure.py | 138 ++++--- jmclient/jmclient/wallet_rpc.py | 5 +- jmclient/test/test_client_protocol.py | 4 +- jmdaemon/jmdaemon/daemon_protocol.py | 18 +- jmdaemon/jmdaemon/message_channel.py | 2 +- jmdaemon/jmdaemon/onionmc.py | 551 +++++++++++++++----------- jmdaemon/test/test_daemon_protocol.py | 6 +- scripts/obwatch/ob-watcher.py | 66 +-- test/e2e-coinjoin-test.py | 85 ++-- test/regtest_joinmarket.cfg | 2 +- 14 files changed, 557 insertions(+), 431 deletions(-) diff --git a/docs/onion-message-channels.md b/docs/onion-message-channels.md index deb5af6cb..5926f435e 100644 --- a/docs/onion-message-channels.md +++ b/docs/onion-message-channels.md @@ -14,7 +14,7 @@ This is a new way for Joinmarket bots to communicate, namely by serving and connecting to Tor onion services. This does not introduce any new requirements to your Joinmarket installation, technically, because the use of Payjoin already required the need -to service such onion services, and connecting to IRC used a SOCKS5 proxy (by default, and used by almost all users) over Tor to +to run such onion services, and connecting to IRC used a SOCKS5 proxy (used by almost all users) over Tor to a remote onion service. The purpose of this new type of message channel is as follows: @@ -25,23 +25,57 @@ albeit it was and remains E2E encrypted data, in either case) * the above can lead to better scalability at large numbers * a substantial increase in the speed of transaction negotiation; this is mostly related to the throttling of high bursts of traffic on IRC -The configuration for a user is simple; in their `joinmarket.cfg` they will add a messaging section like this: +The configuration for a user is simple; in their `joinmarket.cfg` they will get a messaging section like this, if they start from scratch: ``` -[MESSAGING:onion1] +[MESSAGING:onion] +# onion based message channels must have the exact type 'onion' +# (while the section name above can be MESSAGING:whatever), and there must +# be only ONE such message channel configured (note the directory servers +# can be multiple, below): type = onion -onion_serving_port = 8082 + +socks5_host = localhost +socks5_port = 9050 + +# the tor control configuration. +# for most people running the tor daemon +# on Linux, no changes are required here: +tor_control_host = localhost +# or, to use a UNIX socket +# tor_control_host = unix:/var/run/tor/control +tor_control_port = 9051 + +# the host/port actually serving the hidden service +# (note the *virtual port*, that the client uses, +# is hardcoded to 80): +onion_serving_host = 127.0.0.1 +onion_serving_port = 8080 + +# directory node configuration +# +# This is mandatory for directory nodes (who must also set their +# own *.onion:port as the only directory in directory_nodes, below), +# but NOT TO BE USED by non-directory nodes (which is you, unless +# you know otherwise!), as it will greatly degrade your privacy. +# (note the default is no value, don't replace it with ""). +hidden_service_dir = +# # This is a comma separated list (comma can be omitted if only one item). -# Each item has format host:port +# Each item has format host:port ; both are required, though port will +# be 80 if created in this code. directory_nodes = rr6f6qtleiiwic45bby4zwmiwjrj3jsbmcvutwpqxjziaydjydkk5iad.onion:80 + +# This setting is ONLY for developer regtest setups, +# running multiple bots at once. Don't alter it otherwise +regtest_count = 0,0 ``` -Here, I have deliberately omitted the several other settings in this section which will almost always be fine as default; -see `jmclient/jmclient/configure.py` for what those defaults are, and the extensive comments explaining. +All of these can be left as default for most users, except the field `directory_nodes`. -The main point is the list of **directory nodes** (the one shown here is one being run on signet, right now), which will +The list of **directory nodes** (the one shown here is one being run on signet, right now), which will be comma separated if multiple directory nodes are configured (we expect there will be 2 or 3 as a normal situation). -The `onion_serving_port` is on which port on the local machine the onion service is served. +The `onion_serving_port` is on which port on the local machine the onion service is served; you won't usually need to use it, but it mustn't conflict with some other usage (so if you have something running on port 8080, change it). The `type` field must always be `onion` in this case, and distinguishes it from IRC message channels and others. ### Can/should I still run IRC message channels? @@ -50,11 +84,24 @@ In short, yes. ### Do I need to configure Tor, and if so, how? -These message channels use both outbound and inbound connections to onion services (or "hidden services"). +To make outbound Tor connections to other onions in the network, you will need to configure the +SOCKS5 proxy settings (so, only directory nodes may *not* need this; everyone else does). +This is identical to what we already do for IRC, except that in this case, we disallow clearnet connections. + +#### Running/testing as a maker + +A maker will additionally allow *inbound* connections to an onion service. +This onion service will be ephemeral, that is, it will have a different onion address every time +you restart. This should work automatically, using your existing Tor daemon (here, we are using +the same code as we use when running the `receive-payjoin` script, essentially). + +#### Running/testing as other bots (taker) + +A taker will not attempt to serve an onion; it will only use outbound connections, first to directory +nodes and then, as according to need, to individual makers, also. -As previously mentioned, both of these features were already in use in Joinmarket. If you never served an -onion service before, it should work fine as long as you have the Tor service running in the background, -and the default control port 9051 (if not, change that value in the `joinmarket.cfg`, see above. +As previously mentioned, both of these features - inbound and outbound, to onion, Tor connections - were already in use in Joinmarket. If you want to run/test as a maker bot, but never served an onion service before, it should work fine as long as you have the Tor service running in the background, +and the default control port 9051 (if not, change that value in the `joinmarket.cfg`, see above). #### Why not use Lightning based onions? @@ -85,7 +132,7 @@ and pay attention to the settings in `regtest_joinmarket.cfg`.) There is no separate/special configuration for signet other than the configuration that is already needed for running Joinmarket against a signet backend (so e.g. RPC port of 38332). -Add the `[MESSAGING:onion1]` message channel section to your `joinmarket.cfg`, as listed above, including the +Add the `[MESSAGING:onion]` message channel section to your `joinmarket.cfg`, as listed above, including the signet directory node listed above (rr6f6qtleiiwic45bby4zwmiwjrj3jsbmcvutwpqxjziaydjydkk5iad.onion:80), and, for the simplest test, remove the other `[MESSAGING:*]` sections that you have. @@ -101,19 +148,19 @@ who would like to help by running a directory node. You can ignore it if that do This requires a long running bot. It should be on a server you can keep running permanently, so perhaps a VPS, but in any case, very high uptime. For reliability it also makes sense to configure to run as a systemd service. -A note: in this early stage, the usage of Lightning is only really network-layer stuff, and the usage of bitcoin, is none; feel free to add elements that remove any need for a backend bitcoin blockchain, but beware: future upgrades *could* mean that the directory node really does need the bitcoin backend. +A note: the most natural way to run the directory is as a Joinmarket *maker* bot, i.e. run `yg-privacyenhanced.py`, with configuration as described below. For now it will actually offer to do coinjoins - we will want to fix this in future so no coins are needed (but it can just be a trivial size). #### Joinmarket-specific configuration -Add `hidden_service_dir` to your `[MESSAGING:onion1]` with a directory accessible to your user. You may want to lock this down +Add `hidden_service_dir` to your `[MESSAGING:onion]` with a directory accessible to your user. You may want to lock this down a bit! The point to understand is: Joinmarket's `jmbase.JMHiddenService` will, if configured with a non-empty `hidden_service_dir` field, actually start an *independent* instance of Tor specifically for serving this, under the current user. -(our tor interface library `txtorcon` needs read access to the Tor HS dir, so it's troublesome to do this another way). +(our Tor interface library `txtorcon` needs read access to the Tor HS dir, so it's troublesome to do this another way). ##### Question: How to configure the `directory-nodes` list in our `joinmarket.cfg` for this directory node bot? -Answer: **you must only enter your own node in this list!** (otherwise you may find your bot infinitely rebroadcasting messages). +Answer: **you must only enter your own node in this list!**. This way your bot will recognize that it is a directory node and it avoids weird edge case behaviour (so don't add *other* known directory nodes; you won't be talking to them). #### Suggested setup of a service: @@ -143,7 +190,7 @@ WantedBy=multi-user.target ``` This is deliberately a super-basic setup (see above). Don't forget to setup your `bitcoin.conf` as usual, -for the bitcoin user, and make it match (specifically in terms of RPC) what you set up for Lightning below. +for the bitcoin user, and make it match (specifically in terms of RPC) what you set up for Joinmarket below. 2. @@ -167,7 +214,6 @@ To state the obvious, the idea here is that this second service will run the JM to ensure they start up in the correct order. Re: password echo, obviously this kind of password entry is bad; -for now we needn't worry as these nodes don't need to carry any real coins (and it's better they don't!). -Later we may need to change that (though of course you can use standard measures to protect the box). +for now we needn't worry as these nodes don't need to carry significant coins (and it's much better they don't!). TODO: add some material on network hardening/firewalls here, I guess. diff --git a/jmbase/jmbase/commands.py b/jmbase/jmbase/commands.py index 498b96ee0..d75721c91 100644 --- a/jmbase/jmbase/commands.py +++ b/jmbase/jmbase/commands.py @@ -27,11 +27,11 @@ class JMInit(JMCommand): """Communicates the client's required setup configuration. Blockchain source is communicated only as a naming - tag for messagechannels (currently IRC 'realname' field). + tag for messagechannels (for IRC, 'realname' field). """ arguments = [(b'bcsource', Unicode()), (b'network', Unicode()), - (b'irc_configs', JsonEncodable()), + (b'chan_configs', JsonEncodable()), (b'minmakers', Integer()), (b'maker_timeout_sec', Integer()), (b'dust_threshold', Integer()), diff --git a/jmbase/test/test_commands.py b/jmbase/test/test_commands.py index c6e4100b6..0c05e9582 100644 --- a/jmbase/test/test_commands.py +++ b/jmbase/test/test_commands.py @@ -43,9 +43,9 @@ def end_test(): class JMTestServerProtocol(JMBaseProtocol): @JMInit.responder - def on_JM_INIT(self, bcsource, network, irc_configs, minmakers, + def on_JM_INIT(self, bcsource, network, chan_configs, minmakers, maker_timeout_sec, dust_threshold, blacklist_location): - show_receipt("JMINIT", bcsource, network, irc_configs, minmakers, + show_receipt("JMINIT", bcsource, network, chan_configs, minmakers, maker_timeout_sec, dust_threshold, blacklist_location) d = self.callRemote(JMInitProto, nick_hash_length=1, @@ -137,7 +137,7 @@ def clientStart(self): d = self.callRemote(JMInit, bcsource="dummyblockchain", network="dummynetwork", - irc_configs=['dummy', 'irc', 'config'], + chan_configs=['dummy', 'irc', 'config'], minmakers=7, maker_timeout_sec=8, dust_threshold=1500, diff --git a/jmclient/jmclient/client_protocol.py b/jmclient/jmclient/client_protocol.py index 01b00b8e7..180932d5b 100644 --- a/jmclient/jmclient/client_protocol.py +++ b/jmclient/jmclient/client_protocol.py @@ -434,7 +434,7 @@ def clientStart(self): "blockchain_source") #needed only for channel naming convention network = jm_single().config.get("BLOCKCHAIN", "network") - irc_configs = self.factory.get_mchannels() + chan_configs = self.factory.get_mchannels(mode="MAKER") #only here because Init message uses this field; not used by makers TODO minmakers = jm_single().config.getint("POLICY", "minimum_makers") maker_timeout_sec = jm_single().maker_timeout_sec @@ -442,7 +442,7 @@ def clientStart(self): d = self.callRemote(commands.JMInit, bcsource=blockchain_source, network=network, - irc_configs=irc_configs, + chan_configs=chan_configs, minmakers=minmakers, maker_timeout_sec=maker_timeout_sec, dust_threshold=jm_single().DUST_THRESHOLD, @@ -601,7 +601,7 @@ def clientStart(self): "blockchain_source") #needed only for channel naming convention network = jm_single().config.get("BLOCKCHAIN", "network") - irc_configs = self.factory.get_mchannels() + chan_configs = self.factory.get_mchannels(mode="TAKER") minmakers = jm_single().config.getint("POLICY", "minimum_makers") maker_timeout_sec = jm_single().maker_timeout_sec @@ -614,7 +614,7 @@ def clientStart(self): d = self.callRemote(commands.JMInit, bcsource=blockchain_source, network=network, - irc_configs=irc_configs, + chan_configs=chan_configs, minmakers=minmakers, maker_timeout_sec=maker_timeout_sec, dust_threshold=jm_single().DUST_THRESHOLD, @@ -789,19 +789,20 @@ def __init__(self, client, proto_type="TAKER"): def setClient(self, client): self.proto_client = client + def getClient(self): return self.proto_client def buildProtocol(self, addr): return self.protocol(self, self.client) - def get_mchannels(self): + def get_mchannels(self, mode): """ A transparent wrapper that allows override, so that a script can return a customised set of message channel configs; currently used for testing multiple bots on regtest. """ - return get_mchannels() + return get_mchannels(mode) def start_reactor(host, port, factory=None, snickerfactory=None, bip78=False, jm_coinjoin=True, ish=True, diff --git a/jmclient/jmclient/configure.py b/jmclient/jmclient/configure.py index 0c3ff6b10..ea04d0d5c 100644 --- a/jmclient/jmclient/configure.py +++ b/jmclient/jmclient/configure.py @@ -137,27 +137,7 @@ def jm_single(): # information. rpc_wallet_file = -## SERVER 1/3) Darkscience IRC (Tor, IP) -################################################################################ -[MESSAGING:server1] -# by default the legacy format without a `type` field is -# understood to be IRC, but you can, optionally, add it: -# type = irc -channel = joinmarket-pit -port = 6697 -usessl = true - -# For traditional IP (default): -host = irc.darkscience.net -socks5 = false - -# For Tor (recommended as clearnet alternative): -#host = darkirc6tqgpnwd3blln3yfv5ckl47eg7llfxkmtovrv7c7iwohhb6ad.onion -#socks5 = true -#socks5_host = localhost -#socks5_port = 9050 - -[MESSAGING:onion1] +[MESSAGING:onion] # onion based message channels must have the exact type 'onion' # (while the section name above can be MESSAGING:whatever), and there must # be only ONE such message channel configured (note the directory servers @@ -193,23 +173,59 @@ def jm_single(): # This is a comma separated list (comma can be omitted if only one item). # Each item has format host:port ; both are required, though port will # be 80 if created in this code. -directory_nodes = rr6f6qtleiiwic45bby4zwmiwjrj3jsbmcvutwpqxjziaydjydkk5iad.onion:80 +directory_nodes = rr6f6qtleiiwic45bby4zwmiwjrj3jsbmcvutwpqxjziaydjydkk5iad.onion:80,k74oyetjqgcamsyhlym2vgbjtvhcrbxr4iowd4nv4zk5sehw4v665jad.onion:80 # This setting is ONLY for developer regtest setups, # running multiple bots at once. Don't alter it otherwise regtest_count = 0,0 -## SERVER 3/3) ILITA IRC (Tor - disabled by default) +## IRC SERVER 1: Darkscience IRC (Tor, IP) +################################################################################ +[MESSAGING:server1] +# by default the legacy format without a `type` field is +# understood to be IRC, but you can, optionally, add it: +# type = irc +channel = joinmarket-pit +port = 6697 +usessl = true + +# For traditional IP: +#host = irc.darkscience.net +#socks5 = false + +# For Tor (recommended as clearnet alternative): +host = darkirc6tqgpnwd3blln3yfv5ckl47eg7llfxkmtovrv7c7iwohhb6ad.onion +socks5 = true +socks5_host = localhost +socks5_port = 9050 + +## IRC SERVER 2: ILITA IRC (optional IRC alternate, Tor only) +################################################################################ +[MESSAGING:server2] +channel = joinmarket-pit +port = 6667 +usessl = false +socks5 = true +socks5_host = localhost + +host = ilitafrzzgxymv6umx2ux7kbz3imyeko6cnqkvy4nisjjj4qpqkrptid.onion +socks5_port = 9050 + +## IRC SERVER 3) (backup) hackint IRC (Tor, IP) ################################################################################ #[MESSAGING:server3] -#channel = joinmarket-pit +# channel = joinmarket-pit +# For traditional IP: +## host = irc.hackint.org +## port = 6697 +## usessl = true +## socks5 = false +# For Tor (default): +#host = ncwkrwxpq2ikcngxq3dy2xctuheniggtqeibvgofixpzvrwpa77tozqd.onion #port = 6667 #usessl = false #socks5 = true #socks5_host = localhost - -# For Tor (recommended): -#host = ilitafrzzgxymv6umx2ux7kbz3imyeko6cnqkvy4nisjjj4qpqkrptid.onion #socks5_port = 9050 [LOGGING] @@ -510,7 +526,7 @@ def set_config(cfg, bcint=None): global_singleton.bc_interface = bcint -def get_mchannels(): +def get_mchannels(mode="TAKER"): SECTION_NAME = 'MESSAGING' # FIXME: remove in future release if jm_single().config.has_section(SECTION_NAME): @@ -521,65 +537,64 @@ def get_mchannels(): return _get_irc_mchannels_old() SECTION_NAME += ':' - sections = [] - for s in jm_single().config.sections(): - if s.startswith(SECTION_NAME): - sections.append(s) - assert sections irc_fields = [("host", str), ("port", int), ("channel", str), ("usessl", str), - ("socks5", str), ("socks5_host", str), ("socks5_port", str)] + ("socks5", str), ("socks5_host", str), ("socks5_port", int)] onion_fields = [("type", str), ("directory_nodes", str), ("regtest_count", str), ("socks5_host", str), ("socks5_port", int), ("tor_control_host", str), ("tor_control_port", int), ("onion_serving_host", str), ("onion_serving_port", int), ("hidden_service_dir", str)] - configs = [] - - # processing the IRC sections: - for section in sections: - if jm_single().config.has_option(section, "type"): - # legacy IRC configs do not have "type" but just - # in case, we'll allow the "irc" type: - if not jm_single().config.get(section, "type").lower( - ) == "irc": - break + def get_irc_section(s): server_data = {} - # check if socks5 is enabled for tor and load relevant config if so try: - server_data["socks5"] = jm_single().config.get(section, "socks5") + server_data["socks5"] = jm_single().config.get(s, "socks5") except NoOptionError: server_data["socks5"] = "false" if server_data["socks5"].lower() == 'true': - server_data["socks5_host"] = jm_single().config.get(section, "socks5_host") - server_data["socks5_port"] = jm_single().config.get(section, "socks5_port") + server_data["socks5_host"] = jm_single().config.get(s, "socks5_host") + server_data["socks5_port"] = jm_single().config.get(s, "socks5_port") for option, otype in irc_fields: - val = jm_single().config.get(section, option) + val = jm_single().config.get(s, option) server_data[option] = otype(val) server_data['btcnet'] = get_network() - configs.append(server_data) + return server_data - # processing the onion sections: - for section in sections: - if not jm_single().config.has_option(section, "type") or \ - not jm_single().config.get(section, "type").lower() == "onion": - continue + def get_onion_section(s): onion_data = {} for option, otype in onion_fields: try: - val = jm_single().config.get(section, option) + val = jm_single().config.get(s, option) except NoOptionError: continue onion_data[option] = otype(val) + # the onion messaging section must specify whether + # to serve an onion: + onion_data["serving"] = mode == "MAKER" onion_data['btcnet'] = get_network() # Just to allow a dynamic set of var: - onion_data["section-name"] = section - configs.append(onion_data) + onion_data["section-name"] = s + return onion_data - return configs + onion_sections = [] + irc_sections = [] + for section in jm_single().config.sections(): + if not section.startswith(SECTION_NAME): + continue + if jm_single().config.has_option(section, "type"): + channel_type = jm_single().config.get(section, "type").lower() + if channel_type == "onion": + onion_sections.append(get_onion_section(section)) + elif channel_type == "irc": + irc_sections.append(get_irc_section(section)) + else: + irc_sections.append(get_irc_section(section)) + assert irc_sections or onion_sections + assert len(onion_sections) < 2 + return irc_sections + onion_sections def _get_irc_mchannels_old(): fields = [("host", str), ("port", int), ("channel", str), ("usessl", str), @@ -777,11 +792,6 @@ def load_program_config(config_path="", bs=None, plugin_services=[]): if not os.path.exists(plogsdir): os.makedirs(plogsdir) p.set_log_dir(plogsdir) - # Check if a onion message channel was configured, and if so, - # check there is only 1; multiple directory nodes will be inside the config. - chans = get_mchannels() - onion_chans = [x for x in chans if "type" in x and x["type"] == "onion"] - assert len(onion_chans) < 2 def load_test_config(**kwargs): if "config_path" not in kwargs: diff --git a/jmclient/jmclient/wallet_rpc.py b/jmclient/jmclient/wallet_rpc.py index 68c8f895d..dd50b72b0 100644 --- a/jmclient/jmclient/wallet_rpc.py +++ b/jmclient/jmclient/wallet_rpc.py @@ -423,8 +423,7 @@ def dummy_restart_callback(msg): walletname=self.wallet_name, token=self.cookie) - def taker_finished(self, res, fromtx=False, - waittime=0.0, txdetails=None): + def taker_finished(self, res, fromtx=False, waittime=0.0, txdetails=None): # This is a slimmed down version compared with what is seen in # the CLI code, since that code encompasses schedules with multiple # entries; for now, the RPC only supports single joins. @@ -1007,7 +1006,7 @@ def dummy_user_callback(rel, abs): self.taker = Taker(self.services["wallet"], schedule, max_cj_fee = max_cj_fee, callbacks=(self.filter_orders_callback, - None, self.taker_finished)) + None, self.taker_finished)) # TODO ; this makes use of a pre-existing hack to allow # selectively disabling the stallMonitor function that checks # if transactions went through or not; here we want to cleanly diff --git a/jmclient/test/test_client_protocol.py b/jmclient/test/test_client_protocol.py index d60adcbff..d1f07f9f5 100644 --- a/jmclient/test/test_client_protocol.py +++ b/jmclient/test/test_client_protocol.py @@ -167,9 +167,9 @@ def end_test(): class JMTestServerProtocol(JMBaseProtocol): @JMInit.responder - def on_JM_INIT(self, bcsource, network, irc_configs, minmakers, + def on_JM_INIT(self, bcsource, network, chan_configs, minmakers, maker_timeout_sec, dust_threshold, blacklist_location): - show_receipt("JMINIT", bcsource, network, irc_configs, minmakers, + show_receipt("JMINIT", bcsource, network, chan_configs, minmakers, maker_timeout_sec, dust_threshold, blacklist_location) d = self.callRemote(JMInitProto, nick_hash_length=1, diff --git a/jmdaemon/jmdaemon/daemon_protocol.py b/jmdaemon/jmdaemon/daemon_protocol.py index d84bbb514..9fdd641d6 100644 --- a/jmdaemon/jmdaemon/daemon_protocol.py +++ b/jmdaemon/jmdaemon/daemon_protocol.py @@ -475,7 +475,7 @@ def __init__(self, factory): self.factory = factory self.jm_state = 0 self.restart_mc_required = False - self.irc_configs = None + self.chan_configs = None self.mcc = None #Default role is TAKER; must be overriden to MAKER in JMSetup message. self.role = "TAKER" @@ -504,7 +504,7 @@ def defaultCallbacks(self, d): d.addErrback(self.defaultErrback) @JMInit.responder - def on_JM_INIT(self, bcsource, network, irc_configs, minmakers, + def on_JM_INIT(self, bcsource, network, chan_configs, minmakers, maker_timeout_sec, dust_threshold, blacklist_location): """Reads in required configuration from client for a new session; feeds back joinmarket messaging protocol constants @@ -518,25 +518,25 @@ def on_JM_INIT(self, bcsource, network, irc_configs, minmakers, self.dust_threshold = int(dust_threshold) #(bitcoin) network only referenced in channel name construction self.network = network - if irc_configs == self.irc_configs: + if chan_configs == self.chan_configs: self.restart_mc_required = False log.msg("New init received did not require a new message channel" " setup.") else: - if self.irc_configs: + if self.chan_configs: #close the existing connections self.mc_shutdown() - self.irc_configs = irc_configs + self.chan_configs = chan_configs self.restart_mc_required = True mcs = [] - for c in self.irc_configs: + for c in self.chan_configs: if "type" in c and c["type"] == "onion": mcs.append(OnionMessageChannel(c, daemon=self)) else: # default is IRC; TODO allow others mcs.append(IRCMessageChannel(c, - daemon=self, - realname='btcint=' + bcsource)) + daemon=self, + realname='btcint=' + bcsource)) self.mcc = MessageChannelCollection(mcs) OrderbookWatch.set_msgchan(self, self.mcc) #register taker-specific msgchan callbacks here @@ -952,7 +952,7 @@ def init_connections(self, nick): for a new transaction; effectively means any previous incomplete transaction is wiped. """ - self.jm_state = 0 #uninited + self.jm_state = 0 self.mcc.set_nick(nick) if self.restart_mc_required: self.mcc.run() diff --git a/jmdaemon/jmdaemon/message_channel.py b/jmdaemon/jmdaemon/message_channel.py index 9549f193d..4b46d817e 100644 --- a/jmdaemon/jmdaemon/message_channel.py +++ b/jmdaemon/jmdaemon/message_channel.py @@ -259,7 +259,7 @@ def privmsg(self, nick, cmd, message, mc=None): for x in self.available_channels() if mc == x.hostid] if len(matching_channels) != 1: #pragma: no cover - #this can happen if an IRC goes down shortly before a message + #this can happen if a m-channel goes down shortly before a message #is supposed to be sent. There used to be an exception raise. #to prevent a crash (especially in makers), we just inform #the user about it for now diff --git a/jmdaemon/jmdaemon/onionmc.py b/jmdaemon/jmdaemon/onionmc.py index a426674bb..3d8bf8fc8 100644 --- a/jmdaemon/jmdaemon/onionmc.py +++ b/jmdaemon/jmdaemon/onionmc.py @@ -1,17 +1,24 @@ from jmdaemon.message_channel import MessageChannel from jmdaemon.protocol import COMMAND_PREFIX, JM_VERSION -from jmbase import get_log, JM_APP_NAME, JMHiddenService +from jmbase import get_log, JM_APP_NAME, JMHiddenService, stop_reactor import json import copy -from typing import Callable, Union +from typing import Callable, Union, Tuple, List from twisted.internet import reactor, task, protocol from twisted.protocols import basic +from twisted.application.internet import ClientService from twisted.internet.endpoints import TCP4ClientEndpoint from twisted.internet.address import IPv4Address, IPv6Address from txtorcon.socks import TorSocksEndpoint log = get_log() + +NOT_SERVING_ONION_HOSTNAME = "NOT-SERVING-ONION" + +def location_tuple_to_str(t: Tuple[str, int]) -> str: + return f"{t[0]}:{t[1]}" + def network_addr_to_string(location: Union[IPv4Address, IPv4Address]) -> str: if isinstance(location, (IPv4Address, IPv6Address)): host = location.host @@ -19,18 +26,17 @@ def network_addr_to_string(location: Union[IPv4Address, IPv4Address]) -> str: else: # TODO handle other addr types assert False - return host + ":" + str(port) + return location_tuple_to_str((host, port)) # module-level var to control whether we use Tor or not -# (specifically for tests): +# (specifically for tests) testing_mode = False def set_testing_mode(configdata: dict) -> None: """ Toggles testing mode which enables non-Tor network setup: """ global testing_mode - if not "regtest_count" in configdata: - log.debug("Onion message channel is not using regtest mode.") + if "regtest_count" not in configdata: testing_mode = False return try: @@ -39,7 +45,7 @@ def set_testing_mode(configdata: dict) -> None: log.info("Failed to get regtest count settings, error: {}".format(repr(e))) testing_mode = False return - if s == 0 and e == 0: + if s == e == 0: testing_mode = False return testing_mode = True @@ -97,6 +103,9 @@ class OnionPeerConnectionError(OnionPeerError): class OnionCustomMessageDecodingError(Exception): pass +class InvalidLocationStringError(Exception): + pass + class OnionCustomMessage(object): """ Encapsulates the messages passed over the wire to and from other onion peers @@ -105,49 +114,57 @@ def __init__(self, text: str, msgtype: int): self.text = text self.msgtype = msgtype - def encode(self) -> str: + def encode(self) -> bytes: self.encoded = json.dumps({"type": self.msgtype, "line": self.text}).encode("utf-8") return self.encoded @classmethod - def from_string_decode(cls, msg: str) -> 'OnionCustomMessage': + def from_string_decode(cls, msg: bytes) -> 'OnionCustomMessage': """ Build a custom message from a json-ified string. """ try: msg_obj = json.loads(msg) text = msg_obj["line"] msgtype = msg_obj["type"] + # we insist on integer but not a valid msgtype, + # crudely 'syntax, not semantics': + # semantics is the job of the OnionMessageChannel object. + assert isinstance(msgtype, int) + assert isinstance(text, str) except: + # this blanket catch and re-raise: + # we must handle untrusted input bytes without + # crashing under any circumstance. raise OnionCustomMessageDecodingError return cls(text, msgtype) class OnionLineProtocol(basic.LineReceiver): def connectionMade(self): self.factory.register_connection(self) + basic.LineReceiver.connectionMade(self) def connectionLost(self, reason): self.factory.register_disconnection(self) + basic.LineReceiver.connectionLost(self, reason) - def lineReceived(self, line: str) -> None: - #print("received", repr(line)) + def lineReceived(self, line: bytes) -> None: try: msg = OnionCustomMessage.from_string_decode(line) except OnionCustomMessageDecodingError: - log.debug("Received invalid message, dropping connection.") + log.debug("Received invalid message: {}, " + "dropping connection.".format(line)) self.transport.loseConnection() return self.factory.receive_message(msg, self) def message(self, message: OnionCustomMessage) -> None: - #log.info("in OnionLineProtocol, about to send message: {} to peer {}".format(message.encode(), self.transport.getPeer())) - self.transport.write(message.encode() + self.delimiter) + self.sendLine(message.encode()) class OnionLineProtocolFactory(protocol.ServerFactory): """ This factory allows us to start up instances of the LineReceiver protocol that are instantiated towards us. - As such, it is responsible for keeping track """ protocol = OnionLineProtocol @@ -159,15 +176,15 @@ def register_connection(self, p: OnionLineProtocol) -> None: # make a local control message registering # the new connection peer_location = network_addr_to_string(p.transport.getPeer()) - self.client.register_connection(peer_location, direction=0) self.peers[peer_location] = p + self.client.register_connection(peer_location, direction=0) def register_disconnection(self, p: OnionLineProtocol) -> None: # make a local control message registering - # the new connection + # the disconnection peer_location = network_addr_to_string(p.transport.getPeer()) self.client.register_disconnection(peer_location) - if not peer_location in self.peers: + if peer_location not in self.peers: log.warn("Disconnection event registered for non-existent peer.") return del self.peers[peer_location] @@ -178,16 +195,15 @@ def receive_message(self, message: OnionCustomMessage, p.transport.getPeer())) def send(self, message: OnionCustomMessage, destination: str) -> bool: - #print("trying to send in OnionLineProtocolFactory.") - #print("message: {}, destination: {}".format(message.encode(), destination)) - if not (destination in self.peers): - print("sending message {}, destination {} was not in peers {}".format(message.encode(), destination, self.peers)) + if destination not in self.peers: + log.warn("sending message {}, destination {} was not in peers {}".format( + message.encode(), destination, self.peers)) return False proto = self.peers[destination] proto.message(message) return True -class OnionClientFactory(protocol.ServerFactory): +class OnionClientFactory(protocol.ReconnectingClientFactory): """ We define a distinct protocol factory for outbound connections. Notably, this factory supports only *one* protocol instance at a time. """ @@ -195,7 +211,9 @@ class OnionClientFactory(protocol.ServerFactory): def __init__(self, message_receive_callback: Callable, connection_callback: Callable, - disconnection_callback: Callable): + disconnection_callback: Callable, + directory: bool, + mc: 'OnionMessageChannel'): self.proto_client = None # callback takes OnionCustomMessage as arg and returns None self.message_receive_callback = message_receive_callback @@ -203,9 +221,32 @@ def __init__(self, message_receive_callback: Callable, self.connection_callback = connection_callback # disconnection the same self.disconnection_callback = disconnection_callback + # is this connection to a directory? + self.directory = directory + # to keep track of state of overall messagechannel + self.mc = mc + + def clientConnectionLost(self, connector, reason): + log.debug('Onion client connection lost: ' + str(reason)) + # persistent reconnection is reserved for directories; + # for makers, it isn't logical to keep trying; they may + # well have just shut down the onion permanently, and we can + # reach them via directory anyway. + if self.directory and not self.mc.give_up: + if reactor.running: + log.info('Attempting to reconnect...') + protocol.ReconnectingClientFactory.clientConnectionLost(self, + connector, reason) + def clientConnectionFailed(self, connector, reason): + log.info('Onion client connection failed: ' + str(reason)) + # reasoning here exactly as for clientConnectionLost + if self.directory and not self.mc.give_up: + if reactor.running: + log.info('Attempting to reconnect...') + protocol.ReconnectingClientFactory.clientConnectionFailed(self, + connector, reason) def register_connection(self, p: OnionLineProtocol) -> None: - #print("in OnionClientFactory, registered a connection, proto instance: ", p) self.proto_client = p self.connection_callback() @@ -215,33 +256,19 @@ def register_disconnection(self, p: OnionLineProtocol) -> None: def send(self, msg: OnionCustomMessage) -> bool: self.proto_client.message(msg) + # Unlike the serving protocol, the client protocol + # is never in a condition of not knowing the counterparty + return True def receive_message(self, message: OnionCustomMessage, p: OnionLineProtocol) -> None: self.message_receive_callback(message) - """ - def clientConnectionLost(self, connector, reason): - log.debug('Connection to peer lost: {}, reason: {}'.format(connector, reason)) - if reactor.running: - log.info('Attempting to reconnect...') - protocol.ReconnectingClientFactory.clientConnectionLost( - self, connector, reason) - - def clientConnectionFailed(self, connector, reason): - log.debug('Connection to peer failed: {}, reason: {}'.format( - connector, reason)) - if reactor.running: - log.info('Attempting to reconnect...') - protocol.ReconnectingClientFactory.clientConnectionFailed( - self, connector, reason) - """ - class OnionPeer(object): def __init__(self, messagechannel: 'OnionMessageChannel', socks5_host: str, socks5_port: int, - hostname: str=None, port: int=-1, + location_tuple: Tuple[str, int], directory: bool=False, nick: str="", handshake_callback: Callable=None): # reference to the managing OnionMessageChannel instance is @@ -253,9 +280,19 @@ def __init__(self, messagechannel: 'OnionMessageChannel', self.socks5_host = socks5_host self.socks5_port = socks5_port # remote net config: - self.hostname = hostname - self.port = port - if directory and not (self.hostname): + self.hostname = location_tuple[0] + self.port = location_tuple[1] + # alternate location strings are used for inbound + # connections for this peer (these will be used by + # directories and onion-serving peers, sending + # messages backwards on a connection created towards them). + self.alternate_location = "" + if self.hostname != NOT_SERVING_ONION_HOSTNAME: + # There is no harm in always setting it by default; + # it only gets used if we don't have an outbound. + self.set_alternate_location(location_tuple_to_str( + location_tuple)) + if directory and not self.hostname: raise OnionPeerDirectoryWithoutHostError() self.directory = directory self._status = PEER_STATUS_UNCONNECTED @@ -267,13 +304,11 @@ def __init__(self, messagechannel: 'OnionMessageChannel', # to the remote peer. Note that this won't always be used, # if we have an inbound connection from this peer: self.factory = None - # alternate location strings are used for inbound - # connections for this peer (these will be used first - # and foremost by directories, sending messages backwards - # on a connection created towards them). - self.alternate_location = "" + # the reconnecting service allows auto-reconnection to + # some peers: + self.reconnecting_service = None - def set_alternate_location(self, location_string: str): + def set_alternate_location(self, location_string: str) -> None: self.alternate_location = location_string def update_status(self, destn_status: int) -> None: @@ -284,8 +319,7 @@ def update_status(self, destn_status: int) -> None: assert destn_status in range(4) ignored_updates = [] if self._status == PEER_STATUS_UNCONNECTED: - allowed_updates = [PEER_STATUS_CONNECTED, - PEER_STATUS_DISCONNECTED] + allowed_updates = [PEER_STATUS_CONNECTED] elif self._status == PEER_STATUS_CONNECTED: # updates from connected->connected are harmless allowed_updates = [PEER_STATUS_CONNECTED, @@ -298,7 +332,6 @@ def update_status(self, destn_status: int) -> None: allowed_updates = [PEER_STATUS_CONNECTED] ignored_updates = [PEER_STATUS_DISCONNECTED] if destn_status in ignored_updates: - # TODO: this happens sometimes from 2->1; why? log.debug("Attempt to update status of peer from {} " "to {} ignored.".format(self._status, destn_status)) return @@ -307,7 +340,8 @@ def update_status(self, destn_status: int) -> None: self._status = destn_status # the handshakes are always initiated by a client: if destn_status == PEER_STATUS_CONNECTED: - log.info("We, {}, are calling the handshake callback as client.".format(self.messagechannel.self_as_peer.peer_location())) + log.info("We, {}, are calling the handshake callback as client.".format( + self.messagechannel.self_as_peer.peer_location())) self.handshake_callback(self) def status(self) -> int: @@ -336,26 +370,31 @@ def from_location_string(cls, mc: 'OnionMessageChannel', connection information given by the network interface. TODO: special handling for inbound is needed. """ - host, port = location.split(":") - return cls(mc, socks5_host, socks5_port, hostname=host, - port=int(port), directory=directory, + try: + host, port = location.split(":") + portint = int(port) + except: + raise InvalidLocationStringError(location) + return cls(mc, socks5_host, socks5_port, + (host, portint), directory=directory, handshake_callback=handshake_callback) - def set_host_port(self, hostname: str, port: int) -> None: - """ If the connection info is discovered - after this peer was already added to our list, - we can set it with this method. - """ - self.hostname = hostname - self.port = port - def set_location(self, location_string: str) -> bool: """ Allows setting location from an unchecked - input string argument; if the string does not have - the required format, - will return False, otherwise self.hostname, self.port are + input string argument. + If the location is specified as the 'no serving' case, + we put the currently existing inbound connection as the alternate + location, and the NOT_SERVING const as the 'location', returning True. + If the string does not have the required format, will return False, + otherwise self.hostname, self.port are updated for future `peer_location` calls, and True is returned. """ + if location_string == NOT_SERVING_ONION_HOSTNAME: + self.set_alternate_location(location_tuple_to_str( + (self.hostname, self.port))) + self.hostname = NOT_SERVING_ONION_HOSTNAME + self.port = -1 + return True try: host, port = location_string.split(":") portint = int(port) @@ -369,21 +408,23 @@ def set_location(self, location_string: str) -> bool: return True def peer_location(self) -> str: - assert (self.hostname and self.port > 0) - return self.hostname + ":" + str(self.port) + if self.hostname == NOT_SERVING_ONION_HOSTNAME: + # special case for non-reachable peers, which can include + # self_as_peer: we just return this string constant + return NOT_SERVING_ONION_HOSTNAME + # in every other case we need a sensible port/host combo: + assert (self.port > 0 and self.hostname) + return location_tuple_to_str((self.hostname, self.port)) def send(self, message: OnionCustomMessage) -> bool: """ If the message can be sent on either an inbound or outbound connection, True is returned, else False. """ if not self.factory: - #print("We are: {}. peer, wich was directory {}, did not have factory, so we send via mc".format( - # self.messagechannel.self_as_peer.peer_location(), self.directory)) # we try to send via the overall message channel serving # protocol, i.e. we assume the connection was made inbound: - #print("and to this location: ", self.peer_location()) - return self.messagechannel.proto_factory.send(message, self.alternate_location) - #print("peer which was directory {} did have factory {}, we send via that".format(self.directory, self.factory)) + return self.messagechannel.proto_factory.send(message, + self.alternate_location) return self.factory.send(message) def receive_message(self, message: OnionCustomMessage) -> None: @@ -392,8 +433,6 @@ def receive_message(self, message: OnionCustomMessage) -> None: def connect(self) -> None: """ This method is called to connect, over Tor, to the remote peer at the given onion host/port. - The connection is 'persistent' in the sense that we use a - ReconnectingClientFactory. """ if self._status in [PEER_STATUS_HANDSHAKED, PEER_STATUS_CONNECTED]: return @@ -402,18 +441,25 @@ def connect(self) -> None: "Cannot connect without host, port info") self.factory = OnionClientFactory(self.receive_message, - self.register_connection, self.register_disconnection) + self.register_connection, self.register_disconnection, + self.directory, self.messagechannel) if testing_mode: - print("{} is making a tcp connection to {}, {}, {},".format( - self.messagechannel.self_as_peer.peer_location(), self.hostname, self.port, self.factory)) - self.tcp_connector = reactor.connectTCP(self.hostname, self.port, self.factory) + log.debug("{} is making a tcp connection to {}, {}, {},".format( + self.messagechannel.self_as_peer.peer_location(), self.hostname, + self.port, self.factory)) + self.tcp_connector = reactor.connectTCP(self.hostname, self.port, + self.factory) else: - torEndpoint = TCP4ClientEndpoint(reactor, self.socks5_host, self.socks5_port) - onionEndpoint = TorSocksEndpoint(torEndpoint, self.hostname, self.port) - onionEndpoint.connect(self.factory) + torEndpoint = TCP4ClientEndpoint(reactor, self.socks5_host, + self.socks5_port) + onionEndpoint = TorSocksEndpoint(torEndpoint, self.hostname, + self.port) + self.reconnecting_service = ClientService(onionEndpoint, self.factory) + self.reconnecting_service.startService() def register_connection(self) -> None: - self.messagechannel.register_connection(self.peer_location(), direction=1) + self.messagechannel.register_connection(self.peer_location(), + direction=1) def register_disconnection(self) -> None: self.messagechannel.register_disconnection(self.peer_location()) @@ -425,6 +471,8 @@ def try_to_connect(self) -> None: try: self.connect() except OnionPeerConnectionError as e: + # Note that this will happen naturally for non-serving peers. + # TODO remove message or change it. log.debug("Tried to connect but failed: {}".format(repr(e))) except Exception as e: log.warn("Got unexpected exception in connect attempt: {}".format( @@ -436,12 +484,12 @@ def disconnect(self) -> None: if not (self.hostname and self.port > 0): raise OnionPeerConnectionError( "Cannot disconnect without host, port info") - d = self.reconnecting_service.stopService() + d = self.factory.proto_client.transport.loseConnection() d.addCallback(self.complete_disconnection) d.addErrback(log.warn, "Failed to disconnect from peer {}.".format( self.peer_location())) - def complete_disconnection(self): + def complete_disconnection(self) -> None: log.debug("Disconnected from peer: {}".format(self.peer_location())) self.update_status(PEER_STATUS_DISCONNECTED) self.factory = None @@ -462,11 +510,15 @@ def try_to_connect(self) -> None: except OnionPeerConnectionError: reactor.callLater(self.delay, self.try_to_connect) + class OnionMessageChannel(MessageChannel): - """ Receives messages via a Torv3 hidden/onion service. - Sends messages to other nodes of the same type over Tor + """ Sends messages to other nodes of the same type over Tor via SOCKS5. - Uses one or more configured "directory nodes" + *Optionally*: Receives messages via a Torv3 hidden/onion service. + If no onion service, it means we only have connections outbound + to other onion services (directory nodes first, others if and + when they send us a privmsg.). + Uses one or more configured "directory nodes" (which could be us) to access a list of current active nodes, and updates dynamically from messages seen. """ @@ -478,54 +530,69 @@ def __init__(self, # hostid is a feature to avoid replay attacks across message channels; # TODO investigate, but for now, treat onion-based as one "server". self.hostid = "onion-network" + # receives notification that we are shutting down + self.give_up = False + # for backwards compat: make sure MessageChannel log can refer to + # this in dynamic switch message: + self.serverport = self.hostid self.tor_control_host = configdata["tor_control_host"] - self.tor_control_port = int(configdata["tor_control_port"]) + self.tor_control_port = configdata["tor_control_port"] self.onion_serving_host=configdata["onion_serving_host"] - self.onion_serving_port=int(configdata["onion_serving_port"]) - self.hidden_service_dir = configdata["hidden_service_dir"] + self.onion_serving = configdata["serving"] + if self.onion_serving: + self.onion_serving_port = configdata["onion_serving_port"] + self.hidden_service_dir = configdata["hidden_service_dir"] # client side config: - self.socks5_host = "127.0.0.1" - self.socks5_port = 9050 + self.socks5_host = configdata["socks5_host"] + self.socks5_port = configdata["socks5_port"] # we use the setting in the config sent over from # the client, to decide whether to set up our connections # over localhost (if testing), without Tor: set_testing_mode(configdata) - log.info("after call to testing_mode, it is: {}".format(testing_mode)) # keep track of peers. the list will be instances # of OnionPeer: self.peers = set() - for dn in configdata["directory_nodes"].split(","): + for dn in [x.strip() for x in configdata["directory_nodes"].split(",")]: # note we don't use a nick for directories: - self.peers.add(OnionDirectoryPeer.from_location_string( - self, dn, self.socks5_host, self.socks5_port, - directory=True, handshake_callback=self.handshake_as_client)) + try: + self.peers.add(OnionDirectoryPeer.from_location_string( + self, dn, self.socks5_host, self.socks5_port, + directory=True, handshake_callback=self.handshake_as_client)) + except InvalidLocationStringError as e: + log.error("Failed to load directory nodes: {}".format(repr(e))) + stop_reactor() + return # we can direct messages via the protocol factory, which # will index protocol connections by peer location: self.proto_factory = OnionLineProtocolFactory(self) - if testing_mode: - # we serve over TCP: - self.testing_serverconn = reactor.listenTCP(self.onion_serving_port, - self.proto_factory, interface="localhost") - self.onion_hostname = "127.0.0.1" + if self.onion_serving: + if testing_mode: + # we serve over TCP: + self.testing_serverconn = reactor.listenTCP(self.onion_serving_port, + self.proto_factory, interface="localhost") + self.onion_hostname = "127.0.0.1" + else: + self.hs = JMHiddenService(self.proto_factory, + self.info_callback, + self.setup_error_callback, + self.onion_hostname_callback, + self.tor_control_host, + self.tor_control_port, + self.onion_serving_host, + self.onion_serving_port, + shutdown_callback=self.shutdown_callback, + hidden_service_dir=self.hidden_service_dir) + # this call will start bringing up the HS; when it's finished, + # it will fire the `onion_hostname_callback`, or if it fails, + # it'll fire the `setup_error_callback`. + self.hs.start_tor() + + # This will serve as our unique identifier, indicating + # that we are ready to communicate (in both directions) over Tor. + self.onion_hostname = None else: - self.hs = JMHiddenService(self.proto_factory, - self.info_callback, - self.setup_error_callback, - self.onion_hostname_callback, - self.tor_control_host, - self.tor_control_port, - self.onion_serving_host, - self.onion_serving_port, - shutdown_callback=self.shutdown_callback, - hidden_service_dir=self.hidden_service_dir) - # this call will start bringing up the HS; when it's finished, - # it will fire the `onion_hostname_callback`, or if it fails, - # it'll fire the `setup_error_callback`. - self.hs.start_tor() - - # This will serve as our unique identifier, indicating - # that we are ready to communicate (in both directions) over Tor. - self.onion_hostname = None + # dummy 'hostname' to indicate we can start running immediately: + self.onion_hostname = NOT_SERVING_ONION_HOSTNAME # intended to represent the special case of 'we are the # only directory node known', however for now dns don't interact @@ -537,23 +604,21 @@ def __init__(self, # the rpc connection calls are not using twisted) self.wait_for_directories_loop = None - def info_callback(self, msg): + def info_callback(self, msg: str) -> None: log.info(msg) - def setup_error_callback(self, msg): + def setup_error_callback(self, msg: str) -> None: log.error(msg) - def shutdown_callback(self, msg): + def shutdown_callback(self, msg: str) -> None: log.info("in shutdown callback: {}".format(msg)) - def onion_hostname_callback(self, hostname): + def onion_hostname_callback(self, hostname: str) -> None: """ This entrypoint marks the start of the OnionMessageChannel running, since we need this unique identifier as our name before we can start working (we need to compare it with the configured directory nodes). """ - print("hostname: ", hostname) - print("type: ", type(hostname)) log.info("setting onion hostname to : {}".format(hostname)) self.onion_hostname = hostname @@ -562,6 +627,12 @@ def run(self) -> None: self.hs_up_loop = task.LoopingCall(self.check_onion_hostname) self.hs_up_loop.start(0.5) + def shutdown(self) -> None: + self.give_up = True + for p in self.peers: + if p.reconnecting_service: + p.reconnecting_service.stopService() + def get_pubmsg(self, msg:str, source_nick:str ="") -> str: """ Converts a message into the known format for pubmsgs; if we are not sending this (because we @@ -572,7 +643,7 @@ def get_pubmsg(self, msg:str, source_nick:str ="") -> str: return nick + COMMAND_PREFIX + "PUBLIC" + msg def get_privmsg(self, nick: str, cmd: str, message: str, - source_nick=None) -> None: + source_nick=None) -> str: """ See `get_pubmsg` for comment on `source_nick`. """ from_nick = source_nick if source_nick else self.nick @@ -584,34 +655,36 @@ def _pubmsg(self, msg:str) -> None: send the message to every known directory node, with the PUBLIC message type and nick. """ - peerids = self.get_directory_peers() + dps = self.get_directory_peers() msg = OnionCustomMessage(self.get_pubmsg(msg), JM_MESSAGE_TYPES["pubmsg"]) - for peerid in peerids: + for dp in dps: # currently a directory node can send its own # pubmsgs (act as maker or taker); this will # probably be removed but is useful in testing: - if peerid == self.self_as_peer.peer_location(): + if dp == self.self_as_peer: self.receive_msg(msg, "00") else: - self._send(self.get_peer_by_id(peerid), msg) + self._send(dp, msg) def _privmsg(self, nick: str, cmd: str, msg:str) -> None: - log.debug("Privmsging to: {}, {}, {}".format(nick, cmd, msg)) + # in certain test scenarios the directory may try to transfer + # commitments to itself: + if nick == self.nick: + log.debug("Not sending message to ourselves: {}, {}, {}".format( + nick, cmd, msg)) + return encoded_privmsg = OnionCustomMessage(self.get_privmsg(nick, cmd, msg), JM_MESSAGE_TYPES["privmsg"]) - peerid = self.get_peerid_by_nick(nick) - if peerid: - peer = self.get_peer_by_id(peerid) - # notice the order matters here!: - if not peerid or not peer or not peer.status() == PEER_STATUS_HANDSHAKED: + peer = self.get_peer_by_nick(nick) + if not peer or peer.status() != PEER_STATUS_HANDSHAKED: # If we are trying to message a peer via their nick, we # may not yet have a connection; then we just # forward via directory nodes. log.debug("Privmsg peer: {} but don't have peerid; " "sending via directory.".format(nick)) try: - # TODO change this to redundant or switching? + # TODO change this to redundant or switching peer = self.get_connected_directory_peers()[0] except Exception as e: log.warn("Failed to send privmsg because no " @@ -625,7 +698,7 @@ def _announce_orders(self, offerlist: list) -> None: # End ABC implementation section - def check_onion_hostname(self): + def check_onion_hostname(self) -> None: if not self.onion_hostname: return self.hs_up_loop.stop() @@ -638,30 +711,37 @@ def check_onion_hostname(self): # start sending us messages. reactor.callLater(0.0, self.connect_to_directories) + def get_my_location_tuple(self) -> Tuple[str, int]: + if self.onion_hostname == NOT_SERVING_ONION_HOSTNAME: + return (self.onion_hostname, -1) + elif testing_mode: + return (self.onion_hostname, self.onion_serving_port) + else: + return (self.onion_hostname, 80) + def get_our_peer_info(self) -> None: """ Create a special OnionPeer object, outside of our peerlist, to refer to ourselves. """ - dp = self.get_directory_peers() + dps = self.get_directory_peers() self_dir = False - # only for publically exposed onion does the 'virtual port' exist; + # only for publicly exposed onion does the 'virtual port' exist; # for local tests we always connect to an actual machine port: - port_to_check = 80 if not testing_mode else self.onion_serving_port - my_location_str = self.onion_hostname + ":" + str(port_to_check) - log.info("To check if we are genesis, we compare {} with {}".format(my_location_str, dp)) - if [my_location_str] == dp: + my_location_tuple = self.get_my_location_tuple() + my_location_str = location_tuple_to_str(my_location_tuple) + if [my_location_str] == [d.peer_location() for d in dps]: log.info("This is the genesis node: {}".format(self.onion_hostname)) self.genesis_node = True self_dir = True - elif my_location_str in dp: + elif my_location_str in dps: # Here we are just one of many directory nodes, # which should be fine, we should just be careful # to not query ourselves. self_dir = True self.self_as_peer = OnionPeer(self, self.socks5_host, self.socks5_port, - self.onion_hostname, self.onion_serving_port, - self_dir, nick=self.nick, - handshake_callback=None) + my_location_tuple, + self_dir, nick=self.nick, + handshake_callback=None) def connect_to_directories(self) -> None: if self.genesis_node: @@ -686,32 +766,33 @@ def connect_to_directories(self) -> None: def handshake_as_client(self, peer: OnionPeer) -> None: assert peer.status() == PEER_STATUS_CONNECTED if self.self_as_peer.directory: - log.debug("Not sending client handshake to {} because we are directory.".format(peer.peer_location())) + log.debug("Not sending client handshake to {} because we " + "are directory.".format(peer.peer_location())) return our_hs = copy.deepcopy(client_handshake_json) our_hs["location-string"] = self.self_as_peer.peer_location() our_hs["nick"] = self.nick - # We fire and forget the handshake; successful setting - # of the `is_handshaked` var in the Peer object will depend - # on a valid/success return via the custommsg hook in the plugin. - log.info("Sending this handshake: {} to peer {}".format(json.dumps(our_hs), peer.peer_location())) - self._send(peer, OnionCustomMessage(json.dumps(our_hs), + our_hs_json = json.dumps(our_hs) + log.info("Sending this handshake: {} to peer {}".format( + our_hs_json, peer.peer_location())) + self._send(peer, OnionCustomMessage(our_hs_json, CONTROL_MESSAGE_TYPES["handshake"])) def handshake_as_directory(self, peer: OnionPeer, our_hs: dict) -> None: assert peer.status() == PEER_STATUS_CONNECTED - log.info("Sending this handshake as directory: {}".format(json.dumps(our_hs))) - self._send(peer, OnionCustomMessage(json.dumps(our_hs), + our_hs_json = json.dumps(our_hs) + log.info("Sending this handshake as directory: {}".format( + our_hs_json)) + self._send(peer, OnionCustomMessage(our_hs_json, CONTROL_MESSAGE_TYPES["dn-handshake"])) def get_directory_peers(self) -> list: - return [ p.peer_location() for p in self.peers if p.directory is True] + return [p for p in self.peers if p.directory is True] - def get_peerid_by_nick(self, nick:str) -> Union[OnionPeer, None]: + def get_peer_by_nick(self, nick:str) -> Union[OnionPeer, None]: for p in self.get_all_connected_peers(): if p.nick == nick: - return p.peer_location() - return None + return p def _send(self, peer: OnionPeer, message: OnionCustomMessage) -> bool: try: @@ -723,10 +804,6 @@ def _send(self, peer: OnionPeer, message: OnionCustomMessage) -> bool: peer.peer_location(), repr(e))) return False - def shutdown(self): - """ TODO - """ - def receive_msg(self, message: OnionCustomMessage, peer_location: str) -> None: """ Messages from peers and also connection related control messages. These messages either come via OnionPeer or via @@ -734,7 +811,8 @@ def receive_msg(self, message: OnionCustomMessage, peer_location: str) -> None: inbound connections. """ if self.self_as_peer.directory: - print("received message as directory: ", message.encode()) + # TODO remove, useful while testing + log.debug("received message as directory: {}".format(message.encode())) peer = self.get_peer_by_id(peer_location) if not peer: log.warn("Received message but could not find peer: {}".format(peer_location)) @@ -752,7 +830,7 @@ def receive_msg(self, message: OnionCustomMessage, peer_location: str) -> None: return # ignore non-JM messages: - if not msgtype in JM_MESSAGE_TYPES.values(): + if msgtype not in JM_MESSAGE_TYPES.values(): log.debug("Invalid message type, ignoring: {}".format(msgtype)) return @@ -762,8 +840,6 @@ def receive_msg(self, message: OnionCustomMessage, peer_location: str) -> None: from_nick, to_nick = nicks_msgs[:2] msg = COMMAND_PREFIX + COMMAND_PREFIX.join(nicks_msgs[2:]) if to_nick == "PUBLIC": - #log.debug("A pubmsg is being processed by {} from {}; it " - # "is {}".format(self.self_as_peer.nick, from_nick, msg)) self.on_pubmsg(from_nick, msg) if self.self_as_peer.directory: self.forward_pubmsg_to_peers(msg, from_nick) @@ -775,9 +851,8 @@ def receive_msg(self, message: OnionCustomMessage, peer_location: str) -> None: else: self.on_privmsg(from_nick, msg) except Exception as e: - log.debug("Invalid joinmarket message: {}, error was: {}".format( + log.debug("Invalid Joinmarket message: {}, error was: {}".format( msgval, repr(e))) - return def forward_pubmsg_to_peers(self, msg: str, from_nick: str) -> None: """ Used by directory nodes currently. Takes a received @@ -806,35 +881,37 @@ def forward_pubmsg_to_peers(self, msg: str, from_nick: str) -> None: def forward_privmsg_to_peer(self, nick: str, message: str, from_nick: str) -> None: assert self.self_as_peer.directory - peerid = self.get_peerid_by_nick(nick) - if not peerid: + peer = self.get_peer_by_nick(nick) + if not peer: log.debug("We were asked to send a message from {} to {}, " "but {} is not connected.".format(from_nick, nick, nick)) return # The `message` passed in has format COMMAND_PREFIX||command||" "||msg # we need to parse out cmd, message for sending. - _, cmdmsg = message.split(COMMAND_PREFIX) - cmdmsglist = cmdmsg.split(" ") + # second argument for split means only one split allowed. + cmdsmsgs = message.split(COMMAND_PREFIX, 1)[1] + cmdmsglist = cmdsmsgs.split(" ") cmd = cmdmsglist[0] msg = " ".join(cmdmsglist[1:]) privmsg = self.get_privmsg(nick, cmd, msg, source_nick=from_nick) - #log.debug("Sending out privmsg: {} to peer: {}".format(privmsg, peerid)) encoded_msg = OnionCustomMessage(privmsg, JM_MESSAGE_TYPES["privmsg"]) - self._send(self.get_peer_by_id(peerid), encoded_msg) + self._send(peer, encoded_msg) # If possible, we forward the from-nick's network location # to the to-nick peer, so they can just talk directly next time. - peerid_from = self.get_peerid_by_nick(from_nick) - if not peerid_from: + peer_from = self.get_peer_by_nick(from_nick) + if not peer_from: return - peer_to = self.get_peer_by_id(peerid) - self.send_peers(peer_to, peerid_filter=[peerid_from]) + self.send_peers(peer, peer_filter=[peer_from]) def process_control_message(self, peerid: str, msgtype: int, msgval: str) -> bool: """ Triggered by a directory node feeding us peers, or by a connect/disconnect hook; this is our housekeeping to try to create, and keep track of, useful connections. + The returned boolean indicates whether we succeeded in processing + the message or whether it must be analyzed again (note e.g. that + we return True for a rejected message!) """ all_ctrl = list(LOCAL_CONTROL_MESSAGE_TYPES.values( )) + list(CONTROL_MESSAGE_TYPES.values()) @@ -846,9 +923,12 @@ def process_control_message(self, peerid: str, msgtype: int, #log.debug("received control message: {},{}".format(msgtype, msgval)) if msgtype == CONTROL_MESSAGE_TYPES["peerlist"]: # This is the base method of seeding connections; - # a directory node can send this any time. We may well - # need to control this; for now it just gets processed, - # whereever it came from: + # a directory node can send this any time. + # These messages can only be accepted from directory peers + # (which we have configured ourselves): + peer = self.get_peer_by_id(peerid) + if not peer or not peer.directory: + return True try: peerlist = msgval.split(",") for peer in peerlist: @@ -858,25 +938,14 @@ def process_control_message(self, peerid: str, msgtype: int, except Exception as e: log.debug("Incorrectly formatted peer list: {}, " "ignoring, {}".format(msgval, e)) - # returning True either way, because although it was an - # invalid message, it *was* a control message, and should - # not be processed as something else. + # returning True whether raised or not - see docstring return True elif msgtype == CONTROL_MESSAGE_TYPES["getpeerlist"]: - # getpeerlist must be accompanied by a full node - # locator, and nick; - # add that peer before returning our peer list. - p = self.add_peer(msgval, connection=True, - overwrite_connection=True, with_nick=True) - try: - self.send_peers(p) - except OnionPeerConnectionError: - pass - # comment much as above; if we can't connect, it's none - # of our business. + log.warn("getpeerlist request received, currently not supported.") return True elif msgtype == CONTROL_MESSAGE_TYPES["handshake"]: - # sent by non-directory peers on startup + # sent by non-directory peers on startup, also to + # other non-dn peers during tx flow self.process_handshake(peerid, msgval) return True elif msgtype == CONTROL_MESSAGE_TYPES["dn-handshake"]: @@ -896,6 +965,12 @@ def process_control_message(self, peerid: str, msgtype: int, msgval = self.get_peer_by_id(msgval).peer_location() self.add_peer(msgval, connection=False, overwrite_connection=True) + # bubble up the disconnection event to the abstract + # message channel logic: + if self.on_nick_leave: + p = self.get_peer_by_id(msgval) + if p and p.nick: + reactor.callLater(0.0, self.on_nick_leave, p.nick, self) else: assert False # If we got here it is *not* a non-local control message; @@ -919,7 +994,6 @@ def process_handshake(self, peerid: str, message: str, peerid, peer.status())) return if dn: - print("We, {}, are processing a handshake with dn {} from peer {}".format(self.self_as_peer.peer_location(), dn, peerid)) # it means, we are a non-dn and we are expecting # a returned `dn-handshake` message: # (currently dns don't talk to other dns): @@ -946,8 +1020,9 @@ def process_handshake(self, peerid: str, message: str, assert isinstance(features, dict) assert isinstance(nick, str) except Exception as e: - log.warn("Invalid handshake message from: {}, exception: {}, message: {}," - "ignoring".format(peerid, repr(e), message)) + log.warn("Invalid handshake message from: {}," + " exception: {}, message: {},ignoring".format( + peerid, repr(e), message)) return # currently we are not using any features, but the intention # is forwards compatibility, so we don't check its contents @@ -964,7 +1039,6 @@ def process_handshake(self, peerid: str, message: str, peer.update_status(PEER_STATUS_HANDSHAKED) peer.set_nick(nick) else: - print("We, {}, are processing a handshake with dn {} from peer {}".format(self.self_as_peer.peer_location(), dn, peerid)) # it means, we are receiving an initial handshake # message from a 'client' (non-dn) peer. # dns don't talk to each other: @@ -982,9 +1056,11 @@ def process_handshake(self, peerid: str, message: str, assert isinstance(features, dict) assert isinstance(nick, str) except Exception as e: - log.warn("(not dn) Invalid handshake message from: {}, exception: {}, message: {}," - "ignoring".format(peerid, repr(e), message)) - accepted = False + log.warn("(not dn) Invalid handshake message from: {}, " + "exception: {}, message: {}, ignoring".format( + peerid, repr(e), message)) + # just ignore, since a syntax failure could lead to a crash + return if not (app_name == JM_APP_NAME and proto_ver == JM_VERSION \ and not is_directory): log.warn("Invalid handshake name/version data: {}, from peer: " @@ -992,13 +1068,12 @@ def process_handshake(self, peerid: str, message: str, accepted = False # If accepted, we should update the peer to have the full # location which in general will not yet be present, so as to - # allow publishing their location via `getpeerlist`: + # allow publishing their location via `getpeerlist`. Note + # that if the peer declares itself as not serving, we do + # nothing here: if not peer.set_location(full_location_string): accepted = False - if not peerid == full_location_string: - print("we are reading a handshake from location {} but they sent" - "us full location string {}, setting an alternate".format( - peerid, full_location_string)) + if peerid != full_location_string: peer.set_alternate_location(peerid) peer.set_nick(nick) # client peer's handshake message was valid; send ours, and @@ -1018,8 +1093,10 @@ def get_peer_by_id(self, p: str) -> Union[OnionPeer, bool]: if p == "00": return self.self_as_peer for x in self.peers: - if x.peer_location() == p: + if x.peer_location() == p and p != NOT_SERVING_ONION_HOSTNAME: return x + # non-reachable peers can only match on their inbound + # connection port if x.alternate_location == p: return x return False @@ -1046,7 +1123,7 @@ def register_disconnection(self, peer_location: str) -> None: self.receive_msg(msg, "00") def add_peer(self, peerdata: str, connection: bool=False, - overwrite_connection: bool=False, with_nick=False) -> None: + overwrite_connection: bool=False, with_nick=False) -> Union[OnionPeer, None]: """ add non-directory peer from (nick, peer) serialization `peerdata`, where "peer" is host:port; return the created OnionPeer object. Or, with_nick=False means @@ -1084,14 +1161,15 @@ def add_peer(self, peerdata: str, connection: bool=False, log.warn("Failed to add peer: {}, exception: {}".format(peer, repr(e))) return if not self.get_peer_by_id(temp_p.peer_location()): + self.peers.add(temp_p) if connection: log.info("Updating status of peer: {} to connected.".format(temp_p.peer_location())) temp_p.update_status(PEER_STATUS_CONNECTED) else: - temp_p.update_status(PEER_STATUS_DISCONNECTED) + if overwrite_connection: + temp_p.update_status(PEER_STATUS_DISCONNECTED) if with_nick: temp_p.set_nick(nick) - self.peers.add(temp_p) if not connection: # Here, we are not currently connected. We # try to connect asynchronously. We don't pay attention @@ -1130,7 +1208,6 @@ def wait_for_directories(self) -> None: # Notice this is checking for *handshaked* dps; # the handshake will have been initiated once a # connection was seen: - log.warn("in the wait for directories loop, this is the connected dps: {}".format(self.get_connected_directory_peers())) if len(self.get_connected_directory_peers()) == 0: return # This is what triggers the start of taker/maker workflows. @@ -1142,38 +1219,46 @@ def wait_for_directories(self) -> None: """ CONTROL MESSAGES SENT BY US """ def send_peers(self, requesting_peer: OnionPeer, - peerid_filter: list=[]) -> None: - """ This message is sent by directory peers on request - by non-directory peers. - If peerid_filter is specified, only peers whose peerid is in - this list will be sent. (TODO this is inefficient). + peer_filter: List[OnionPeer]) -> None: + """ This message is sent by directory peers, currently + only when a privmsg has to be forwarded to them. It + could also be sent by directories to non-directory peers + according to some other algorithm. + If peer_filter is specified, only those peers will be sent. The peerlist message should have this format: (1) entries comma separated (2) each entry is serialized nick then the NICK_PEERLOCATOR_SEPARATOR - then *either* 66 char hex peerid, *or* peerid@host:port - (3) However this message might be long enough to exceed a 1300 byte limit, - if we don't use a filter, so we may need to split it into multiple - messages (TODO). + then host:port + (3) Peers that do not have a reachable location are not sent. """ if not requesting_peer.status() == PEER_STATUS_HANDSHAKED: raise OnionPeerConnectionError( "Cannot send peer list to unhandshaked peer") peerlist = set() + peer_filter_exists = len(peer_filter) > 0 for p in self.get_connected_nondirectory_peers(): # don't send a peer to itself - if p.peer_location() == requesting_peer.peer_location(): + if p == requesting_peer: continue - if len(peerid_filter) > 0 and p.peer_location() not in peerid_filter: + if peer_filter_exists and p not in peer_filter: continue - if not p.status() == PEER_STATUS_HANDSHAKED: + if p.status() != PEER_STATUS_HANDSHAKED: # don't advertise what is not online. continue # peers that haven't sent their nick yet are not # privmsg-reachable; don't send them if p.nick == "": continue + if p.peer_location() == NOT_SERVING_ONION_HOSTNAME: + # if a connection has no reachable destination, + # don't forward it + continue peerlist.add(p.get_nick_peerlocation_ser()) # For testing: dns won't usually participate: peerlist.add(self.self_as_peer.get_nick_peerlocation_ser()) + # don't send an empty set (will not be possible unless + # above dn add is removed). + if len(peerlist) == 0: + return self._send(requesting_peer, OnionCustomMessage(",".join( peerlist), CONTROL_MESSAGE_TYPES["peerlist"])) diff --git a/jmdaemon/test/test_daemon_protocol.py b/jmdaemon/test/test_daemon_protocol.py index f9dbf390e..f8e9a6d14 100644 --- a/jmdaemon/test/test_daemon_protocol.py +++ b/jmdaemon/test/test_daemon_protocol.py @@ -59,11 +59,11 @@ def connectionMade(self): def clientStart(self): self.sigs_received = 0 - irc = [get_mchannels()[0]] + chan_configs = [get_mchannels()[0]] d = self.callRemote(JMInit, bcsource="dummyblockchain", network="dummynetwork", - irc_configs=irc, + chan_configs=chan_configs, minmakers=2, maker_timeout_sec=3, dust_threshold=27300, @@ -212,7 +212,7 @@ def on_JM_REQUEST_OFFERS(self): return super().on_JM_REQUEST_OFFERS() @JMInit.responder - def on_JM_INIT(self, bcsource, network, irc_configs, minmakers, + def on_JM_INIT(self, bcsource, network, chan_configs, minmakers, maker_timeout_sec, dust_threshold, blacklist_location): self.maker_timeout_sec = maker_timeout_sec self.dust_threshold = int(dust_threshold) diff --git a/scripts/obwatch/ob-watcher.py b/scripts/obwatch/ob-watcher.py index b6a2950ed..e5598144e 100755 --- a/scripts/obwatch/ob-watcher.py +++ b/scripts/obwatch/ob-watcher.py @@ -45,7 +45,8 @@ from jmclient import jm_single, load_program_config, calc_cj_fee, \ get_mchannels, add_base_options -from jmdaemon import OrderbookWatch, MessageChannelCollection, IRCMessageChannel +from jmdaemon import (OrderbookWatch, MessageChannelCollection, + OnionMessageChannel, IRCMessageChannel) #TODO this is only for base58, find a solution for a client without jmbitcoin import jmbitcoin as btc from jmdaemon.protocol import * @@ -737,32 +738,32 @@ def on_welcome(self): def request_orderbook(self): self.msgchan.request_orderbook() -class ObIRCMessageChannel(IRCMessageChannel): - """A customisation of the message channel - to allow receipt of privmsgs without the - verification hooks in client-daemon communication.""" - def on_privmsg(self, nick, message): - if len(message) < 2: - return - - if message[0] != COMMAND_PREFIX: - log.debug('message not a cmd') - return - cmd_string = message[1:].split(' ')[0] - if cmd_string not in offername_list: - log.debug('non-offer ignored') - return - #Ignore sigs (TODO better to include check) - sig = message[1:].split(' ')[-2:] - #reconstruct original message without cmd pref - rawmessage = ' '.join(message[1:].split(' ')[:-2]) - for command in rawmessage.split(COMMAND_PREFIX): - _chunks = command.split(" ") - try: - self.check_for_orders(nick, _chunks) - self.check_for_fidelity_bond(nick, _chunks) - except: - pass + +"""An override for MessageChannel classes, +to allow receipt of privmsgs without the +verification hooks in client-daemon communication.""" +def on_privmsg(inst, nick, message): + if len(message) < 2: + return + + if message[0] != COMMAND_PREFIX: + log.debug('message not a cmd') + return + cmd_string = message[1:].split(' ')[0] + if cmd_string not in offername_list: + log.debug('non-offer ignored') + return + #Ignore sigs (TODO better to include check) + sig = message[1:].split(' ')[-2:] + #reconstruct original message without cmd pref + rawmessage = ' '.join(message[1:].split(' ')[:-2]) + for command in rawmessage.split(COMMAND_PREFIX): + _chunks = command.split(" ") + try: + inst.check_for_orders(nick, _chunks) + inst.check_for_fidelity_bond(nick, _chunks) + except: + pass def get_dummy_nick(): @@ -804,7 +805,16 @@ def main(): (options, args) = parser.parse_args() load_program_config(config_path=options.datadir) hostport = (options.host, options.port) - mcs = [ObIRCMessageChannel(c) for c in get_mchannels()] + mcs = [] + chan_configs = get_mchannels() + for c in chan_configs: + if "type" in c and c["type"] == "onion": + mcs.append(OnionMessageChannel(c)) + else: + # default is IRC; TODO allow others + mcs.append(IRCMessageChannel(c)) + IRCMessageChannel.on_privmsg = on_privmsg + OnionMessageChannel.on_privmsg = on_privmsg mcc = MessageChannelCollection(mcs) mcc.set_nick(get_dummy_nick()) taker = ObBasic(mcc, hostport) diff --git a/test/e2e-coinjoin-test.py b/test/e2e-coinjoin-test.py index 600d6ecd5..81296af9f 100644 --- a/test/e2e-coinjoin-test.py +++ b/test/e2e-coinjoin-test.py @@ -11,7 +11,7 @@ pytest \ --btcroot=/path/to/bitcoin/bin/ \ --btcpwd=123456abcdef --btcconf=/blah/bitcoin.conf \ - -s test/ln-ygrunner.py + -s test/e2e-coinjoin-test.py ''' from twisted.internet import reactor, defer from twisted.web.client import readBody, Headers @@ -21,7 +21,7 @@ import json from datetime import datetime from jmbase import (get_nontor_agent, BytesProducer, jmprint, - get_log, stop_reactor, hextobin, bintohex) + get_log, stop_reactor) from jmclient import (YieldGeneratorBasic, load_test_config, jm_single, JMClientProtocolFactory, start_reactor, SegwitWallet, get_mchannels, SegwitLegacyWallet, JMWalletDaemon) @@ -45,8 +45,7 @@ directory_node_indices = [1] -# -def get_onion_messaging_config_regtest(run_num: int, dns=[1], hsd=""): +def get_onion_messaging_config_regtest(run_num: int, dns=[1], hsd="", mode="TAKER"): """ Sets a onion messaging channel section for a regtest instance indexed by `run_num`. The indices to be used as directory nodes should be passed as `dns`, as a list of ints. @@ -72,6 +71,10 @@ def location_string(directory_node_run_num): "hidden_service_dir": "", "directory_nodes": dn_nodes_list, "regtest_count": "1, 1"} + if mode == "MAKER": + cf["serving"] = True + else: + cf["serving"] = False if run_num in dns: # only directories need to use fixed hidden service directories: cf["hidden_service_dir"] = hsd @@ -85,11 +88,11 @@ def set_directory_nodes(self, dns): # for this test: self.dns = dns - def get_mchannels(self): - # swaps out any existing lightning configs + def get_mchannels(self, mode="TAKER"): + # swaps out any existing onionmc configs # in the config settings on startup, for one # that's indexed to the regtest counter var: - default_chans = get_mchannels() + default_chans = get_mchannels(mode=mode) new_chans = [] onion_found = False hsd = "" @@ -103,7 +106,7 @@ def get_mchannels(self): new_chans.append(c) if onion_found: new_chans.append(get_onion_messaging_config_regtest( - self.i, self.dns, hsd)) + self.i, self.dns, hsd, mode=mode)) return new_chans class JMWalletDaemonT(JMWalletDaemon): @@ -183,7 +186,7 @@ def test_start_yg_and_taker_setup(setup_onion_ygrunner): walletclass = SegwitLegacyWallet start_bot_num, end_bot_num = [int(x) for x in jm_single().config.get( - "MESSAGING:onion1", "regtest_count").split(",")] + "MESSAGING:onion", "regtest_count").split(",")] num_ygs = end_bot_num - start_bot_num # specify the number of wallets and bots of each type: wallet_services = make_wallets(num_ygs + 1, @@ -254,21 +257,21 @@ def test_start_yg_and_taker_setup(setup_onion_ygrunner): # This ensures that this bot knows which other bots are directory nodes: clientfactory.set_directory_nodes(directory_node_indices) nodaemon = jm_single().config.getint("DAEMON", "no_daemon") - daemon = True if nodaemon == 1 else False + daemon = bool(nodaemon) #rs = True if i == num_ygs - 1 else False start_reactor(jm_single().config.get("DAEMON", "daemon_host"), jm_single().config.getint("DAEMON", "daemon_port"), clientfactory, daemon=daemon, rs=False) - reactor.callLater(1.0, start_test_taker, wallet_services[end_bot_num - 1]['wallet'], end_bot_num) + reactor.callLater(1.0, start_test_taker, wallet_services[end_bot_num - 1]['wallet'], end_bot_num, num_ygs) reactor.run() @defer.inlineCallbacks -def start_test_taker(wallet_service, i): +def start_test_taker(wallet_service, i, num_ygs): # this rpc manager has auth disabled, # and the wallet_service is set manually, # so no unlock etc. mgr = TWalletRPCManager() - mgr.daemon.wallet_service = wallet_service + mgr.daemon.services["wallet"] = wallet_service # because we are manually setting the wallet_service # of the JMWalletDaemon instance, we do not follow the # usual flow of `initialize_wallet_service`, we do not set @@ -276,11 +279,9 @@ def start_test_taker(wallet_service, i): # sync the wallet, including bypassing any restart callback: def dummy_restart_callback(msg): log.warn("Ignoring rescan request from backend wallet service: " + msg) - mgr.daemon.wallet_service.add_restart_callback(dummy_restart_callback) + mgr.daemon.services["wallet"].add_restart_callback(dummy_restart_callback) mgr.daemon.wallet_name = wallet_name - while not mgr.daemon.wallet_service.synced: - mgr.daemon.wallet_service.sync_wallet(fast=True) - mgr.daemon.wallet_service.startService() + mgr.daemon.services["wallet"].startService() def get_client_factory(): clientfactory = RegtestJMClientProtocolFactory(mgr.daemon.taker, proto_type="TAKER") @@ -290,18 +291,25 @@ def get_client_factory(): mgr.daemon.get_client_factory = get_client_factory # before preparing the RPC call to the wallet daemon, - # we decide a coinjoin destination and amount. Choosing - # a destination in the wallet is a bit easier because + # we decide a coinjoin destination, counterparty count and amount. + # Choosing a destination in the wallet is a bit easier because # we can query the mixdepth balance at the end. - coinjoin_destination = mgr.daemon.wallet_service.get_internal_addr(4) + coinjoin_destination = mgr.daemon.services["wallet"].get_internal_addr(4) cj_amount = 22000000 + def n_cps_from_n_ygs(n): + if n > 4: + return n - 2 + if n > 2: + return 2 + assert False, "Need at least 3 yield generators to test" + n_cps = n_cps_from_n_ygs(num_ygs) # once the taker is finished we sanity check before # shutting down: def dummy_taker_finished(res, fromtx=False, waittime=0.0, txdetails=None): jmprint("Taker is finished") # check that the funds have arrived. - mbal = mgr.daemon.wallet_service.get_balance_by_mixdepth()[4] + mbal = mgr.daemon.services["wallet"].get_balance_by_mixdepth()[4] assert mbal == cj_amount jmprint("Funds: {} sats successfully arrived into mixdepth 4.".format(cj_amount)) stop_reactor() @@ -315,7 +323,7 @@ def dummy_taker_finished(res, fromtx=False, addr = addr.encode() body = BytesProducer(json.dumps({"mixdepth": "1", "amount_sats": cj_amount, - "counterparties": "2", + "counterparties": str(n_cps), "destination": coinjoin_destination}).encode()) yield mgr.do_request(agent, b"POST", addr, body, process_coinjoin_response) @@ -324,39 +332,6 @@ def process_coinjoin_response(response): json_body = json.loads(response.decode("utf-8")) print("coinjoin response: {}".format(json_body)) -def get_addr_and_fund(yg): - """ This function allows us to create - and publish a fidelity bond for a particular - yield generator object after the wallet has reached - a synced state and is therefore ready to serve up - timelock addresses. We create the TL address, fund it, - refresh the wallet and then republish our offers, which - will also publish the new FB. - """ - if not yg.wallet_service.synced: - return - if yg.wallet_service.timelock_funded: - return - addr = wallet_gettimelockaddress(yg.wallet_service.wallet, "2021-11") - print("Got timelockaddress: {}".format(addr)) - - # pay into it; amount is randomized for now. - # Note that grab_coins already mines 1 block. - fb_amt = random.randint(1, 5) - jm_single().bc_interface.grab_coins(addr, fb_amt) - - # we no longer have to run this loop (TODO kill with nonlocal) - yg.wallet_service.timelock_funded = True - - # force wallet to check for the new coins so the new - # yg offers will include them: - yg.wallet_service.transaction_monitor() - - # publish a new offer: - yg.offerlist = yg.create_my_orders() - yg.fidelity_bond = yg.get_fidelity_bond_template() - jmprint('updated offerlist={}'.format(yg.offerlist)) - @pytest.fixture(scope="module") def setup_onion_ygrunner(): load_test_config() diff --git a/test/regtest_joinmarket.cfg b/test/regtest_joinmarket.cfg index 3345e29ff..ab0742aac 100644 --- a/test/regtest_joinmarket.cfg +++ b/test/regtest_joinmarket.cfg @@ -37,7 +37,7 @@ socks5 = false socks5_host = localhost socks5_port = 9150 -[MESSAGING:onion1] +[MESSAGING:onion] # onion based message channels must have the exact type 'onion' # (while the section name above can be MESSAGING:whatever), and there must # be only ONE such message channel configured (note the directory servers From 5cc16951546ea9f5bcd803f40ff3922ee5a9a4b2 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Wed, 30 Mar 2022 16:04:19 +0100 Subject: [PATCH 3/3] Disconnection logic fixes, add btcnet to handshake Also, exports JMMakerClientProtocol for custom directory node scripts (stored in the custom-scripts repo). Modify default config with 2 signet and mainnet directory nodes to start. Handles unreachable directory nodes with a human readable error and adjusts connection timeouts to be realistic. Changes wording in Qt notifications from "IRC" to message channel. Updates docs, new directory node information. --- docs/onion-message-channels.md | 82 ++++++++++------------- jmclient/jmclient/__init__.py | 2 +- jmclient/jmclient/configure.py | 8 ++- jmdaemon/jmdaemon/onionmc.py | 118 +++++++++++++++++++++++++++++---- scripts/joinmarket-qt.py | 4 +- test/e2e-coinjoin-test.py | 3 +- 6 files changed, 153 insertions(+), 64 deletions(-) diff --git a/docs/onion-message-channels.md b/docs/onion-message-channels.md index 5926f435e..e883de5c2 100644 --- a/docs/onion-message-channels.md +++ b/docs/onion-message-channels.md @@ -17,6 +17,8 @@ introduce any new requirements to your Joinmarket installation, technically, bec to run such onion services, and connecting to IRC used a SOCKS5 proxy (used by almost all users) over Tor to a remote onion service. +(Note however that taker bots will *not* be required to serve onions; they will only make outbound SOCKS connections, as they currently do on IRC). + The purpose of this new type of message channel is as follows: * less reliance on any service external to Joinmarket @@ -25,7 +27,7 @@ albeit it was and remains E2E encrypted data, in either case) * the above can lead to better scalability at large numbers * a substantial increase in the speed of transaction negotiation; this is mostly related to the throttling of high bursts of traffic on IRC -The configuration for a user is simple; in their `joinmarket.cfg` they will get a messaging section like this, if they start from scratch: +The configuration for a user is simple; in their `joinmarket.cfg` they will get a new `[MESSAGING]` section like this, if they start from scratch: ``` [MESSAGING:onion] @@ -64,23 +66,26 @@ hidden_service_dir = # This is a comma separated list (comma can be omitted if only one item). # Each item has format host:port ; both are required, though port will # be 80 if created in this code. -directory_nodes = rr6f6qtleiiwic45bby4zwmiwjrj3jsbmcvutwpqxjziaydjydkk5iad.onion:80 +# for MAINNET: +directory_nodes = 3kxw6lf5vf6y26emzwgibzhrzhmhqiw6ekrek3nqfjjmhwznb2moonad.onion,qqd22cwgygaxcy6vdw6mzwkyaxg5urb4ptbc5d74nrj25phspajxjbqd.onion + +# for SIGNET (testing network): +# directory_nodes = rr6f6qtleiiwic45bby4zwmiwjrj3jsbmcvutwpqxjziaydjydkk5iad.onion:80,k74oyetjqgcamsyhlym2vgbjtvhcrbxr4iowd4nv4zk5sehw4v665jad.onion:80 # This setting is ONLY for developer regtest setups, # running multiple bots at once. Don't alter it otherwise regtest_count = 0,0 ``` -All of these can be left as default for most users, except the field `directory_nodes`. +All of these can be left as default for most users - but most importantly, pay attention to: -The list of **directory nodes** (the one shown here is one being run on signet, right now), which will -be comma separated if multiple directory nodes are configured (we expect there will be 2 or 3 as a normal situation). -The `onion_serving_port` is on which port on the local machine the onion service is served; you won't usually need to use it, but it mustn't conflict with some other usage (so if you have something running on port 8080, change it). +* The list of `directory_nodes`, which will be comma separated if multiple directory nodes are configured (we expect there will be 2 or 3 as a normal situation). Make sure to choose the ones for your network (mainnet by default, or signet or otherwise); if it's wrong your bot will just get auto-disconnected. +* The `onion_serving_port` is the port on the local machine on which the onion service is served; you won't usually need to use it, but it mustn't conflict with some other usage (so if you have something running on port 8080, change it). The `type` field must always be `onion` in this case, and distinguishes it from IRC message channels and others. ### Can/should I still run IRC message channels? -In short, yes. +In short, yes, at least for now, though you are free to disable any message channel you like. ### Do I need to configure Tor, and if so, how? @@ -95,7 +100,7 @@ This onion service will be ephemeral, that is, it will have a different onion ad you restart. This should work automatically, using your existing Tor daemon (here, we are using the same code as we use when running the `receive-payjoin` script, essentially). -#### Running/testing as other bots (taker) +#### Running/testing as other bots (taker, ob-watcher) A taker will not attempt to serve an onion; it will only use outbound connections, first to directory nodes and then, as according to need, to individual makers, also. @@ -132,9 +137,7 @@ and pay attention to the settings in `regtest_joinmarket.cfg`.) There is no separate/special configuration for signet other than the configuration that is already needed for running Joinmarket against a signet backend (so e.g. RPC port of 38332). -Add the `[MESSAGING:onion]` message channel section to your `joinmarket.cfg`, as listed above, including the -signet directory node listed above (rr6f6qtleiiwic45bby4zwmiwjrj3jsbmcvutwpqxjziaydjydkk5iad.onion:80), and, -for the simplest test, remove the other `[MESSAGING:*]` sections that you have. +You can just uncomment the `directory_nodes` entry listed as SIGNET, and comment out the one for MAINNET. Then just make sure your bot has some signet coins and try running as maker or taker or both. @@ -148,12 +151,17 @@ who would like to help by running a directory node. You can ignore it if that do This requires a long running bot. It should be on a server you can keep running permanently, so perhaps a VPS, but in any case, very high uptime. For reliability it also makes sense to configure to run as a systemd service. -A note: the most natural way to run the directory is as a Joinmarket *maker* bot, i.e. run `yg-privacyenhanced.py`, with configuration as described below. For now it will actually offer to do coinjoins - we will want to fix this in future so no coins are needed (but it can just be a trivial size). +The currently suggested way to run a directory node is to use the script found [here](https://github.com/JoinMarket-Org/custom-scripts/blob/0eda6154265e012b907c43e2ecdacb895aa9e3ab/start-dn.py); you can place it in your `joinmarket-clientserver/scripts` directory and run it *without* arguments, but with one option flag: `--datadir=/your/chosen/datadir` (as you'll see below). + +This slightly unobvious approach is based on the following ideas: we run a Joinmarket script, with a Joinmarket python virtualenv, so that we are able to parse messages; this means that the directory node *can* be a bot, e.g. a maker bot, but need not be - and here it is basically a "crippled" maker bot that cannot do anything. This 'crippling' is actually very useful because (a) we use the `no-blockchain` argument (it is forced in-code; you don't need to set it) so we don't need a running Bitcoin node (of whatever flavour), and (b) we don't need a wallet either. #### Joinmarket-specific configuration -Add `hidden_service_dir` to your `[MESSAGING:onion]` with a directory accessible to your user. You may want to lock this down -a bit! +Add a non-empty `hidden_service_dir` entry to your `[MESSAGING:onion]` with a directory accessible to your user. You may want to lock this down +a bit, but be careful changing permissions from what is created by the script, because Tor is very finicky about this. + +The hostname for your onion service will not change and will be stored permanently in that directory. + The point to understand is: Joinmarket's `jmbase.JMHiddenService` will, if configured with a non-empty `hidden_service_dir` field, actually start an *independent* instance of Tor specifically for serving this, under the current user. (our Tor interface library `txtorcon` needs read access to the Tor HS dir, so it's troublesome to do this another way). @@ -162,58 +170,40 @@ field, actually start an *independent* instance of Tor specifically for serving Answer: **you must only enter your own node in this list!**. This way your bot will recognize that it is a directory node and it avoids weird edge case behaviour (so don't add *other* known directory nodes; you won't be talking to them). +A natural retort is: but I don't know my own node's onion service hostname before I start it the first time. Indeed. So, just run it once with the default `directory_nodes` entries, then note down the new onion service hostname you created, and insert that as the only entry in the list. -#### Suggested setup of a service: -You will need two components: bitcoind, and Joinmarket itself, which you can run as a yg. -Since this task is going to be attempted by someone with significant technical knowledge, -only an outline is provided here; several details will need to be filled in. -Here is a sketch of how the systemd service files can be set up for signet: +#### Suggested setup of a systemd service: -If someone wants to put together a docker setup of this for a more "one-click install", that would be great. - -1. bitcoin-signet.service +The most basic bare-bones service seems to work fine here: ``` [Unit] -Description=bitcoind signet +Description=My JM signet directory node +Requires=network-online.target After=network-online.target -Wants=network-online.target [Service] Type=simple -ExecStart=/usr/local/bin/bitcoind -signet +ExecStart=/bin/bash -c 'cd /path/to/joinmarket-clientserver && source jmvenv/bin/activate && cd scripts && python start-dn.py --datadir=/path/to/chosen/datadir' User=user [Install] WantedBy=multi-user.target ``` -This is deliberately a super-basic setup (see above). Don't forget to setup your `bitcoin.conf` as usual, -for the bitcoin user, and make it match (specifically in terms of RPC) what you set up for Joinmarket below. +... however, you need to kind of 'bootstrap' it the first time. For example: +* run once with systemctl start -2. +* look at log with `journalctl`, service fails due to default `joinmarket.cfg` and quit. +* go to that cfg file. Remove the IRC settings, they serve no purpose here. Change the `hidden_service_dir` to `/yourlocation/hidserv` (the actual directory need not exist, it's better if it doesn't, this first time). Edit the `network` field in `BLOCKCHAIN` to whatever network (mainnet, signet) you intend to support - it can be only one for one directory node, for now. -``` -[Unit] -Description=joinmarket directory node on signet -Requires=bitcoin-signet.service -After=bitcoin-signet.service +* `systemctl start` again, now note the onion hostname created from the log or the directory -[Service] -Type=simple -ExecStart=/bin/bash -c 'cd /path/to/joinmarket-clientserver && source jmvenv/bin/activate && cd scripts && echo -n "password" | python yg-privacyenhanced.py --wallet-password-stdin --datadir=/custom/joinmarket-datadir some-signet-wallet.jmdat' -User=user - -[Install] -WantedBy=multi-user.target -``` +* set that hostname in `directory_nodes` in `joinmarket.cfg` -To state the obvious, the idea here is that this second service will run the JM directory node and have a dependency on the previous one, -to ensure they start up in the correct order. - -Re: password echo, obviously this kind of password entry is bad; -for now we needn't worry as these nodes don't need to carry significant coins (and it's much better they don't!). +* now the service should start correctly TODO: add some material on network hardening/firewalls here, I guess. + diff --git a/jmclient/jmclient/__init__.py b/jmclient/jmclient/__init__.py index ca91ace64..086509c87 100644 --- a/jmclient/jmclient/__init__.py +++ b/jmclient/jmclient/__init__.py @@ -32,7 +32,7 @@ from .snicker_receiver import SNICKERError, SNICKERReceiver from .client_protocol import (JMTakerClientProtocol, JMClientProtocolFactory, start_reactor, SNICKERClientProtocolFactory, - BIP78ClientProtocolFactory, + BIP78ClientProtocolFactory, JMMakerClientProtocol, get_daemon_serving_params) from .podle import (set_commitment_file, get_commitment_file, add_external_commitments, diff --git a/jmclient/jmclient/configure.py b/jmclient/jmclient/configure.py index ea04d0d5c..08b0e71c6 100644 --- a/jmclient/jmclient/configure.py +++ b/jmclient/jmclient/configure.py @@ -173,7 +173,11 @@ def jm_single(): # This is a comma separated list (comma can be omitted if only one item). # Each item has format host:port ; both are required, though port will # be 80 if created in this code. -directory_nodes = rr6f6qtleiiwic45bby4zwmiwjrj3jsbmcvutwpqxjziaydjydkk5iad.onion:80,k74oyetjqgcamsyhlym2vgbjtvhcrbxr4iowd4nv4zk5sehw4v665jad.onion:80 +# for MAINNET: +directory_nodes = 3kxw6lf5vf6y26emzwgibzhrzhmhqiw6ekrek3nqfjjmhwznb2moonad.onion:80,qqd22cwgygaxcy6vdw6mzwkyaxg5urb4ptbc5d74nrj25phspajxjbqd.onion:80 + +# for SIGNET (testing network): +# directory_nodes = rr6f6qtleiiwic45bby4zwmiwjrj3jsbmcvutwpqxjziaydjydkk5iad.onion:80,k74oyetjqgcamsyhlym2vgbjtvhcrbxr4iowd4nv4zk5sehw4v665jad.onion:80 # This setting is ONLY for developer regtest setups, # running multiple bots at once. Don't alter it otherwise @@ -211,7 +215,7 @@ def jm_single(): host = ilitafrzzgxymv6umx2ux7kbz3imyeko6cnqkvy4nisjjj4qpqkrptid.onion socks5_port = 9050 -## IRC SERVER 3) (backup) hackint IRC (Tor, IP) +## IRC SERVER 3: (backup) hackint IRC (Tor, IP) ################################################################################ #[MESSAGING:server3] # channel = joinmarket-pit diff --git a/jmdaemon/jmdaemon/onionmc.py b/jmdaemon/jmdaemon/onionmc.py index 3d8bf8fc8..400053fed 100644 --- a/jmdaemon/jmdaemon/onionmc.py +++ b/jmdaemon/jmdaemon/onionmc.py @@ -9,13 +9,17 @@ from twisted.application.internet import ClientService from twisted.internet.endpoints import TCP4ClientEndpoint from twisted.internet.address import IPv4Address, IPv6Address -from txtorcon.socks import TorSocksEndpoint +from txtorcon.socks import TorSocksEndpoint, HostUnreachableError log = get_log() NOT_SERVING_ONION_HOSTNAME = "NOT-SERVING-ONION" +# How many seconds to wait before treating an onion +# as unreachable +CONNECT_TO_ONION_TIMEOUT = 10 + def location_tuple_to_str(t: Tuple[str, int]) -> str: return f"{t[0]}:{t[1]}" @@ -65,14 +69,15 @@ def set_testing_mode(configdata: dict) -> None: # Used for some control message construction, as detailed below. NICK_PEERLOCATOR_SEPARATOR = ";" -# location_string and nick must be set before sending, +# location_string, nick and network must be set before sending, # otherwise invalid: client_handshake_json = {"app-name": JM_APP_NAME, "directory": False, "location-string": "", "proto-ver": JM_VERSION, "features": {}, - "nick": "" + "nick": "", + "network": "" } # default acceptance false; code must switch it on: @@ -83,6 +88,7 @@ def set_testing_mode(configdata: dict) -> None: "features": {}, "accepted": False, "nick": "", + "network": "", "motd": "Default MOTD, replace with information for the directory." } @@ -189,6 +195,13 @@ def register_disconnection(self, p: OnionLineProtocol) -> None: return del self.peers[peer_location] + def disconnect_inbound_peer(self, inbound_peer_str: str) -> None: + if inbound_peer_str not in self.peers: + log.warn("cannot disconnect peer at {}, not found".format( + inbound_peer_str)) + proto = self.peers[inbound_peer_str] + proto.transport.loseConnection() + def receive_message(self, message: OnionCustomMessage, p: OnionLineProtocol) -> None: self.client.receive_msg(message, network_addr_to_string( @@ -212,6 +225,7 @@ class OnionClientFactory(protocol.ReconnectingClientFactory): def __init__(self, message_receive_callback: Callable, connection_callback: Callable, disconnection_callback: Callable, + message_not_sendable_callback: Callable, directory: bool, mc: 'OnionMessageChannel'): self.proto_client = None @@ -221,6 +235,9 @@ def __init__(self, message_receive_callback: Callable, self.connection_callback = connection_callback # disconnection the same self.disconnection_callback = disconnection_callback + # a callback that can be fired if we are not able to send messages, + # no args, returns None + self.message_not_sendable_callback = message_not_sendable_callback # is this connection to a directory? self.directory = directory # to keep track of state of overall messagechannel @@ -255,6 +272,11 @@ def register_disconnection(self, p: OnionLineProtocol) -> None: self.disconnection_callback() def send(self, msg: OnionCustomMessage) -> bool: + # we may be sending at the time the counterparty + # disconnected + if not self.proto_client: + self.message_not_sendable_callback() + return False self.proto_client.message(msg) # Unlike the serving protocol, the client protocol # is never in a condition of not knowing the counterparty @@ -430,6 +452,15 @@ def send(self, message: OnionCustomMessage) -> bool: def receive_message(self, message: OnionCustomMessage) -> None: self.messagechannel.receive_msg(message, self.peer_location()) + def notify_message_unsendable(self): + """ Triggered by a failure to send a message on the network, + by the encapsulated ClientFactory. Just used to notify calling + code; no action is triggered. + """ + name = "directory" if self.directory else "peer" + log.warn("Failure to send message to {}: {}.".format( + name, self.peer_location())) + def connect(self) -> None: """ This method is called to connect, over Tor, to the remote peer at the given onion host/port. @@ -442,7 +473,7 @@ def connect(self) -> None: self.factory = OnionClientFactory(self.receive_message, self.register_connection, self.register_disconnection, - self.directory, self.messagechannel) + self.notify_message_unsendable, self.directory, self.messagechannel) if testing_mode: log.debug("{} is making a tcp connection to {}, {}, {},".format( self.messagechannel.self_as_peer.peer_location(), self.hostname, @@ -450,13 +481,37 @@ def connect(self) -> None: self.tcp_connector = reactor.connectTCP(self.hostname, self.port, self.factory) else: + # non-default timeout; needs to be much lower than our + # 'wait at least a minute for the IRC connections to come up', + # which is used for *all* message channels, together. torEndpoint = TCP4ClientEndpoint(reactor, self.socks5_host, - self.socks5_port) + self.socks5_port, + timeout=CONNECT_TO_ONION_TIMEOUT) onionEndpoint = TorSocksEndpoint(torEndpoint, self.hostname, self.port) self.reconnecting_service = ClientService(onionEndpoint, self.factory) + # if we want to actually do something about an unreachable host, + # we have to force t.a.i.ClientService to give up after the timeout: + d = self.reconnecting_service.whenConnected(failAfterFailures=1) + d.addErrback(self.respond_to_connection_failure) self.reconnecting_service.startService() + def respond_to_connection_failure(self, failure): + # the error should be of this type specifically, if the onion + # is down, or was configured wrong: + failure.trap(HostUnreachableError) + # if this is a non-dir reachable peer, we just record + # the failure and explicitly give up: + if not self.directory: + log.info("We failed to connect to peer {}; giving up".format( + self.peer_location())) + self.reconnecting_service.stopService() + else: + # in this case, the still-running ClientService will + # just keep trying: + log.warn("We failed to connect to directory {}; trying " + "again.".format(self.peer_location())) + def register_connection(self) -> None: self.messagechannel.register_connection(self.peer_location(), direction=1) @@ -484,12 +539,14 @@ def disconnect(self) -> None: if not (self.hostname and self.port > 0): raise OnionPeerConnectionError( "Cannot disconnect without host, port info") - d = self.factory.proto_client.transport.loseConnection() - d.addCallback(self.complete_disconnection) - d.addErrback(log.warn, "Failed to disconnect from peer {}.".format( - self.peer_location())) + if self.factory: + d = self.reconnecting_service.stopService() + d.addCallback(self.complete_disconnection) + else: + self.messagechannel.proto_factory.disconnect_inbound_peer( + self.alternate_location) - def complete_disconnection(self) -> None: + def complete_disconnection(self, r) -> None: log.debug("Disconnected from peer: {}".format(self.peer_location())) self.update_status(PEER_STATUS_DISCONNECTED) self.factory = None @@ -530,6 +587,7 @@ def __init__(self, # hostid is a feature to avoid replay attacks across message channels; # TODO investigate, but for now, treat onion-based as one "server". self.hostid = "onion-network" + self.btc_network = configdata["btcnet"] # receives notification that we are shutting down self.give_up = False # for backwards compat: make sure MessageChannel log can refer to @@ -759,9 +817,10 @@ def connect_to_directories(self) -> None: # do not trigger on_welcome event until all directories # configured are ready: self.on_welcome_sent = False + self.directory_wait_counter = 0 self.wait_for_directories_loop = task.LoopingCall( self.wait_for_directories) - self.wait_for_directories_loop.start(10.0) + self.wait_for_directories_loop.start(2.0) def handshake_as_client(self, peer: OnionPeer) -> None: assert peer.status() == PEER_STATUS_CONNECTED @@ -772,6 +831,7 @@ def handshake_as_client(self, peer: OnionPeer) -> None: our_hs = copy.deepcopy(client_handshake_json) our_hs["location-string"] = self.self_as_peer.peer_location() our_hs["nick"] = self.nick + our_hs["network"] = self.btc_network our_hs_json = json.dumps(our_hs) log.info("Sending this handshake: {} to peer {}".format( our_hs_json, peer.peer_location())) @@ -780,6 +840,7 @@ def handshake_as_client(self, peer: OnionPeer) -> None: def handshake_as_directory(self, peer: OnionPeer, our_hs: dict) -> None: assert peer.status() == PEER_STATUS_CONNECTED + our_hs["network"] = self.btc_network our_hs_json = json.dumps(our_hs) log.info("Sending this handshake as directory: {}".format( our_hs_json)) @@ -1015,10 +1076,12 @@ def process_handshake(self, peerid: str, message: str, features = handshake_json["features"] accepted = handshake_json["accepted"] nick = handshake_json["nick"] + net = handshake_json["network"] assert isinstance(proto_max, int) assert isinstance(proto_min, int) assert isinstance(features, dict) assert isinstance(nick, str) + assert isinstance(net, str) except Exception as e: log.warn("Invalid handshake message from: {}," " exception: {}, message: {},ignoring".format( @@ -1029,11 +1092,19 @@ def process_handshake(self, peerid: str, message: str, # at all. if not accepted: log.warn("Directory: {} rejected our handshake.".format(peerid)) + # explicitly choose to disconnect (if other side already did, + # this is no-op). + peer.disconnect() return if not (app_name == JM_APP_NAME and is_directory and JM_VERSION \ <= proto_max and JM_VERSION >= proto_min and accepted): log.warn("Handshake from directory is incompatible or " "rejected: {}".format(handshake_json)) + peer.disconnect() + return + if not net == self.btc_network: + log.warn("Handshake from directory is on an incompatible " + "network: {}".format(net)) return # We received a valid, accepting dn-handshake. Update the peer. peer.update_status(PEER_STATUS_HANDSHAKED) @@ -1052,9 +1123,11 @@ def process_handshake(self, peerid: str, message: str, features = handshake_json["features"] full_location_string = handshake_json["location-string"] nick = handshake_json["nick"] + net = handshake_json["network"] assert isinstance(proto_ver, int) assert isinstance(features, dict) assert isinstance(nick, str) + assert isinstance(net, str) except Exception as e: log.warn("(not dn) Invalid handshake message from: {}, " "exception: {}, message: {}, ignoring".format( @@ -1066,6 +1139,10 @@ def process_handshake(self, peerid: str, message: str, log.warn("Invalid handshake name/version data: {}, from peer: " "{}, rejecting.".format(message, peerid)) accepted = False + if not net == self.btc_network: + log.warn("Handshake from peer is on an incompatible " + "network: {}".format(net)) + accepted = False # If accepted, we should update the peer to have the full # location which in general will not yet be present, so as to # allow publishing their location via `getpeerlist`. Note @@ -1207,10 +1284,27 @@ def get_connected_nondirectory_peers(self) -> list: def wait_for_directories(self) -> None: # Notice this is checking for *handshaked* dps; # the handshake will have been initiated once a - # connection was seen: + # connection was seen. + # Note also that this is *only* called on startup, + # so we are guaranteed to have only directory peers. + if len(self.get_connected_directory_peers()) < len(self.peers): + self.directory_wait_counter += 1 + # < 2*11 = 22 seconds; compare with CONNECT_TO_ONION_TIMEOUT; + # with current vals, we get to try twice before entirely + # giving up. + if self.directory_wait_counter < 11: + return if len(self.get_connected_directory_peers()) == 0: + # at least one handshake must have succeeded, for us + # to continue. + log.error("We failed to connect and handshake with " + "ANY directories; onion messaging is not functioning.") + self.wait_for_directories_loop.stop() return # This is what triggers the start of taker/maker workflows. + # Note that even if the preceding (max) 50 seconds failed to + # connect all our configured dps, we will keep trying and they + # can still be used. if not self.on_welcome_sent: self.on_welcome(self) self.on_welcome_sent = True diff --git a/scripts/joinmarket-qt.py b/scripts/joinmarket-qt.py index 69858b6d9..aed08dfc8 100755 --- a/scripts/joinmarket-qt.py +++ b/scripts/joinmarket-qt.py @@ -912,10 +912,10 @@ def startJoin(self): daemon=daemon, gui=True) else: - #This will re-use IRC connections in background (daemon), no restart + #This will re-use message channels in background (daemon), no restart self.clientfactory.getClient().client = self.taker self.clientfactory.getClient().clientStart() - mainWindow.statusBar().showMessage("Connecting to IRC ...") + mainWindow.statusBar().showMessage("Connecting to message channels ...") def takerInfo(self, infotype, infomsg): if infotype == "INFO": diff --git a/test/e2e-coinjoin-test.py b/test/e2e-coinjoin-test.py index 81296af9f..d5e9818c3 100644 --- a/test/e2e-coinjoin-test.py +++ b/test/e2e-coinjoin-test.py @@ -32,7 +32,7 @@ # For quicker testing, restrict the range of timelock # addresses to avoid slow load of multiple bots. -# Note: no need to revert this change as ygrunner runs +# Note: no need to revert this change as test runs # in isolation. from jmclient import FidelityBondMixin FidelityBondMixin.TIMELOCK_ERA_YEARS = 2 @@ -62,6 +62,7 @@ def location_string(directory_node_run_num): dn_nodes_list = ",".join(dns_to_use) log.info("For node: {}, set dn list to: {}".format(run_num, dn_nodes_list)) cf = {"type": "onion", + "btcnet": "testnet", "socks5_host": "127.0.0.1", "socks5_port": 9050, "tor_control_host": "127.0.0.1",