feat: enabling pubsub and ipns-pubsub via config flags (#8510)

* Allow pubsub and namesys-pubsub to be enabled via config

Signed-off-by: Joe Holden <jwh@zorins.us>
Co-authored-by: Marcin Rataj <lidel@lidel.org>
Co-authored-by: Adin Schmahmann <adin.schmahmann@gmail.com>

Author: 3 people

cmd/ipfs/daemon.go

@@ -178,8 +178,8 @@ Headers.
 		cmds.BoolOption(enableGCKwd, "Enable automatic periodic repo garbage collection"),
 		cmds.BoolOption(adjustFDLimitKwd, "Check and raise file descriptor limits if needed").WithDefault(true),
 		cmds.BoolOption(migrateKwd, "If true, assume yes at the migrate prompt. If false, assume no."),
-		cmds.BoolOption(enablePubSubKwd, "Instantiate the ipfs daemon with the experimental pubsub feature enabled."),
-		cmds.BoolOption(enableIPNSPubSubKwd, "Enable IPNS record distribution through pubsub; enables pubsub."),
+		cmds.BoolOption(enablePubSubKwd, "Enable experimental pubsub feature. Overrides Pubsub.Enabled config."),
+		cmds.BoolOption(enableIPNSPubSubKwd, "Enable IPNS over pubsub. Implicitly enables pubsub, overrides Ipns.UsePubsub config."),
 		cmds.BoolOption(enableMultiplexKwd, "DEPRECATED"),
 		cmds.StringOption(agentVersionSuffix, "Optional suffix to the AgentVersion presented by `ipfs id` and also advertised through BitSwap."),
 
@@ -365,13 +365,26 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment
 	defer repo.Close()
 
 	offline, _ := req.Options[offlineKwd].(bool)
-	ipnsps, _ := req.Options[enableIPNSPubSubKwd].(bool)
-	pubsub, _ := req.Options[enablePubSubKwd].(bool)
+	ipnsps, ipnsPsSet := req.Options[enableIPNSPubSubKwd].(bool)
+	pubsub, psSet := req.Options[enablePubSubKwd].(bool)
+
 	if _, hasMplex := req.Options[enableMultiplexKwd]; hasMplex {
 		log.Errorf("The mplex multiplexer has been enabled by default and the experimental %s flag has been removed.")
 		log.Errorf("To disable this multiplexer, please configure `Swarm.Transports.Multiplexers'.")
 	}
 
+	cfg, err := repo.Config()
+	if err != nil {
+		return err
+	}
+
+	if !psSet {
+		pubsub = cfg.Pubsub.Enabled.WithDefault(false)
+	}
+	if !ipnsPsSet {
+		ipnsps = cfg.Ipns.UsePubsub.WithDefault(false)
+	}
+
 	// Start assembling node config
 	ncfg := &core.BuildCfg{
 		Repo:                        repo,
@@ -387,11 +400,6 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment
 
 	routingOption, _ := req.Options[routingOptionKwd].(string)
 	if routingOption == routingOptionDefaultKwd {
-		cfg, err := repo.Config()
-		if err != nil {
-			return err
-		}
-
 		routingOption = cfg.Routing.Type
 		if routingOption == "" {
 			routingOption = routingOptionDHTKwd

docs/config.md

@@ -69,6 +69,7 @@ config file at runtime.
     - [`Ipns.RepublishPeriod`](#ipnsrepublishperiod)
     - [`Ipns.RecordLifetime`](#ipnsrecordlifetime)
     - [`Ipns.ResolveCacheSize`](#ipnsresolvecachesize)
+    - [`Ipns.UsePubsub`](#ipnsusepubsub)
   - [`Migration`](#migration)
     - [`Migration.DownloadSources`](#migrationdownloadsources)
     - [`Migration.Keep`](#migrationkeep)
@@ -87,6 +88,7 @@ config file at runtime.
           - [`Pinning.RemoteServices: Policies.MFS.PinName`](#pinningremoteservices-policiesmfspinname)
           - [`Pinning.RemoteServices: Policies.MFS.RepinInterval`](#pinningremoteservices-policiesmfsrepininterval)
   - [`Pubsub`](#pubsub)
+    - [`Pubsub.Enabled`](#pubsubenabled)
     - [`Pubsub.Router`](#pubsubrouter)
     - [`Pubsub.DisableSigning`](#pubsubdisablesigning)
   - [`Peering`](#peering)
@@ -946,6 +948,16 @@ Default: `128`
 
 Type: `integer` (non-negative, 0 means the default)
 
+### `Ipns.UsePubsub`
+
+Enables IPFS over pubsub experiment for publishing IPNS records in real time.
+
+**EXPERIMENTAL:**  read about current limitations at [experimental-features.md#ipns-pubsub](./experimental-features.md#ipns-pubsub).
+
+Default: `disabled`
+
+Type: `flag`
+
 ## `Migration`
 
 Migration configures how migrations are downloaded and if the downloads are added to IPFS locally.
@@ -1078,7 +1090,18 @@ Type: `duration`
 ## `Pubsub`
 
 Pubsub configures the `ipfs pubsub` subsystem. To use, it must be enabled by
-passing the `--enable-pubsub-experiment` flag to the daemon.
+passing the `--enable-pubsub-experiment` flag to the daemon
+or via the `Pubsub.Enabled` flag below.
+
+### `Pubsub.Enabled`
+
+**EXPERIMENTAL:** read about current limitations at [experimental-features.md#ipfs-pubsub](./experimental-features.md#ipfs-pubsub).
+
+Enables the pubsub system.
+
+Default: `false`
+
+Type: `flag`
 
 ### `Pubsub.Router`
 

docs/experimental-features.md

@@ -38,14 +38,22 @@ Candidate, disabled by default but will be enabled by default in 0.6.0.
 
 ### In Version
 
-0.4.5
+0.4.5 (`--enable-pubsub-experiment`)
+0.11.0 (`Pubsub.Enabled` flag in config)
 
 ### How to enable
 
-run your daemon with the `--enable-pubsub-experiment` flag. Then use the
-`ipfs pubsub` commands.
+Run your daemon with the `--enable-pubsub-experiment` flag
+or modify your ipfs config and restart the daemon:
+```
+ipfs config --json Pubsub.Enabled true
+```
+
+Then use the `ipfs pubsub` commands.
+
+NOTE: `--enable-pubsub-experiment` CLI flag overrides `Pubsub.Enabled` config.
 
-Configuration documentation can be found in [./config.md]()
+Configuration documentation can be found in [go-ipfs/docs/config.md](./config.md#pubsub)
 
 ### Road to being a real feature
 
@@ -426,12 +434,15 @@ go-ipfs now automatically shards when directory block is bigger than 256KB, ensu
 0.4.14 :
   - Introduced
 
-0.5.0 : 
+0.5.0 :
    - No longer needs to use the DHT for the first resolution
    - When discovering PubSub peers via the DHT, the DHT key is different from previous versions
       - This leads to 0.5 IPNS pubsub peers and 0.4 IPNS pubsub peers not being able to find each other in the DHT
    - Robustness improvements
 
+0.11.0 :
+  - Can be enabled via `Ipns.UsePubsub` flag in config
+
 ### State
 
 Experimental, default-disabled.
@@ -452,7 +463,15 @@ Users interested in this feature should upgrade to at least 0.5.0
 
 ### How to enable
 
-run your daemon with the `--enable-namesys-pubsub` flag; enables pubsub.
+Run your daemon with the `--enable-namesys-pubsub` flag
+or modify your ipfs config and restart the daemon:
+```
+ipfs config --json Ipns.UsePubsub true
+```
+
+NOTE:
+- This feature implicitly enables [ipfs pubsub](#ipfs-pubsub).
+- Passing `--enable-namesys-pubsub` CLI flag overrides `Ipns.UsePubsub` config.
 
 ### Road to being a real feature
 

test/sharness/t0183-namesys-pubsub.sh

@@ -10,79 +10,116 @@ test_expect_success 'init iptb' '
     iptb testbed create -type localipfs -count $NUM_NODES -init
 '
 
-startup_cluster $NUM_NODES --enable-namesys-pubsub
-
-test_expect_success 'peer ids' '
-    PEERID_0_BASE36=$(ipfsi 0 key list --ipns-base=base36 -l | grep self | head -n 1 | cut -d " " -f1) &&
-    PEERID_0_B58MH=$(ipfsi 0 key list --ipns-base=b58mh -l | grep self | head -n 1 | cut -d " " -f1)
-'
-
-test_expect_success 'check namesys pubsub state' '
-    echo enabled > expected &&
-    ipfsi 0 name pubsub state > state0 &&
-    ipfsi 1 name pubsub state > state1 &&
-    ipfsi 2 name pubsub state > state2 &&
-    test_cmp expected state0 &&
-    test_cmp expected state1 &&
-    test_cmp expected state2
-'
-
-# These commands are *expected* to fail. We haven't published anything yet.
-test_expect_success 'subscribe nodes to the publisher topic' '
-    ipfsi 1 name resolve /ipns/$PEERID_0_BASE36 --timeout=1s;
-    ipfsi 2 name resolve /ipns/$PEERID_0_BASE36 --timeout=1s;
-    true
+run_ipnspubsub_tests() {
+
+    test_expect_success 'peer ids' '
+        PEERID_0_BASE36=$(ipfsi 0 key list --ipns-base=base36 -l | grep self | head -n 1 | cut -d " " -f1) &&
+        PEERID_0_B58MH=$(ipfsi 0 key list --ipns-base=b58mh -l | grep self | head -n 1 | cut -d " " -f1)
+    '
+
+    test_expect_success 'check namesys pubsub state' '
+        echo enabled > expected &&
+        ipfsi 0 name pubsub state > state0 &&
+        ipfsi 1 name pubsub state > state1 &&
+        ipfsi 2 name pubsub state > state2 &&
+        test_cmp expected state0 &&
+        test_cmp expected state1 &&
+        test_cmp expected state2
+    '
+
+    # These commands are *expected* to fail. We haven't published anything yet.
+    test_expect_success 'subscribe nodes to the publisher topic' '
+        ipfsi 1 name resolve /ipns/$PEERID_0_BASE36 --timeout=1s;
+        ipfsi 2 name resolve /ipns/$PEERID_0_BASE36 --timeout=1s;
+        true
+    '
+
+    test_expect_success 'check subscriptions' '
+        echo /ipns/$PEERID_0_BASE36 > expected_base36 &&
+        echo /ipns/$PEERID_0_B58MH > expected_b58mh &&
+        ipfsi 1 name pubsub subs > subs1 &&
+        ipfsi 2 name pubsub subs > subs2 &&
+        ipfsi 1 name pubsub subs --ipns-base=b58mh > subs1_b58mh &&
+        ipfsi 2 name pubsub subs --ipns-base=b58mh > subs2_b58mh &&
+        test_cmp expected_base36 subs1 &&
+        test_cmp expected_base36 subs2 &&
+        test_cmp expected_b58mh subs1_b58mh &&
+        test_cmp expected_b58mh subs2_b58mh
+    '
+
+    test_expect_success 'add an object on publisher node' '
+        echo "ipns is super fun" > file &&
+        HASH_FILE=$(ipfsi 0 add -q file)
+    '
+
+    test_expect_success 'publish that object as an ipns entry' '
+        ipfsi 0 name publish $HASH_FILE
+    '
+
+    test_expect_success 'wait for the flood' '
+        sleep 1
+    '
+
+    test_expect_success 'resolve name in subscriber nodes' '
+        echo "/ipfs/$HASH_FILE" > expected &&
+        ipfsi 1 name resolve /ipns/$PEERID_0_BASE36 > name1 &&
+        ipfsi 2 name resolve /ipns/$PEERID_0_BASE36 > name2 &&
+        test_cmp expected name1 &&
+        test_cmp expected name2
+    '
+
+    test_expect_success 'cancel subscriptions to the publisher topic' '
+        ipfsi 1 name pubsub cancel /ipns/$PEERID_0_BASE36 &&
+        ipfsi 2 name pubsub cancel /ipns/$PEERID_0_BASE36
+    '
+
+    test_expect_success 'check subscriptions' '
+        rm -f expected && touch expected &&
+        ipfsi 1 name pubsub subs > subs1 &&
+        ipfsi 2 name pubsub subs > subs2 &&
+        test_cmp expected subs1 &&
+        test_cmp expected subs2
+    '
+
+    test_expect_success "shut down iptb" '
+        iptb stop
+    '
+
+}
+
+# Test everything with ipns-pubsub enabled via config
+test_expect_success 'enable ipns over pubsub' '
+  iptb run -- ipfs config --json Ipns.UsePubsub true
 '
 
-test_expect_success 'check subscriptions' '
-    echo /ipns/$PEERID_0_BASE36 > expected_base36 &&
-    echo /ipns/$PEERID_0_B58MH > expected_b58mh &&
-    ipfsi 1 name pubsub subs > subs1 &&
-    ipfsi 2 name pubsub subs > subs2 &&
-    ipfsi 1 name pubsub subs --ipns-base=b58mh > subs1_b58mh &&
-    ipfsi 2 name pubsub subs --ipns-base=b58mh > subs2_b58mh &&
-    test_cmp expected_base36 subs1 &&
-    test_cmp expected_base36 subs2 &&
-    test_cmp expected_b58mh subs1_b58mh &&
-    test_cmp expected_b58mh subs2_b58mh
-'
-
-test_expect_success 'add an object on publisher node' '
-    echo "ipns is super fun" > file &&
-    HASH_FILE=$(ipfsi 0 add -q file)
-'
+startup_cluster $NUM_NODES
+run_ipnspubsub_tests
 
-test_expect_success 'publish that object as an ipns entry' '
-    ipfsi 0 name publish $HASH_FILE
+# Test again, this time CLI parameter override the config
+test_expect_success 'enable ipns over pubsub' '
+  iptb run -- ipfs config --json Ipns.UsePubsub false
 '
+startup_cluster $NUM_NODES --enable-namesys-pubsub
+run_ipnspubsub_tests
 
-test_expect_success 'wait for the flood' '
-    sleep 1
-'
+# Confirm negative CLI flag takes precedence over positive config
 
-test_expect_success 'resolve name in subscriber nodes' '
-    echo "/ipfs/$HASH_FILE" > expected &&
-    ipfsi 1 name resolve /ipns/$PEERID_0_BASE36 > name1 &&
-    ipfsi 2 name resolve /ipns/$PEERID_0_BASE36 > name2 &&
-    test_cmp expected name1 &&
-    test_cmp expected name2
+test_expect_success 'enable the pubsub-ipns via config' '
+  iptb run -- ipfs config --json Ipns.UsePubsub true
 '
+startup_cluster $NUM_NODES --enable-namesys-pubsub=false
 
-test_expect_success 'cancel subscriptions to the publisher topic' '
-    ipfsi 1 name pubsub cancel /ipns/$PEERID_0_BASE36 &&
-    ipfsi 2 name pubsub cancel /ipns/$PEERID_0_BASE36
+test_expect_success 'ipns pubsub cmd fails because it was disabled via cli flag' '
+  test_expect_code 1 ipfsi 1 name pubsub subs 2> pubsubipns_cmd_out
 '
 
-test_expect_success 'check subscriptions' '
-    rm -f expected && touch expected &&
-    ipfsi 1 name pubsub subs > subs1 &&
-    ipfsi 2 name pubsub subs > subs2 &&
-    test_cmp expected subs1 &&
-    test_cmp expected subs2
-'
+test_expect_success "ipns pubsub cmd produces error" "
+  echo -e \"Error: IPNS pubsub subsystem is not enabled\nUse 'ipfs name pubsub subs --help' for information about this command\" > expected &&
+  test_cmp expected pubsubipns_cmd_out
+"
 
-test_expect_success "shut down iptb" '
-    iptb stop
+test_expect_success 'stop iptb' '
+  iptb stop
 '
 
 test_done