Merge #15935: Add <datadir>/settings.json persistent settings storage

9c69cfe Add <datadir>/settings.json persistent settings storage. (Russell Yanofsky)
eb682c5 util: Add ReadSettings and WriteSettings functions (Russell Yanofsky)

Pull request description:

  Persistent settings are used in followup PRs #15936 to unify gui settings between bitcoin-qt and bitcoind, and #15937 to add a load_on_startup flag to the loadwallet RPC and maintain a dynamic list of wallets that should be loaded on startup that also can be shared between bitcoind and bitcoin-qt.

ACKs for top commit:
  MarcoFalke:
    Approach re-ACK 9c69cfe 🌾
  jnewbery:
    utACK 9c69cfe

Tree-SHA512: 39fcc6051717117c9141e934de1d0d3f739484be4685cdf97d54de967c8c816502b4fd0de12114433beaa5c5b7060c810fd8ae4e2b3ce7c371eb729ac01ba2e1

Author: MarcoFalke

doc/files.md

@@ -50,14 +50,15 @@ Subdirectory       | File(s)               | Description
 `indexes/blockfilter/basic/`    | `fltrNNNNN.dat`<sup>[\[2\]](#note2)</sup> | Blockfilter index filters for the basic filtertype; *optional*, used if `-blockfilterindex=basic`
 `wallets/`         |                       | [Contains wallets](#multi-wallet-environment); can be specified by `-walletdir` option; if `wallets/` subdirectory does not exist, a wallet resides in the data directory
 `./`               | `banlist.dat`         | Stores the IPs/subnets of banned nodes
-`./`               | `bitcoin.conf`        | Contains [configuration settings](bitcoin-conf.md) for `bitcoind` or `bitcoin-qt`; can be specified by `-conf` option
+`./`               | `bitcoin.conf`        | User-defined [configuration settings](bitcoin-conf.md) for `bitcoind` or `bitcoin-qt`. File is not written to by the software and must be created manually. Path can be specified by `-conf` option
 `./`               | `bitcoind.pid`        | Stores the process ID (PID) of `bitcoind` or `bitcoin-qt` while running; created at start and deleted on shutdown; can be specified by `-pid` option
 `./`               | `debug.log`           | Contains debug information and general logging generated by `bitcoind` or `bitcoin-qt`; can be specified by `-debuglogfile` option
 `./`               | `fee_estimates.dat`   | Stores statistics used to estimate minimum transaction fees and priorities required for confirmation
 `./`               | `guisettings.ini.bak` | Backup of former [GUI settings](#gui-settings) after `-resetguisettings` option is used
 `./`               | `mempool.dat`         | Dump of the mempool's transactions
 `./`               | `onion_private_key`   | Cached Tor hidden service private key for `-listenonion` option
 `./`               | `peers.dat`           | Peer IP address database (custom format)
+`./`               | `settings.json`       | Read-write settings set through GUI or RPC interfaces, augmenting manual settings from [bitcoin.conf](bitcoin-conf.md). File is created automatically if read-write settings storage is not disabled with `-nosettings` option. Path can be specified with `-settings` option
 `./`               | `.cookie`             | Session RPC authentication cookie; if used, created at start and deleted on shutdown; can be specified by `-rpccookiefile` option
 `./`               | `.lock`               | Data directory lock file
 

src/bitcoind.cpp

@@ -101,6 +101,11 @@ static bool AppInit(int argc, char* argv[])
             }
         }
 
+        if (!gArgs.InitSettings(error)) {
+            InitError(Untranslated(error));
+            return false;
+        }
+
         // -server defaults to true for bitcoind but not for the GUI so do this here
         gArgs.SoftSetBoolArg("-server", true);
         // Set this early so that parameter interactions go to console

src/init.cpp

@@ -397,7 +397,7 @@ void SetupServerArgs(NodeContext& node)
 #endif
     gArgs.AddArg("-blockreconstructionextratxn=<n>", strprintf("Extra transactions to keep in memory for compact block reconstructions (default: %u)", DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     gArgs.AddArg("-blocksonly", strprintf("Whether to reject transactions from network peers. Automatic broadcast and rebroadcast of any transactions from inbound peers is disabled, unless the peer has the 'forcerelay' permission. RPC transactions are not affected. (default: %u)", DEFAULT_BLOCKSONLY), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
-    gArgs.AddArg("-conf=<file>", strprintf("Specify configuration file. Relative paths will be prefixed by datadir location. (default: %s)", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
+    gArgs.AddArg("-conf=<file>", strprintf("Specify path to read-only configuration file. Relative paths will be prefixed by datadir location. (default: %s)", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     gArgs.AddArg("-datadir=<dir>", "Specify data directory", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     gArgs.AddArg("-dbbatchsize", strprintf("Maximum database write batch size in bytes (default: %u)", nDefaultDbBatchSize), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::OPTIONS);
     gArgs.AddArg("-dbcache=<n>", strprintf("Maximum database cache size <n> MiB (%d to %d, default: %d). In addition, unused mempool memory is shared for this cache (see -maxmempool).", nMinDbCache, nMaxDbCache, nDefaultDbCache), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
@@ -418,6 +418,7 @@ void SetupServerArgs(NodeContext& node)
             "(default: 0 = disable pruning blocks, 1 = allow manual pruning via RPC, >=%u = automatically prune block files to stay under the specified target size in MiB)", MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     gArgs.AddArg("-reindex", "Rebuild chain state and block index from the blk*.dat files on disk", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     gArgs.AddArg("-reindex-chainstate", "Rebuild chain state from the currently indexed blocks. When in pruning mode or if blocks on disk might be corrupted, use full -reindex instead.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
+    gArgs.AddArg("-settings=<file>", strprintf("Specify path to dynamic settings data file. Can be disabled with -nosettings. File is written at runtime and not meant to be edited by users (use %s instead for custom settings). Relative paths will be prefixed by datadir location. (default: %s)", BITCOIN_CONF_FILENAME, BITCOIN_SETTINGS_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
 #ifndef WIN32
     gArgs.AddArg("-sysperms", "Create new files with system default permissions, instead of umask 077 (only effective with disabled wallet functionality)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
 #else

src/interfaces/node.cpp

@@ -66,6 +66,7 @@ class NodeImpl : public Node
     bool softSetArg(const std::string& arg, const std::string& value) override { return gArgs.SoftSetArg(arg, value); }
     bool softSetBoolArg(const std::string& arg, bool value) override { return gArgs.SoftSetBoolArg(arg, value); }
     void selectParams(const std::string& network) override { SelectParams(network); }
+    bool initSettings(std::string& error) override { return gArgs.InitSettings(error); }
     uint64_t getAssumedBlockchainSize() override { return Params().AssumedBlockchainSize(); }
     uint64_t getAssumedChainStateSize() override { return Params().AssumedChainStateSize(); }
     std::string getNetwork() override { return Params().NetworkIDString(); }

src/interfaces/node.h

@@ -66,6 +66,11 @@ class Node
     //! Choose network parameters.
     virtual void selectParams(const std::string& network) = 0;
 
+    //! Read and update <datadir>/settings.json file with saved settings. This
+    //! needs to be called after selectParams() because the settings file
+    //! location is network-specific.
+    virtual bool initSettings(std::string& error) = 0;
+
     //! Get the (assumed) blockchain size.
     virtual uint64_t getAssumedBlockchainSize() = 0;
 

src/qt/bitcoin.cpp

@@ -528,6 +528,11 @@ int GuiMain(int argc, char* argv[])
     // Parse URIs on command line -- this can affect Params()
     PaymentServer::ipcParseCommandLine(*node, argc, argv);
 #endif
+    if (!node->initSettings(error)) {
+        node->initError(Untranslated(error));
+        QMessageBox::critical(nullptr, PACKAGE_NAME, QObject::tr("Error initializing settings: %1").arg(QString::fromStdString(error)));
+        return EXIT_FAILURE;
+    }
 
     QScopedPointer<const NetworkStyle> networkStyle(NetworkStyle::instantiate(Params().NetworkIDString()));
     assert(!networkStyle.isNull());

src/test/settings_tests.cpp

@@ -12,10 +12,90 @@
 #include <univalue.h>
 #include <util/strencodings.h>
 #include <util/string.h>
+#include <util/system.h>
 #include <vector>
 
+inline bool operator==(const util::SettingsValue& a, const util::SettingsValue& b)
+{
+    return a.write() == b.write();
+}
+
+inline std::ostream& operator<<(std::ostream& os, const util::SettingsValue& value)
+{
+    os << value.write();
+    return os;
+}
+
+inline std::ostream& operator<<(std::ostream& os, const std::pair<std::string, util::SettingsValue>& kv)
+{
+    util::SettingsValue out(util::SettingsValue::VOBJ);
+    out.__pushKV(kv.first, kv.second);
+    os << out.write();
+    return os;
+}
+
+inline void WriteText(const fs::path& path, const std::string& text)
+{
+    fsbridge::ofstream file;
+    file.open(path);
+    file << text;
+}
+
 BOOST_FIXTURE_TEST_SUITE(settings_tests, BasicTestingSetup)
 
+BOOST_AUTO_TEST_CASE(ReadWrite)
+{
+    fs::path path = GetDataDir() / "settings.json";
+
+    WriteText(path, R"({
+        "string": "string",
+        "num": 5,
+        "bool": true,
+        "null": null
+    })");
+
+    std::map<std::string, util::SettingsValue> expected{
+        {"string", "string"},
+        {"num", 5},
+        {"bool", true},
+        {"null", {}},
+    };
+
+    // Check file read.
+    std::map<std::string, util::SettingsValue> values;
+    std::vector<std::string> errors;
+    BOOST_CHECK(util::ReadSettings(path, values, errors));
+    BOOST_CHECK_EQUAL_COLLECTIONS(values.begin(), values.end(), expected.begin(), expected.end());
+    BOOST_CHECK(errors.empty());
+
+    // Check no errors if file doesn't exist.
+    fs::remove(path);
+    BOOST_CHECK(util::ReadSettings(path, values, errors));
+    BOOST_CHECK(values.empty());
+    BOOST_CHECK(errors.empty());
+
+    // Check duplicate keys not allowed
+    WriteText(path, R"({
+        "dupe": "string",
+        "dupe": "dupe"
+    })");
+    BOOST_CHECK(!util::ReadSettings(path, values, errors));
+    std::vector<std::string> dup_keys = {strprintf("Found duplicate key dupe in settings file %s", path.string())};
+    BOOST_CHECK_EQUAL_COLLECTIONS(errors.begin(), errors.end(), dup_keys.begin(), dup_keys.end());
+
+    // Check non-kv json files not allowed
+    WriteText(path, R"("non-kv")");
+    BOOST_CHECK(!util::ReadSettings(path, values, errors));
+    std::vector<std::string> non_kv = {strprintf("Found non-object value \"non-kv\" in settings file %s", path.string())};
+    BOOST_CHECK_EQUAL_COLLECTIONS(errors.begin(), errors.end(), non_kv.begin(), non_kv.end());
+
+    // Check invalid json not allowed
+    WriteText(path, R"(invalid json)");
+    BOOST_CHECK(!util::ReadSettings(path, values, errors));
+    std::vector<std::string> fail_parse = {strprintf("Unable to parse settings file %s", path.string())};
+    BOOST_CHECK_EQUAL_COLLECTIONS(errors.begin(), errors.end(), fail_parse.begin(), fail_parse.end());
+}
+
 //! Check settings struct contents against expected json strings.
 static void CheckValues(const util::Settings& settings, const std::string& single_val, const std::string& list_val)
 {

src/test/util_tests.cpp

@@ -9,6 +9,7 @@
 #include <key.h>  // For CKey
 #include <optional.h>
 #include <sync.h>
+#include <test/util/logging.h>
 #include <test/util/setup_common.h>
 #include <test/util/str.h>
 #include <uint256.h>
@@ -1138,6 +1139,28 @@ BOOST_FIXTURE_TEST_CASE(util_ChainMerge, ChainMergeTestingSetup)
     BOOST_CHECK_EQUAL(out_sha_hex, "f0b3a3c29869edc765d579c928f7f1690a71fbb673b49ccf39cbc4de18156a0d");
 }
 
+BOOST_AUTO_TEST_CASE(util_ReadWriteSettings)
+{
+    // Test writing setting.
+    TestArgsManager args1;
+    args1.LockSettings([&](util::Settings& settings) { settings.rw_settings["name"] = "value"; });
+    args1.WriteSettingsFile();
+
+    // Test reading setting.
+    TestArgsManager args2;
+    args2.ReadSettingsFile();
+    args2.LockSettings([&](util::Settings& settings) { BOOST_CHECK_EQUAL(settings.rw_settings["name"].get_str(), "value"); });
+
+    // Test error logging, and remove previously written setting.
+    {
+        ASSERT_DEBUG_LOG("Failed renaming settings file");
+        fs::remove(GetDataDir() / "settings.json");
+        fs::create_directory(GetDataDir() / "settings.json");
+        args2.WriteSettingsFile();
+        fs::remove(GetDataDir() / "settings.json");
+    }
+}
+
 BOOST_AUTO_TEST_CASE(util_FormatMoney)
 {
     BOOST_CHECK_EQUAL(FormatMoney(0), "0.00");

src/util/settings.cpp

@@ -4,6 +4,7 @@
 
 #include <util/settings.h>
 
+#include <tinyformat.h>
 #include <univalue.h>
 
 namespace util {
@@ -12,12 +13,13 @@ namespace {
 enum class Source {
    FORCED,
    COMMAND_LINE,
+   RW_SETTINGS,
    CONFIG_FILE_NETWORK_SECTION,
    CONFIG_FILE_DEFAULT_SECTION
 };
 
 //! Merge settings from multiple sources in precedence order:
-//! Forced config > command line > config file network-specific section > config file default section
+//! Forced config > command line > read-write settings file > config file network-specific section > config file default section
 //!
 //! This function is provided with a callback function fn that contains
 //! specific logic for how to merge the sources.
@@ -32,6 +34,10 @@ static void MergeSettings(const Settings& settings, const std::string& section,
     if (auto* values = FindKey(settings.command_line_options, name)) {
         fn(SettingsSpan(*values), Source::COMMAND_LINE);
     }
+    // Merge in the read-write settings
+    if (const SettingsValue* value = FindKey(settings.rw_settings, name)) {
+        fn(SettingsSpan(*value), Source::RW_SETTINGS);
+    }
     // Merge in the network-specific section of the config file
     if (!section.empty()) {
         if (auto* map = FindKey(settings.ro_config, section)) {
@@ -49,6 +55,62 @@ static void MergeSettings(const Settings& settings, const std::string& section,
 }
 } // namespace
 
+bool ReadSettings(const fs::path& path, std::map<std::string, SettingsValue>& values, std::vector<std::string>& errors)
+{
+    values.clear();
+    errors.clear();
+
+    fsbridge::ifstream file;
+    file.open(path);
+    if (!file.is_open()) return true; // Ok for file not to exist.
+
+    SettingsValue in;
+    if (!in.read(std::string{std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>()})) {
+        errors.emplace_back(strprintf("Unable to parse settings file %s", path.string()));
+        return false;
+    }
+
+    if (file.fail()) {
+        errors.emplace_back(strprintf("Failed reading settings file %s", path.string()));
+        return false;
+    }
+    file.close(); // Done with file descriptor. Release while copying data.
+
+    if (!in.isObject()) {
+        errors.emplace_back(strprintf("Found non-object value %s in settings file %s", in.write(), path.string()));
+        return false;
+    }
+
+    const std::vector<std::string>& in_keys = in.getKeys();
+    const std::vector<SettingsValue>& in_values = in.getValues();
+    for (size_t i = 0; i < in_keys.size(); ++i) {
+        auto inserted = values.emplace(in_keys[i], in_values[i]);
+        if (!inserted.second) {
+            errors.emplace_back(strprintf("Found duplicate key %s in settings file %s", in_keys[i], path.string()));
+        }
+    }
+    return errors.empty();
+}
+
+bool WriteSettings(const fs::path& path,
+    const std::map<std::string, SettingsValue>& values,
+    std::vector<std::string>& errors)
+{
+    SettingsValue out(SettingsValue::VOBJ);
+    for (const auto& value : values) {
+        out.__pushKV(value.first, value.second);
+    }
+    fsbridge::ofstream file;
+    file.open(path);
+    if (file.fail()) {
+        errors.emplace_back(strprintf("Error: Unable to open settings file %s for writing", path.string()));
+        return false;
+    }
+    file << out.write(/* prettyIndent= */ 1, /* indentLevel= */ 4) << std::endl;
+    file.close();
+    return true;
+}
+
 SettingsValue GetSetting(const Settings& settings,
     const std::string& section,
     const std::string& name,

src/util/settings.h

@@ -5,6 +5,8 @@
 #ifndef BITCOIN_UTIL_SETTINGS_H
 #define BITCOIN_UTIL_SETTINGS_H
 
+#include <fs.h>
+
 #include <map>
 #include <string>
 #include <vector>
@@ -24,19 +26,31 @@ namespace util {
 //!       https://github.com/bitcoin/bitcoin/pull/15934/files#r337691812)
 using SettingsValue = UniValue;
 
-//! Stored bitcoin settings. This struct combines settings from the command line
-//! and a read-only configuration file.
+//! Stored settings. This struct combines settings from the command line, a
+//! read-only configuration file, and a read-write runtime settings file.
 struct Settings {
     //! Map of setting name to forced setting value.
     std::map<std::string, SettingsValue> forced_settings;
     //! Map of setting name to list of command line values.
     std::map<std::string, std::vector<SettingsValue>> command_line_options;
+    //! Map of setting name to read-write file setting value.
+    std::map<std::string, SettingsValue> rw_settings;
     //! Map of config section name and setting name to list of config file values.
     std::map<std::string, std::map<std::string, std::vector<SettingsValue>>> ro_config;
 };
 
+//! Read settings file.
+bool ReadSettings(const fs::path& path,
+    std::map<std::string, SettingsValue>& values,
+    std::vector<std::string>& errors);
+
+//! Write settings file.
+bool WriteSettings(const fs::path& path,
+    const std::map<std::string, SettingsValue>& values,
+    std::vector<std::string>& errors);
+
 //! Get settings value from combined sources: forced settings, command line
-//! arguments and the read-only config file.
+//! arguments, runtime read-write settings, and the read-only config file.
 //!
 //! @param ignore_default_section_config - ignore values in the default section
 //!                                        of the config file (part before any