Merge pull request #1469 from Expensify/main

Update expensify_prod branch

BedrockServer.cpp

@@ -108,7 +108,7 @@ void BedrockServer::sync()
     // If still no value, use the number of cores on the machine, if available.
     workerThreads = workerThreads ? workerThreads : max(1u, thread::hardware_concurrency());
 
-    // A minumum of *2* worker threads are required. One for blocking writes, one for other commands.
+    // A minimum of *2* worker threads are required. One for blocking writes, one for other commands.
     if (workerThreads < 2) {
         workerThreads = 2;
     }
@@ -119,7 +119,7 @@ void BedrockServer::sync()
     // We use fewer FDs on test machines that have other resource restrictions in place.
     int fdLimit = args.isSet("-live") ? 25'000 : 250;
     SINFO("Setting dbPool size to: " << fdLimit);
-    _dbPool = make_shared<SQLitePool>(fdLimit, args["-db"], args.calc("-cacheSize"), args.calc("-maxJournalSize"), workerThreads, args["-synchronous"], mmapSizeGB);
+    _dbPool = make_shared<SQLitePool>(fdLimit, args["-db"], args.calc("-cacheSize"), args.calc("-maxJournalSize"), workerThreads, args["-synchronous"], mmapSizeGB, args.isSet("-hctree"));
     SQLite& db = _dbPool->getBase();
 
     // Initialize the command processor.

main.cpp

@@ -312,6 +312,8 @@ int main(int argc, char* argv[]) {
     } else if (args.isSet("-bootstrap")) {
         // Allow for bootstraping a node with no database file in place.
         SINFO("Loading in bootstrap mode, skipping check for database existance.");
+    } else if (args.isSet("-hctree")) {
+        SINFO("Starting in hctree mode, skipping check for database existance.");
     } else {
         // Otherwise verify the database exists
         SDEBUG("Verifying database exists");

sqlitecluster/SQLite.cpp

@@ -32,13 +32,16 @@ string SQLite::initializeFilename(const string& filename) {
         char resolvedPath[PATH_MAX];
         char* result = realpath(filename.c_str(), resolvedPath);
         if (!result) {
+            if (errno == ENOENT) {
+                return filename;
+            }
             SERROR("Couldn't resolve pathname for: " << filename);
         }
         return resolvedPath;
     }
 }
 
-SQLite::SharedData& SQLite::initializeSharedData(sqlite3* db, const string& filename, const vector<string>& journalNames) {
+SQLite::SharedData& SQLite::initializeSharedData(sqlite3* db, const string& filename, const vector<string>& journalNames, bool hctree) {
     static struct SharedDataLookupMapType {
         map<string, SharedData*> m;
         ~SharedDataLookupMapType() {
@@ -61,7 +64,7 @@ SQLite::SharedData& SQLite::initializeSharedData(sqlite3* db, const string& file
         bool isDBCurrentlyUsingWAL2 = result.rows.size() && result.rows[0][0] == "wal2";
 
         // If the intended wal setting doesn't match the existing wal setting, change it.
-        if (!isDBCurrentlyUsingWAL2) {
+        if (!hctree && !isDBCurrentlyUsingWAL2) {
             SASSERT(!SQuery(db, "", "PRAGMA journal_mode = delete;", result));
             SASSERT(!SQuery(db, "", "PRAGMA journal_mode = WAL2;", result));
         }
@@ -91,11 +94,17 @@ SQLite::SharedData& SQLite::initializeSharedData(sqlite3* db, const string& file
     }
 }
 
-sqlite3* SQLite::initializeDB(const string& filename, int64_t mmapSizeGB) {
+sqlite3* SQLite::initializeDB(const string& filename, int64_t mmapSizeGB, bool hctree) {
     // Open the DB in read-write mode.
     SINFO((SFileExists(filename) ? "Opening" : "Creating") << " database '" << filename << "'.");
     sqlite3* db;
-    SASSERT(!sqlite3_open_v2(filename.c_str(), &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_NOMUTEX, NULL));
+    string completeFilename = filename;
+    if (hctree) {
+        // Per the docs here: https://sqlite.org/hctree/doc/hctree/doc/hctree/index.html
+        // We only need to specify the full URL when creating new DBs. Existing DBs will be auto-detected as HC-Tree or not.
+        completeFilename = "file://" + completeFilename + "?hctree=1";
+    }
+    SASSERT(!sqlite3_open_v2(completeFilename.c_str(), &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_URI, NULL));
 
     // PRAGMA legacy_file_format=OFF sets the default for creating new databases, so it must be called before creating
     // any tables to be effective.
@@ -167,14 +176,16 @@ uint64_t SQLite::initializeJournalSize(sqlite3* db, const vector<string>& journa
     return max - min;
 }
 
-void SQLite::commonConstructorInitialization() {
+void SQLite::commonConstructorInitialization(bool hctree) {
     // Perform sanity checks.
     SASSERT(!_filename.empty());
     SASSERT(_cacheSize > 0);
     SASSERT(_maxJournalSize > 0);
 
     // WAL is what allows simultaneous read/writing.
-    SASSERT(!SQuery(_db, "enabling write ahead logging (wal2)", "PRAGMA journal_mode = wal2;"));
+    if (!hctree) {
+        SASSERT(!SQuery(_db, "enabling write ahead logging (wal2)", "PRAGMA journal_mode = wal2;"));
+    }
 
     if (_mmapSizeGB) {
         SASSERT(!SQuery(_db, "enabling memory-mapped I/O", "PRAGMA mmap_size=" + to_string(_mmapSizeGB * 1024 * 1024 * 1024) + ";"));
@@ -206,32 +217,33 @@ void SQLite::commonConstructorInitialization() {
 }
 
 SQLite::SQLite(const string& filename, int cacheSize, int maxJournalSize,
-               int minJournalTables, const string& synchronous, int64_t mmapSizeGB) :
+               int minJournalTables, const string& synchronous, int64_t mmapSizeGB, bool hctree) :
     _filename(initializeFilename(filename)),
     _maxJournalSize(maxJournalSize),
-    _db(initializeDB(_filename, mmapSizeGB)),
+    _db(initializeDB(_filename, mmapSizeGB, hctree)),
     _journalNames(initializeJournal(_db, minJournalTables)),
-    _sharedData(initializeSharedData(_db, _filename, _journalNames)),
+    _sharedData(initializeSharedData(_db, _filename, _journalNames, hctree)),
     _journalSize(initializeJournalSize(_db, _journalNames)),
     _cacheSize(cacheSize),
     _synchronous(synchronous),
     _mmapSizeGB(mmapSizeGB)
 {
-    commonConstructorInitialization();
+    commonConstructorInitialization(hctree);
 }
 
 SQLite::SQLite(const SQLite& from) :
     _filename(from._filename),
     _maxJournalSize(from._maxJournalSize),
-    _db(initializeDB(_filename, from._mmapSizeGB)), // Create a *new* DB handle from the same filename, don't copy the existing handle.
+    _db(initializeDB(_filename, from._mmapSizeGB, false)), // Create a *new* DB handle from the same filename, don't copy the existing handle.
     _journalNames(from._journalNames),
     _sharedData(from._sharedData),
     _journalSize(from._journalSize),
     _cacheSize(from._cacheSize),
     _synchronous(from._synchronous),
     _mmapSizeGB(from._mmapSizeGB)
 {
-    commonConstructorInitialization();
+    // This can always pass "true" because the copy constructor does not need to set the DB to WAL2 mode, it would have been set in the object being copied.
+    commonConstructorInitialization(true);
 }
 
 int SQLite::_progressHandlerCallback(void* arg) {
@@ -644,9 +656,12 @@ int SQLite::commit(const string& description, function<void()>* preCheckpointCal
 
         // Similarly, record WAL file size.
         sqlite3_file *pWal = 0;
-        sqlite3_int64 sz;
+        sqlite3_int64 sz = 0;
         sqlite3_file_control(_db, "main", SQLITE_FCNTL_JOURNAL_POINTER, &pWal);
-        pWal->pMethods->xFileSize(pWal, &sz);
+        if (pWal) {
+            // This is not set for HC-tree DBs.
+            pWal->pMethods->xFileSize(pWal, &sz);
+        }
 
         _commitElapsed += STimeNow() - before;
         _journalSize = newJournalSize;

sqlitecluster/SQLite.h

@@ -56,7 +56,7 @@ class SQLite {
     //
     // mmapSizeGB: address space to use for memory-mapped IO, in GB.
     SQLite(const string& filename, int cacheSize, int maxJournalSize, int minJournalTables,
-           const string& synchronous = "", int64_t mmapSizeGB = 0);
+           const string& synchronous = "", int64_t mmapSizeGB = 0, bool hctree = false);
 
     // Compatibility constructor. Remove when AuthTester::getStripeSQLiteDB no longer uses this outdated version.
     SQLite(const string& filename, int cacheSize, int maxJournalSize, int minJournalTables, int synchronous) :
@@ -313,11 +313,11 @@ class SQLite {
 
     // Initializers to support RAII-style allocation in constructors.
     static string initializeFilename(const string& filename);
-    static SharedData& initializeSharedData(sqlite3* db, const string& filename, const vector<string>& journalNames);
-    static sqlite3* initializeDB(const string& filename, int64_t mmapSizeGB);
+    static SharedData& initializeSharedData(sqlite3* db, const string& filename, const vector<string>& journalNames, bool hctree);
+    static sqlite3* initializeDB(const string& filename, int64_t mmapSizeGB, bool hctree);
     static vector<string> initializeJournal(sqlite3* db, int minJournalTables);
     static uint64_t initializeJournalSize(sqlite3* db, const vector<string>& journalNames);
-    void commonConstructorInitialization();
+    void commonConstructorInitialization(bool hctree = false);
 
     // The filename of this DB, canonicalized to its full path on disk.
     const string _filename;

sqlitecluster/SQLitePool.cpp

@@ -8,9 +8,10 @@ SQLitePool::SQLitePool(size_t maxDBs,
                        int maxJournalSize,
                        int minJournalTables,
                        const string& synchronous,
-                       int64_t mmapSizeGB)
+                       int64_t mmapSizeGB,
+                       bool hctree)
 : _maxDBs(max(maxDBs, 1ul)),
-  _baseDB(filename, cacheSize, maxJournalSize, minJournalTables, synchronous, mmapSizeGB),
+  _baseDB(filename, cacheSize, maxJournalSize, minJournalTables, synchronous, mmapSizeGB, hctree),
   _objects(_maxDBs, nullptr)
 {
 }

sqlitecluster/SQLitePool.h

@@ -7,7 +7,7 @@ class SQLitePool {
   public:
     // Create a pool of DB handles.
     SQLitePool(size_t maxDBs, const string& filename, int cacheSize, int maxJournalSize, int minJournalTables,
-               const string& synchronous = "", int64_t mmapSizeGB = 0);
+               const string& synchronous = "", int64_t mmapSizeGB = 0, bool hctree = false);
     ~SQLitePool();
 
     // Get the base object (the first one created, which uses the `journal` table). Note that if called by multiple

test/lib/BedrockTester.cpp

@@ -8,13 +8,15 @@
 
 #include <libstuff/SData.h>
 #include <libstuff/SFastBuffer.h>
+#include <libstuff/SQResult.h>
 #include <sqlitecluster/SQLite.h>
 #include <test/lib/BedrockTester.h>
 #include <test/lib/tpunit++.hpp>
 
 PortMap BedrockTester::ports;
 mutex BedrockTester::_testersMutex;
 set<BedrockTester*> BedrockTester::_testers;
+const bool BedrockTester::ENABLE_HCTREE{false};
 
 string BedrockTester::getTempFileName(string prefix) {
     string templateStr = "/tmp/" + prefix + "bedrocktest_XXXXXX.db";
@@ -61,8 +63,9 @@ BedrockTester::BedrockTester(const map<string, string>& args,
         serverName = bedrockBinary;
     }
 
+    string dbFileName = getTempFileName();
     map <string, string> defaultArgs = {
-        {"-db", getTempFileName()},
+        {"-db", dbFileName},
         {"-serverHost", "127.0.0.1:" + to_string(_serverPort)},
         {"-nodeName", "bedrock_test"},
         {"-nodeHost", "localhost:" + to_string(_nodePort)},
@@ -84,6 +87,10 @@ BedrockTester::BedrockTester(const map<string, string>& args,
         {"-testName", currentTestName},
     };
 
+    if (ENABLE_HCTREE) {
+        defaultArgs["-hctree"] = "";
+    }
+
     // Set defaults.
     for (auto& row : defaultArgs) {
         _args[row.first] = row.second;
@@ -93,18 +100,22 @@ BedrockTester::BedrockTester(const map<string, string>& args,
     for (auto& row : args) {
         _args[row.first] = row.second;
     }
-    
+
     // If the DB file doesn't exist, create it.
     if (!SFileExists(_args["-db"])) {
         SFileSave(_args["-db"], "");
     }
-
+    
     // Run any supplied queries on the DB.
     // We don't use SQLite here, because we specifically want to avoid dealing with journal tables.
     if (queries.size()) {
         sqlite3* db;
         sqlite3_initialize();
-        sqlite3_open_v2(_args["-db"].c_str(), &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_NOMUTEX, NULL);
+        string completeFilename = dbFileName;
+        if (ENABLE_HCTREE) {
+            completeFilename = "file://" + completeFilename + "?hctree=1";
+        }
+        sqlite3_open_v2(completeFilename.c_str(), &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_URI, NULL);
         for (string query : queries) {
             int error = sqlite3_exec(db, query.c_str(), 0, 0, 0);
             if (error) {
@@ -521,20 +532,47 @@ SQLite& BedrockTester::getSQLiteDB()
 
 string BedrockTester::readDB(const string& query)
 {
-    SQLite& db = getSQLiteDB();
-    db.beginTransaction();
-    string result = db.read(query);
-    db.rollback();
-    return result;
+    if (ENABLE_HCTREE) {
+        SData command("Query");
+        command["Query"] = query;
+        command["Format"] = "JSON";
+        auto row0 = SParseJSONObject(executeWaitMultipleData({command})[0].content)["rows"];
+        return SParseJSONArray(SParseJSONArray(row0).front()).front();
+    } else {
+        SQLite& db = getSQLiteDB();
+        db.beginTransaction();
+        string result = db.read(query);
+        db.rollback();
+        return result;
+    }
 }
 
 bool BedrockTester::readDB(const string& query, SQResult& result)
 {
-    SQLite& db = getSQLiteDB();
-    db.beginTransaction();
-    bool success = db.read(query, result);
-    db.rollback();
-    return success;
+    if (ENABLE_HCTREE) {
+        result.clear();
+        SData command("Query");
+        command["Query"] = query;
+        command["Format"] = "JSON";
+        auto row0 = SParseJSONObject(executeWaitMultipleData({command})[0].content)["rows"];
+
+        list<string> rows = SParseJSONArray(row0);
+        for (const string& rowStr : rows) {
+            list<string> vals = SParseJSONArray(rowStr);
+            vector<string> row;
+            for (auto& v : vals) {
+                row.push_back(v);
+            }
+            result.rows.push_back(row);
+        }
+        return true;
+    } else {
+        SQLite& db = getSQLiteDB();
+        db.beginTransaction();
+        bool success = db.read(query, result);
+        db.rollback();
+        return success;
+    }
 }
 
 bool BedrockTester::waitForStatusTerm(const string& term, const string& testValue, uint64_t timeoutUS) {

test/lib/BedrockTester.h

@@ -8,6 +8,8 @@ class SQLite;
 
 class BedrockTester {
   public:
+    static const bool ENABLE_HCTREE;
+
     // This is an allocator for TCP ports, new servers can get new port numbers from this object and return them when
     // they're done.
     static PortMap ports;