Feature/apm 173 (#14830)

Author: cpjulia

CHANGELOG

@@ -1,6 +1,18 @@
 devel
 -----
 
+* APM-173: Now, arangobench, arangodump and arangorestore support multiple 
+  coordinators, so the flag `--server.endpoint` can be used multiple times, 
+  as in the example below:
+
+  arangobench \
+    --server.endpoint tcp://[::1]::8529 \
+    --server.endpoint tcp://[::1]::8530 \
+    --server.endpoint tcp://[::1]::8531 
+
+  This does not compromise the use of the other client tools, which preserve
+  the behavior of having one coordinator.
+
 * The server now has two flags to control the escaping control and Unicode 
   characters in the log. The flag `--log.escape` is now deprecated and, instead, 
   the new flags `--log.escape-control-chars` and `--log.escape-unicode-chars` 

arangosh/Backup/BackupFeature.cpp

@@ -162,7 +162,7 @@ arangodb::Result waitForRestart(arangodb::ClientManager& clientManager,
   while (timeSinceStart() < maxWaitForRestart) {
     std::unique_ptr<arangodb::httpclient::SimpleHttpClient> client;
     try {
-      result = clientManager.getConnectedClient(client, true, false, false, true);
+      result = clientManager.getConnectedClient(client, true, false, false, true, 0);
       if (result.ok() && client != nullptr) {
         double uptime = 0.0;
         result = ::getUptime(*client, uptime);
@@ -860,7 +860,7 @@ void BackupFeature::validateOptions(std::shared_ptr<options::ProgramOptions> opt
 void BackupFeature::start() {
   Result result;
   std::unique_ptr<httpclient::SimpleHttpClient> client =
-      _clientManager.getConnectedClient(false, true, true);
+      _clientManager.getConnectedClient(false, true, true, 0);
   if (_options.operation == OperationList) {
     result = ::executeList(*client, _options);
   } else if (_options.operation == OperationCreate) {

arangosh/Benchmark/BenchFeature.cpp

@@ -29,8 +29,8 @@
 #include "BenchFeature.h"
 
 #include <ctime>
-#include <iomanip>
 #include <fstream>
+#include <iomanip>
 #include <iostream>
 
 #ifdef TRI_HAVE_UNISTD_H
@@ -39,19 +39,20 @@
 
 #include "ApplicationFeatures/ApplicationServer.h"
 #include "Basics/FileUtils.h"
+#include "Basics/NumberOfCores.h"
 #include "Basics/StaticStrings.h"
 #include "Basics/StringUtils.h"
 #include "Basics/Utf8Helper.h"
 #include "Basics/application-exit.h"
 #include "Basics/files.h"
 #include "Basics/system-functions.h"
-#include "Logger/Logger.h"
-#include "Logger/LogMacros.h"
 #include "Benchmark/BenchmarkCounter.h"
 #include "Benchmark/BenchmarkOperation.h"
 #include "Benchmark/BenchmarkStats.h"
 #include "Benchmark/BenchmarkThread.h"
 #include "FeaturePhases/BasicFeaturePhaseClient.h"
+#include "Logger/LogMacros.h"
+#include "Logger/Logger.h"
 #include "ProgramOptions/ProgramOptions.h"
 #include "ProgramOptions/Section.h"
 #include "Shell/ClientFeature.h"
@@ -75,7 +76,7 @@ using namespace arangodb::rest;
 
 BenchFeature::BenchFeature(application_features::ApplicationServer& server, int* result)
     : ApplicationFeature(server, "Bench"),
-      _concurrency(1),
+      _concurrency(NumberOfCores::getValue()),
       _operations(1000),
       _realOperations(0),
       _batchSize(0),
@@ -139,11 +140,10 @@ void BenchFeature::collectOptions(std::shared_ptr<ProgramOptions> options) {
   options->addOption("--histogram.num-intervals",
                      "number of buckets (resolution)",
                      new UInt64Parameter(&_histogramNumIntervals));
-  options->addOption("--histogram.percentiles",
-                     "which percentiles to calculate",
+  options->addOption("--histogram.percentiles", "which percentiles to calculate",
                      new VectorParameter<DoubleParameter>(&_percentiles),
                      arangodb::options::makeDefaultFlags(options::Flags::FlushOnFirst));
-  
+
   options->addOption("--async", "send asynchronous requests", new BooleanParameter(&_async));
 
   options->addOption("--concurrency",
@@ -178,7 +178,8 @@ void BenchFeature::collectOptions(std::shared_ptr<ProgramOptions> options) {
                      new BooleanParameter(&_waitForSync));
 
   options->addOption("--create-database",
-                     "whether we should create the database specified via the server connection",
+                     "whether we should create the database specified via the "
+                     "server connection",
                      new BooleanParameter(&_createDatabase));
 
   options->addOption("--duration",
@@ -216,16 +217,25 @@ void BenchFeature::collectOptions(std::shared_ptr<ProgramOptions> options) {
   options->addOption("--progress", "log intermediate progress",
                      new BooleanParameter(&_progress));
 
-  options->addOption("--custom-query", "the query to be used in the 'custom-query' testcase",
-                     new StringParameter(&_customQuery)).setIntroducedIn(30800);
-                     
-  options->addOption("--custom-query-file", "path to a file with the query to be used in the 'custom-query' testcase. "
-                     "If --custom-query is specified as well, it has higher priority.",
-                     new StringParameter(&_customQueryFile)).setIntroducedIn(30800);
-                     
+  options
+      ->addOption("--custom-query",
+                  "the query to be used in the 'custom-query' testcase",
+                  new StringParameter(&_customQuery))
+      .setIntroducedIn(30800);
+
+  options
+      ->addOption(
+          "--custom-query-file",
+          "path to a file with the query to be used in the 'custom-query' "
+          "testcase. "
+          "If --custom-query is specified as well, it has higher priority.",
+          new StringParameter(&_customQueryFile))
+      .setIntroducedIn(30800);
+
   options->addOption("--quiet", "suppress status messages", new BooleanParameter(&_quiet));
-  
-  options->addObsoleteOption("--verbose", "print out replies if the HTTP header indicates DB errors", false);
+
+  options->addObsoleteOption(
+      "--verbose", "print out replies if the HTTP header indicates DB errors", false);
 }
 
 void BenchFeature::status(std::string const& value) {
@@ -241,8 +251,8 @@ void BenchFeature::updateStartCounter() { ++_started; }
 int BenchFeature::getStartCounter() { return _started; }
 
 void BenchFeature::start() {
-  sort(_percentiles.begin(), _percentiles.end());
-  
+  std::sort(_percentiles.begin(), _percentiles.end());
+
   if (!_jsonReportFile.empty() && FileUtils::exists(_jsonReportFile)) {
     LOG_TOPIC("ee2a4", FATAL, arangodb::Logger::BENCH)
         << "file already exists: '" << _jsonReportFile << "' - won't overwrite it.";
@@ -261,14 +271,13 @@ void BenchFeature::start() {
     b.openObject();
     b.add("name", VPackValue(normalizeUtf8ToNFC(connectDB)));
     b.close();
-      
+
     std::unordered_map<std::string, std::string> headers;
     headers.emplace(StaticStrings::ContentTypeHeader, StaticStrings::MimeTypeVPack);
 
     auto result = createDbClient->request(rest::RequestType::POST,
-                                          "/_api/database",
-                                          b.slice().startAs<char>(), b.slice().byteSize(),
-                                          headers);
+                                          "/_api/database", b.slice().startAs<char>(),
+                                          b.slice().byteSize(), headers);
 
     if (!result || result->wasHttpError()) {
       std::string msg;
@@ -278,7 +287,7 @@ void BenchFeature::start() {
       }
 
       LOG_TOPIC("5cda8", FATAL, arangodb::Logger::BENCH)
-        << "failed to create the specified database: " << msg;
+          << "failed to create the specified database: " << msg;
       FATAL_ERROR_EXIT();
     }
 
@@ -289,7 +298,8 @@ void BenchFeature::start() {
 
   *_result = ret;
 
-  std::unique_ptr<BenchmarkOperation> benchmark = BenchmarkOperation::createBenchmark(_testCase, *this);
+  std::unique_ptr<BenchmarkOperation> benchmark =
+      BenchmarkOperation::createBenchmark(_testCase, *this);
 
   if (benchmark == nullptr) {
     LOG_TOPIC("ee2a5", FATAL, arangodb::Logger::BENCH)
@@ -298,10 +308,11 @@ void BenchFeature::start() {
   }
 
   LOG_TOPIC("69091", INFO, arangodb::Logger::BENCH)
-      << "Running test case '" <<  _testCase << "': " << benchmark->getDescription();
+      << "Running test case '" << _testCase << "': " << benchmark->getDescription();
   if (benchmark->isDeprecated()) {
     LOG_TOPIC("caf8a", WARN, arangodb::Logger::BENCH)
-        << "Please note: this test case is deprecated and will be removed in a future version.";
+        << "Please note: this test case is deprecated and will be removed in a "
+           "future version.";
   }
 
   if (_duration != 0) {
@@ -323,10 +334,10 @@ void BenchFeature::start() {
   // add some more offset so we don't get into trouble with threads of different
   // speed
   realStep += 10000;
-  
+
   // aggregated stats for all runs
   BenchmarkStats totalStats;
-  
+
   auto builder = std::make_shared<VPackBuilder>();
   builder->openObject();
   builder->add("histogram", VPackValue(VPackValueType::Object));
@@ -336,7 +347,8 @@ void BenchFeature::start() {
   std::stringstream pp;
   pp << "Interval/Percentile:";
   for (auto percentile : _percentiles) {
-    pp << std::fixed << std::right << std::setw(12) << std::setprecision(2) << percentile << "%";
+    pp << std::fixed << std::right << std::setw(12) << std::setprecision(2)
+       << percentile << "%";
   }
   pp << std::endl;
 
@@ -356,11 +368,9 @@ void BenchFeature::start() {
 
     for (uint64_t i = 0; i < _concurrency; ++i) {
       auto thread = std::make_unique<BenchmarkThread>(
-          server(), benchmark.get(), &startCondition,
-          &BenchFeature::updateStartCounter, static_cast<int>(i),
-          _batchSize, &operationsCounter,
-          client, _keepAlive, _async, 
-          _histogramIntervalSize, _histogramNumIntervals);
+          server(), benchmark.get(), &startCondition, &BenchFeature::updateStartCounter,
+          static_cast<int>(i), _batchSize, &operationsCounter, client,
+          _keepAlive, _async, _histogramIntervalSize, _histogramNumIntervals);
       thread->setOffset(i * realStep);
       thread->start();
       threads.push_back(std::move(thread));
@@ -400,10 +410,11 @@ void BenchFeature::start() {
       }
 
       if (_progress && numOperations >= nextReportValue) {
-        LOG_TOPIC("c3604", INFO, arangodb::Logger::BENCH) << "number of operations: " << nextReportValue;
+        LOG_TOPIC("c3604", INFO, arangodb::Logger::BENCH)
+            << "number of operations: " << nextReportValue;
         nextReportValue += stepValue;
       }
-      
+
       std::this_thread::sleep_for(std::chrono::milliseconds(5));
     }
 
@@ -476,7 +487,9 @@ void BenchFeature::start() {
   *_result = ret;
 }
 
-bool BenchFeature::report(ClientFeature& client, std::vector<BenchRunResult> results, BenchmarkStats const& stats, std::string const& histogram, VPackBuilder& builder) {
+bool BenchFeature::report(ClientFeature& client, std::vector<BenchRunResult> results,
+                          BenchmarkStats const& stats,
+                          std::string const& histogram, VPackBuilder& builder) {
   std::cout << std::endl;
 
   std::cout << "Total number of operations: " << _realOperations << ", runs: " << _runs
@@ -505,7 +518,6 @@ bool BenchFeature::report(ClientFeature& client, std::vector<BenchRunResult> res
   builder.add("database", VPackValue(client.databaseName()));
   builder.add("collection", VPackValue(_collection));
 
-  
   std::sort(results.begin(), results.end(),
             [](BenchRunResult a, BenchRunResult b) { return a._time < b._time; });
 
@@ -516,11 +528,11 @@ bool BenchFeature::report(ClientFeature& client, std::vector<BenchRunResult> res
     std::cout << std::endl;
     std::cout << "Printing fastest result" << std::endl;
     std::cout << "=======================" << std::endl;
-    
+
     builder.add("fastestResults", VPackValue(VPackValueType::Object));
     printResult(results[0], builder);
     builder.close();
-    
+
     std::cout << "Printing slowest result" << std::endl;
     std::cout << "=======================" << std::endl;
 
@@ -549,16 +561,19 @@ bool BenchFeature::report(ClientFeature& client, std::vector<BenchRunResult> res
   printResult(output, builder);
   builder.close();
 
-  std::cout <<
-    "Min request time: " << std::setprecision(6) << (stats.min * 1000) << "ms" << std::endl <<
-    "Avg request time: " << std::setprecision(6) << (stats.avg() * 1000) << "ms" << std::endl <<
-    "Max request time: " << std::setprecision(6) << (stats.max * 1000) << "ms" << std::endl << std::endl;
+  std::cout << "Min request time: " << std::setprecision(6)
+            << (stats.min * 1000) << "ms" << std::endl
+            << "Avg request time: " << std::setprecision(6)
+            << (stats.avg() * 1000) << "ms" << std::endl
+            << "Max request time: " << std::setprecision(6)
+            << (stats.max * 1000) << "ms" << std::endl
+            << std::endl;
 
   std::cout << histogram;
 
   builder.add("min", VPackValue(stats.min));
   builder.add("avg", VPackValue(stats.avg()));
-  builder.add("max", VPackValue(stats.max)); 
+  builder.add("max", VPackValue(stats.max));
   builder.close();
 
   if (!_jsonReportFile.empty()) {
@@ -617,25 +632,26 @@ void BenchFeature::printResult(BenchRunResult const& result, VPackBuilder& build
   builder.add("requestTime", VPackValue(result._requestTime));
   std::cout << "Request/response duration (per thread): " << std::fixed
             << (result._requestTime / (double)_concurrency) << " s" << std::endl;
-  builder.add("requestResponseDurationPerThread", VPackValue(result._requestTime / (double)_concurrency));
+  builder.add("requestResponseDurationPerThread",
+              VPackValue(result._requestTime / (double)_concurrency));
 
   std::cout << "Time needed per operation: " << std::fixed
             << (result._time / _realOperations) << " s" << std::endl;
   builder.add("timeNeededPerOperation", VPackValue(result._time / _realOperations));
-  
+
   std::cout << "Time needed per operation per thread: " << std::fixed
             << (result._time / (double)_realOperations * (double)_concurrency)
             << " s" << std::endl;
-  builder.add("timeNeededPerOperationPerThread", VPackValue(
-                result._time / (double)_realOperations * (double)_concurrency));
-  
+  builder.add("timeNeededPerOperationPerThread",
+              VPackValue(result._time / (double)_realOperations * (double)_concurrency));
+
   std::cout << "Operations per second rate: " << std::fixed
             << ((double)_realOperations / result._time) << std::endl;
-  builder.add("operationsPerSecondRate", 
+  builder.add("operationsPerSecondRate",
               VPackValue((double)_realOperations / result._time));
-  
-  std::cout << "Elapsed time since start: " << std::fixed << result._time << " s"
-            << std::endl
+
+  std::cout << "Elapsed time since start: " << std::fixed << result._time
+            << " s" << std::endl
             << std::endl;
   builder.add("timeSinceStart", VPackValue(result._time));
 

arangosh/Benchmark/BenchmarkOperation.h

@@ -67,7 +67,7 @@ struct BenchmarkOperation {
   /// @brief build the HTTP request and set it's values (url, type and payload) 
   /// inside RequestData.
   /// The caller must provide it and clean its values before the call
-  virtual void buildRequest(int threadNumber, size_t threadCounter, size_t globalCounter, RequestData&) const = 0;
+  virtual void buildRequest(size_t threadNumber, size_t threadCounter, size_t globalCounter, RequestData&) const = 0;
 
   using BenchmarkFactory = std::function<std::unique_ptr<BenchmarkOperation>(BenchFeature&)>;