Merge branch 'master' of github.com:apple/foundationdb into features/boost70

Author: Markus Pilman

CMakeLists.txt

@@ -179,9 +179,6 @@ set(SEED "0x${SEED_}" CACHE STRING "Random seed for testing")
 ################################################################################
 
 include(CompileBoost)
-if(WITH_TLS)
-  add_subdirectory(FDBLibTLS)
-endif()
 add_subdirectory(flow)
 add_subdirectory(fdbrpc)
 add_subdirectory(fdbclient)
@@ -192,15 +189,15 @@ if(NOT WIN32)
 else()
   add_subdirectory(fdbservice)
 endif()
+add_subdirectory(fdbbackup)
+add_subdirectory(contrib)
+add_subdirectory(tests)
 if(WITH_PYTHON)
   add_subdirectory(bindings)
 endif()
-add_subdirectory(fdbbackup)
-add_subdirectory(tests)
 if(WITH_DOCUMENTATION)
   add_subdirectory(documentation)
 endif()
-add_subdirectory(contrib/monitoring)
 
 if(WIN32)
   add_subdirectory(packaging/msi)

Makefile

@@ -51,7 +51,7 @@ ifeq ($(PLATFORM),Linux)
   CXXFLAGS += -std=c++17
 
   BOOST_BASEDIR ?= /opt
-  TLS_LIBDIR ?= /usr/local/lib
+  TLS_LIBDIR ?= /usr/local/lib64
   DLEXT := so
   java_DLEXT := so
   TARGET_LIBC_VERSION ?= 2.11
@@ -67,7 +67,7 @@ else ifeq ($(PLATFORM),Darwin)
   .LIBPATTERNS := lib%.dylib lib%.a
 
   BOOST_BASEDIR ?= ${HOME}
-  TLS_LIBDIR ?= /usr/local/lib
+  TLS_LIBDIR ?= /usr/local/lib64
   DLEXT := dylib
   java_DLEXT := jnilib
 else
@@ -112,8 +112,8 @@ CFLAGS += -DTLS_DISABLED
 FDB_TLS_LIB :=
 TLS_LIBS :=
 else
-FDB_TLS_LIB := lib/libFDBLibTLS.a
-TLS_LIBS += $(addprefix $(TLS_LIBDIR)/,libtls.a libssl.a libcrypto.a)
+FDB_TLS_LIB :=
+TLS_LIBS += $(addprefix $(TLS_LIBDIR)/,libssl.a libcrypto.a)
 endif
 
 CXXFLAGS += -Wno-deprecated -DBOOST_ERROR_CODE_HEADER_ONLY -DBOOST_SYSTEM_NO_DEPRECATED
@@ -126,9 +126,6 @@ VPATH += $(addprefix :,$(filter-out lib,$(patsubst -L%,%,$(filter -L%,$(LDFLAGS)
 
 CS_PROJECTS := flow/actorcompiler flow/coveragetool fdbclient/vexillographer
 CPP_PROJECTS := flow fdbrpc fdbclient fdbbackup fdbserver fdbcli bindings/c bindings/java fdbmonitor bindings/flow/tester bindings/flow
-ifndef TLS_DISABLED
-CPP_PROJECTS += FDBLibTLS
-endif
 OTHER_PROJECTS := bindings/python bindings/ruby bindings/go
 
 CS_MK_GENERATED := $(CS_PROJECTS:=/generated.mk)

README.md

@@ -133,9 +133,9 @@ If you want to create a package you have to tell cmake what platform it is for.
 And then you can build by simply calling `cpack`. So for debian, call:
 
 ```
-cmake -DINSTALL_LAYOUT=DEB  <FDB_SOURCE_DIR>
+cmake <FDB_SOURCE_DIR>
 make
-cpack
+cpack -G DEB
 ```
 
 For RPM simply replace `DEB` with `RPM`.
@@ -155,9 +155,9 @@ To generate a installable package, you have to call CMake with the corresponding
 arguments and then use cpack to generate the package:
 
 ```sh
-cmake -DINSTALL_LAYOUT=OSX  <FDB_SOURCE_DIR>
+cmake <FDB_SOURCE_DIR>
 make
-cpack
+cpack -G productbuild
 ```
 
 ### Windows

bindings/CMakeLists.txt

@@ -13,3 +13,6 @@ endif()
 if(WITH_RUBY)
   add_subdirectory(ruby)
 endif()
+if(NOT WIN32 AND NOT OPEN_FOR_IDE)
+  package_bindingtester()
+endif()

bindings/bindingtester/run_binding_tester.sh

@@ -25,7 +25,8 @@ BREAKONERROR="${BREAKONERROR:-0}"
 RUNSCRIPTS="${RUNSCRIPTS:-1}"
 RUNTESTS="${RUNTESTS:-1}"
 RANDOMTEST="${RANDOMTEST:-0}"
-BINDINGTESTS="${BINDINGTESTS:-python python3 java java_async ruby go flow}"
+# BINDINGTESTS="${BINDINGTESTS:-python python3 java java_async ruby go flow}"
+BINDINGTESTS="${BINDINGTESTS:-python python3 java java_async go flow}"
 LOGLEVEL="${LOGLEVEL:-INFO}"
 _BINDINGTESTS=(${BINDINGTESTS})
 DISABLEDTESTS=()

bindings/bindingtester/spec/bindingApiTester.md

@@ -164,6 +164,13 @@ futures must apply the following rules to the result:
     database using the get() method. May optionally push a future onto the
     stack.
 
+#### GET_ESTIMATED_RANGE_SIZE
+
+    Pops the top two items off of the stack as BEGIN_KEY and END_KEY to 
+    construct a key range. Then call the `getEstimatedRangeSize` API of 
+    the language binding. Make sure the API returns without error. Finally 
+    push the string "GOT_ESTIMATED_RANGE_SIZE" onto the stack.
+
 #### GET_KEY (_SNAPSHOT, _DATABASE)
 
     Pops the top four items off of the stack as KEY, OR_EQUAL, OFFSET, PREFIX

bindings/bindingtester/tests/api.py

@@ -157,6 +157,7 @@ def generate(self, args, thread_number):
         read_conflicts = ['READ_CONFLICT_RANGE', 'READ_CONFLICT_KEY']
         write_conflicts = ['WRITE_CONFLICT_RANGE', 'WRITE_CONFLICT_KEY', 'DISABLE_WRITE_CONFLICT']
         txn_sizes = ['GET_APPROXIMATE_SIZE']
+        storage_metrics = ['GET_ESTIMATED_RANGE_SIZE']
 
         op_choices += reads
         op_choices += mutations
@@ -170,6 +171,7 @@ def generate(self, args, thread_number):
         op_choices += write_conflicts
         op_choices += resets
         op_choices += txn_sizes
+        op_choices += storage_metrics
 
         idempotent_atomic_ops = ['BIT_AND', 'BIT_OR', 'MAX', 'MIN', 'BYTE_MIN', 'BYTE_MAX']
         atomic_ops = idempotent_atomic_ops + ['ADD', 'BIT_XOR', 'APPEND_IF_FITS']
@@ -536,6 +538,21 @@ def generate(self, args, thread_number):
                 instructions.push_args(d)
                 instructions.append(op)
                 self.add_strings(1)
+            elif op == 'GET_ESTIMATED_RANGE_SIZE':
+                # Protect against inverted range and identical keys
+                key1 = self.workspace.pack(self.random.random_tuple(1))
+                key2 = self.workspace.pack(self.random.random_tuple(1))
+
+                while key1 == key2:
+                    key1 = self.workspace.pack(self.random.random_tuple(1))
+                    key2 = self.workspace.pack(self.random.random_tuple(1))
+
+                if key1 > key2:
+                    key1, key2 = key2, key1
+
+                instructions.push_args(key1, key2)
+                instructions.append(op)
+                self.add_strings(1)
 
             else:
                 assert False, 'Unknown operation: ' + op

bindings/c/CMakeLists.txt

@@ -38,6 +38,21 @@ else()
 endif()
 add_dependencies(fdb_c fdb_c_generated fdb_c_options)
 target_link_libraries(fdb_c PUBLIC $<BUILD_INTERFACE:fdbclient>)
+if(APPLE)
+  set(symbols ${CMAKE_CURRENT_BINARY_DIR}/fdb_c.symbols)
+  add_custom_command(OUTPUT ${symbols}
+    COMMAND $<TARGET_FILE:Python::Interpreter> ${CMAKE_CURRENT_SOURCE_DIR}/symbolify.py
+        ${CMAKE_CURRENT_SOURCE_DIR}/foundationdb/fdb_c.h
+        ${symbols}
+    DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/symbolify.py ${CMAKE_CURRENT_SOURCE_DIR}/foundationdb/fdb_c.h
+    COMMENT "Generate exported_symbols_list")
+  add_custom_target(exported_symbols_list DEPENDS ${symbols})
+  add_dependencies(fdb_c exported_symbols_list)
+  target_link_options(fdb_c PRIVATE "LINKER:-no_weak_exports,-exported_symbols_list,${symbols}")
+elseif(WIN32)
+else()
+  target_link_options(fdb_c PRIVATE "LINKER:--version-script=${CMAKE_CURRENT_SOURCE_DIR}/fdb_c.map,-z,nodelete")
+endif()
 target_include_directories(fdb_c PUBLIC
   $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
   $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>

bindings/c/symbolify.py

@@ -0,0 +1,10 @@
+if __name__ == '__main__':
+    import re
+    import sys
+    r = re.compile('DLLEXPORT[^(]*(fdb_[^(]*)[(]')
+    (fdb_c_h, symbols_file) = sys.argv[1:]
+    with open(fdb_c_h, 'r') as f:
+        symbols = sorted(set('_' + m.group(1) for m in r.finditer(f.read())))
+    with open(symbols_file, 'w') as f:
+        f.write('\n'.join(symbols))
+        f.write('\n')

bindings/flow/fdb_flow.actor.cpp

@@ -25,6 +25,7 @@
 
 #include "flow/DeterministicRandom.h"
 #include "flow/SystemMonitor.h"
+#include "flow/TLSConfig.actor.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 using namespace FDB;
@@ -82,7 +83,7 @@ void fdb_flow_test() {
 	fdb->setupNetwork();
 	startThread(networkThread, fdb);
 
-	g_network = newNet2( false );
+	g_network = newNet2(TLSConfig());
 
 	openTraceFile(NetworkAddress(), 1000000, 1000000, ".");
 	systemMonitor();

bindings/flow/tester/Tester.actor.cpp

@@ -28,6 +28,7 @@
 #include "bindings/flow/FDBLoanerTypes.h"
 #include "fdbrpc/fdbrpc.h"
 #include "flow/DeterministicRandom.h"
+#include "flow/TLSConfig.actor.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 // Otherwise we have to type setupNetwork(), FDB::open(), etc.
@@ -429,9 +430,8 @@ struct LogStackFunc : InstructionFunc {
 				wait(logStack(data, entries, prefix));
 				entries.clear();
 			}
-
-			wait(logStack(data, entries, prefix));
 		}
+		wait(logStack(data, entries, prefix));
 
 		return Void();
 	}
@@ -638,6 +638,29 @@ struct GetFunc : InstructionFunc {
 const char* GetFunc::name = "GET";
 REGISTER_INSTRUCTION_FUNC(GetFunc);
 
+struct GetEstimatedRangeSize : InstructionFunc {
+	static const char* name;
+
+	ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
+		state std::vector<StackItem> items = data->stack.pop(2);
+		if (items.size() != 2)
+			return Void();
+
+		Standalone<StringRef> s1 = wait(items[0].value);
+		state Standalone<StringRef> beginKey = Tuple::unpack(s1).getString(0);
+
+		Standalone<StringRef> s2 = wait(items[1].value);
+		state Standalone<StringRef> endKey = Tuple::unpack(s2).getString(0);
+		Future<int64_t> fsize = instruction->tr->getEstimatedRangeSizeBytes(KeyRangeRef(beginKey, endKey));
+		int64_t size = wait(fsize);
+		data->stack.pushTuple(LiteralStringRef("GOT_ESTIMATED_RANGE_SIZE"));
+
+		return Void();
+	}
+};
+const char* GetEstimatedRangeSize::name = "GET_ESTIMATED_RANGE_SIZE";
+REGISTER_INSTRUCTION_FUNC(GetEstimatedRangeSize);
+
 struct GetKeyFunc : InstructionFunc {
 	static const char* name;
 
@@ -1748,7 +1771,7 @@ ACTOR void startTest(std::string clusterFilename, StringRef prefix, int apiVersi
 		populateOpsThatCreateDirectories(); // FIXME
 
 		// This is "our" network
-		g_network = newNet2(false);
+		g_network = newNet2(TLSConfig());
 
 		ASSERT(!API::isAPIVersionSelected());
 		try {
@@ -1791,7 +1814,7 @@ ACTOR void startTest(std::string clusterFilename, StringRef prefix, int apiVersi
 
 ACTOR void _test_versionstamp() {
 	try {
-		g_network = newNet2(false);
+		g_network = newNet2(TLSConfig());
 
 		API *fdb = FDB::API::selectAPIVersion(700);
 

bindings/flow/tester/local.mk

@@ -23,6 +23,7 @@
 fdb_flow_tester_CFLAGS := -Ibindings/c $(fdbrpc_CFLAGS)
 fdb_flow_tester_LDFLAGS := -Llib $(fdbrpc_LDFLAGS) -lfdb_c
 fdb_flow_tester_LIBS := lib/libfdb_flow.a lib/libflow.a lib/libfdb_c.$(DLEXT)
+fdb_flow_tester_STATIC_LIBS := $(TLS_LIBS)
 
 fdb_flow_tester: lib/libfdb_c.$(DLEXT)
 	@mkdir -p bindings/flow/bin

bindings/go/src/_stacktester/stacktester.go

@@ -569,6 +569,16 @@ func (sm *StackMachine) processInst(idx int, inst tuple.Tuple) {
 		}
 
 		sm.store(idx, res.(fdb.FutureByteSlice))
+	case op == "GET_ESTIMATED_RANGE_SIZE":
+		r := sm.popKeyRange()
+		_, e := rt.ReadTransact(func(rtr fdb.ReadTransaction) (interface{}, error) {
+			_ = rtr.GetEstimatedRangeSizeBytes(r).MustGet()
+			sm.store(idx, []byte("GOT_ESTIMATED_RANGE_SIZE"))
+			return nil, nil
+		})
+		if e != nil {
+			panic(e)
+		}
 	case op == "COMMIT":
 		sm.store(idx, sm.currentTransaction().Commit())
 	case op == "RESET":

bindings/go/src/fdb/generated.go

@@ -88,6 +88,13 @@ func (o NetworkOptions) SetTraceFormat(param string) error {
 	return o.setOpt(34, []byte(param))
 }
 
+// Select clock source for trace files. now (default) or realtime are supported.
+//
+// Parameter: Trace clock source
+func (o NetworkOptions) SetTraceClockSource(param string) error {
+	return o.setOpt(35, []byte(param))
+}
+
 // Set internal tuning or debugging knobs
 //
 // Parameter: knob_name=knob_value
@@ -323,7 +330,7 @@ func (o DatabaseOptions) SetTransactionCausalReadRisky() error {
 	return o.setOpt(504, nil)
 }
 
-// Addresses returned by get_addresses_for_key include the port when enabled. This will be enabled by default in api version 700, and this option will be deprecated.
+// Addresses returned by get_addresses_for_key include the port when enabled. As of api version 700, this option is enabled by default and setting this has no effect.
 func (o DatabaseOptions) SetTransactionIncludePortInAddress() error {
 	return o.setOpt(505, nil)
 }
@@ -343,7 +350,7 @@ func (o TransactionOptions) SetCausalReadDisable() error {
 	return o.setOpt(21, nil)
 }
 
-// Addresses returned by get_addresses_for_key include the port when enabled. This will be enabled by default in api version 700, and this option will be deprecated.
+// Addresses returned by get_addresses_for_key include the port when enabled. As of api version 700, this option is enabled by default and setting this has no effect.
 func (o TransactionOptions) SetIncludePortInAddress() error {
 	return o.setOpt(23, nil)
 }
@@ -505,13 +512,14 @@ const (
 	// small portion of data is transferred to the client initially (in order to
 	// minimize costs if the client doesn't read the entire range), and as the
 	// caller iterates over more items in the range larger batches will be
-	// transferred in order to minimize latency.
+	// transferred in order to minimize latency. After enough iterations, the
+	// iterator mode will eventually reach the same byte limit as ``WANT_ALL``
 	StreamingModeIterator StreamingMode = 0
 
 	// Infrequently used. The client has passed a specific row limit and wants
 	// that many rows delivered in a single batch. Because of iterator operation
 	// in client drivers make request batches transparent to the user, consider
-	// “WANT_ALL“ StreamingMode instead. A row limit must be specified if this
+	// ``WANT_ALL`` StreamingMode instead. A row limit must be specified if this
 	// mode is used.
 	StreamingModeExact StreamingMode = 1
 
@@ -628,15 +636,15 @@ type ErrorPredicate int
 
 const (
 
-	// Returns “true“ if the error indicates the operations in the transactions
-	// should be retried because of transient error.
+	// Returns ``true`` if the error indicates the operations in the
+	// transactions should be retried because of transient error.
 	ErrorPredicateRetryable ErrorPredicate = 50000
 
-	// Returns “true“ if the error indicates the transaction may have succeeded,
-	// though not in a way the system can verify.
+	// Returns ``true`` if the error indicates the transaction may have
+	// succeeded, though not in a way the system can verify.
 	ErrorPredicateMaybeCommitted ErrorPredicate = 50001
 
-	// Returns “true“ if the error indicates the transaction has not committed,
-	// though in a way that can be retried.
+	// Returns ``true`` if the error indicates the transaction has not
+	// committed, though in a way that can be retried.
 	ErrorPredicateRetryableNotCommitted ErrorPredicate = 50002
 )

bindings/go/src/fdb/range.go

@@ -54,7 +54,8 @@ type RangeOptions struct {
 	// Reverse indicates that the read should be performed in lexicographic
 	// (false) or reverse lexicographic (true) order. When Reverse is true and
 	// Limit is non-zero, the last Limit key-value pairs in the range are
-	// returned.
+	// returned. Reading ranges in reverse is supported natively by the
+	// database and should have minimal extra cost.
 	Reverse bool
 }
 

bindings/go/src/fdb/transaction.go

@@ -318,8 +318,8 @@ func (t *transaction) getEstimatedRangeSizeBytes(beginKey Key, endKey Key) Futur
 	}
 }
 
-// GetEstimatedRangeSizeBytes will get the byte size of the key range based on the
-// byte sample collected by FDB
+// GetEstimatedRangeSizeBytes will get an estimate for the number of bytes
+// stored in the given range.
 func (t Transaction) GetEstimatedRangeSizeBytes(r ExactRange) FutureInt64 {
 	beginKey, endKey := r.FDBRangeKeys()
 	return t.getEstimatedRangeSizeBytes(