Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/main' into read-write-fungible
Browse files Browse the repository at this point in the history
  • Loading branch information
@sfc-gh-tclinkenbeard
sfc-gh-tclinkenbeard committed Sep 27, 2022
2 parents 6b094c1 + 1b8512e commit ba8fbc9
Show file tree
Hide file tree
Showing 33 changed files with 923 additions and 175 deletions.
1 change: 1 addition & 0 deletions FDBLibTLS/FDBLibTLSPolicy.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
#include <algorithm>
#include <exception>
#include <map>
#include <stdexcept>
#include <string>
#include <vector>

Expand Down
1 change: 1 addition & 0 deletions FDBLibTLS/FDBLibTLSVerify.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
#include <algorithm>
#include <exception>
#include <cstring>
#include <stdexcept>

static int hexValue(char c) {
static char const digits[] = "0123456789ABCDEF";
Expand Down
53 changes: 47 additions & 6 deletions bindings/c/test/apitester/TesterBlobGranuleCorrectnessWorkload.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,8 @@ class ApiBlobGranuleCorrectnessWorkload : public ApiWorkload {
OP_GET_GRANULES,
OP_SUMMARIZE,
OP_GET_BLOB_RANGES,
OP_LAST = OP_GET_BLOB_RANGES
OP_VERIFY,
OP_LAST = OP_VERIFY
};
std::vector<OpType> excludedOpTypes;

Expand Down Expand Up @@ -156,6 +157,12 @@ class ApiBlobGranuleCorrectnessWorkload : public ApiWorkload {
}

void randomSummarizeOp(TTaskFct cont, std::optional<int> tenantId) {
if (!seenReadSuccess) {
// tester can't handle this throwing bg_txn_too_old, so just don't call it unless we have already seen a
// read success
schedule(cont);
return;
}
fdb::Key begin = randomKeyName();
fdb::Key end = randomKeyName();
if (begin > end) {
Expand All @@ -174,11 +181,9 @@ class ApiBlobGranuleCorrectnessWorkload : public ApiWorkload {
true);
},
[this, begin, end, results, cont]() {
if (seenReadSuccess) {
ASSERT(results->size() > 0);
ASSERT(results->front().keyRange.beginKey <= begin);
ASSERT(results->back().keyRange.endKey >= end);
}
ASSERT(results->size() > 0);
ASSERT(results->front().keyRange.beginKey <= begin);
ASSERT(results->back().keyRange.endKey >= end);

for (int i = 0; i < results->size(); i++) {
// TODO: could do validation of subsequent calls and ensure snapshot version never decreases
Expand Down Expand Up @@ -254,6 +259,39 @@ class ApiBlobGranuleCorrectnessWorkload : public ApiWorkload {
/* failOnError = */ false);
}

void randomVerifyOp(TTaskFct cont) {
fdb::Key begin = randomKeyName();
fdb::Key end = randomKeyName();
if (begin > end) {
std::swap(begin, end);
}

auto verifyVersion = std::make_shared<int64_t>(false);
// info("Verify op starting");

execOperation(
[begin, end, verifyVersion](auto ctx) {
fdb::Future f = ctx->db().verifyBlobRange(begin, end, -2 /* latest version*/).eraseType();
ctx->continueAfter(f, [ctx, verifyVersion, f]() {
*verifyVersion = f.get<fdb::future_var::Int64>();
ctx->done();
});
},
[this, begin, end, verifyVersion, cont]() {
if (*verifyVersion == -1) {
ASSERT(!seenReadSuccess);
} else {
if (!seenReadSuccess) {
info("BlobGranuleCorrectness::randomVerifyOp first success");
}
seenReadSuccess = true;
}
// info(fmt::format("verify op done @ {}", *verifyVersion));
schedule(cont);
},
/* failOnError = */ false);
}

void randomOperation(TTaskFct cont) {
std::optional<int> tenantId = randomTenant();

Expand Down Expand Up @@ -284,6 +322,9 @@ class ApiBlobGranuleCorrectnessWorkload : public ApiWorkload {
case OP_GET_BLOB_RANGES:
randomGetBlobRangesOp(cont);
break;
case OP_VERIFY:
randomVerifyOp(cont);
break;
}
}
};
Expand Down
185 changes: 178 additions & 7 deletions bindings/c/test/apitester/TesterBlobGranuleErrorsWorkload.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,21 @@ class BlobGranuleErrorsWorkload : public ApiWorkload {
OP_READ_NO_MATERIALIZE,
OP_READ_FILE_LOAD_ERROR,
OP_READ_TOO_OLD,
OP_CANCEL_RANGES,
OP_LAST = OP_CANCEL_RANGES
OP_PURGE_UNALIGNED,
OP_BLOBBIFY_UNALIGNED,
OP_UNBLOBBIFY_UNALIGNED,
OP_CANCEL_GET_GRANULES,
OP_CANCEL_GET_RANGES,
OP_CANCEL_VERIFY,
OP_CANCEL_SUMMARIZE,
OP_CANCEL_BLOBBIFY,
OP_CANCEL_UNBLOBBIFY,
OP_CANCEL_PURGE,
OP_LAST = OP_CANCEL_PURGE
};

// could add summarize too old and verify too old as ops if desired but those are lower value

// Allow reads at the start to get blob_granule_transaction_too_old if BG data isn't initialized yet
// FIXME: should still guarantee a read succeeds eventually somehow
bool seenReadSuccess = false;
Expand Down Expand Up @@ -74,9 +85,6 @@ class BlobGranuleErrorsWorkload : public ApiWorkload {
error(fmt::format("Operation succeeded in error test!"));
}
ASSERT(err.code() != error_code_success);
if (err.code() != error_code_blob_granule_transaction_too_old) {
seenReadSuccess = true;
}
if (err.code() != expectedError) {
info(fmt::format("incorrect error. Expected {}, Got {}", expectedError, err.code()));
if (err.code() == error_code_blob_granule_transaction_too_old) {
Expand All @@ -86,6 +94,9 @@ class BlobGranuleErrorsWorkload : public ApiWorkload {
ctx->onError(err);
}
} else {
if (err.code() != error_code_blob_granule_transaction_too_old) {
seenReadSuccess = true;
}
ctx->done();
}
},
Expand All @@ -107,7 +118,55 @@ class BlobGranuleErrorsWorkload : public ApiWorkload {
doErrorOp(cont, "", true, 1, error_code_blob_granule_transaction_too_old);
}

void randomCancelGetRangesOp(TTaskFct cont) {
void randomPurgeUnalignedOp(TTaskFct cont) {
// blobbify/unblobbify need to be aligned to blob range boundaries, so this should always fail
fdb::Key begin = randomKeyName();
fdb::Key end = randomKeyName();
if (begin > end) {
std::swap(begin, end);
}
execOperation(
[this, begin, end](auto ctx) {
fdb::Future f = ctx->db().purgeBlobGranules(begin, end, -2, false).eraseType();
ctx->continueAfter(
f,
[this, ctx, f]() {
info(fmt::format("unaligned purge got {}", f.error().code()));
ASSERT(f.error().code() == error_code_unsupported_operation);
ctx->done();
},
true);
},
[this, cont]() { schedule(cont); });
}

void randomBlobbifyUnalignedOp(bool blobbify, TTaskFct cont) {
// blobbify/unblobbify need to be aligned to blob range boundaries, so this should always return false
fdb::Key begin = randomKeyName();
fdb::Key end = randomKeyName();
if (begin > end) {
std::swap(begin, end);
}
auto success = std::make_shared<bool>(false);
execOperation(
[begin, end, blobbify, success](auto ctx) {
fdb::Future f = blobbify ? ctx->db().blobbifyRange(begin, end).eraseType()
: ctx->db().unblobbifyRange(begin, end).eraseType();
ctx->continueAfter(
f,
[ctx, f, success]() {
*success = f.get<fdb::future_var::Bool>();
ctx->done();
},
true);
},
[this, cont, success]() {
ASSERT(!(*success));
schedule(cont);
});
}

void randomCancelGetGranulesOp(TTaskFct cont) {
fdb::Key begin = randomKeyName();
fdb::Key end = randomKeyName();
if (begin > end) {
Expand All @@ -121,6 +180,90 @@ class BlobGranuleErrorsWorkload : public ApiWorkload {
[this, cont]() { schedule(cont); });
}

void randomCancelGetRangesOp(TTaskFct cont) {
fdb::Key begin = randomKeyName();
fdb::Key end = randomKeyName();
if (begin > end) {
std::swap(begin, end);
}
execOperation(
[begin, end](auto ctx) {
fdb::Future f = ctx->db().listBlobbifiedRanges(begin, end, 1000).eraseType();
ctx->done();
},
[this, cont]() { schedule(cont); });
}

void randomCancelVerifyOp(TTaskFct cont) {
fdb::Key begin = randomKeyName();
fdb::Key end = randomKeyName();
if (begin > end) {
std::swap(begin, end);
}
execOperation(
[begin, end](auto ctx) {
fdb::Future f = ctx->db().verifyBlobRange(begin, end, -2 /* latest version*/).eraseType();
ctx->done();
},
[this, cont]() { schedule(cont); });
}

void randomCancelSummarizeOp(TTaskFct cont) {
fdb::Key begin = randomKeyName();
fdb::Key end = randomKeyName();
if (begin > end) {
std::swap(begin, end);
}
execTransaction(
[begin, end](auto ctx) {
fdb::Future f = ctx->tx().summarizeBlobGranules(begin, end, -2, 1000).eraseType();
ctx->done();
},
[this, cont]() { schedule(cont); });
}

void randomCancelBlobbifyOp(TTaskFct cont) {
fdb::Key begin = randomKeyName();
fdb::Key end = randomKeyName();
if (begin > end) {
std::swap(begin, end);
}
execOperation(
[begin, end](auto ctx) {
fdb::Future f = ctx->db().blobbifyRange(begin, end).eraseType();
ctx->done();
},
[this, cont]() { schedule(cont); });
}

void randomCancelUnblobbifyOp(TTaskFct cont) {
fdb::Key begin = randomKeyName();
fdb::Key end = randomKeyName();
if (begin > end) {
std::swap(begin, end);
}
execOperation(
[begin, end](auto ctx) {
fdb::Future f = ctx->db().unblobbifyRange(begin, end).eraseType();
ctx->done();
},
[this, cont]() { schedule(cont); });
}

void randomCancelPurgeOp(TTaskFct cont) {
fdb::Key begin = randomKeyName();
fdb::Key end = randomKeyName();
if (begin > end) {
std::swap(begin, end);
}
execOperation(
[begin, end](auto ctx) {
fdb::Future f = ctx->db().purgeBlobGranules(begin, end, -2, false).eraseType();
ctx->done();
},
[this, cont]() { schedule(cont); });
}

void randomOperation(TTaskFct cont) override {
OpType txType = (OpType)Random::get().randomInt(0, OP_LAST);
switch (txType) {
Expand All @@ -133,9 +276,37 @@ class BlobGranuleErrorsWorkload : public ApiWorkload {
case OP_READ_TOO_OLD:
randomOpReadTooOld(cont);
break;
case OP_CANCEL_RANGES:
case OP_PURGE_UNALIGNED:
// gets the correct error but it doesn't propagate properly in the test
// randomPurgeUnalignedOp(cont);
break;
case OP_BLOBBIFY_UNALIGNED:
randomBlobbifyUnalignedOp(true, cont);
break;
case OP_UNBLOBBIFY_UNALIGNED:
randomBlobbifyUnalignedOp(false, cont);
break;
case OP_CANCEL_GET_GRANULES:
randomCancelGetGranulesOp(cont);
break;
case OP_CANCEL_GET_RANGES:
randomCancelGetRangesOp(cont);
break;
case OP_CANCEL_VERIFY:
randomCancelVerifyOp(cont);
break;
case OP_CANCEL_SUMMARIZE:
randomCancelSummarizeOp(cont);
break;
case OP_CANCEL_BLOBBIFY:
randomCancelBlobbifyOp(cont);
break;
case OP_CANCEL_UNBLOBBIFY:
randomCancelUnblobbifyOp(cont);
break;
case OP_CANCEL_PURGE:
randomCancelPurgeOp(cont);
break;
}
}
};
Expand Down
42 changes: 41 additions & 1 deletion bindings/c/test/fdb_api.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,13 @@ struct None {
struct Type {};
static Error extract(native::FDBFuture*, Type&) noexcept { return Error(0); }
};
struct Bool {
using Type = native::fdb_bool_t;
static Error extract(native::FDBFuture* f, Type& out) noexcept {
auto err = native::fdb_future_get_bool(f, &out);
return Error(err);
}
};
struct Int64 {
using Type = int64_t;
static Error extract(native::FDBFuture* f, Type& out) noexcept {
Expand Down Expand Up @@ -775,10 +782,43 @@ class Database {

TypedFuture<future_var::KeyRangeRefArray> listBlobbifiedRanges(KeyRef begin, KeyRef end, int rangeLimit) {
if (!db)
throw std::runtime_error("list_blobbified_ranges from null database");
throw std::runtime_error("listBlobbifiedRanges from null database");
return native::fdb_database_list_blobbified_ranges(
db.get(), begin.data(), intSize(begin), end.data(), intSize(end), rangeLimit);
}

TypedFuture<future_var::Int64> verifyBlobRange(KeyRef begin, KeyRef end, int64_t version) {
if (!db)
throw std::runtime_error("verifyBlobRange from null database");
return native::fdb_database_verify_blob_range(
db.get(), begin.data(), intSize(begin), end.data(), intSize(end), version);
}

TypedFuture<future_var::Bool> blobbifyRange(KeyRef begin, KeyRef end) {
if (!db)
throw std::runtime_error("blobbifyRange from null database");
return native::fdb_database_blobbify_range(db.get(), begin.data(), intSize(begin), end.data(), intSize(end));
}

TypedFuture<future_var::Bool> unblobbifyRange(KeyRef begin, KeyRef end) {
if (!db)
throw std::runtime_error("unblobbifyRange from null database");
return native::fdb_database_unblobbify_range(db.get(), begin.data(), intSize(begin), end.data(), intSize(end));
}

TypedFuture<future_var::KeyRef> purgeBlobGranules(KeyRef begin, KeyRef end, int64_t version, bool force) {
if (!db)
throw std::runtime_error("purgeBlobGranules from null database");
native::fdb_bool_t forceBool = force;
return native::fdb_database_purge_blob_granules(
db.get(), begin.data(), intSize(begin), end.data(), intSize(end), version, forceBool);
}

TypedFuture<future_var::None> waitPurgeGranulesComplete(KeyRef purgeKey) {
if (!db)
throw std::runtime_error("purgeBlobGranules from null database");
return native::fdb_database_wait_purge_granules_complete(db.get(), purgeKey.data(), intSize(purgeKey));
}
};

inline Error selectApiVersionNothrow(int version) {
Expand Down
Loading

0 comments on commit ba8fbc9

Please sign in to comment.