update bulk operation benchmark programs

berkeleydbtest.c

@@ -100,19 +100,26 @@ static void exit_env(DB_ENV *env)
 	env->close(env, 0);
 }
 
+/* Shared by all benchmark threads */
+static struct BDB *bdb;
+
 static void *open_db(struct benchmark_config *config)
 {
 	int ret;
-	struct BDB *bdb = xmalloc(sizeof(*bdb));
 	DB *db;
 	DB_ENV *dbenv;
 
+	if (bdb)
+		return bdb;
+
+	bdb = xmalloc(sizeof(*bdb));
+
 	dbenv = init_env(config->path);
 
 	ret = db_create(&db, dbenv, 0);
 	if (ret) {
 		dbenv->err(dbenv, ret, "db_create");
-		return NULL;
+		exit(1);
 	}
 	db->set_errfile(db, stderr);
 
@@ -123,7 +130,7 @@ static void *open_db(struct benchmark_config *config)
 	ret = db->open(db, NULL, "data.db", NULL, DB_BTREE, DB_CREATE, 0664);
 	if (ret) {
 		db->err(db, ret, "open");
-		return NULL;
+		exit(1);
 	}
 
 	bdb->dbenv = dbenv;
@@ -142,9 +149,11 @@ static void *open_db(struct benchmark_config *config)
 
 static void close_db(void *db)
 {
-	struct BDB *bdb = db;
 	DB_ENV *dbenv = bdb->dbenv;
 
+	if (bdb != db)
+		return;
+
 	pthread_mutex_lock(&bdb->stop_mutex);
 	bdb->stop = true;
 	pthread_mutex_unlock(&bdb->stop_mutex);
@@ -158,6 +167,7 @@ static void close_db(void *db)
 	exit_env(dbenv);
 
 	free(bdb);
+	bdb = NULL;
 }
 
 #define KSIZ KEYGEN_KEY_SIZE
@@ -473,7 +483,6 @@ struct benchmark_config config = {
 	.consumer_thnum = 1,
 	.debug = false,
 	.verbose = 1,
-	.share = 1,
 	.ops = {
 		.open_db = open_db,
 		.close_db = close_db,
@@ -492,8 +501,6 @@ int main(int argc, char **argv)
 {
 	parse_options(&config, argc, argv);
 
-	if (config.share != 1)
-		die("Invalid -share option");
 	debug = config.debug;
 
 	benchmark(&config);

kyototycoontest.cc

@@ -56,13 +56,11 @@ static void get_test(void *db, int num, int vsiz, unsigned int seed)
 
 	for (i = 0; i < num; i++) {
 		string key(keygen_next_key(&keygen));
-		string *value;
+		string value;
 
-		value = rdb->get(key);
-		if (debug && vsiz != value->size())
-			die("Unexpected value size: %d", value->size());
-
-		delete value;
+		rdb->get(key, &value);
+		if (debug && vsiz != value.size())
+			die("Unexpected value size: %d", value.size());
 	}
 }
 
@@ -79,7 +77,7 @@ static void putlist_bin_test(void *db, const char *command, int num, int vsiz,
 
 	for (i = 0; i < num; i++) {
 		string key(keygen_next_key(&keygen));
-		RemoteDB::BulkRecord rec = { 0, key, value, INT64_MAX };
+		RemoteDB::BulkRecord rec = { 0, key, value, kc::INT64MAX };
 
 		bulkrecs.push_back(rec);
 
@@ -279,26 +277,24 @@ static void range_test(void *db, const char *command, int num, int vsiz,
 	RemoteDB *rdb = (RemoteDB *)db;
 	struct keygen keygen;
 	char prefix[KEYGEN_PREFIX_SIZE + 1];
-	pair<string, string> *rec;
+	string key, value;
 	int nrecs = 0;
 
 	keygen_init(&keygen, seed);
 	keygen_prefix(&keygen, prefix);
 	RemoteDB::Cursor *cur = rdb->cursor();
 	cur->jump(prefix, strlen(prefix));
 
-	while ((rec = cur->get_pair(NULL, true)) != NULL) {
-		if (strncmp(rec->first.data(), prefix, strlen(prefix))) {
-			delete rec;
+	while (cur->get(&key, &value)) {
+		if (strncmp(key.data(), prefix, strlen(prefix))) {
 			break;
 		}
-		if (debug && vsiz != rec->second.size())
-			die("Unexpected value size: %d", rec->second.size());
+		if (debug && vsiz != value.size())
+			die("Unexpected value size: %d", value.size());
 		if (debug && strncmp(keygen_next_key(&keygen),
-				rec->first.data(), rec->first.size()))
+				key.data(), key.size()))
 			die("Unexpected key");
 		nrecs++;
-		delete rec;
 	}
 	if (debug && num != nrecs)
 		die("Unexpected record num: %d", nrecs);
@@ -374,7 +370,6 @@ int main(int argc, char **argv)
 	config.consumer_thnum = 1;
 	config.debug = false;
 	config.verbose = 1;
-	config.share = 0;
 	config.ops.open_db = open_db;
 	config.ops.close_db = close_db;
 	config.ops.put_test = put_test;

testutil.c

@@ -134,8 +134,6 @@ static void fixup_config(struct benchmark_config *config)
 		config->producer_thnum = 1;
 	if (config->consumer_thnum < 1)
 		config->consumer_thnum = 1;
-	if (config->share < 1)
-		config->share = INT_MAX;
 	if (config->num_works < 1)
 		config->num_works = config->producer_thnum;
 }
@@ -181,8 +179,6 @@ void parse_options(struct benchmark_config *config, int argc, char **argv)
 			config->debug = true;
 		} else if (!strcmp(argv[i], "-verbose")) {
 			config->verbose = atoi(argv[++i]);
-		} else if (!strcmp(argv[i], "-share")) {
-			config->share = atoi(argv[++i]);
 		} else {
 			die("Invalid command option: %s", argv[i]);
 		}
@@ -281,6 +277,11 @@ struct worker_info {
 	struct benchmark_config *config;
 };
 
+static int strstartswith(const char *str, const char *prefix)
+{
+	return !strncmp(str, prefix, strlen(prefix));
+}
+
 static void handle_work(struct worker_info *data, struct work *work)
 {
 	const char *command = data->command;
@@ -293,38 +294,38 @@ static void handle_work(struct worker_info *data, struct work *work)
 
 	start = stopwatch_start();
 
-	if (!strcmp(command, "putlist") || !strcmp(command, "putlist2")) {
+	if (strstartswith(command, "putlist")) {
 		bops->putlist_test(data->db, command, config->num,
 				config->vsiz, config->batch, work->seed);
 	} else if (!strcmp(command, "fwmkeys")) {
 		bops->fwmkeys_test(data->db, config->num, work->seed);
-	} else if (!strcmp(command, "range") || !strcmp(command, "range2")) {
+	} else if (!strcmp(command, "range") || !strcmp(command, "range_atomic")) {
 		bops->range_test(data->db, command, config->num,
 				config->vsiz, config->batch, work->seed);
-	} else if (!strcmp(command, "rangeout")) {
+	} else if (!strcmp(command, "rangeout_atomic")) {
 		bops->rangeout_test(data->db, command, config->num,
 				config->vsiz, config->batch, work->seed);
-	} else if (!strcmp(command, "getlist") || !strcmp(command, "getlist2")) {
+	} else if (strstartswith(command, "getlist")) {
 		bops->getlist_test(data->db, command, config->num,
 				config->vsiz, config->batch, work->seed);
 	} else if (!strcmp(command, "fwmkeys-getlist")) {
 		bops->fwmkeys_test(data->db, config->num, work->seed);
 		bops->getlist_test(data->db, "getlist", config->num,
 				config->vsiz, config->batch, work->seed);
-	} else if (!strcmp(command, "fwmkeys-getlist2")) {
+	} else if (!strcmp(command, "fwmkeys-getlist_atomic")) {
 		bops->fwmkeys_test(data->db, config->num, work->seed);
-		bops->getlist_test(data->db, "getlist2", config->num,
+		bops->getlist_test(data->db, "getlist_atomic", config->num,
 				config->vsiz, config->batch, work->seed);
-	} else if (!strcmp(command, "outlist") || !strcmp(command, "outlist2")) {
+	} else if (strstartswith(command, "outlist")) {
 		bops->outlist_test(data->db, command, config->num,
 					config->batch, work->seed);
 	} else if (!strcmp(command, "fwmkeys-outlist")) {
 		bops->fwmkeys_test(data->db, config->num, work->seed);
 		bops->outlist_test(data->db, "outlist", config->num,
 					config->batch, work->seed);
-	} else if (!strcmp(command, "fwmkeys-outlist2")) {
+	} else if (!strcmp(command, "fwmkeys-outlist_atomic")) {
 		bops->fwmkeys_test(data->db, config->num, work->seed);
-		bops->outlist_test(data->db, "outlist2", config->num,
+		bops->outlist_test(data->db, "outlist_atomic", config->num,
 					config->batch, work->seed);
 	} else if (!strcmp(command, "put")) {
 		bops->put_test(data->db, config->num, config->vsiz, work->seed);
@@ -363,11 +364,7 @@ static struct worker_info *create_workers(struct benchmark_config *config,
 	int i;
 
 	for (i = 0; i < thnum; i++) {
-		if ((i % config->share) == i)
-			data[i].db = config->ops.open_db(config);
-		else
-			data[i].db = data[i % config->share].db;
-
+		data[i].db = config->ops.open_db(config);
 		data[i].config = config;
 		data[i].command = command;
 		data[i].in_queue = in_queue;
@@ -393,8 +390,7 @@ static void destroy_workers(struct worker_info *data, int thnum)
 	int i;
 
 	for (i = 0; i < thnum; i++) {
-		if ((i % config->share) == i)
-			config->ops.close_db(data[i].db);
+		config->ops.close_db(data[i].db);
 	}
 	free(data);
 }
@@ -478,7 +474,6 @@ void benchmark(struct benchmark_config *config)
 	consumers = create_workers(config, config->consumer_thnum,
 				config->consumer, &queue_to_consumer,
 				&trash_queue);
-
 	start = stopwatch_start();
 
 	for (i = 0; i < config->num_works; i++) {

testutil.h

@@ -66,7 +66,6 @@ struct benchmark_config {
 	int num_works;
 	bool debug;
 	int verbose;
-	int share;
 	struct benchmark_operations ops;
 };
 

tokyocabinettest.c

@@ -5,9 +5,13 @@
 
 static bool debug = false;
 
+/* Shared by all benchmark threads */
+static TCADB *adb;
+
 static void *open_db(struct benchmark_config *config)
 {
-	TCADB *adb;
+	if (adb)
+		return adb;
 
 	adb = tcadbnew();
 
@@ -19,12 +23,14 @@ static void *open_db(struct benchmark_config *config)
 
 static void close_db(void *db)
 {
-	TCADB *adb = db;
+	if (adb != db)
+		return;
 
-	if (!tcadbclose(adb)){
+	if (!tcadbclose(adb))
 		die("close error: %s", tcadbpath(adb));
-	}
+
 	tcadbdel(adb);
+	adb = NULL;
 }
 
 static void put_test(void *db, int num, int vsiz, unsigned int seed)
@@ -203,7 +209,7 @@ static void getlist_test(void *db, const char *command, int num, int vsiz,
 	tclistdel(list);
 }
 
-static void range1_test(void *db, int num, int vsiz, int batch,
+static void range_nonatomic_test(void *db, int num, int vsiz, int batch,
 			unsigned int seed)
 {
 	TCADB *adb = db;
@@ -233,19 +239,19 @@ static void range1_test(void *db, int num, int vsiz, int batch,
 		if (!num_recs)
 			break;
 
-		check_records(recs, &keygen, vsiz, num_recs);
+		check_records(recs, &keygen, vsiz, num < batch ? num : batch);
 		/* overwrite start_key by the last one + '\0' */
 		tclistover(args, 0, tclistval2(recs, 2 * (num_recs - 1)), KEYGEN_KEY_SIZE + 1);
 		tclistdel(recs);
 		num -= num_recs;
 	}
 	if (debug && num)
-		die("Unexpected record num");
+		die("Unexpected record num: %d", num);
 
 	tclistdel(args);
 }
 
-static void range2_test(void *db, int num, int vsiz, int batch,
+static void range_atomic_test(void *db, int num, int vsiz, int batch,
 			unsigned int seed)
 {
 	TCADB *adb = db;
@@ -273,18 +279,18 @@ static void range2_test(void *db, int num, int vsiz, int batch,
 		TCLIST *recs;
 		int num_recs;
 
-		recs = do_tcadbmisc(adb, "range2", args);
+		recs = do_tcadbmisc(adb, "range_atomic", args);
 		num_recs = tclistnum(recs) / 2;
 		if (!num_recs)
 			break;
 
-		check_records(recs, &keygen, vsiz, num_recs);
+		check_records(recs, &keygen, vsiz, num < batch ? num : batch);
 		tclistover2(args, 0, tclistval2(recs, 2 * (num_recs - 1)));
 		tclistdel(recs);
 		num -= num_recs;
 	}
 	if (debug && num)
-		die("Unexpected record num");
+		die("Unexpected record num: %d", num);
 
 	tclistdel(args);
 }
@@ -293,9 +299,9 @@ static void range_test(void *db, const char *command, int num, int vsiz,
 			int batch, unsigned int seed)
 {
 	if (!strcmp(command, "range"))
-		return range1_test(db, num, vsiz, batch, seed);
-	else if (!strcmp(command, "range2"))
-		return range2_test(db, num, vsiz, batch, seed);
+		return range_nonatomic_test(db, num, vsiz, batch, seed);
+	else if (!strcmp(command, "range_atomic"))
+		return range_atomic_test(db, num, vsiz, batch, seed);
 
 	die("invalid range command");
 }
@@ -381,7 +387,6 @@ struct benchmark_config config = {
 	.consumer_thnum = 1,
 	.debug = false,
 	.verbose = 1,
-	.share = 1,
 	.ops = {
 		.open_db = open_db,
 		.close_db = close_db,
@@ -400,8 +405,6 @@ int main(int argc, char **argv)
 {
 	parse_options(&config, argc, argv);
 	debug = config.debug;
-	if (config.share != 1)
-		die("Invalid -share option");
 	benchmark(&config);
 
 	return 0;