obj: wrap pool opens/closes in a crude global mutex

Prevents crashes (or more likely, valgrind gripe) if someone tries to
do that in parallel.  Concurrent read of pool info have been safe since
early 2019.

We're close to fully concurrent opens/closes as well -- IIRC the main
blocker is the use of globals for confit parsing -- but until that part
is fixed, let's at least serialize and be safe.

src/libpmemobj/obj.c

@@ -64,6 +64,7 @@ static struct critnib *pools_ht; /* hash table used for searching by UUID */
 static struct critnib *pools_tree; /* tree used for searching by address */
 
 int _pobj_cache_invalidate;
+static os_mutex_t pools_mutex;
 
 #ifndef _WIN32
 
@@ -267,6 +268,8 @@ obj_init(void)
 
 	COMPILE_ERROR_ON(PMEMOBJ_F_MEM_NOFLUSH != PMEM_F_MEM_NOFLUSH);
 
+	os_mutex_init(&pools_mutex);
+
 #ifdef _WIN32
 	/* XXX - temporary implementation (see above) */
 	os_once(&Cached_pool_key_once, _Cached_pool_key_alloc);
@@ -302,6 +305,8 @@ obj_fini(void)
 	lane_info_destroy();
 	util_remote_fini();
 
+	os_mutex_destroy(&pools_mutex);
+
 #ifdef _WIN32
 	(void) os_tls_key_delete(Cached_pool_key);
 #endif
@@ -1323,6 +1328,8 @@ pmemobj_createU(const char *path, const char *layout,
 		return NULL;
 	}
 
+	os_mutex_lock(&pools_mutex);
+
 	/*
 	 * A number of lanes available at runtime equals the lowest value
 	 * from all reported by remote replicas hosts. In the single host mode
@@ -1344,6 +1351,7 @@ pmemobj_createU(const char *path, const char *layout,
 			PMEMOBJ_MIN_PART, &adj_pool_attr, &runtime_nlanes,
 			REPLICAS_ENABLED) != 0) {
 		LOG(2, "cannot create pool or pool set");
+		os_mutex_unlock(&pools_mutex);
 		return NULL;
 	}
 
@@ -1397,6 +1405,7 @@ pmemobj_createU(const char *path, const char *layout,
 	util_poolset_fdclose(set);
 
 	LOG(3, "pop %p", pop);
+	os_mutex_unlock(&pools_mutex);
 
 	return pop;
 
@@ -1406,6 +1415,7 @@ pmemobj_createU(const char *path, const char *layout,
 	if (set->remote)
 		obj_cleanup_remote(pop);
 	util_poolset_close(set, DELETE_CREATED_PARTS);
+	os_mutex_unlock(&pools_mutex);
 	errno = oerrno;
 	return NULL;
 }
@@ -1707,6 +1717,8 @@ obj_open_common(const char *path, const char *layout, unsigned flags, int boot)
 	PMEMobjpool *pop = NULL;
 	struct pool_set *set;
 
+	os_mutex_lock(&pools_mutex);
+
 	/*
 	 * A number of lanes available at runtime equals the lowest value
 	 * from all reported by remote replicas hosts. In the single host mode
@@ -1715,8 +1727,10 @@ obj_open_common(const char *path, const char *layout, unsigned flags, int boot)
 	 * environment variable whichever is lower.
 	 */
 	unsigned runtime_nlanes = obj_get_nlanes();
-	if (obj_pool_open(&set, path, flags, &runtime_nlanes))
+	if (obj_pool_open(&set, path, flags, &runtime_nlanes)) {
+		os_mutex_unlock(&pools_mutex);
 		return NULL;
+	}
 
 	/* pop is master replica from now on */
 	pop = set->replica[0]->part[0].addr;
@@ -1769,6 +1783,7 @@ obj_open_common(const char *path, const char *layout, unsigned flags, int boot)
 #endif
 
 	util_poolset_fdclose(set);
+	os_mutex_unlock(&pools_mutex);
 
 	LOG(3, "pop %p", pop);
 
@@ -1781,6 +1796,7 @@ obj_open_common(const char *path, const char *layout, unsigned flags, int boot)
 	obj_replicas_fini(set);
 replicas_init:
 	obj_pool_close(set);
+	os_mutex_unlock(&pools_mutex);
 	return NULL;
 }
 
@@ -1941,6 +1957,7 @@ pmemobj_close(PMEMobjpool *pop)
 	LOG(3, "pop %p", pop);
 	PMEMOBJ_API_START();
 
+	os_mutex_lock(&pools_mutex);
 	_pobj_cache_invalidate++;
 
 	if (critnib_remove(pools_ht, pop->uuid_lo) != pop) {
@@ -1970,6 +1987,7 @@ pmemobj_close(PMEMobjpool *pop)
 #endif /* _WIN32 */
 
 	obj_pool_cleanup(pop);
+	os_mutex_unlock(&pools_mutex);
 	PMEMOBJ_API_END();
 }