Merge branch '5.x'

BUILD.md

@@ -31,7 +31,6 @@ In addition, there some additional dependencies at runtime:
 * python-eventlet
 * python-werkzeug
 * python-gunicorn
-* python-plyvel
 * python-redis
 * python-requests
 * python-simplejson

all-requirements.txt

@@ -4,7 +4,6 @@ eventlet>=0.18.2,!=0.18.3,!=0.20.1,!=0.21.0,<1.0.0
 gunicorn>=19.4.5,<20.0.0
 lxml<5.0.0
 oiozookeeper>3.5.5
-plyvel>=0.9,<2.0.0
 PyECLib>=1.2.0,<2.0.0
 pyparsing!=2.4.1,!=2.4.1.1
 pyxattr>=0.4.0,<1.0.0

bin/oio-meta2-rebuilder

@@ -78,7 +78,7 @@ if __name__ == '__main__':
     conf['report_interval'] = args.report_interval
     # local
     conf['concurrency'] = args.concurrency
-    conf['items_per_second'] = args.references_per_second
+    conf['items_per_second'] = args.items_per_second
 
     logger = get_logger_from_args(args, default_conf=conf)
 

oio/common/redis_conn.py

@@ -33,7 +33,8 @@ def catch_service_errors(func):
     """
     redis_exc_mod = importlib.import_module('redis.exceptions')
     error_types = (redis_exc_mod.ConnectionError,
-                   redis_exc_mod.InvalidResponse)
+                   redis_exc_mod.InvalidResponse,
+                   redis_exc_mod.TimeoutError)
 
     @wraps(func)
     def catch_service_errors_wrapper(*args, **kwargs):

oio/common/wsgi.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2019 OpenIO SAS, as part of OpenIO SDS
+# Copyright (C) 2015-2020 OpenIO SAS, as part of OpenIO SDS
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -17,8 +17,10 @@
 from gunicorn.glogging import Logger
 from werkzeug.wrappers import Request, Response
 from werkzeug.utils import escape
-from werkzeug.exceptions import HTTPException, InternalServerError
+from werkzeug.exceptions import HTTPException, InternalServerError, \
+    ServiceUnavailable
 
+from oio.common.exceptions import ServiceBusy
 from oio.common.utils import CPU_COUNT
 from oio.common.configuration import read_conf
 from oio.common.logger import get_logger
@@ -109,6 +111,11 @@ def dispatch_request(self, req):
             resp = getattr(self, 'on_' + endpoint)(req, **params)
         except HTTPException as exc:
             resp = exc
+        except ServiceBusy as exc:
+            if self.logger:
+                self.logger.error(str(exc))
+            resp = ServiceUnavailable(
+                "Could not satisfy the request: %s" % exc)
         except Exception as exc:
             if self.logger:
                 self.logger.exception('ERROR Unhandled exception in request')

proxy/m2_actions.c

@@ -753,7 +753,7 @@ _load_alias_from_headers(struct req_args_s *args, GSList **pbeans)
 	do {
 		gchar *s = g_tree_lookup(args->rq->tree_headers,
 								 PROXYD_HEADER_PREFIX "content-meta-policy");
-		if (NULL != s)
+		if (oio_str_is_set(s))
 			CONTENTS_HEADERS_set2_policy(header, s);
 	} while (0);
 
@@ -798,6 +798,12 @@ _load_alias_from_headers(struct req_args_s *args, GSList **pbeans)
 			// TODO(adu) Remove this when all clients will only use `content-meta-size`
 			s = g_tree_lookup(args->rq->tree_headers,
 					PROXYD_HEADER_PREFIX "content-meta-length");
+			if (oio_str_is_set(s)) {
+				GRID_DEBUG("Client is using the deprecated %s header "
+						"(replaced by %s)",
+						PROXYD_HEADER_PREFIX "content-meta-length",
+						PROXYD_HEADER_PREFIX "content-meta-size");
+			}
 		}
 		if (!s) {
 			err = BADREQ("Header: missing content size");
@@ -2693,48 +2699,85 @@ action_m2_content_purge (struct req_args_s *args, struct json_object *j UNUSED)
 // POST /v3.0/{NS}/content/create?acct=<account_name>&ref=<container_name>&path=<file_path>
 // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 //
+// Create a new object. This method does not upload any data, it just
+// registers object metadata in the database. It is supposed to be called
+// after at least one call to the content/prepare2 route and a successful
+// data upload to rawx services.
+//
+// Most of the required information is available in the content/prepare2
+// response. Additional information must be computed by the client (object
+// size and hash).
+//
+// Sample request:
+//
+// .. code-block:: http
+//
+//    POST /v3.0/OPENIO/content/create?acct=my_account&ref=mycontainer&path=mycontent HTTP/1.1
+//    Host: 127.0.0.1:6000
+//    User-Agent: curl/7.47.0
+//    Accept: */*
+//    x-oio-content-meta-id: 2996752DFD7205006B73F17AD315AA2B
+//    x-oio-content-meta-policy: SINGLE
+//    x-oio-content-meta-size: 64
+//    x-oio-content-meta-version: 554086800
+//    x-oio-content-meta-chunk-method: plain/nb_copy=1
+//    x-oio-content-meta-hash: 2996752DFD7205006B73F17AD315AA2B
+//    x-oio-content-meta-mime-type: application/octet-stream
+//    Content-Length: 165
+//    Content-Type: application/x-www-form-urlencoded
+//
 // .. code-block:: json
 //
-//    [
-//      {
-//        "url":"http://127.0.0.1:6012/D3F2...",
-//        "pos":"0",
-//        "size":1048576,
-//        "hash":"00000000000000000000000000000000"
+//    {
+//      "chunks": [
+//        {
+//          "url":"http://127.0.0.1:6012/D3F2...",
+//          "pos":"0",
+//          "size":1048576,
+//          "hash":"00000000000000000000000000000000"
+//        }
+//      ],
+//      "properties": {
+//        "category": "dogs"
 //      }
-//    ]
+//    }
 //
-// You must specify content length and ID on header
+// The following request headers are mandatory:
+// content ID, storage policy, size and version.
 //
 // .. code-block:: text
 //
-//    "x-oio-content-meta-id: 2996752DFD7205006B73F17AD315AA2B"
-//    "x-oio-content-meta-size: 64"
+//    x-oio-content-meta-id: 2996752DFD7205006B73F17AD315AA2B
+//    x-oio-content-meta-policy: SINGLE
+//    x-oio-content-meta-size: 64
+//    x-oio-content-meta-version: 554086800000000
 //
-// You can update system property policy.version of container
+// The following request headers are recommended:
+// content hash (MD5), mime-type, chunk method.
 //
 // .. code-block:: text
-//    "x-oio-force-versioning: -1"
 //
-// You can create this object as if the versioning is enabled
+//    x-oio-content-meta-chunk-method: plain/nb_copy=1
+//    x-oio-content-meta-hash: 2996752DFD7205006B73F17AD315AA2B
+//    x-oio-content-meta-mime-type: application/octet-stream
+//
+// The following header allows to set the versioning policy of the
+// container hosting the object:
 //
 // .. code-block:: text
-//    "x-oio-simulate-versioning: 1"
 //
-// Create a new object. This method does not upload any data, it just
-// registers object metadata in the database.
+//    "x-oio-force-versioning: -1"
 //
-// .. code-block:: http
+// The following header tells that the object must be created
+// as if the versioning was enabled and unlimited
+// (this prevents the automatic garbage collection):
 //
-//    POST /v3.0/OPENIO/content/create?acct=my_account&ref=mycontainer&path=mycontent HTTP/1.1
-//    Host: 127.0.0.1:6000
-//    User-Agent: curl/7.47.0
-//    Accept: */*
-//    x-oio-content-meta-size: 64
-//    x-oio-content-meta-id: 2996752DFD7205006B73F17AD315AA2B
-//    Content-Length: 165
-//    Content-Type: application/x-www-form-urlencoded
+// .. code-block:: text
 //
+//    "x-oio-simulate-versioning: 1"
+//
+//
+// Sample response:
 //
 // .. code-block:: http
 //
@@ -2884,6 +2927,8 @@ enum http_rc_e action_content_prepare (struct req_args_s *args) {
 //
 // Prepare an upload: get URLs of chunks on available rawx.
 //
+// Sample request:
+//
 // .. code-block:: http
 //
 //    POST /v3.0/OPENIO/content/prepare2?acct=my_account&ref=mycontainer&path=mycontent HTTP/1.1
@@ -2901,6 +2946,8 @@ enum http_rc_e action_content_prepare (struct req_args_s *args) {
 //    }
 //
 //
+// Sample response:
+//
 // .. code-block:: http
 //
 //    HTTP/1.1 200 OK

sqlx/sqlx_service.c

@@ -108,6 +108,9 @@ static struct grid_main_option_s common_options[] =
 	{"Replicate", OT_BOOL, {.b = &SRV.flag_replicable},
 		"DO NOT USE THIS. This might disable the replication"},
 
+	{"NoXattrLock", OT_BOOL, {.b = &SRV.flag_nolock},
+		"Set to TRUE to avoid any xattr operation on the repository root."},
+
 	{"CacheEnabled", OT_BOOL, {.b = &SRV.flag_cached_bases},
 		"If set, each base will be cached in a way it won't be accessed"
 			" by several requests in the same time."},