diff --git a/BUILD.md b/BUILD.md index 246e223786..4d8dd2c4a2 100644 --- a/BUILD.md +++ b/BUILD.md @@ -31,7 +31,6 @@ In addition, there some additional dependencies at runtime: * python-eventlet * python-werkzeug * python-gunicorn -* python-plyvel * python-redis * python-requests * python-simplejson diff --git a/all-requirements.txt b/all-requirements.txt index 839b965edd..ab828e6fc1 100644 --- a/all-requirements.txt +++ b/all-requirements.txt @@ -4,7 +4,6 @@ eventlet>=0.18.2,!=0.18.3,!=0.20.1,!=0.21.0,<1.0.0 gunicorn>=19.4.5,<20.0.0 lxml<5.0.0 oiozookeeper>3.5.5 -plyvel>=0.9,<2.0.0 PyECLib>=1.2.0,<2.0.0 pyparsing!=2.4.1,!=2.4.1.1 pyxattr>=0.4.0,<1.0.0 diff --git a/bin/oio-meta2-rebuilder b/bin/oio-meta2-rebuilder index 06ea308429..2d585e29a9 100644 --- a/bin/oio-meta2-rebuilder +++ b/bin/oio-meta2-rebuilder @@ -78,7 +78,7 @@ if __name__ == '__main__': conf['report_interval'] = args.report_interval # local conf['concurrency'] = args.concurrency - conf['items_per_second'] = args.references_per_second + conf['items_per_second'] = args.items_per_second logger = get_logger_from_args(args, default_conf=conf) diff --git a/oio/common/redis_conn.py b/oio/common/redis_conn.py index cd3594a64a..7d44536cf5 100644 --- a/oio/common/redis_conn.py +++ b/oio/common/redis_conn.py @@ -33,7 +33,8 @@ def catch_service_errors(func): """ redis_exc_mod = importlib.import_module('redis.exceptions') error_types = (redis_exc_mod.ConnectionError, - redis_exc_mod.InvalidResponse) + redis_exc_mod.InvalidResponse, + redis_exc_mod.TimeoutError) @wraps(func) def catch_service_errors_wrapper(*args, **kwargs): diff --git a/oio/common/wsgi.py b/oio/common/wsgi.py index 1e0086bf92..58dde7b14b 100644 --- a/oio/common/wsgi.py +++ b/oio/common/wsgi.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2019 OpenIO SAS, as part of OpenIO SDS +# Copyright (C) 2015-2020 OpenIO SAS, as part of OpenIO SDS # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -17,8 +17,10 @@ from gunicorn.glogging import Logger from werkzeug.wrappers import Request, Response from werkzeug.utils import escape -from werkzeug.exceptions import HTTPException, InternalServerError +from werkzeug.exceptions import HTTPException, InternalServerError, \ + ServiceUnavailable +from oio.common.exceptions import ServiceBusy from oio.common.utils import CPU_COUNT from oio.common.configuration import read_conf from oio.common.logger import get_logger @@ -109,6 +111,11 @@ def dispatch_request(self, req): resp = getattr(self, 'on_' + endpoint)(req, **params) except HTTPException as exc: resp = exc + except ServiceBusy as exc: + if self.logger: + self.logger.error(str(exc)) + resp = ServiceUnavailable( + "Could not satisfy the request: %s" % exc) except Exception as exc: if self.logger: self.logger.exception('ERROR Unhandled exception in request') diff --git a/proxy/m2_actions.c b/proxy/m2_actions.c index 5b21bffa4a..461ad88909 100644 --- a/proxy/m2_actions.c +++ b/proxy/m2_actions.c @@ -753,7 +753,7 @@ _load_alias_from_headers(struct req_args_s *args, GSList **pbeans) do { gchar *s = g_tree_lookup(args->rq->tree_headers, PROXYD_HEADER_PREFIX "content-meta-policy"); - if (NULL != s) + if (oio_str_is_set(s)) CONTENTS_HEADERS_set2_policy(header, s); } while (0); @@ -798,6 +798,12 @@ _load_alias_from_headers(struct req_args_s *args, GSList **pbeans) // TODO(adu) Remove this when all clients will only use `content-meta-size` s = g_tree_lookup(args->rq->tree_headers, PROXYD_HEADER_PREFIX "content-meta-length"); + if (oio_str_is_set(s)) { + GRID_DEBUG("Client is using the deprecated %s header " + "(replaced by %s)", + PROXYD_HEADER_PREFIX "content-meta-length", + PROXYD_HEADER_PREFIX "content-meta-size"); + } } if (!s) { err = BADREQ("Header: missing content size"); @@ -2693,48 +2699,85 @@ action_m2_content_purge (struct req_args_s *args, struct json_object *j UNUSED) // POST /v3.0/{NS}/content/create?acct=&ref=&path= // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // +// Create a new object. This method does not upload any data, it just +// registers object metadata in the database. It is supposed to be called +// after at least one call to the content/prepare2 route and a successful +// data upload to rawx services. +// +// Most of the required information is available in the content/prepare2 +// response. Additional information must be computed by the client (object +// size and hash). +// +// Sample request: +// +// .. code-block:: http +// +// POST /v3.0/OPENIO/content/create?acct=my_account&ref=mycontainer&path=mycontent HTTP/1.1 +// Host: 127.0.0.1:6000 +// User-Agent: curl/7.47.0 +// Accept: */* +// x-oio-content-meta-id: 2996752DFD7205006B73F17AD315AA2B +// x-oio-content-meta-policy: SINGLE +// x-oio-content-meta-size: 64 +// x-oio-content-meta-version: 554086800 +// x-oio-content-meta-chunk-method: plain/nb_copy=1 +// x-oio-content-meta-hash: 2996752DFD7205006B73F17AD315AA2B +// x-oio-content-meta-mime-type: application/octet-stream +// Content-Length: 165 +// Content-Type: application/x-www-form-urlencoded +// // .. code-block:: json // -// [ -// { -// "url":"http://127.0.0.1:6012/D3F2...", -// "pos":"0", -// "size":1048576, -// "hash":"00000000000000000000000000000000" +// { +// "chunks": [ +// { +// "url":"http://127.0.0.1:6012/D3F2...", +// "pos":"0", +// "size":1048576, +// "hash":"00000000000000000000000000000000" +// } +// ], +// "properties": { +// "category": "dogs" // } -// ] +// } // -// You must specify content length and ID on header +// The following request headers are mandatory: +// content ID, storage policy, size and version. // // .. code-block:: text // -// "x-oio-content-meta-id: 2996752DFD7205006B73F17AD315AA2B" -// "x-oio-content-meta-size: 64" +// x-oio-content-meta-id: 2996752DFD7205006B73F17AD315AA2B +// x-oio-content-meta-policy: SINGLE +// x-oio-content-meta-size: 64 +// x-oio-content-meta-version: 554086800000000 // -// You can update system property policy.version of container +// The following request headers are recommended: +// content hash (MD5), mime-type, chunk method. // // .. code-block:: text -// "x-oio-force-versioning: -1" // -// You can create this object as if the versioning is enabled +// x-oio-content-meta-chunk-method: plain/nb_copy=1 +// x-oio-content-meta-hash: 2996752DFD7205006B73F17AD315AA2B +// x-oio-content-meta-mime-type: application/octet-stream +// +// The following header allows to set the versioning policy of the +// container hosting the object: // // .. code-block:: text -// "x-oio-simulate-versioning: 1" // -// Create a new object. This method does not upload any data, it just -// registers object metadata in the database. +// "x-oio-force-versioning: -1" // -// .. code-block:: http +// The following header tells that the object must be created +// as if the versioning was enabled and unlimited +// (this prevents the automatic garbage collection): // -// POST /v3.0/OPENIO/content/create?acct=my_account&ref=mycontainer&path=mycontent HTTP/1.1 -// Host: 127.0.0.1:6000 -// User-Agent: curl/7.47.0 -// Accept: */* -// x-oio-content-meta-size: 64 -// x-oio-content-meta-id: 2996752DFD7205006B73F17AD315AA2B -// Content-Length: 165 -// Content-Type: application/x-www-form-urlencoded +// .. code-block:: text // +// "x-oio-simulate-versioning: 1" +// +// +// Sample response: // // .. code-block:: http // @@ -2884,6 +2927,8 @@ enum http_rc_e action_content_prepare (struct req_args_s *args) { // // Prepare an upload: get URLs of chunks on available rawx. // +// Sample request: +// // .. code-block:: http // // POST /v3.0/OPENIO/content/prepare2?acct=my_account&ref=mycontainer&path=mycontent HTTP/1.1 @@ -2901,6 +2946,8 @@ enum http_rc_e action_content_prepare (struct req_args_s *args) { // } // // +// Sample response: +// // .. code-block:: http // // HTTP/1.1 200 OK diff --git a/sqlx/sqlx_service.c b/sqlx/sqlx_service.c index f00943099c..98de9f075f 100644 --- a/sqlx/sqlx_service.c +++ b/sqlx/sqlx_service.c @@ -108,6 +108,9 @@ static struct grid_main_option_s common_options[] = {"Replicate", OT_BOOL, {.b = &SRV.flag_replicable}, "DO NOT USE THIS. This might disable the replication"}, + {"NoXattrLock", OT_BOOL, {.b = &SRV.flag_nolock}, + "Set to TRUE to avoid any xattr operation on the repository root."}, + {"CacheEnabled", OT_BOOL, {.b = &SRV.flag_cached_bases}, "If set, each base will be cached in a way it won't be accessed" " by several requests in the same time."},