Merge remote-tracking branch 'gh/jewel'

configure.ac

@@ -8,7 +8,7 @@ AC_PREREQ(2.59)
 # VERSION define is not used by the code.  It gets a version string
 # from 'git describe'; see src/ceph_ver.[ch]
 
-AC_INIT([ceph], [10.0.5], [ceph-devel@vger.kernel.org])
+AC_INIT([ceph], [10.1.0], [ceph-devel@vger.kernel.org])
 
 AX_CXX_COMPILE_STDCXX_11(, mandatory)
 

debian/changelog

@@ -1,3 +1,9 @@
+ceph (10.1.0-1) stable; urgency=medium
+
+  * New upstream release
+
+ -- Alfredo Deza <adeza@redhat.com>  Thu, 24 Mar 2016 10:53:47 +0000
+
 ceph (10.0.5) stable; urgency=low
 
   * New upstream release (just fixing changelog)

debian/control

@@ -57,7 +57,7 @@ Build-Depends: autoconf,
                python-nose,
 	       python-setuptools,
                python-sphinx,
-               python-virtualenv,
+               virtualenv | python-virtualenv,
                sdparm | hdparm,
                uuid-runtime,
                valgrind,

qa/workunits/ceph-helpers.sh

@@ -1313,7 +1313,7 @@ function main() {
     shopt -s -o xtrace
     PS4='${BASH_SOURCE[0]}:$LINENO: ${FUNCNAME[0]}:  '
 
-    export PATH=ceph-disk/virtualenv/bin:ceph-detect-init/virtualenv/bin:.:$PATH # make sure program from sources are prefered
+    export PATH=ceph-disk/ceph-disk-virtualenv/bin:ceph-detect-init/ceph-detect-init-virtualenv/bin:.:$PATH # make sure program from sources are prefered
 
     export CEPH_CONF=/dev/null
     unset CEPH_ARGS
@@ -1335,7 +1335,7 @@ function run_tests() {
     shopt -s -o xtrace
     PS4='${BASH_SOURCE[0]}:$LINENO: ${FUNCNAME[0]}:  '
 
-    export PATH=ceph-disk/virtualenv/bin:ceph-detect-init/virtualenv/bin:.:$PATH # make sure program from sources are prefered
+    export PATH=ceph-disk/ceph-disk-virtualenv/bin:ceph-detect-init/ceph-detect-init-virtualenv/bin:.:$PATH # make sure program from sources are prefered
 
     export CEPH_MON="127.0.0.1:7109" # git grep '\<7109\>' : there must be only one
     export CEPH_ARGS

src/Makefile-env.am

@@ -297,5 +297,7 @@ LIBCIVETWEB_DEPS =
 DENCODER_SOURCES =
 DENCODER_DEPS =
 
+# put virtualenvs in this directory for build
+CEPH_BUILD_VIRTUALENV="/tmp/"
 
 radoslibdir = $(libdir)/rados-classes

src/ceph-detect-init/Makefile.am

@@ -53,13 +53,15 @@ EXTRA_DIST += \
 	ceph-detect-init/tests/test_all.py \
 	ceph-detect-init/tox.ini
 
-ceph-detect-init-all: ceph-detect-init/virtualenv
+export CEPH_DETECT_INIT_VIRTUALENV = ${CEPH_BUILD_VIRTUALENV}ceph-detect-init-virtualenv
 
-ceph-detect-init/virtualenv:
-	cd $(srcdir)/ceph-detect-init ; ../tools/setup-virtualenv.sh ; test -d wheelhouse && export NO_INDEX=--no-index ; virtualenv/bin/pip install $$NO_INDEX --use-wheel --find-links=file://$$(pwd)/wheelhouse -e .
+ceph-detect-init-all: ${CEPH_DETECT_INIT_VIRTUALENV}
+
+${CEPH_DETECT_INIT_VIRTUALENV}:
+	cd $(srcdir)/ceph-detect-init ; ../tools/setup-virtualenv.sh ${CEPH_DETECT_INIT_VIRTUALENV} ; test -d wheelhouse && export NO_INDEX=--no-index ; ${CEPH_DETECT_INIT_VIRTUALENV}/bin/pip install $$NO_INDEX --use-wheel --find-links=file://$$(pwd)/wheelhouse -e .
 
 ceph-detect-init-clean:
-	cd $(srcdir)/ceph-detect-init ; python setup.py clean ; rm -fr wheelhouse .tox build virtualenv .coverage *.egg-info
+	cd $(srcdir)/ceph-detect-init ; python setup.py clean ; rm -fr wheelhouse .tox build ${CEPH_DETECT_INIT_VIRTUALENV} .coverage *.egg-info
 
 ceph-detect-init-install-data:
 	cd $(srcdir)/ceph-detect-init ; \

src/ceph-detect-init/run-tox.sh

@@ -18,9 +18,10 @@
 #
 
 # run from the ceph-detect-init directory or from its parent
+: ${CEPH_DETECT_INIT_VIRTUALENV:=ceph-detect-init-virtualenv}
 test -d ceph-detect-init && cd ceph-detect-init
-source virtualenv/bin/activate
-tox > virtualenv/tox.out 2>&1
+source ${CEPH_DETECT_INIT_VIRTUALENV}/bin/activate
+tox > ${CEPH_DETECT_INIT_VIRTUALENV}/tox.out 2>&1
 status=$?
-grep -v InterpreterNotFound < virtualenv/tox.out
+grep -v InterpreterNotFound < ${CEPH_DETECT_INIT_VIRTUALENV}/tox.out
 exit $status

src/ceph-disk/Makefile.am

@@ -29,13 +29,15 @@ EXTRA_DIST += \
 	ceph-disk/tests/test_main.py \
 	ceph-disk/tox.ini
 
-ceph-disk-all: ceph-disk/virtualenv
+export CEPH_DISK_VIRTUALENV = ${CEPH_BUILD_VIRTUALENV}ceph-disk-virtualenv
 
-ceph-disk/virtualenv:
-	cd $(srcdir)/ceph-disk ; ../tools/setup-virtualenv.sh ; test -d wheelhouse && export NO_INDEX=--no-index ; virtualenv/bin/pip install $$NO_INDEX --use-wheel --find-links=file://$$(pwd)/wheelhouse -e .
+ceph-disk-all: ${CEPH_DISK_VIRTUALENV}
+
+${CEPH_DISK_VIRTUALENV}:
+	cd $(srcdir)/ceph-disk ; ../tools/setup-virtualenv.sh ${CEPH_DISK_VIRTUALENV} ; test -d wheelhouse && export NO_INDEX=--no-index ; ${CEPH_DISK_VIRTUALENV}/bin/pip install $$NO_INDEX --use-wheel --find-links=file://$$(pwd)/wheelhouse -e .
 
 ceph-disk-clean:
-	cd $(srcdir)/ceph-disk ; python setup.py clean ; rm -fr wheelhouse .tox build virtualenv .coverage *.egg-info
+	cd $(srcdir)/ceph-disk ; python setup.py clean ; rm -fr wheelhouse .tox build ${CEPH_DISK_VIRTUALENV} .coverage *.egg-info
 
 ceph-disk-install-data:
 	cd $(srcdir)/ceph-disk ; \

src/ceph-disk/run-tox.sh

@@ -16,9 +16,10 @@
 #
 
 # run from the ceph-disk directory or from its parent
+: ${CEPH_DISK_VIRTUALENV:=ceph-disk-virtualenv}
 test -d ceph-disk && cd ceph-disk
-source virtualenv/bin/activate
-tox > virtualenv/tox.out 2>&1
+source ${CEPH_DISK_VIRTUALENV}/bin/activate
+tox > ${CEPH_DISK_VIRTUALENV}/tox.out 2>&1
 status=$?
-grep -v InterpreterNotFound < virtualenv/tox.out
+grep -v InterpreterNotFound < ${CEPH_DISK_VIRTUALENV}/tox.out
 exit $status

src/client/Client.cc

@@ -372,7 +372,10 @@ void Client::tear_down_cache()
 
 inodeno_t Client::get_root_ino()
 {
-  return root->ino;
+  if (use_faked_inos())
+    return root->faked_ino;
+  else
+    return root->ino;
 }
 
 Inode *Client::get_root()

src/client/fuse_ll.cc

@@ -1067,6 +1067,9 @@ int CephFuse::Handle::loop()
 
 uint64_t CephFuse::Handle::fino_snap(uint64_t fino)
 {
+  if (fino == FUSE_ROOT_ID)
+    return CEPH_NOSNAP;
+
   if (client->use_faked_inos()) {
     vinodeno_t vino  = client->map_faked_ino(fino);
     return vino.snapid;
@@ -1080,11 +1083,12 @@ uint64_t CephFuse::Handle::fino_snap(uint64_t fino)
 
 Inode * CephFuse::Handle::iget(fuse_ino_t fino)
 {
+  if (fino == FUSE_ROOT_ID)
+    return client->get_root();
+
   if (client->use_faked_inos()) {
     return client->ll_get_inode((ino_t)fino);
   } else {
-    if (fino == 1)
-      fino = inodeno_t(client->get_root_ino());
     vinodeno_t vino(FINO_INO(fino), fino_snap(fino));
     return client->ll_get_inode(vino);
   }
@@ -1099,8 +1103,14 @@ uint64_t CephFuse::Handle::make_fake_ino(inodeno_t ino, snapid_t snapid)
 {
   if (client->use_faked_inos()) {
     // already faked by libcephfs
+    if (ino == client->get_root_ino())
+      return FUSE_ROOT_ID;
+
     return ino;
   } else {
+    if (snapid == CEPH_NOSNAP && ino == client->get_root_ino())
+      return FUSE_ROOT_ID;
+
     Mutex::Locker l(stag_lock);
     uint64_t stag;
     if (snap_stag_map.count(snapid) == 0) {

src/common/TrackedOp.cc

@@ -87,14 +87,23 @@ void OpHistory::dump_ops(utime_t now, Formatter *f)
   f->close_section();
 }
 
-void OpTracker::dump_historic_ops(Formatter *f)
+bool OpTracker::dump_historic_ops(Formatter *f)
 {
+  RWLock::RLocker l(lock);
+  if (!tracking_enabled)
+    return false;
+
   utime_t now = ceph_clock_now(cct);
   history.dump_ops(now, f);
+  return true;
 }
 
-void OpTracker::dump_ops_in_flight(Formatter *f, bool print_only_blocked)
+bool OpTracker::dump_ops_in_flight(Formatter *f, bool print_only_blocked)
 {
+  RWLock::RLocker l(lock);
+  if (!tracking_enabled)
+    return false;
+
   f->open_object_section("ops_in_flight"); // overall dump
   uint64_t total_ops_in_flight = 0;
   f->open_array_section("ops"); // list of TrackedOps
@@ -119,12 +128,14 @@ void OpTracker::dump_ops_in_flight(Formatter *f, bool print_only_blocked)
   } else
     f->dump_int("num_ops", total_ops_in_flight);
   f->close_section(); // overall dump
+  return true;
 }
 
-void OpTracker::register_inflight_op(xlist<TrackedOp*>::item *i)
+bool OpTracker::register_inflight_op(xlist<TrackedOp*>::item *i)
 {
-  // caller checks;
-  assert(tracking_enabled);
+  RWLock::RLocker l(lock);
+  if (!tracking_enabled)
+    return false;
 
   uint64_t current_seq = seq.inc();
   uint32_t shard_index = current_seq % num_optracker_shards;
@@ -135,6 +146,7 @@ void OpTracker::register_inflight_op(xlist<TrackedOp*>::item *i)
     sdata->ops_in_flight_sharded.push_back(i);
     sdata->ops_in_flight_sharded.back()->seq = current_seq;
   }
+  return true;
 }
 
 void OpTracker::unregister_inflight_op(TrackedOp *i)
@@ -312,6 +324,9 @@ void TrackedOp::mark_event(const string &event)
 
 void TrackedOp::dump(utime_t now, Formatter *f) const
 {
+  // Ignore if still in the constructor
+  if (!is_tracked)
+    return;
   stringstream name;
   _dump_op_descriptor_unlocked(name);
   f->dump_string("description", name.str().c_str()); // this TrackedOp

src/common/TrackedOp.h

@@ -22,6 +22,7 @@
 #include "msg/Message.h"
 #include "include/memory.h"
 #include "common/RWLock.h"
+#include <atomic>
 
 class TrackedOp;
 typedef ceph::shared_ptr<TrackedOp> TrackedOpRef;
@@ -74,16 +75,16 @@ class OpTracker {
   float complaint_time;
   int log_threshold;
   void _mark_event(TrackedOp *op, const string &evt, utime_t now);
+  bool tracking_enabled;
+  RWLock       lock;
 
 public:
-  bool tracking_enabled;
   CephContext *cct;
-  RWLock       lock;
   OpTracker(CephContext *cct_, bool tracking, uint32_t num_shards) : seq(0), 
                                      num_optracker_shards(num_shards),
 				     complaint_time(0), log_threshold(0),
-				     tracking_enabled(tracking), cct(cct_),
-				     lock("OpTracker::lock") {
+				     tracking_enabled(tracking),
+				     lock("OpTracker::lock"), cct(cct_) {
 
     for (uint32_t i = 0; i < num_optracker_shards; i++) {
       char lock_name[32] = {0};
@@ -104,9 +105,9 @@ class OpTracker {
     RWLock::WLocker l(lock);
     tracking_enabled = enable;
   }
-  void dump_ops_in_flight(Formatter *f, bool print_only_blocked=false);
-  void dump_historic_ops(Formatter *f);
-  void register_inflight_op(xlist<TrackedOp*>::item *i);
+  bool dump_ops_in_flight(Formatter *f, bool print_only_blocked=false);
+  bool dump_historic_ops(Formatter *f);
+  bool register_inflight_op(xlist<TrackedOp*>::item *i);
   void unregister_inflight_op(TrackedOp *i);
 
   void get_age_ms_histogram(pow2_hist_t *h);
@@ -139,6 +140,7 @@ class OpTracker {
   {
     typename T::Ref retval(new T(params, this),
 			   RemoveOnDelete(this));
+    retval->tracking_start();
     return retval;
   }
 };
@@ -158,7 +160,8 @@ class TrackedOp {
   uint64_t seq; /// a unique value set by the OpTracker
 
   uint32_t warn_interval_multiplier; // limits output of a given op warning
-  bool is_tracked; //whether in tracker
+  // Transitions from false -> true without locks being held
+  atomic<bool> is_tracked; //whether in tracker and out of constructor
   TrackedOp(OpTracker *_tracker, const utime_t& initiated) :
     xitem(this),
     tracker(_tracker),
@@ -167,14 +170,7 @@ class TrackedOp {
     seq(0),
     warn_interval_multiplier(1),
     is_tracked(false)
-  {
-    RWLock::RLocker l(tracker->lock);
-    if (tracker->tracking_enabled) {
-      tracker->register_inflight_op(&xitem);
-      events.push_back(make_pair(initiated_at, "initiated"));
-      is_tracked = true;
-    }
-  }
+  { }
 
   /// output any type-specific data you want to get when dump() is called
   virtual void _dump(utime_t now, Formatter *f) const {}
@@ -193,6 +189,7 @@ class TrackedOp {
   }
 
   double get_duration() const {
+    Mutex::Locker l(lock);
     if (!events.empty() && events.rbegin()->second.compare("done") == 0)
       return events.rbegin()->first - get_initiated();
     else
@@ -201,9 +198,16 @@ class TrackedOp {
 
   void mark_event(const string &event);
   virtual const char *state_string() const {
+    Mutex::Locker l(lock);
     return events.rbegin()->second.c_str();
   }
   void dump(utime_t now, Formatter *f) const;
+  void tracking_start() {
+    if (tracker->register_inflight_op(&xitem)) {
+      events.push_back(make_pair(initiated_at, "initiated"));
+      is_tracked = true;
+    }
+  }
 };
 
 #endif

src/mds/CInode.cc

@@ -4173,12 +4173,10 @@ void CInode::scrub_initialize(CDentry *scrub_parent,
 			      MDSInternalContextBase *f)
 {
   dout(20) << __func__ << " with scrub_version " << get_version() << dendl;
-  assert(!scrub_infop || !scrub_infop->scrub_in_progress);
+  assert(!scrub_is_in_progress());
   scrub_info();
   if (!scrub_infop)
     scrub_infop = new scrub_info_t();
-  else
-    assert(!scrub_infop->scrub_in_progress);
 
   if (get_projected_inode()->is_dir()) {
     // fill in dirfrag_stamps with initial state
@@ -4210,7 +4208,7 @@ void CInode::scrub_initialize(CDentry *scrub_parent,
 int CInode::scrub_dirfrag_next(frag_t* out_dirfrag)
 {
   dout(20) << __func__ << dendl;
-  assert(scrub_infop && scrub_infop->scrub_in_progress);
+  assert(scrub_is_in_progress());
 
   if (!is_dir()) {
     return -ENOTDIR;
@@ -4258,7 +4256,7 @@ void CInode::scrub_dirfrags_scrubbing(list<frag_t>* out_dirfrags)
 void CInode::scrub_dirfrag_finished(frag_t dirfrag)
 {
   dout(20) << __func__ << " on frag " << dirfrag << dendl;
-  assert(scrub_infop && scrub_infop->scrub_in_progress);
+  assert(scrub_is_in_progress());
 
   std::map<frag_t, scrub_stamp_info_t>::iterator i =
       scrub_infop->dirfrag_stamps.find(dirfrag);
@@ -4271,7 +4269,7 @@ void CInode::scrub_dirfrag_finished(frag_t dirfrag)
 
 void CInode::scrub_finished(MDSInternalContextBase **c) {
   dout(20) << __func__ << dendl;
-  assert(scrub_info()->scrub_in_progress);
+  assert(scrub_is_in_progress());
   for (std::map<frag_t, scrub_stamp_info_t>::iterator i =
       scrub_infop->dirfrag_stamps.begin();
       i != scrub_infop->dirfrag_stamps.end();

src/mds/CInode.h

@@ -288,6 +288,10 @@ class CInode : public MDSCacheObject, public InodeStoreBase {
       scrub_info_create();
     return scrub_infop;
   }
+
+  bool scrub_is_in_progress() const {
+    return (scrub_infop && scrub_infop->scrub_in_progress);
+  }
   /**
    * Start scrubbing on this inode. That could be very short if it's
    * a file, or take a long time if we're recursively scrubbing a directory.