Skip to content

Commit

Permalink
ceph: force sending a cap update msg back to MDS for revoke op
Browse files Browse the repository at this point in the history 
If a client sends out a cap update dropping caps with the prior 'seq'
just before an incoming cap revoke request, then the client may drop
the revoke because it believes it's already released the requested
capabilities.

This causes the MDS to wait indefinitely for the client to respond
to the revoke. It's therefore always a good idea to ack the cap
revoke request with the bumped up 'seq'.

Currently if the cap->issued equals to the newcaps the check_caps()
will do nothing, we should force flush the caps.

Cc: stable@vger.kernel.org
Link: https://tracker.ceph.com/issues/61782
Signed-off-by: Xiubo Li <xiubli@redhat.com>
Reviewed-by: Venky Shankar <vshankar@redhat.com>
Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
  • Loading branch information
@lxbsz @idryomov
lxbsz authored and idryomov committed Aug 1, 2024
1 parent 8400291 commit 31634d7
Show file tree
Hide file tree
Showing 2 changed files with 28 additions and 14 deletions.
35 changes: 24 additions & 11 deletions fs/ceph/caps.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2016,6 +2016,8 @@ bool __ceph_should_report_size(struct ceph_inode_info *ci)
* CHECK_CAPS_AUTHONLY - we should only check the auth cap
* CHECK_CAPS_FLUSH - we should flush any dirty caps immediately, without
* further delay.
* CHECK_CAPS_FLUSH_FORCE - we should flush any caps immediately, without
* further delay.
*/
void ceph_check_caps(struct ceph_inode_info *ci, int flags)
{
Expand Down Expand Up @@ -2097,15 +2099,16 @@ void ceph_check_caps(struct ceph_inode_info *ci, int flags)
}

doutc(cl, "%p %llx.%llx file_want %s used %s dirty %s "
"flushing %s issued %s revoking %s retain %s %s%s%s\n",
"flushing %s issued %s revoking %s retain %s %s%s%s%s\n",
inode, ceph_vinop(inode), ceph_cap_string(file_wanted),
ceph_cap_string(used), ceph_cap_string(ci->i_dirty_caps),
ceph_cap_string(ci->i_flushing_caps),
ceph_cap_string(issued), ceph_cap_string(revoking),
ceph_cap_string(retain),
(flags & CHECK_CAPS_AUTHONLY) ? " AUTHONLY" : "",
(flags & CHECK_CAPS_FLUSH) ? " FLUSH" : "",
(flags & CHECK_CAPS_NOINVAL) ? " NOINVAL" : "");
(flags & CHECK_CAPS_NOINVAL) ? " NOINVAL" : "",
(flags & CHECK_CAPS_FLUSH_FORCE) ? " FLUSH_FORCE" : "");

/*
* If we no longer need to hold onto old our caps, and we may
Expand Down Expand Up @@ -2180,6 +2183,11 @@ void ceph_check_caps(struct ceph_inode_info *ci, int flags)
queue_writeback = true;
}

if (flags & CHECK_CAPS_FLUSH_FORCE) {
doutc(cl, "force to flush caps\n");
goto ack;
}

if (cap == ci->i_auth_cap &&
(cap->issued & CEPH_CAP_FILE_WR)) {
/* request larger max_size from MDS? */
Expand Down Expand Up @@ -3510,6 +3518,8 @@ static void handle_cap_grant(struct inode *inode,
bool queue_invalidate = false;
bool deleted_inode = false;
bool fill_inline = false;
bool revoke_wait = false;
int flags = 0;

/*
* If there is at least one crypto block then we'll trust
Expand Down Expand Up @@ -3705,16 +3715,18 @@ static void handle_cap_grant(struct inode *inode,
ceph_cap_string(cap->issued), ceph_cap_string(newcaps),
ceph_cap_string(revoking));
if (S_ISREG(inode->i_mode) &&
(revoking & used & CEPH_CAP_FILE_BUFFER))
(revoking & used & CEPH_CAP_FILE_BUFFER)) {
writeback = true; /* initiate writeback; will delay ack */
else if (queue_invalidate &&
revoke_wait = true;
} else if (queue_invalidate &&
revoking == CEPH_CAP_FILE_CACHE &&
(newcaps & CEPH_CAP_FILE_LAZYIO) == 0)
; /* do nothing yet, invalidation will be queued */
else if (cap == ci->i_auth_cap)
(newcaps & CEPH_CAP_FILE_LAZYIO) == 0) {
revoke_wait = true; /* do nothing yet, invalidation will be queued */
} else if (cap == ci->i_auth_cap) {
check_caps = 1; /* check auth cap only */
else
} else {
check_caps = 2; /* check all caps */
}
/* If there is new caps, try to wake up the waiters */
if (~cap->issued & newcaps)
wake = true;
Expand All @@ -3741,8 +3753,9 @@ static void handle_cap_grant(struct inode *inode,
BUG_ON(cap->issued & ~cap->implemented);

/* don't let check_caps skip sending a response to MDS for revoke msgs */
if (le32_to_cpu(grant->op) == CEPH_CAP_OP_REVOKE) {
if (!revoke_wait && le32_to_cpu(grant->op) == CEPH_CAP_OP_REVOKE) {
cap->mds_wanted = 0;
flags |= CHECK_CAPS_FLUSH_FORCE;
if (cap == ci->i_auth_cap)
check_caps = 1; /* check auth cap only */
else
Expand Down Expand Up @@ -3798,9 +3811,9 @@ static void handle_cap_grant(struct inode *inode,

mutex_unlock(&session->s_mutex);
if (check_caps == 1)
ceph_check_caps(ci, CHECK_CAPS_AUTHONLY | CHECK_CAPS_NOINVAL);
ceph_check_caps(ci, flags | CHECK_CAPS_AUTHONLY | CHECK_CAPS_NOINVAL);
else if (check_caps == 2)
ceph_check_caps(ci, CHECK_CAPS_NOINVAL);
ceph_check_caps(ci, flags | CHECK_CAPS_NOINVAL);
}

/*
Expand Down
7 changes: 4 additions & 3 deletions fs/ceph/super.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,9 +200,10 @@ struct ceph_cap {
struct list_head caps_item;
};

#define CHECK_CAPS_AUTHONLY 1 /* only check auth cap */
#define CHECK_CAPS_FLUSH 2 /* flush any dirty caps */
#define CHECK_CAPS_NOINVAL 4 /* don't invalidate pagecache */
#define CHECK_CAPS_AUTHONLY 1 /* only check auth cap */
#define CHECK_CAPS_FLUSH 2 /* flush any dirty caps */
#define CHECK_CAPS_NOINVAL 4 /* don't invalidate pagecache */
#define CHECK_CAPS_FLUSH_FORCE 8 /* force flush any caps */

struct ceph_cap_flush {
u64 tid;
Expand Down

0 comments on commit 31634d7

Please sign in to comment.