History

ceph: force a cap message when a deferred revoke can't be acked immediately

When the MDS revokes capabilities, handle_cap_grant() normally guarantees a response by setting `CHECK_CAPS_FLUSH_FORCE` (see commit 31634d7597d8 ("ceph: force sending a cap update msg back to MDS for revoke op")), so ceph_check_caps() sends a cap message even if the client would otherwise decide it has nothing to do. That guarantee is skipped whenever the revoke has to be deferred (via revoke_wait): revoking Fb while dirty data is still buffered (writeback is queued first) or revoking Fc while pages are cached (async invalidation is queued first). In those cases, the ack is left to the deferred completion (ceph_put_wrbuffer_cap_refs() after writeback, or the invalidate worker after invalidation); both of which call ceph_check_caps(ci,0) i.e. without `CHECK_CAPS_FLUSH_FORCE`. Nothing gets sent under one of the following conditions: - the inode is retaining caps because the file was used recently (file_wanted != 0; retain |= CEPH_CAP_ANY) - the revoked cap is still used because the page was re-cached (e.g. a file being re-read) - the MDS has meanwhile re-granted, so `issued==implemented` and the client sees nothing being revoked The client then never emits the cap message which the MDS is waiting for. The MDS blocks on the revoke indefinitely and logs, for minutes or hours: client.NNN isn't responding to mclientcaps(revoke), ino 0x... pending pAsxLsXsxFsxcrwb issued pAsxLsXsxFsxcrwb, sent 964.899182 seconds ago The client-side state at that point shows the full cap set still issued, nothing in the revoking/flushing sets. Thus nothing gets sent. This patch fixes it by remembering that a forced response is expected. When a revoke is deferred, set `CEPH_I_FLUSH_FORCE` on the inode. ceph_check_caps() replays it as `CHECK_CAPS_FLUSH_FORCE`, so whichever path re-checks the inode next (the writeback/invalidate completion, the delayed worker, or any other caller) is guaranteed to send a cap message to the MDS. __prep_cap() clears the flag once a message is actually built. This is the deferred-path counterpart of the existing `CHECK_CAPS_FLUSH_FORCE` handling; a normal (non-deferred) revoke still forces the response inline as before.

Max Kellermann committed 2 months ago

Commit 8fdf9461 parent 3cde4a82 files · +61 −7Browse files

Unified · Split

2 files
  1. fs/ceph/caps.c +56 −7
  2. fs/ceph/super.h +5 −0
fs/ceph/caps.c+56−7
@@ -987,6 +987,27 @@ int __ceph_caps_revoking_other(struct ceph_inode_info *ci, · 986 unmodified lines
987 return 0;987 return 0;
988}988}
989 989
990/*
991 * Return true if any cap of this inode holds caps which the MDS has
992 * revoked, but which we have not released yet.
993 */
994static bool __ceph_is_any_revoking(const struct ceph_inode_info *ci)
995{
996 const struct rb_node *p;
997
998 lockdep_assert_held(&ci->i_ceph_lock);
999
1000 for (p = rb_first(&ci->i_caps); p; p = rb_next(p)) {
1001 const struct ceph_cap *cap =
1002 rb_entry(p, struct ceph_cap, ci_node);
1003
1004 if (cap->implemented & ~cap->issued)
1005 return true;
1006 }
1007
1008 return false;
1009}
1010
990int __ceph_caps_used(struct ceph_inode_info *ci)1011int __ceph_caps_used(struct ceph_inode_info *ci)
991{1012{
992 int used = 0;1013 int used = 0;
@@ -1431,6 +1452,9 @@ static void __prep_cap(struct cap_msg_args *arg, struct ceph_inode_info *ci, · 438 unmodified lines
1431 cap->implemented &= cap->issued | used;1452 cap->implemented &= cap->issued | used;
1432 cap->mds_wanted = want;1453 cap->mds_wanted = want;
1433 1454
1455 if ((ci->i_ceph_flags & CEPH_I_FLUSH_FORCE) != 0 && !__ceph_is_any_revoking(ci))
1456 clear_bit(CEPH_I_FLUSH_FORCE_BIT, &ci->i_ceph_flags);
1457
1434 arg->session = cap->session;1458 arg->session = cap->session;
1435 arg->ino = ceph_vino(inode).ino;1459 arg->ino = ceph_vino(inode).ino;
1436 arg->cid = cap->cap_id;1460 arg->cid = cap->cap_id;
@@ -2048,6 +2072,14 @@ void ceph_check_caps(struct ceph_inode_info *ci, int flags) · 611 unmodified lines
2048 2072
2049 if (ci->i_ceph_flags & CEPH_I_FLUSH)2073 if (ci->i_ceph_flags & CEPH_I_FLUSH)
2050 flags |= CHECK_CAPS_FLUSH;2074 flags |= CHECK_CAPS_FLUSH;
2075 /*
2076 * A revoke whose response was deferred (see handle_cap_grant()) must
2077 * still be acknowledged. Replay the forced flush here so that even a
2078 * check triggered by writeback/invalidation completion sends a cap
2079 * message to the MDS.
2080 */
2081 if (ci->i_ceph_flags & CEPH_I_FLUSH_FORCE)
2082 flags |= CHECK_CAPS_FLUSH_FORCE;
2051retry:2083retry:
2052 /* Caps wanted by virtue of active open files. */2084 /* Caps wanted by virtue of active open files. */
2053 file_wanted = __ceph_caps_file_wanted(ci);2085 file_wanted = __ceph_caps_file_wanted(ci);
@@ -3774,13 +3806,30 @@ static void handle_cap_grant(struct inode *inode, · 1720 unmodified lines
3774 BUG_ON(cap->issued & ~cap->implemented);3806 BUG_ON(cap->issued & ~cap->implemented);
3775 3807
3776 /* don't let check_caps skip sending a response to MDS for revoke msgs */3808 /* don't let check_caps skip sending a response to MDS for revoke msgs */
3777 if (!revoke_wait && le32_to_cpu(grant->op) == CEPH_CAP_OP_REVOKE) {3809 if (le32_to_cpu(grant->op) == CEPH_CAP_OP_REVOKE) {
3778 cap->mds_wanted = 0;3810 if (revoke_wait) {
3779 flags |= CHECK_CAPS_FLUSH_FORCE;3811 /*
3780 if (cap == ci->i_auth_cap)3812 * We can't ack the revoke yet: the response is deferred
3781 check_caps = 1; /* check auth cap only */3813 * until the writeback or cache invalidation queued above
3782 else3814 * completes. Set the CEPH_I_FLUSH_FORCE flag to remember
3783 check_caps = 2; /* check all caps */3815 * that a forced cap message is owed so that deferred
3816 * completion (ceph_put_wrbuffer_cap_refs() or the
3817 * invalidate worker, both of which call ceph_check_caps())
3818 * actually sends one, even if by then the revoked caps look
3819 * unused, the inode is retaining caps, or the MDS has
3820 * re-granted them. Without this, the cap message is never
3821 * sent and the MDS hangs ("isn't responding to
3822 * mclientcaps(revoke)").
3823 */
3824 set_bit(CEPH_I_FLUSH_FORCE_BIT, &ci->i_ceph_flags);
3825 } else {
3826 cap->mds_wanted = 0;
3827 flags |= CHECK_CAPS_FLUSH_FORCE;
3828 if (cap == ci->i_auth_cap)
3829 check_caps = 1; /* check auth cap only */
3830 else
3831 check_caps = 2; /* check all caps */
3832 }
3784 }3833 }
3785 3834
3786 if (extra_info->inline_version > 0 &&3835 if (extra_info->inline_version > 0 &&
fs/ceph/super.h+5−0
@@ -709,6 +709,10 @@ static inline struct inode *ceph_find_inode(struct super_block *sb, · 708 unmodified lines
709#define CEPH_I_ASYNC_CREATE_BIT (12) /* async create in flight for this */709#define CEPH_I_ASYNC_CREATE_BIT (12) /* async create in flight for this */
710#define CEPH_I_SHUTDOWN_BIT (13) /* inode is no longer usable */710#define CEPH_I_SHUTDOWN_BIT (13) /* inode is no longer usable */
711#define CEPH_I_ASYNC_CHECK_CAPS_BIT (14) /* check caps after async creating finishes */711#define CEPH_I_ASYNC_CHECK_CAPS_BIT (14) /* check caps after async creating finishes */
712#define CEPH_I_FLUSH_FORCE_BIT (15) /* a revoke's response was deferred;
713 * force a cap message to the MDS once
714 * the deferred work completes
715 */
712 716
713#define CEPH_I_DIR_ORDERED (1 << CEPH_I_DIR_ORDERED_BIT)717#define CEPH_I_DIR_ORDERED (1 << CEPH_I_DIR_ORDERED_BIT)
714#define CEPH_I_FLUSH (1 << CEPH_I_FLUSH_BIT)718#define CEPH_I_FLUSH (1 << CEPH_I_FLUSH_BIT)
@@ -721,6 +725,7 @@ static inline struct inode *ceph_find_inode(struct super_block *sb, · 6 unmodified lines
721#define CEPH_I_ODIRECT (1 << CEPH_I_ODIRECT_BIT)725#define CEPH_I_ODIRECT (1 << CEPH_I_ODIRECT_BIT)
722#define CEPH_I_ASYNC_CREATE (1 << CEPH_I_ASYNC_CREATE_BIT)726#define CEPH_I_ASYNC_CREATE (1 << CEPH_I_ASYNC_CREATE_BIT)
723#define CEPH_I_SHUTDOWN (1 << CEPH_I_SHUTDOWN_BIT)727#define CEPH_I_SHUTDOWN (1 << CEPH_I_SHUTDOWN_BIT)
728#define CEPH_I_FLUSH_FORCE (1 << CEPH_I_FLUSH_FORCE_BIT)
724 729
725/*730/*
726 * Masks of ceph inode work.731 * Masks of ceph inode work.

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t