History

Merge tag 'ceph-for-5.5-rc2' of git://github.com/ceph/ceph-client

Pull ceph fixes from Ilya Dryomov: "A fix to avoid a corner case when scheduling cap reclaim in batches from Xiubo, a patch to add some observability into cap waiters from Jeff and a couple of cleanups" * tag 'ceph-for-5.5-rc2' of git://github.com/ceph/ceph-client: ceph: add more debug info when decoding mdsmap ceph: switch to global cap helper ceph: trigger the reclaim work once there has enough pending caps ceph: show tasks waiting on caps in debugfs caps file ceph: convert int fields in ceph_mount_options to unsigned int

Linus Torvalds committed 6 years ago

Commit 37d4e842 parents ae4b064 da08e1e7 files · +85 −42Browse files

Unified · Split

7 files
  1. fs/ceph/caps.c +27 −14
  2. fs/ceph/debugfs.c +13 −0
  3. fs/ceph/mds_client.c +5 −3
  4. fs/ceph/mds_client.h +9 −0
  5. fs/ceph/mdsmap.c +8 −4
  6. fs/ceph/super.c +15 −13
  7. fs/ceph/super.h +8 −8
fs/ceph/caps.c+27−14
@@ -1011,18 +1011,13 @@ static int __ceph_is_single_caps(struct ceph_inode_info *ci) · 1010 unmodified lines
1011 return rb_first(&ci->i_caps) == rb_last(&ci->i_caps);1011 return rb_first(&ci->i_caps) == rb_last(&ci->i_caps);
1012}1012}
1013 1013
1014static int __ceph_is_any_caps(struct ceph_inode_info *ci)
1015{
1016 return !RB_EMPTY_ROOT(&ci->i_caps);
1017}
1018
1019int ceph_is_any_caps(struct inode *inode)1014int ceph_is_any_caps(struct inode *inode)
1020{1015{
1021 struct ceph_inode_info *ci = ceph_inode(inode);1016 struct ceph_inode_info *ci = ceph_inode(inode);
1022 int ret;1017 int ret;
1023 1018
1024 spin_lock(&ci->i_ceph_lock);1019 spin_lock(&ci->i_ceph_lock);
1025 ret = __ceph_is_any_caps(ci);1020 ret = __ceph_is_any_real_caps(ci);
1026 spin_unlock(&ci->i_ceph_lock);1021 spin_unlock(&ci->i_ceph_lock);
1027 1022
1028 return ret;1023 return ret;
@@ -1099,15 +1094,16 @@ void __ceph_remove_cap(struct ceph_cap *cap, bool queue_release) · 70 unmodified lines
1099 if (removed)1094 if (removed)
1100 ceph_put_cap(mdsc, cap);1095 ceph_put_cap(mdsc, cap);
1101 1096
1102 /* when reconnect denied, we remove session caps forcibly,1097 if (!__ceph_is_any_real_caps(ci)) {
1103 * i_wr_ref can be non-zero. If there are ongoing write,1098 /* when reconnect denied, we remove session caps forcibly,
1104 * keep i_snap_realm.1099 * i_wr_ref can be non-zero. If there are ongoing write,
1105 */1100 * keep i_snap_realm.
1106 if (!__ceph_is_any_caps(ci) && ci->i_wr_ref == 0 && ci->i_snap_realm)1101 */
1107 drop_inode_snap_realm(ci);1102 if (ci->i_wr_ref == 0 && ci->i_snap_realm)
1103 drop_inode_snap_realm(ci);
1108 1104
1109 if (!__ceph_is_any_real_caps(ci))
1110 __cap_delay_cancel(mdsc, ci);1105 __cap_delay_cancel(mdsc, ci);
1106 }
1111}1107}
1112 1108
1113struct cap_msg_args {1109struct cap_msg_args {
@@ -2764,7 +2760,19 @@ int ceph_get_caps(struct file *filp, int need, int want, · 1650 unmodified lines
2764 if (ret == -EAGAIN)2760 if (ret == -EAGAIN)
2765 continue;2761 continue;
2766 if (!ret) {2762 if (!ret) {
2763 struct ceph_mds_client *mdsc = fsc->mdsc;
2764 struct cap_wait cw;
2767 DEFINE_WAIT_FUNC(wait, woken_wake_function);2765 DEFINE_WAIT_FUNC(wait, woken_wake_function);
2766
2767 cw.ino = inode->i_ino;
2768 cw.tgid = current->tgid;
2769 cw.need = need;
2770 cw.want = want;
2771
2772 spin_lock(&mdsc->caps_list_lock);
2773 list_add(&cw.list, &mdsc->cap_wait_list);
2774 spin_unlock(&mdsc->caps_list_lock);
2775
2768 add_wait_queue(&ci->i_cap_wq, &wait);2776 add_wait_queue(&ci->i_cap_wq, &wait);
2769 2777
2770 flags |= NON_BLOCKING;2778 flags |= NON_BLOCKING;
@@ -2778,6 +2786,11 @@ int ceph_get_caps(struct file *filp, int need, int want, · 7 unmodified lines
2778 }2786 }
2779 2787
2780 remove_wait_queue(&ci->i_cap_wq, &wait);2788 remove_wait_queue(&ci->i_cap_wq, &wait);
2789
2790 spin_lock(&mdsc->caps_list_lock);
2791 list_del(&cw.list);
2792 spin_unlock(&mdsc->caps_list_lock);
2793
2781 if (ret == -EAGAIN)2794 if (ret == -EAGAIN)
2782 continue;2795 continue;
2783 }2796 }
@@ -2928,7 +2941,7 @@ void ceph_put_cap_refs(struct ceph_inode_info *ci, int had) · 144 unmodified lines
2928 ci->i_head_snapc = NULL;2941 ci->i_head_snapc = NULL;
2929 }2942 }
2930 /* see comment in __ceph_remove_cap() */2943 /* see comment in __ceph_remove_cap() */
2931 if (!__ceph_is_any_caps(ci) && ci->i_snap_realm)2944 if (!__ceph_is_any_real_caps(ci) && ci->i_snap_realm)
2932 drop_inode_snap_realm(ci);2945 drop_inode_snap_realm(ci);
2933 }2946 }
2934 spin_unlock(&ci->i_ceph_lock);2947 spin_unlock(&ci->i_ceph_lock);
fs/ceph/debugfs.c+13−0
@@ -139,6 +139,7 @@ static int caps_show(struct seq_file *s, void *p) · 138 unmodified lines
139 struct ceph_fs_client *fsc = s->private;139 struct ceph_fs_client *fsc = s->private;
140 struct ceph_mds_client *mdsc = fsc->mdsc;140 struct ceph_mds_client *mdsc = fsc->mdsc;
141 int total, avail, used, reserved, min, i;141 int total, avail, used, reserved, min, i;
142 struct cap_wait *cw;
142 143
143 ceph_reservation_status(fsc, &total, &avail, &used, &reserved, &min);144 ceph_reservation_status(fsc, &total, &avail, &used, &reserved, &min);
144 seq_printf(s, "total\t\t%d\n"145 seq_printf(s, "total\t\t%d\n"
@@ -166,6 +167,18 @@ static int caps_show(struct seq_file *s, void *p) · 21 unmodified lines
166 }167 }
167 mutex_unlock(&mdsc->mutex);168 mutex_unlock(&mdsc->mutex);
168 169
170 seq_printf(s, "\n\nWaiters:\n--------\n");
171 seq_printf(s, "tgid ino need want\n");
172 seq_printf(s, "-----------------------------------------------------\n");
173
174 spin_lock(&mdsc->caps_list_lock);
175 list_for_each_entry(cw, &mdsc->cap_wait_list, list) {
176 seq_printf(s, "%-13d0x%-17lx%-17s%-17s\n", cw->tgid, cw->ino,
177 ceph_cap_string(cw->need),
178 ceph_cap_string(cw->want));
179 }
180 spin_unlock(&mdsc->caps_list_lock);
181
169 return 0;182 return 0;
170}183}
171 184
fs/ceph/mds_client.c+5−3
@@ -2015,7 +2015,7 @@ void ceph_reclaim_caps_nr(struct ceph_mds_client *mdsc, int nr) · 2014 unmodified lines
2015 if (!nr)2015 if (!nr)
2016 return;2016 return;
2017 val = atomic_add_return(nr, &mdsc->cap_reclaim_pending);2017 val = atomic_add_return(nr, &mdsc->cap_reclaim_pending);
2018 if (!(val % CEPH_CAPS_PER_RELEASE)) {2018 if ((val % CEPH_CAPS_PER_RELEASE) < nr) {
2019 atomic_set(&mdsc->cap_reclaim_pending, 0);2019 atomic_set(&mdsc->cap_reclaim_pending, 0);
2020 ceph_queue_cap_reclaim_work(mdsc);2020 ceph_queue_cap_reclaim_work(mdsc);
2021 }2021 }
@@ -2032,12 +2032,13 @@ int ceph_alloc_readdir_reply_buffer(struct ceph_mds_request *req, · 10 unmodified lines
2032 struct ceph_mds_reply_info_parsed *rinfo = &req->r_reply_info;2032 struct ceph_mds_reply_info_parsed *rinfo = &req->r_reply_info;
2033 struct ceph_mount_options *opt = req->r_mdsc->fsc->mount_options;2033 struct ceph_mount_options *opt = req->r_mdsc->fsc->mount_options;
2034 size_t size = sizeof(struct ceph_mds_reply_dir_entry);2034 size_t size = sizeof(struct ceph_mds_reply_dir_entry);
2035 int order, num_entries;2035 unsigned int num_entries;
2036 int order;
2036 2037
2037 spin_lock(&ci->i_ceph_lock);2038 spin_lock(&ci->i_ceph_lock);
2038 num_entries = ci->i_files + ci->i_subdirs;2039 num_entries = ci->i_files + ci->i_subdirs;
2039 spin_unlock(&ci->i_ceph_lock);2040 spin_unlock(&ci->i_ceph_lock);
2040 num_entries = max(num_entries, 1);2041 num_entries = max(num_entries, 1U);
2041 num_entries = min(num_entries, opt->max_readdir);2042 num_entries = min(num_entries, opt->max_readdir);
2042 2043
2043 order = get_order(size * num_entries);2044 order = get_order(size * num_entries);
@@ -4168,6 +4169,7 @@ int ceph_mdsc_init(struct ceph_fs_client *fsc) · 2124 unmodified lines
4168 INIT_DELAYED_WORK(&mdsc->delayed_work, delayed_work);4169 INIT_DELAYED_WORK(&mdsc->delayed_work, delayed_work);
4169 mdsc->last_renew_caps = jiffies;4170 mdsc->last_renew_caps = jiffies;
4170 INIT_LIST_HEAD(&mdsc->cap_delay_list);4171 INIT_LIST_HEAD(&mdsc->cap_delay_list);
4172 INIT_LIST_HEAD(&mdsc->cap_wait_list);
4171 spin_lock_init(&mdsc->cap_delay_lock);4173 spin_lock_init(&mdsc->cap_delay_lock);
4172 INIT_LIST_HEAD(&mdsc->snap_flush_list);4174 INIT_LIST_HEAD(&mdsc->snap_flush_list);
4173 spin_lock_init(&mdsc->snap_flush_lock);4175 spin_lock_init(&mdsc->snap_flush_lock);
fs/ceph/mds_client.h+9−0
@@ -340,6 +340,14 @@ struct ceph_quotarealm_inode { · 339 unmodified lines
340 struct inode *inode;340 struct inode *inode;
341};341};
342 342
343struct cap_wait {
344 struct list_head list;
345 unsigned long ino;
346 pid_t tgid;
347 int need;
348 int want;
349};
350
343/*351/*
344 * mds client state352 * mds client state
345 */353 */
@@ -416,6 +424,7 @@ struct ceph_mds_client { · 70 unmodified lines
416 spinlock_t caps_list_lock;424 spinlock_t caps_list_lock;
417 struct list_head caps_list; /* unused (reserved or425 struct list_head caps_list; /* unused (reserved or
418 unreserved) */426 unreserved) */
427 struct list_head cap_wait_list;
419 int caps_total_count; /* total caps allocated */428 int caps_total_count; /* total caps allocated */
420 int caps_use_count; /* in use */429 int caps_use_count; /* in use */
421 int caps_use_max; /* max used caps */430 int caps_use_max; /* max used caps */
fs/ceph/mdsmap.c+8−4
@@ -158,6 +158,7 @@ struct ceph_mdsmap *ceph_mdsmap_decode(void **p, void *end) · 157 unmodified lines
158 void *pexport_targets = NULL;158 void *pexport_targets = NULL;
159 struct ceph_timespec laggy_since;159 struct ceph_timespec laggy_since;
160 struct ceph_mds_info *info;160 struct ceph_mds_info *info;
161 bool laggy;
161 162
162 ceph_decode_need(p, end, sizeof(u64) + 1, bad);163 ceph_decode_need(p, end, sizeof(u64) + 1, bad);
163 global_id = ceph_decode_64(p);164 global_id = ceph_decode_64(p);
@@ -190,6 +191,7 @@ struct ceph_mdsmap *ceph_mdsmap_decode(void **p, void *end) · 26 unmodified lines
190 if (err)191 if (err)
191 goto corrupt;192 goto corrupt;
192 ceph_decode_copy(p, &laggy_since, sizeof(laggy_since));193 ceph_decode_copy(p, &laggy_since, sizeof(laggy_since));
194 laggy = laggy_since.tv_sec != 0 || laggy_since.tv_nsec != 0;
193 *p += sizeof(u32);195 *p += sizeof(u32);
194 ceph_decode_32_safe(p, end, namelen, bad);196 ceph_decode_32_safe(p, end, namelen, bad);
195 *p += namelen;197 *p += namelen;
@@ -207,10 +209,11 @@ struct ceph_mdsmap *ceph_mdsmap_decode(void **p, void *end) · 11 unmodified lines
207 *p = info_end;209 *p = info_end;
208 }210 }
209 211
210 dout("mdsmap_decode %d/%d %lld mds%d.%d %s %s\n",212 dout("mdsmap_decode %d/%d %lld mds%d.%d %s %s%s\n",
211 i+1, n, global_id, mds, inc,213 i+1, n, global_id, mds, inc,
212 ceph_pr_addr(&addr),214 ceph_pr_addr(&addr),
213 ceph_mds_state_name(state));215 ceph_mds_state_name(state),
216 laggy ? "(laggy)" : "");
214 217
215 if (mds < 0 || state <= 0)218 if (mds < 0 || state <= 0)
216 continue;219 continue;
@@ -230,8 +233,7 @@ struct ceph_mdsmap *ceph_mdsmap_decode(void **p, void *end) · 13 unmodified lines
230 info->global_id = global_id;233 info->global_id = global_id;
231 info->state = state;234 info->state = state;
232 info->addr = addr;235 info->addr = addr;
233 info->laggy = (laggy_since.tv_sec != 0 ||236 info->laggy = laggy;
234 laggy_since.tv_nsec != 0);
235 info->num_export_targets = num_export_targets;237 info->num_export_targets = num_export_targets;
236 if (num_export_targets) {238 if (num_export_targets) {
237 info->export_targets = kcalloc(num_export_targets,239 info->export_targets = kcalloc(num_export_targets,
@@ -355,6 +357,8 @@ struct ceph_mdsmap *ceph_mdsmap_decode(void **p, void *end) · 117 unmodified lines
355 m->m_damaged = false;357 m->m_damaged = false;
356 }358 }
357bad_ext:359bad_ext:
360 dout("mdsmap_decode m_enabled: %d, m_damaged: %d, m_num_laggy: %d\n",
361 !!m->m_enabled, !!m->m_damaged, m->m_num_laggy);
358 *p = end;362 *p = end;
359 dout("mdsmap_decode success epoch %u\n", m->m_epoch);363 dout("mdsmap_decode success epoch %u\n", m->m_epoch);
360 return m;364 return m;
fs/ceph/super.c+15−13
@@ -172,10 +172,10 @@ static const struct fs_parameter_enum ceph_mount_param_enums[] = { · 171 unmodified lines
172static const struct fs_parameter_spec ceph_mount_param_specs[] = {172static const struct fs_parameter_spec ceph_mount_param_specs[] = {
173 fsparam_flag_no ("acl", Opt_acl),173 fsparam_flag_no ("acl", Opt_acl),
174 fsparam_flag_no ("asyncreaddir", Opt_asyncreaddir),174 fsparam_flag_no ("asyncreaddir", Opt_asyncreaddir),
175 fsparam_u32 ("caps_max", Opt_caps_max),175 fsparam_s32 ("caps_max", Opt_caps_max),
176 fsparam_u32 ("caps_wanted_delay_max", Opt_caps_wanted_delay_max),176 fsparam_u32 ("caps_wanted_delay_max", Opt_caps_wanted_delay_max),
177 fsparam_u32 ("caps_wanted_delay_min", Opt_caps_wanted_delay_min),177 fsparam_u32 ("caps_wanted_delay_min", Opt_caps_wanted_delay_min),
178 fsparam_s32 ("write_congestion_kb", Opt_congestion_kb),178 fsparam_u32 ("write_congestion_kb", Opt_congestion_kb),
179 fsparam_flag_no ("copyfrom", Opt_copyfrom),179 fsparam_flag_no ("copyfrom", Opt_copyfrom),
180 fsparam_flag_no ("dcache", Opt_dcache),180 fsparam_flag_no ("dcache", Opt_dcache),
181 fsparam_flag_no ("dirstat", Opt_dirstat),181 fsparam_flag_no ("dirstat", Opt_dirstat),
@@ -187,8 +187,8 @@ static const struct fs_parameter_spec ceph_mount_param_specs[] = { · 5 unmodified lines
187 fsparam_flag_no ("quotadf", Opt_quotadf),187 fsparam_flag_no ("quotadf", Opt_quotadf),
188 fsparam_u32 ("rasize", Opt_rasize),188 fsparam_u32 ("rasize", Opt_rasize),
189 fsparam_flag_no ("rbytes", Opt_rbytes),189 fsparam_flag_no ("rbytes", Opt_rbytes),
190 fsparam_s32 ("readdir_max_bytes", Opt_readdir_max_bytes),190 fsparam_u32 ("readdir_max_bytes", Opt_readdir_max_bytes),
191 fsparam_s32 ("readdir_max_entries", Opt_readdir_max_entries),191 fsparam_u32 ("readdir_max_entries", Opt_readdir_max_entries),
192 fsparam_enum ("recover_session", Opt_recover_session),192 fsparam_enum ("recover_session", Opt_recover_session),
193 fsparam_flag_no ("require_active_mds", Opt_require_active_mds),193 fsparam_flag_no ("require_active_mds", Opt_require_active_mds),
194 fsparam_u32 ("rsize", Opt_rsize),194 fsparam_u32 ("rsize", Opt_rsize),
@@ -328,7 +328,9 @@ static int ceph_parse_mount_param(struct fs_context *fc, · 133 unmodified lines
328 fsopt->caps_wanted_delay_max = result.uint_32;328 fsopt->caps_wanted_delay_max = result.uint_32;
329 break;329 break;
330 case Opt_caps_max:330 case Opt_caps_max:
331 fsopt->caps_max = result.uint_32;331 if (result.int_32 < 0)
332 goto out_of_range;
333 fsopt->caps_max = result.int_32;
332 break;334 break;
333 case Opt_readdir_max_entries:335 case Opt_readdir_max_entries:
334 if (result.uint_32 < 1)336 if (result.uint_32 < 1)
@@ -547,25 +549,25 @@ static int ceph_show_options(struct seq_file *m, struct dentry *root) · 212 unmodified lines
547 seq_show_option(m, "recover_session", "clean");549 seq_show_option(m, "recover_session", "clean");
548 550
549 if (fsopt->wsize != CEPH_MAX_WRITE_SIZE)551 if (fsopt->wsize != CEPH_MAX_WRITE_SIZE)
550 seq_printf(m, ",wsize=%d", fsopt->wsize);552 seq_printf(m, ",wsize=%u", fsopt->wsize);
551 if (fsopt->rsize != CEPH_MAX_READ_SIZE)553 if (fsopt->rsize != CEPH_MAX_READ_SIZE)
552 seq_printf(m, ",rsize=%d", fsopt->rsize);554 seq_printf(m, ",rsize=%u", fsopt->rsize);
553 if (fsopt->rasize != CEPH_RASIZE_DEFAULT)555 if (fsopt->rasize != CEPH_RASIZE_DEFAULT)
554 seq_printf(m, ",rasize=%d", fsopt->rasize);556 seq_printf(m, ",rasize=%u", fsopt->rasize);
555 if (fsopt->congestion_kb != default_congestion_kb())557 if (fsopt->congestion_kb != default_congestion_kb())
556 seq_printf(m, ",write_congestion_kb=%d", fsopt->congestion_kb);558 seq_printf(m, ",write_congestion_kb=%u", fsopt->congestion_kb);
557 if (fsopt->caps_max)559 if (fsopt->caps_max)
558 seq_printf(m, ",caps_max=%d", fsopt->caps_max);560 seq_printf(m, ",caps_max=%d", fsopt->caps_max);
559 if (fsopt->caps_wanted_delay_min != CEPH_CAPS_WANTED_DELAY_MIN_DEFAULT)561 if (fsopt->caps_wanted_delay_min != CEPH_CAPS_WANTED_DELAY_MIN_DEFAULT)
560 seq_printf(m, ",caps_wanted_delay_min=%d",562 seq_printf(m, ",caps_wanted_delay_min=%u",
561 fsopt->caps_wanted_delay_min);563 fsopt->caps_wanted_delay_min);
562 if (fsopt->caps_wanted_delay_max != CEPH_CAPS_WANTED_DELAY_MAX_DEFAULT)564 if (fsopt->caps_wanted_delay_max != CEPH_CAPS_WANTED_DELAY_MAX_DEFAULT)
563 seq_printf(m, ",caps_wanted_delay_max=%d",565 seq_printf(m, ",caps_wanted_delay_max=%u",
564 fsopt->caps_wanted_delay_max);566 fsopt->caps_wanted_delay_max);
565 if (fsopt->max_readdir != CEPH_MAX_READDIR_DEFAULT)567 if (fsopt->max_readdir != CEPH_MAX_READDIR_DEFAULT)
566 seq_printf(m, ",readdir_max_entries=%d", fsopt->max_readdir);568 seq_printf(m, ",readdir_max_entries=%u", fsopt->max_readdir);
567 if (fsopt->max_readdir_bytes != CEPH_MAX_READDIR_BYTES_DEFAULT)569 if (fsopt->max_readdir_bytes != CEPH_MAX_READDIR_BYTES_DEFAULT)
568 seq_printf(m, ",readdir_max_bytes=%d", fsopt->max_readdir_bytes);570 seq_printf(m, ",readdir_max_bytes=%u", fsopt->max_readdir_bytes);
569 if (strcmp(fsopt->snapdir_name, CEPH_SNAPDIRNAME_DEFAULT))571 if (strcmp(fsopt->snapdir_name, CEPH_SNAPDIRNAME_DEFAULT))
570 seq_show_option(m, "snapdirname", fsopt->snapdir_name);572 seq_show_option(m, "snapdirname", fsopt->snapdir_name);
571 573
fs/ceph/super.h+8−8
@@ -73,16 +73,16 @@ · 72 unmodified lines
73#define CEPH_CAPS_WANTED_DELAY_MAX_DEFAULT 60 /* cap release delay */73#define CEPH_CAPS_WANTED_DELAY_MAX_DEFAULT 60 /* cap release delay */
74 74
75struct ceph_mount_options {75struct ceph_mount_options {
76 int flags;76 unsigned int flags;
77 77
78 int wsize; /* max write size */78 unsigned int wsize; /* max write size */
79 int rsize; /* max read size */79 unsigned int rsize; /* max read size */
80 int rasize; /* max readahead */80 unsigned int rasize; /* max readahead */
81 int congestion_kb; /* max writeback in flight */81 unsigned int congestion_kb; /* max writeback in flight */
82 int caps_wanted_delay_min, caps_wanted_delay_max;82 unsigned int caps_wanted_delay_min, caps_wanted_delay_max;
83 int caps_max;83 int caps_max;
84 int max_readdir; /* max readdir result (entires) */84 unsigned int max_readdir; /* max readdir result (entries) */
85 int max_readdir_bytes; /* max readdir result (bytes) */85 unsigned int max_readdir_bytes; /* max readdir result (bytes) */
86 86
87 /*87 /*
88 * everything above this point can be memcmp'd; everything below88 * everything above this point can be memcmp'd; everything below

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t