History

Merge tag 'dlm-6.11' of git://git.kernel.org/pub/scm/linux/kernel/git/teigland/linux-dlm

Pull dlm updates from David Teigland: - New flag DLM_LSFL_SOFTIRQ_SAFE can be set by code using dlm to indicate callbacks can be run from softirq - Change md-cluster to set DLM_LSFL_SOFTIRQ_SAFE - Clean up for previous changes, e.g. unused code and parameters - Remove custom pre-allocation of rsb structs which is unnecessary with kmem caches - Change idr to xarray for lkb structs in use - Change idr to xarray for rsb structs being recovered - Change outdated naming related to internal rsb states - Fix some incorrect add/remove of rsb on scan list - Use rcu to free rsb structs * tag 'dlm-6.11' of git://git.kernel.org/pub/scm/linux/kernel/git/teigland/linux-dlm: dlm: add rcu_barrier before destroy kmem cache dlm: remove DLM_LSFL_SOFTIRQ from exflags fs: dlm: remove unused struct 'dlm_processed_nodes' md-cluster: use DLM_LSFL_SOFTIRQ for dlm_new_lockspace() dlm: implement LSFL_SOFTIRQ_SAFE dlm: introduce DLM_LSFL_SOFTIRQ_SAFE dlm: use LSFL_FS to check for kernel lockspace dlm: use rcu to avoid an extra rsb struct lookup dlm: fix add_scan and del_scan usage dlm: change list and timer names dlm: move recover idr to xarray datastructure dlm: move lkb idr to xarray datastructure dlm: drop own rsb pre allocation mechanism dlm: remove ls_local_handle from struct dlm_ls dlm: remove unused parameter in dlm_midcomms_addr dlm: don't kref_init rsbs created for toss list dlm: remove scand leftovers

Linus Torvalds committed 2 years ago

Commit f097ef02 parents 586f14a 89b019121 files · +584 −562Browse files

Unified · Split

21 files
  1. drivers/md/md-cluster.c +1 −1
  2. fs/dlm/ast.c +111 −61
  3. fs/dlm/ast.h +5 −6
  4. fs/dlm/config.c +1 −1
  5. fs/dlm/debug_fs.c +5 −5
  6. fs/dlm/dlm_internal.h +22 −38
  7. fs/dlm/lock.c +279 −289
  8. fs/dlm/lock.h +3 −4
  9. fs/dlm/lockspace.c +58 −73
  10. fs/dlm/lowcomms.c +1 −7
  11. fs/dlm/lowcomms.h +1 −1
  12. fs/dlm/member.c +1 −1
  13. fs/dlm/memory.c +9 −1
  14. fs/dlm/midcomms.c +2 −2
  15. fs/dlm/midcomms.h +1 −1
  16. fs/dlm/recover.c +40 −38
  17. fs/dlm/recover.h +1 −1
  18. fs/dlm/recoverd.c +7 −7
  19. fs/dlm/user.c +18 −24
  20. include/linux/dlm.h +16 −1
  21. include/uapi/linux/dlm.h +2 −0
drivers/md/md-cluster.c+1−1
@@ -887,7 +887,7 @@ static int join(struct mddev *mddev, int nodes) · 886 unmodified lines
887887 memset(str, 0, 64);
888888 sprintf(str, "%pU", mddev->uuid);
889889 ret = dlm_new_lockspace(str, mddev->bitmap_info.cluster_name,
890− 0, LVB_SIZE, &md_ls_ops, mddev,
890+ DLM_LSFL_SOFTIRQ, LVB_SIZE, &md_ls_ops, mddev,
891891 &ops_rv, &cinfo->lockspace);
892892 if (ret)
893893 goto err;
fs/dlm/ast.c+111−61
@@ -18,35 +18,52 @@ · 17 unmodified lines
1818 #include "user.h"
1919 #include "ast.h"
2020
21−static void dlm_callback_work(struct work_struct *work)
21+static void dlm_run_callback(uint32_t ls_id, uint32_t lkb_id, int8_t mode,
22+ uint32_t flags, uint8_t sb_flags, int sb_status,
23+ struct dlm_lksb *lksb,
24+ void (*astfn)(void *astparam),
25+ void (*bastfn)(void *astparam, int mode),
26+ void *astparam, const char *res_name,
27+ size_t res_length)
2228 {
23− struct dlm_callback *cb = container_of(work, struct dlm_callback, work);
24−
25− if (cb->flags & DLM_CB_BAST) {
26− trace_dlm_bast(cb->ls_id, cb->lkb_id, cb->mode, cb->res_name,
27− cb->res_length);
28− cb->bastfn(cb->astparam, cb->mode);
29− } else if (cb->flags & DLM_CB_CAST) {
30− trace_dlm_ast(cb->ls_id, cb->lkb_id, cb->sb_status,
31− cb->sb_flags, cb->res_name, cb->res_length);
32− cb->lkb_lksb->sb_status = cb->sb_status;
33− cb->lkb_lksb->sb_flags = cb->sb_flags;
34− cb->astfn(cb->astparam);
29+ if (flags & DLM_CB_BAST) {
30+ trace_dlm_bast(ls_id, lkb_id, mode, res_name, res_length);
31+ bastfn(astparam, mode);
32+ } else if (flags & DLM_CB_CAST) {
33+ trace_dlm_ast(ls_id, lkb_id, sb_status, sb_flags, res_name,
34+ res_length);
35+ lksb->sb_status = sb_status;
36+ lksb->sb_flags = sb_flags;
37+ astfn(astparam);
3538 }
39+}
3640
41+static void dlm_do_callback(struct dlm_callback *cb)
42+{
43+ dlm_run_callback(cb->ls_id, cb->lkb_id, cb->mode, cb->flags,
44+ cb->sb_flags, cb->sb_status, cb->lkb_lksb,
45+ cb->astfn, cb->bastfn, cb->astparam,
46+ cb->res_name, cb->res_length);
3747 dlm_free_cb(cb);
3848 }
3949
40−int dlm_queue_lkb_callback(struct dlm_lkb *lkb, uint32_t flags, int mode,
41− int status, uint32_t sbflags,
42− struct dlm_callback **cb)
50+static void dlm_callback_work(struct work_struct *work)
51+{
52+ struct dlm_callback *cb = container_of(work, struct dlm_callback, work);
53+
54+ dlm_do_callback(cb);
55+}
56+
57+bool dlm_may_skip_callback(struct dlm_lkb *lkb, uint32_t flags, int mode,
58+ int status, uint32_t sbflags, int *copy_lvb)
4359 {
4460 struct dlm_rsb *rsb = lkb->lkb_resource;
45− int rv = DLM_ENQUEUE_CALLBACK_SUCCESS;
4661 struct dlm_ls *ls = rsb->res_ls;
47− int copy_lvb = 0;
4862 int prev_mode;
4963
64+ if (copy_lvb)
65+ *copy_lvb = 0;
66+
5067 if (flags & DLM_CB_BAST) {
5168 /* if cb is a bast, it should be skipped if the blocking mode is
5269 * compatible with the last granted mode
@@ -56,7 +73,7 @@ int dlm_queue_lkb_callback(struct dlm_lkb *lkb, uint32_t flags, int mode, · 3 unmodified lines
5673 log_debug(ls, "skip %x bast mode %d for cast mode %d",
5774 lkb->lkb_id, mode,
5875 lkb->lkb_last_cast_cb_mode);
59− goto out;
76+ return true;
6077 }
6178 }
6279
@@ -74,7 +91,7 @@ int dlm_queue_lkb_callback(struct dlm_lkb *lkb, uint32_t flags, int mode, · 11 unmodified lines
7491 (prev_mode > mode && prev_mode > DLM_LOCK_PR)) {
7592 log_debug(ls, "skip %x add bast mode %d for bast mode %d",
7693 lkb->lkb_id, mode, prev_mode);
77− goto out;
94+ return true;
7895 }
7996 }
8097
@@ -85,8 +102,10 @@ int dlm_queue_lkb_callback(struct dlm_lkb *lkb, uint32_t flags, int mode, · 4 unmodified lines
85102 prev_mode = lkb->lkb_last_cast_cb_mode;
86103
87104 if (!status && lkb->lkb_lksb->sb_lvbptr &&
88− dlm_lvb_operations[prev_mode + 1][mode + 1])
89− copy_lvb = 1;
105+ dlm_lvb_operations[prev_mode + 1][mode + 1]) {
106+ if (copy_lvb)
107+ *copy_lvb = 1;
108+ }
90109 }
91110
92111 lkb->lkb_last_cast_cb_mode = mode;
@@ -96,11 +115,19 @@ int dlm_queue_lkb_callback(struct dlm_lkb *lkb, uint32_t flags, int mode, · 3 unmodified lines
96115 lkb->lkb_last_cb_mode = mode;
97116 lkb->lkb_last_cb_flags = flags;
98117
118+ return false;
119+}
120+
121+int dlm_get_cb(struct dlm_lkb *lkb, uint32_t flags, int mode,
122+ int status, uint32_t sbflags,
123+ struct dlm_callback **cb)
124+{
125+ struct dlm_rsb *rsb = lkb->lkb_resource;
126+ struct dlm_ls *ls = rsb->res_ls;
127+
99128 *cb = dlm_allocate_cb();
100− if (!*cb) {
101− rv = DLM_ENQUEUE_CALLBACK_FAILURE;
102− goto out;
103− }
129+ if (WARN_ON_ONCE(!*cb))
130+ return -ENOMEM;
104131
105132 /* for tracing */
106133 (*cb)->lkb_id = lkb->lkb_id;
@@ -112,19 +139,34 @@ int dlm_queue_lkb_callback(struct dlm_lkb *lkb, uint32_t flags, int mode, · 5 unmodified lines
112139 (*cb)->mode = mode;
113140 (*cb)->sb_status = status;
114141 (*cb)->sb_flags = (sbflags & 0x000000FF);
115− (*cb)->copy_lvb = copy_lvb;
116142 (*cb)->lkb_lksb = lkb->lkb_lksb;
117143
118− rv = DLM_ENQUEUE_CALLBACK_NEED_SCHED;
144+ return 0;
145+}
146+
147+static int dlm_get_queue_cb(struct dlm_lkb *lkb, uint32_t flags, int mode,
148+ int status, uint32_t sbflags,
149+ struct dlm_callback **cb)
150+{
151+ int rv;
152+
153+ rv = dlm_get_cb(lkb, flags, mode, status, sbflags, cb);
154+ if (rv)
155+ return rv;
119156
120−out:
121− return rv;
157+ (*cb)->astfn = lkb->lkb_astfn;
158+ (*cb)->bastfn = lkb->lkb_bastfn;
159+ (*cb)->astparam = lkb->lkb_astparam;
160+ INIT_WORK(&(*cb)->work, dlm_callback_work);
161+
162+ return 0;
122163 }
123164
124165 void dlm_add_cb(struct dlm_lkb *lkb, uint32_t flags, int mode, int status,
125− uint32_t sbflags)
166+ uint32_t sbflags)
126167 {
127− struct dlm_ls *ls = lkb->lkb_resource->res_ls;
168+ struct dlm_rsb *rsb = lkb->lkb_resource;
169+ struct dlm_ls *ls = rsb->res_ls;
128170 struct dlm_callback *cb;
129171 int rv;
130172
@@ -133,34 +175,36 @@ void dlm_add_cb(struct dlm_lkb *lkb, uint32_t flags, int mode, int status, · 2 unmodified lines
133175 return;
134176 }
135177
136− rv = dlm_queue_lkb_callback(lkb, flags, mode, status, sbflags,
137− &cb);
138− switch (rv) {
139− case DLM_ENQUEUE_CALLBACK_NEED_SCHED:
140− cb->astfn = lkb->lkb_astfn;
141− cb->bastfn = lkb->lkb_bastfn;
142− cb->astparam = lkb->lkb_astparam;
143− INIT_WORK(&cb->work, dlm_callback_work);
144−
145− spin_lock_bh(&ls->ls_cb_lock);
146− if (test_bit(LSFL_CB_DELAY, &ls->ls_flags))
178+ if (dlm_may_skip_callback(lkb, flags, mode, status, sbflags, NULL))
179+ return;
180+
181+ spin_lock_bh(&ls->ls_cb_lock);
182+ if (test_bit(LSFL_CB_DELAY, &ls->ls_flags)) {
183+ rv = dlm_get_queue_cb(lkb, flags, mode, status, sbflags, &cb);
184+ if (!rv)
147185 list_add(&cb->list, &ls->ls_cb_delay);
148− else
149− queue_work(ls->ls_callback_wq, &cb->work);
150− spin_unlock_bh(&ls->ls_cb_lock);
151− break;
152− case DLM_ENQUEUE_CALLBACK_SUCCESS:
153− break;
154− case DLM_ENQUEUE_CALLBACK_FAILURE:
155− fallthrough;
156− default:
157− WARN_ON_ONCE(1);
158− break;
186+ } else {
187+ if (test_bit(LSFL_SOFTIRQ, &ls->ls_flags)) {
188+ dlm_run_callback(ls->ls_global_id, lkb->lkb_id, mode, flags,
189+ sbflags, status, lkb->lkb_lksb,
190+ lkb->lkb_astfn, lkb->lkb_bastfn,
191+ lkb->lkb_astparam, rsb->res_name,
192+ rsb->res_length);
193+ } else {
194+ rv = dlm_get_queue_cb(lkb, flags, mode, status, sbflags, &cb);
195+ if (!rv)
196+ queue_work(ls->ls_callback_wq, &cb->work);
197+ }
159198 }
199+ spin_unlock_bh(&ls->ls_cb_lock);
160200 }
161201
162202 int dlm_callback_start(struct dlm_ls *ls)
163203 {
204+ if (!test_bit(LSFL_FS, &ls->ls_flags) ||
205+ test_bit(LSFL_SOFTIRQ, &ls->ls_flags))
206+ return 0;
207+
164208 ls->ls_callback_wq = alloc_ordered_workqueue("dlm_callback",
165209 WQ_HIGHPRI | WQ_MEM_RECLAIM);
166210 if (!ls->ls_callback_wq) {
@@ -178,13 +222,15 @@ void dlm_callback_stop(struct dlm_ls *ls) · 11 unmodified lines
178222
179223 void dlm_callback_suspend(struct dlm_ls *ls)
180224 {
181− if (ls->ls_callback_wq) {
182− spin_lock_bh(&ls->ls_cb_lock);
183− set_bit(LSFL_CB_DELAY, &ls->ls_flags);
184− spin_unlock_bh(&ls->ls_cb_lock);
225+ if (!test_bit(LSFL_FS, &ls->ls_flags))
226+ return;
227+
228+ spin_lock_bh(&ls->ls_cb_lock);
229+ set_bit(LSFL_CB_DELAY, &ls->ls_flags);
230+ spin_unlock_bh(&ls->ls_cb_lock);
185231
232+ if (ls->ls_callback_wq)
186233 flush_workqueue(ls->ls_callback_wq);
187− }
188234 }
189235
190236 #define MAX_CB_QUEUE 25
@@ -195,14 +241,18 @@ void dlm_callback_resume(struct dlm_ls *ls) · 4 unmodified lines
195241 int count = 0, sum = 0;
196242 bool empty;
197243
198− if (!ls->ls_callback_wq)
244+ if (!test_bit(LSFL_FS, &ls->ls_flags))
199245 return;
200246
201247 more:
202248 spin_lock_bh(&ls->ls_cb_lock);
203249 list_for_each_entry_safe(cb, safe, &ls->ls_cb_delay, list) {
204250 list_del(&cb->list);
205− queue_work(ls->ls_callback_wq, &cb->work);
251+ if (test_bit(LSFL_SOFTIRQ, &ls->ls_flags))
252+ dlm_do_callback(cb);
253+ else
254+ queue_work(ls->ls_callback_wq, &cb->work);
255+
206256 count++;
207257 if (count == MAX_CB_QUEUE)
208258 break;
fs/dlm/ast.h+5−6
@@ -11,12 +11,11 @@ · 10 unmodified lines
1111 #ifndef __ASTD_DOT_H__
1212 #define __ASTD_DOT_H__
1313
14−#define DLM_ENQUEUE_CALLBACK_NEED_SCHED 1
15−#define DLM_ENQUEUE_CALLBACK_SUCCESS 0
16−#define DLM_ENQUEUE_CALLBACK_FAILURE -1
17−int dlm_queue_lkb_callback(struct dlm_lkb *lkb, uint32_t flags, int mode,
18− int status, uint32_t sbflags,
19− struct dlm_callback **cb);
14+bool dlm_may_skip_callback(struct dlm_lkb *lkb, uint32_t flags, int mode,
15+ int status, uint32_t sbflags, int *copy_lvb);
16+int dlm_get_cb(struct dlm_lkb *lkb, uint32_t flags, int mode,
17+ int status, uint32_t sbflags,
18+ struct dlm_callback **cb);
2019 void dlm_add_cb(struct dlm_lkb *lkb, uint32_t flags, int mode, int status,
2120 uint32_t sbflags);
2221
fs/dlm/config.c+1−1
@@ -672,7 +672,7 @@ static ssize_t comm_addr_store(struct config_item *item, const char *buf, · 671 unmodified lines
672672
673673 memcpy(addr, buf, len);
674674
675− rv = dlm_midcomms_addr(cm->nodeid, addr, len);
675+ rv = dlm_midcomms_addr(cm->nodeid, addr);
676676 if (rv) {
677677 kfree(addr);
678678 return rv;
fs/dlm/debug_fs.c+5−5
@@ -380,7 +380,7 @@ static const struct seq_operations format4_seq_ops; · 379 unmodified lines
380380
381381 static int table_seq_show(struct seq_file *seq, void *iter_ptr)
382382 {
383− struct dlm_rsb *rsb = list_entry(iter_ptr, struct dlm_rsb, res_rsbs_list);
383+ struct dlm_rsb *rsb = list_entry(iter_ptr, struct dlm_rsb, res_slow_list);
384384
385385 if (seq->op == &format1_seq_ops)
386386 print_format1(rsb, seq);
@@ -409,9 +409,9 @@ static void *table_seq_start(struct seq_file *seq, loff_t *pos) · 22 unmodified lines
409409 }
410410
411411 if (seq->op == &format4_seq_ops)
412− list = &ls->ls_toss;
412+ list = &ls->ls_slow_inactive;
413413 else
414− list = &ls->ls_keep;
414+ list = &ls->ls_slow_active;
415415
416416 read_lock_bh(&ls->ls_rsbtbl_lock);
417417 return seq_list_start(list, *pos);
@@ -423,9 +423,9 @@ static void *table_seq_next(struct seq_file *seq, void *iter_ptr, loff_t *pos) · 5 unmodified lines
423423 struct list_head *list;
424424
425425 if (seq->op == &format4_seq_ops)
426− list = &ls->ls_toss;
426+ list = &ls->ls_slow_inactive;
427427 else
428− list = &ls->ls_keep;
428+ list = &ls->ls_slow_active;
429429
430430 return seq_list_next(iter_ptr, list, pos);
431431 }
fs/dlm/dlm_internal.h+22−38
@@ -36,7 +36,7 @@ · 35 unmodified lines
3636 #include <linux/miscdevice.h>
3737 #include <linux/rhashtable.h>
3838 #include <linux/mutex.h>
39−#include <linux/idr.h>
39+#include <linux/xarray.h>
4040 #include <linux/ratelimit.h>
4141 #include <linux/uaccess.h>
4242
@@ -316,26 +316,24 @@ struct dlm_rsb { · 273 unmodified lines
316316 int res_nodeid;
317317 int res_master_nodeid;
318318 int res_dir_nodeid;
319− int res_id; /* for ls_recover_idr */
319+ unsigned long res_id; /* for ls_recover_xa */
320320 uint32_t res_lvbseq;
321321 uint32_t res_hash;
322322 unsigned long res_toss_time;
323323 uint32_t res_first_lkid;
324324 struct list_head res_lookup; /* lkbs waiting on first */
325− union {
326− struct list_head res_hashchain;
327− struct rhash_head res_node; /* rsbtbl */
328− };
325+ struct rhash_head res_node; /* rsbtbl */
329326 struct list_head res_grantqueue;
330327 struct list_head res_convertqueue;
331328 struct list_head res_waitqueue;
332329
333− struct list_head res_rsbs_list;
330+ struct list_head res_slow_list; /* ls_slow_* */
331+ struct list_head res_scan_list;
334332 struct list_head res_root_list; /* used for recovery */
335333 struct list_head res_masters_list; /* used for recovery */
336334 struct list_head res_recover_list; /* used for recovery */
337− struct list_head res_toss_q_list;
338335 int res_recover_locks_count;
336+ struct rcu_head rcu;
339337
340338 char *res_lvbptr;
341339 char res_name[DLM_RESNAME_MAXLEN+1];
@@ -368,7 +366,8 @@ enum rsb_flags { · 26 unmodified lines
368366 RSB_RECOVER_CONVERT,
369367 RSB_RECOVER_GRANT,
370368 RSB_RECOVER_LVB_INVAL,
371− RSB_TOSS,
369+ RSB_INACTIVE,
370+ RSB_HASHED, /* set while rsb is on ls_rsbtbl */
372371 };
373372
374373 static inline void rsb_set_flag(struct dlm_rsb *r, enum rsb_flags flag)
@@ -559,16 +558,8 @@ struct rcom_lock { · 184 unmodified lines
559558 char rl_lvb[];
560559 };
561560
562−/*
563− * The max number of resources per rsbtbl bucket that shrink will attempt
564− * to remove in each iteration.
565− */
566−
567−#define DLM_REMOVE_NAMES_MAX 8
568−
569561 struct dlm_ls {
570562 struct list_head ls_list; /* list of lockspaces */
571− dlm_lockspace_t *ls_local_handle;
572563 uint32_t ls_global_id; /* global unique lockspace ID */
573564 uint32_t ls_generation;
574565 uint32_t ls_exflags;
@@ -578,26 +569,21 @@ struct dlm_ls { · 3 unmodified lines
578569 wait_queue_head_t ls_count_wait;
579570 int ls_create_count; /* create/release refcount */
580571 unsigned long ls_flags; /* LSFL_ */
581− unsigned long ls_scan_time;
582572 struct kobject ls_kobj;
583573
584− struct idr ls_lkbidr;
585− rwlock_t ls_lkbidr_lock;
574+ struct xarray ls_lkbxa;
575+ rwlock_t ls_lkbxa_lock;
586576
577+ /* an rsb is on rsbtl for primary locking functions,
578+ and on a slow list for recovery/dump iteration */
587579 struct rhashtable ls_rsbtbl;
588− rwlock_t ls_rsbtbl_lock;
580+ rwlock_t ls_rsbtbl_lock; /* for ls_rsbtbl and ls_slow */
581+ struct list_head ls_slow_inactive; /* to iterate rsbtbl */
582+ struct list_head ls_slow_active; /* to iterate rsbtbl */
589583
590− struct list_head ls_toss;
591− struct list_head ls_keep;
592−
593− struct timer_list ls_timer;
594− /* this queue is ordered according the
595− * absolute res_toss_time jiffies time
596− * to mod_timer() with the first element
597− * if necessary.
598− */
599− struct list_head ls_toss_q;
600− spinlock_t ls_toss_q_lock;
584+ struct timer_list ls_scan_timer; /* based on first scan_list rsb toss_time */
585+ struct list_head ls_scan_list; /* rsbs ordered by res_toss_time */
586+ spinlock_t ls_scan_lock;
601587
602588 spinlock_t ls_waiters_lock;
603589 struct list_head ls_waiters; /* lkbs needing a reply */
@@ -605,10 +591,6 @@ struct dlm_ls { · 1 unmodified line
605591 spinlock_t ls_orphans_lock;
606592 struct list_head ls_orphans;
607593
608− spinlock_t ls_new_rsb_spin;
609− int ls_new_rsb_count;
610− struct list_head ls_new_rsb; /* new rsb structs */
611−
612594 struct list_head ls_nodes; /* current nodes in ls */
613595 struct list_head ls_nodes_gone; /* dead node list, recovery */
614596 int ls_num_nodes; /* number of nodes in ls */
@@ -664,8 +646,8 @@ struct dlm_ls { · 49 unmodified lines
664646 struct list_head ls_recover_list;
665647 spinlock_t ls_recover_list_lock;
666648 int ls_recover_list_count;
667− struct idr ls_recover_idr;
668− spinlock_t ls_recover_idr_lock;
649+ struct xarray ls_recover_xa;
650+ spinlock_t ls_recover_xa_lock;
669651 wait_queue_head_t ls_wait_general;
670652 wait_queue_head_t ls_recover_lock_wait;
671653 spinlock_t ls_clear_proc_locks;
@@ -716,6 +698,8 @@ struct dlm_ls { · 44 unmodified lines
716698 #define LSFL_CB_DELAY 9
717699 #define LSFL_NODIR 10
718700 #define LSFL_RECV_MSG_BLOCKED 11
701+#define LSFL_FS 12
702+#define LSFL_SOFTIRQ 13
719703
720704 #define DLM_PROC_FLAGS_CLOSING 1
721705 #define DLM_PROC_FLAGS_COMPAT 2
fs/dlm/lock.c+279−289
@@ -89,7 +89,7 @@ static void __receive_convert_reply(struct dlm_rsb *r, struct dlm_lkb *lkb, · 88 unmodified lines
8989 const struct dlm_message *ms, bool local);
9090 static int receive_extralen(const struct dlm_message *ms);
9191 static void do_purge(struct dlm_ls *ls, int nodeid, int pid);
92−static void toss_rsb(struct kref *kref);
92+static void deactivate_rsb(struct kref *kref);
9393
9494 /*
9595 * Lock compatibilty matrix - thanks Steve
@@ -330,8 +330,8 @@ static inline unsigned long rsb_toss_jiffies(void) · 234 unmodified lines
330330
331331 static inline void hold_rsb(struct dlm_rsb *r)
332332 {
333− /* rsbs in toss state never get referenced */
334− WARN_ON(rsb_flag(r, RSB_TOSS));
333+ /* inactive rsbs are not ref counted */
334+ WARN_ON(rsb_flag(r, RSB_INACTIVE));
335335 kref_get(&r->res_ref);
336336 }
337337
@@ -370,15 +370,12 @@ static inline int dlm_kref_put_write_lock_bh(struct kref *kref, · 32 unmodified lines
370370 return 0;
371371 }
372372
373−/* When all references to the rsb are gone it's transferred to
374− the tossed list for later disposal. */
375−
376373 static void put_rsb(struct dlm_rsb *r)
377374 {
378375 struct dlm_ls *ls = r->res_ls;
379376 int rv;
380377
381− rv = dlm_kref_put_write_lock_bh(&r->res_ref, toss_rsb,
378+ rv = dlm_kref_put_write_lock_bh(&r->res_ref, deactivate_rsb,
382379 &ls->ls_rsbtbl_lock);
383380 if (rv)
384381 write_unlock_bh(&ls->ls_rsbtbl_lock);
@@ -389,82 +386,54 @@ void dlm_put_rsb(struct dlm_rsb *r) · 4 unmodified lines
389386 put_rsb(r);
390387 }
391388
392−static int pre_rsb_struct(struct dlm_ls *ls)
393−{
394− struct dlm_rsb *r1, *r2;
395− int count = 0;
396−
397− spin_lock_bh(&ls->ls_new_rsb_spin);
398− if (ls->ls_new_rsb_count > dlm_config.ci_new_rsb_count / 2) {
399− spin_unlock_bh(&ls->ls_new_rsb_spin);
400− return 0;
401− }
402− spin_unlock_bh(&ls->ls_new_rsb_spin);
403−
404− r1 = dlm_allocate_rsb(ls);
405− r2 = dlm_allocate_rsb(ls);
406−
407− spin_lock_bh(&ls->ls_new_rsb_spin);
408− if (r1) {
409− list_add(&r1->res_hashchain, &ls->ls_new_rsb);
410− ls->ls_new_rsb_count++;
411− }
412− if (r2) {
413− list_add(&r2->res_hashchain, &ls->ls_new_rsb);
414− ls->ls_new_rsb_count++;
415− }
416− count = ls->ls_new_rsb_count;
417− spin_unlock_bh(&ls->ls_new_rsb_spin);
418−
419− if (!count)
420− return -ENOMEM;
421− return 0;
422−}
423−
424389 /* connected with timer_delete_sync() in dlm_ls_stop() to stop
425390 * new timers when recovery is triggered and don't run them
426− * again until a dlm_timer_resume() tries it again.
391+ * again until a resume_scan_timer() tries it again.
427392 */
428−static void __rsb_mod_timer(struct dlm_ls *ls, unsigned long jiffies)
393+static void enable_scan_timer(struct dlm_ls *ls, unsigned long jiffies)
429394 {
430395 if (!dlm_locking_stopped(ls))
431− mod_timer(&ls->ls_timer, jiffies);
396+ mod_timer(&ls->ls_scan_timer, jiffies);
432397 }
433398
434399 /* This function tries to resume the timer callback if a rsb
435− * is on the toss list and no timer is pending. It might that
400+ * is on the scan list and no timer is pending. It might that
436401 * the first entry is on currently executed as timer callback
437402 * but we don't care if a timer queued up again and does
438403 * nothing. Should be a rare case.
439404 */
440−void dlm_timer_resume(struct dlm_ls *ls)
405+void resume_scan_timer(struct dlm_ls *ls)
441406 {
442407 struct dlm_rsb *r;
443408
444− spin_lock_bh(&ls->ls_toss_q_lock);
445− r = list_first_entry_or_null(&ls->ls_toss_q, struct dlm_rsb,
446− res_toss_q_list);
447− if (r && !timer_pending(&ls->ls_timer))
448− __rsb_mod_timer(ls, r->res_toss_time);
449− spin_unlock_bh(&ls->ls_toss_q_lock);
409+ spin_lock_bh(&ls->ls_scan_lock);
410+ r = list_first_entry_or_null(&ls->ls_scan_list, struct dlm_rsb,
411+ res_scan_list);
412+ if (r && !timer_pending(&ls->ls_scan_timer))
413+ enable_scan_timer(ls, r->res_toss_time);
414+ spin_unlock_bh(&ls->ls_scan_lock);
450415 }
451416
452−/* ls_rsbtbl_lock must be held and being sure the rsb is in toss state */
453−static void rsb_delete_toss_timer(struct dlm_ls *ls, struct dlm_rsb *r)
417+/* ls_rsbtbl_lock must be held */
418+
419+static void del_scan(struct dlm_ls *ls, struct dlm_rsb *r)
454420 {
455421 struct dlm_rsb *first;
456422
457− spin_lock_bh(&ls->ls_toss_q_lock);
423+ /* active rsbs should never be on the scan list */
424+ WARN_ON(!rsb_flag(r, RSB_INACTIVE));
425+
426+ spin_lock_bh(&ls->ls_scan_lock);
458427 r->res_toss_time = 0;
459428
460429 /* if the rsb is not queued do nothing */
461− if (list_empty(&r->res_toss_q_list))
430+ if (list_empty(&r->res_scan_list))
462431 goto out;
463432
464433 /* get the first element before delete */
465− first = list_first_entry(&ls->ls_toss_q, struct dlm_rsb,
466− res_toss_q_list);
467− list_del_init(&r->res_toss_q_list);
434+ first = list_first_entry(&ls->ls_scan_list, struct dlm_rsb,
435+ res_scan_list);
436+ list_del_init(&r->res_scan_list);
468437 /* check if the first element was the rsb we deleted */
469438 if (first == r) {
470439 /* try to get the new first element, if the list
@@ -474,70 +443,59 @@ static void rsb_delete_toss_timer(struct dlm_ls *ls, struct dlm_rsb *r) · 3 unmodified lines
474443 * if the list isn't empty and a new first element got
475444 * in place, set the new timer expire time.
476445 */
477− first = list_first_entry_or_null(&ls->ls_toss_q, struct dlm_rsb,
478− res_toss_q_list);
446+ first = list_first_entry_or_null(&ls->ls_scan_list, struct dlm_rsb,
447+ res_scan_list);
479448 if (!first)
480− timer_delete(&ls->ls_timer);
449+ timer_delete(&ls->ls_scan_timer);
481450 else
482− __rsb_mod_timer(ls, first->res_toss_time);
451+ enable_scan_timer(ls, first->res_toss_time);
483452 }
484453
485454 out:
486− spin_unlock_bh(&ls->ls_toss_q_lock);
455+ spin_unlock_bh(&ls->ls_scan_lock);
487456 }
488457
489−/* Caller must held ls_rsbtbl_lock and need to be called every time
490− * when either the rsb enters toss state or the toss state changes
491− * the dir/master nodeid.
492− */
493−static void rsb_mod_timer(struct dlm_ls *ls, struct dlm_rsb *r)
458+static void add_scan(struct dlm_ls *ls, struct dlm_rsb *r)
494459 {
495460 int our_nodeid = dlm_our_nodeid();
496461 struct dlm_rsb *first;
497462
498− /* If we're the directory record for this rsb, and
499− * we're not the master of it, then we need to wait
500− * for the master node to send us a dir remove for
501− * before removing the dir record.
502− */
503− if (!dlm_no_directory(ls) &&
504− (r->res_master_nodeid != our_nodeid) &&
505− (dlm_dir_nodeid(r) == our_nodeid)) {
506− rsb_delete_toss_timer(ls, r);
507− return;
508− }
463+ /* A dir record for a remote master rsb should never be on the scan list. */
464+ WARN_ON(!dlm_no_directory(ls) &&
465+ (r->res_master_nodeid != our_nodeid) &&
466+ (dlm_dir_nodeid(r) == our_nodeid));
467+
468+ /* An active rsb should never be on the scan list. */
469+ WARN_ON(!rsb_flag(r, RSB_INACTIVE));
509470
510− spin_lock_bh(&ls->ls_toss_q_lock);
471+ /* An rsb should not already be on the scan list. */
472+ WARN_ON(!list_empty(&r->res_scan_list));
473+
474+ spin_lock_bh(&ls->ls_scan_lock);
511475 /* set the new rsb absolute expire time in the rsb */
512476 r->res_toss_time = rsb_toss_jiffies();
513− if (list_empty(&ls->ls_toss_q)) {
477+ if (list_empty(&ls->ls_scan_list)) {
514478 /* if the queue is empty add the element and it's
515479 * our new expire time
516480 */
517− list_add_tail(&r->res_toss_q_list, &ls->ls_toss_q);
518− __rsb_mod_timer(ls, r->res_toss_time);
481+ list_add_tail(&r->res_scan_list, &ls->ls_scan_list);
482+ enable_scan_timer(ls, r->res_toss_time);
519483 } else {
520− /* check if the rsb was already queued, if so delete
521− * it from the toss queue
522− */
523− if (!list_empty(&r->res_toss_q_list))
524− list_del(&r->res_toss_q_list);
525−
526484 /* try to get the maybe new first element and then add
527485 * to this rsb with the oldest expire time to the end
528486 * of the queue. If the list was empty before this
529487 * rsb expire time is our next expiration if it wasn't
530488 * the now new first elemet is our new expiration time
531489 */
532− first = list_first_entry_or_null(&ls->ls_toss_q, struct dlm_rsb,
533− res_toss_q_list);
534− list_add_tail(&r->res_toss_q_list, &ls->ls_toss_q);
490+ first = list_first_entry_or_null(&ls->ls_scan_list, struct dlm_rsb,
491+ res_scan_list);
492+ list_add_tail(&r->res_scan_list, &ls->ls_scan_list);
535493 if (!first)
536− __rsb_mod_timer(ls, r->res_toss_time);
494+ enable_scan_timer(ls, r->res_toss_time);
537495 else
538− __rsb_mod_timer(ls, first->res_toss_time);
496+ enable_scan_timer(ls, first->res_toss_time);
539497 }
540− spin_unlock_bh(&ls->ls_toss_q_lock);
498+ spin_unlock_bh(&ls->ls_scan_lock);
541499 }
542500
543501 /* if we hit contention we do in 250 ms a retry to trylock.
@@ -547,9 +505,11 @@ static void rsb_mod_timer(struct dlm_ls *ls, struct dlm_rsb *r) · 3 unmodified lines
547505 */
548506 #define DLM_TOSS_TIMER_RETRY (jiffies + msecs_to_jiffies(250))
549507
550−void dlm_rsb_toss_timer(struct timer_list *timer)
508+/* Called by lockspace scan_timer to free unused rsb's. */
509+
510+void dlm_rsb_scan(struct timer_list *timer)
551511 {
552− struct dlm_ls *ls = from_timer(ls, timer, ls_timer);
512+ struct dlm_ls *ls = from_timer(ls, timer, ls_scan_timer);
553513 int our_nodeid = dlm_our_nodeid();
554514 struct dlm_rsb *r;
555515 int rv;
@@ -557,76 +517,63 @@ void dlm_rsb_toss_timer(struct timer_list *timer) · 1 unmodified line
557517 while (1) {
558518 /* interrupting point to leave iteration when
559519 * recovery waits for timer_delete_sync(), recovery
560− * will take care to delete everything in toss queue.
520+ * will take care to delete everything in scan list.
561521 */
562522 if (dlm_locking_stopped(ls))
563523 break;
564524
565− rv = spin_trylock(&ls->ls_toss_q_lock);
525+ rv = spin_trylock(&ls->ls_scan_lock);
566526 if (!rv) {
567527 /* rearm again try timer */
568− __rsb_mod_timer(ls, DLM_TOSS_TIMER_RETRY);
528+ enable_scan_timer(ls, DLM_TOSS_TIMER_RETRY);
569529 break;
570530 }
571531
572− r = list_first_entry_or_null(&ls->ls_toss_q, struct dlm_rsb,
573− res_toss_q_list);
532+ r = list_first_entry_or_null(&ls->ls_scan_list, struct dlm_rsb,
533+ res_scan_list);
574534 if (!r) {
575− /* nothing to do anymore next rsb queue will
576− * set next mod_timer() expire.
577− */
578− spin_unlock(&ls->ls_toss_q_lock);
535+ /* the next add_scan will enable the timer again */
536+ spin_unlock(&ls->ls_scan_lock);
579537 break;
580538 }
581539
582− /* test if the first rsb isn't expired yet, if
583− * so we stop freeing rsb from toss queue as
584− * the order in queue is ascending to the
585− * absolute res_toss_time jiffies
540+ /*
541+ * If the first rsb is not yet expired, then stop because the
542+ * list is sorted with nearest expiration first.
586543 */
587544 if (time_before(jiffies, r->res_toss_time)) {
588545 /* rearm with the next rsb to expire in the future */
589− __rsb_mod_timer(ls, r->res_toss_time);
590− spin_unlock(&ls->ls_toss_q_lock);
546+ enable_scan_timer(ls, r->res_toss_time);
547+ spin_unlock(&ls->ls_scan_lock);
591548 break;
592549 }
593550
594551 /* in find_rsb_dir/nodir there is a reverse order of this
595552 * lock, however this is only a trylock if we hit some
596553 * possible contention we try it again.
597− *
598− * This lock synchronized while holding ls_toss_q_lock
599− * synchronize everything that rsb_delete_toss_timer()
600− * or rsb_mod_timer() can't run after this timer callback
601− * deletes the rsb from the ls_toss_q. Whereas the other
602− * holders have always a priority to run as this is only
603− * a caching handling and the other holders might to put
604− * this rsb out of the toss state.
605554 */
606555 rv = write_trylock(&ls->ls_rsbtbl_lock);
607556 if (!rv) {
608− spin_unlock(&ls->ls_toss_q_lock);
557+ spin_unlock(&ls->ls_scan_lock);
609558 /* rearm again try timer */
610− __rsb_mod_timer(ls, DLM_TOSS_TIMER_RETRY);
559+ enable_scan_timer(ls, DLM_TOSS_TIMER_RETRY);
611560 break;
612561 }
613562
614− list_del(&r->res_rsbs_list);
563+ list_del(&r->res_slow_list);
615564 rhashtable_remove_fast(&ls->ls_rsbtbl, &r->res_node,
616565 dlm_rhash_rsb_params);
566+ rsb_clear_flag(r, RSB_HASHED);
617567
618− /* not necessary to held the ls_rsbtbl_lock when
619− * calling send_remove()
620− */
568+ /* ls_rsbtbl_lock is not needed when calling send_remove() */
621569 write_unlock(&ls->ls_rsbtbl_lock);
622570
623− /* remove the rsb out of the toss queue its gone
624− * drom DLM now
625− */
626− list_del_init(&r->res_toss_q_list);
627− spin_unlock(&ls->ls_toss_q_lock);
571+ list_del_init(&r->res_scan_list);
572+ spin_unlock(&ls->ls_scan_lock);
628573
629− /* no rsb in this state should ever run a timer */
574+ /* An rsb that is a dir record for a remote master rsb
575+ * cannot be removed, and should not have a timer enabled.
576+ */
630577 WARN_ON(!dlm_no_directory(ls) &&
631578 (r->res_master_nodeid != our_nodeid) &&
632579 (dlm_dir_nodeid(r) == our_nodeid));
@@ -640,7 +587,7 @@ void dlm_rsb_toss_timer(struct timer_list *timer) · 7 unmodified lines
640587 (dlm_dir_nodeid(r) != our_nodeid))
641588 send_remove(r);
642589
643− free_toss_rsb(r);
590+ free_inactive_rsb(r);
644591 }
645592 }
646593
@@ -652,22 +599,10 @@ static int get_rsb_struct(struct dlm_ls *ls, const void *name, int len, · 5 unmodified lines
652599 struct dlm_rsb **r_ret)
653600 {
654601 struct dlm_rsb *r;
655− int count;
656602
657− spin_lock_bh(&ls->ls_new_rsb_spin);
658− if (list_empty(&ls->ls_new_rsb)) {
659− count = ls->ls_new_rsb_count;
660− spin_unlock_bh(&ls->ls_new_rsb_spin);
661− log_debug(ls, "find_rsb retry %d %d %s",
662− count, dlm_config.ci_new_rsb_count,
663− (const char *)name);
664− return -EAGAIN;
665− }
666−
667− r = list_first_entry(&ls->ls_new_rsb, struct dlm_rsb, res_hashchain);
668− list_del(&r->res_hashchain);
669− ls->ls_new_rsb_count--;
670− spin_unlock_bh(&ls->ls_new_rsb_spin);
603+ r = dlm_allocate_rsb(ls);
604+ if (!r)
605+ return -ENOMEM;
671606
672607 r->res_ls = ls;
673608 r->res_length = len;
@@ -679,7 +614,7 @@ static int get_rsb_struct(struct dlm_ls *ls, const void *name, int len, · 5 unmodified lines
679614 INIT_LIST_HEAD(&r->res_convertqueue);
680615 INIT_LIST_HEAD(&r->res_waitqueue);
681616 INIT_LIST_HEAD(&r->res_root_list);
682− INIT_LIST_HEAD(&r->res_toss_q_list);
617+ INIT_LIST_HEAD(&r->res_scan_list);
683618 INIT_LIST_HEAD(&r->res_recover_list);
684619 INIT_LIST_HEAD(&r->res_masters_list);
685620
@@ -702,8 +637,14 @@ int dlm_search_rsb_tree(struct rhashtable *rhash, const void *name, int len, · 16 unmodified lines
702637
703638 static int rsb_insert(struct dlm_rsb *rsb, struct rhashtable *rhash)
704639 {
705− return rhashtable_insert_fast(rhash, &rsb->res_node,
706− dlm_rhash_rsb_params);
640+ int rv;
641+
642+ rv = rhashtable_insert_fast(rhash, &rsb->res_node,
643+ dlm_rhash_rsb_params);
644+ if (!rv)
645+ rsb_set_flag(rsb, RSB_HASHED);
646+
647+ return rv;
707648 }
708649
709650 /*
@@ -733,7 +674,7 @@ static int rsb_insert(struct dlm_rsb *rsb, struct rhashtable *rhash) · 23 unmodified lines
733674 * So, if the given rsb is on the toss list, it is moved to the keep list
734675 * before being returned.
735676 *
736− * toss_rsb() happens when all local usage of the rsb is done, i.e. no
677+ * deactivate_rsb() happens when all local usage of the rsb is done, i.e. no
737678 * more refcounts exist, so the rsb is moved from the keep list to the
738679 * toss list.
739680 *
@@ -781,9 +722,9 @@ static int find_rsb_dir(struct dlm_ls *ls, const void *name, int len, · 41 unmodified lines
781722 *
782723 * If someone sends us a request, we are the dir node, and we do
783724 * not find the rsb anywhere, then recreate it. This happens if
784− * someone sends us a request after we have removed/freed an rsb
785− * from our toss list. (They sent a request instead of lookup
786− * because they are using an rsb from their toss list.)
725+ * someone sends us a request after we have removed/freed an rsb.
726+ * (They sent a request instead of lookup because they are using
727+ * an rsb taken from their scan list.)
787728 */
788729
789730 if (from_local || from_dir ||
@@ -792,15 +733,8 @@ static int find_rsb_dir(struct dlm_ls *ls, const void *name, int len, · 2 unmodified lines
792733 }
793734
794735 retry:
795− if (create) {
796− error = pre_rsb_struct(ls);
797− if (error < 0)
798− goto out;
799− }
800736
801− retry_lookup:
802−
803− /* check if the rsb is in keep state under read lock - likely path */
737+ /* check if the rsb is active under read lock - likely path */
804738 read_lock_bh(&ls->ls_rsbtbl_lock);
805739 error = dlm_search_rsb_tree(&ls->ls_rsbtbl, name, len, &r);
806740 if (error) {
@@ -812,9 +746,9 @@ static int find_rsb_dir(struct dlm_ls *ls, const void *name, int len, · 5 unmodified lines
812746 * rsb is active, so we can't check master_nodeid without lock_rsb.
813747 */
814748
815− if (rsb_flag(r, RSB_TOSS)) {
749+ if (rsb_flag(r, RSB_INACTIVE)) {
816750 read_unlock_bh(&ls->ls_rsbtbl_lock);
817− goto do_toss;
751+ goto do_inactive;
818752 }
819753
820754 kref_get(&r->res_ref);
@@ -822,17 +756,29 @@ static int find_rsb_dir(struct dlm_ls *ls, const void *name, int len, · 1 unmodified line
822756 goto out;
823757
824758
825− do_toss:
759+ do_inactive:
826760 write_lock_bh(&ls->ls_rsbtbl_lock);
827761
828− /* retry lookup under write lock to see if its still in toss state
829− * if not it's in keep state and we relookup - unlikely path.
762+ /*
763+ * The expectation here is that the rsb will have HASHED and
764+ * INACTIVE flags set, and that the rsb can be moved from
765+ * inactive back to active again. However, between releasing
766+ * the read lock and acquiring the write lock, this rsb could
767+ * have been removed from rsbtbl, and had HASHED cleared, to
768+ * be freed. To deal with this case, we would normally need
769+ * to repeat dlm_search_rsb_tree while holding the write lock,
770+ * but rcu allows us to simply check the HASHED flag, because
771+ * the rcu read lock means the rsb will not be freed yet.
772+ * If the HASHED flag is not set, then the rsb is being freed,
773+ * so we add a new rsb struct. If the HASHED flag is set,
774+ * and INACTIVE is not set, it means another thread has
775+ * made the rsb active, as we're expecting to do here, and
776+ * we just repeat the lookup (this will be very unlikely.)
830777 */
831− error = dlm_search_rsb_tree(&ls->ls_rsbtbl, name, len, &r);
832− if (!error) {
833− if (!rsb_flag(r, RSB_TOSS)) {
778+ if (rsb_flag(r, RSB_HASHED)) {
779+ if (!rsb_flag(r, RSB_INACTIVE)) {
834780 write_unlock_bh(&ls->ls_rsbtbl_lock);
835− goto retry_lookup;
781+ goto retry;
836782 }
837783 } else {
838784 write_unlock_bh(&ls->ls_rsbtbl_lock);
@@ -842,14 +788,14 @@ static int find_rsb_dir(struct dlm_ls *ls, const void *name, int len, · 3 unmodified lines
842788 /*
843789 * rsb found inactive (master_nodeid may be out of date unless
844790 * we are the dir_nodeid or were the master) No other thread
845− * is using this rsb because it's on the toss list, so we can
791+ * is using this rsb because it's inactive, so we can
846792 * look at or update res_master_nodeid without lock_rsb.
847793 */
848794
849795 if ((r->res_master_nodeid != our_nodeid) && from_other) {
850796 /* our rsb was not master, and another node (not the dir node)
851797 has sent us a request */
852− log_debug(ls, "find_rsb toss from_other %d master %d dir %d %s",
798+ log_debug(ls, "find_rsb inactive from_other %d master %d dir %d %s",
853799 from_nodeid, r->res_master_nodeid, dir_nodeid,
854800 r->res_name);
855801 write_unlock_bh(&ls->ls_rsbtbl_lock);
@@ -859,7 +805,7 @@ static int find_rsb_dir(struct dlm_ls *ls, const void *name, int len, · 3 unmodified lines
859805
860806 if ((r->res_master_nodeid != our_nodeid) && from_dir) {
861807 /* don't think this should ever happen */
862− log_error(ls, "find_rsb toss from_dir %d master %d",
808+ log_error(ls, "find_rsb inactive from_dir %d master %d",
863809 from_nodeid, r->res_master_nodeid);
864810 dlm_print_rsb(r);
865811 /* fix it and go on */
@@ -876,14 +822,12 @@ static int find_rsb_dir(struct dlm_ls *ls, const void *name, int len, · 10 unmodified lines
876822 r->res_first_lkid = 0;
877823 }
878824
879− list_move(&r->res_rsbs_list, &ls->ls_keep);
880− rsb_clear_flag(r, RSB_TOSS);
881− /* rsb got out of toss state, it becomes alive again
882− * and we reinit the reference counter that is only
883− * valid for keep state rsbs
884− */
885− kref_init(&r->res_ref);
886− rsb_delete_toss_timer(ls, r);
825+ /* A dir record will not be on the scan list. */
826+ if (r->res_dir_nodeid != our_nodeid)
827+ del_scan(ls, r);
828+ list_move(&r->res_slow_list, &ls->ls_slow_active);
829+ rsb_clear_flag(r, RSB_INACTIVE);
830+ kref_init(&r->res_ref); /* ref is now used in active state */
887831 write_unlock_bh(&ls->ls_rsbtbl_lock);
888832
889833 goto out;
@@ -898,9 +842,7 @@ static int find_rsb_dir(struct dlm_ls *ls, const void *name, int len, · 8 unmodified lines
898842 goto out;
899843
900844 error = get_rsb_struct(ls, name, len, &r);
901− if (error == -EAGAIN)
902− goto retry;
903− if (error)
845+ if (WARN_ON_ONCE(error))
904846 goto out;
905847
906848 r->res_hash = hash;
@@ -952,9 +894,9 @@ static int find_rsb_dir(struct dlm_ls *ls, const void *name, int len, · 45 unmodified lines
952894 */
953895 write_unlock_bh(&ls->ls_rsbtbl_lock);
954896 dlm_free_rsb(r);
955− goto retry_lookup;
897+ goto retry;
956898 } else if (!error) {
957− list_add(&r->res_rsbs_list, &ls->ls_keep);
899+ list_add(&r->res_slow_list, &ls->ls_slow_active);
958900 }
959901 write_unlock_bh(&ls->ls_rsbtbl_lock);
960902 out:
@@ -976,13 +918,8 @@ static int find_rsb_nodir(struct dlm_ls *ls, const void *name, int len, · 15 unmodified lines
976918 int error;
977919
978920 retry:
979− error = pre_rsb_struct(ls);
980− if (error < 0)
981− goto out;
982921
983− retry_lookup:
984−
985− /* check if the rsb is in keep state under read lock - likely path */
922+ /* check if the rsb is in active state under read lock - likely path */
986923 read_lock_bh(&ls->ls_rsbtbl_lock);
987924 error = dlm_search_rsb_tree(&ls->ls_rsbtbl, name, len, &r);
988925 if (error) {
@@ -990,9 +927,9 @@ static int find_rsb_nodir(struct dlm_ls *ls, const void *name, int len, · 1 unmodified line
990927 goto do_new;
991928 }
992929
993− if (rsb_flag(r, RSB_TOSS)) {
930+ if (rsb_flag(r, RSB_INACTIVE)) {
994931 read_unlock_bh(&ls->ls_rsbtbl_lock);
995− goto do_toss;
932+ goto do_inactive;
996933 }
997934
998935 /*
@@ -1005,17 +942,14 @@ static int find_rsb_nodir(struct dlm_ls *ls, const void *name, int len, · 6 unmodified lines
1005942 goto out;
1006943
1007944
1008− do_toss:
945+ do_inactive:
1009946 write_lock_bh(&ls->ls_rsbtbl_lock);
1010947
1011− /* retry lookup under write lock to see if its still in toss state
1012− * if not it's in keep state and we relookup - unlikely path.
1013− */
1014− error = dlm_search_rsb_tree(&ls->ls_rsbtbl, name, len, &r);
1015− if (!error) {
1016− if (!rsb_flag(r, RSB_TOSS)) {
948+ /* See comment in find_rsb_dir. */
949+ if (rsb_flag(r, RSB_HASHED)) {
950+ if (!rsb_flag(r, RSB_INACTIVE)) {
1017951 write_unlock_bh(&ls->ls_rsbtbl_lock);
1018− goto retry_lookup;
952+ goto retry;
1019953 }
1020954 } else {
1021955 write_unlock_bh(&ls->ls_rsbtbl_lock);
@@ -1025,14 +959,14 @@ static int find_rsb_nodir(struct dlm_ls *ls, const void *name, int len, · 3 unmodified lines
1025959
1026960 /*
1027961 * rsb found inactive. No other thread is using this rsb because
1028− * it's on the toss list, so we can look at or update
1029− * res_master_nodeid without lock_rsb.
962+ * it's inactive, so we can look at or update res_master_nodeid
963+ * without lock_rsb.
1030964 */
1031965
1032966 if (!recover && (r->res_master_nodeid != our_nodeid) && from_nodeid) {
1033967 /* our rsb is not master, and another node has sent us a
1034968 request; this should never happen */
1035− log_error(ls, "find_rsb toss from_nodeid %d master %d dir %d",
969+ log_error(ls, "find_rsb inactive from_nodeid %d master %d dir %d",
1036970 from_nodeid, r->res_master_nodeid, dir_nodeid);
1037971 dlm_print_rsb(r);
1038972 write_unlock_bh(&ls->ls_rsbtbl_lock);
@@ -1044,21 +978,17 @@ static int find_rsb_nodir(struct dlm_ls *ls, const void *name, int len, · 5 unmodified lines
1044978 (dir_nodeid == our_nodeid)) {
1045979 /* our rsb is not master, and we are dir; may as well fix it;
1046980 this should never happen */
1047− log_error(ls, "find_rsb toss our %d master %d dir %d",
981+ log_error(ls, "find_rsb inactive our %d master %d dir %d",
1048982 our_nodeid, r->res_master_nodeid, dir_nodeid);
1049983 dlm_print_rsb(r);
1050984 r->res_master_nodeid = our_nodeid;
1051985 r->res_nodeid = 0;
1052986 }
1053987
1054− list_move(&r->res_rsbs_list, &ls->ls_keep);
1055− rsb_clear_flag(r, RSB_TOSS);
1056− /* rsb got out of toss state, it becomes alive again
1057− * and we reinit the reference counter that is only
1058− * valid for keep state rsbs
1059− */
988+ list_move(&r->res_slow_list, &ls->ls_slow_active);
989+ rsb_clear_flag(r, RSB_INACTIVE);
1060990 kref_init(&r->res_ref);
1061− rsb_delete_toss_timer(ls, r);
991+ del_scan(ls, r);
1062992 write_unlock_bh(&ls->ls_rsbtbl_lock);
1063993
1064994 goto out;
@@ -1070,10 +1000,7 @@ static int find_rsb_nodir(struct dlm_ls *ls, const void *name, int len, · 5 unmodified lines
10701000 */
10711001
10721002 error = get_rsb_struct(ls, name, len, &r);
1073− if (error == -EAGAIN) {
1074− goto retry;
1075− }
1076− if (error)
1003+ if (WARN_ON_ONCE(error))
10771004 goto out;
10781005
10791006 r->res_hash = hash;
@@ -1090,9 +1017,9 @@ static int find_rsb_nodir(struct dlm_ls *ls, const void *name, int len, · 10 unmodified lines
10901017 */
10911018 write_unlock_bh(&ls->ls_rsbtbl_lock);
10921019 dlm_free_rsb(r);
1093− goto retry_lookup;
1020+ goto retry;
10941021 } else if (!error) {
1095− list_add(&r->res_rsbs_list, &ls->ls_keep);
1022+ list_add(&r->res_slow_list, &ls->ls_slow_active);
10961023 }
10971024 write_unlock_bh(&ls->ls_rsbtbl_lock);
10981025
@@ -1101,12 +1028,54 @@ static int find_rsb_nodir(struct dlm_ls *ls, const void *name, int len, · 2 unmodified lines
11011028 return error;
11021029 }
11031030
1031+/*
1032+ * rsb rcu usage
1033+ *
1034+ * While rcu read lock is held, the rsb cannot be freed,
1035+ * which allows a lookup optimization.
1036+ *
1037+ * Two threads are accessing the same rsb concurrently,
1038+ * the first (A) is trying to use the rsb, the second (B)
1039+ * is trying to free the rsb.
1040+ *
1041+ * thread A thread B
1042+ * (trying to use rsb) (trying to free rsb)
1043+ *
1044+ * A1. rcu read lock
1045+ * A2. rsbtbl read lock
1046+ * A3. look up rsb in rsbtbl
1047+ * A4. rsbtbl read unlock
1048+ * B1. rsbtbl write lock
1049+ * B2. look up rsb in rsbtbl
1050+ * B3. remove rsb from rsbtbl
1051+ * B4. clear rsb HASHED flag
1052+ * B5. rsbtbl write unlock
1053+ * B6. begin freeing rsb using rcu...
1054+ *
1055+ * (rsb is inactive, so try to make it active again)
1056+ * A5. read rsb HASHED flag (safe because rsb is not freed yet)
1057+ * A6. the rsb HASHED flag is not set, which it means the rsb
1058+ * is being removed from rsbtbl and freed, so don't use it.
1059+ * A7. rcu read unlock
1060+ *
1061+ * B7. ...finish freeing rsb using rcu
1062+ * A8. create a new rsb
1063+ *
1064+ * Without the rcu optimization, steps A5-8 would need to do
1065+ * an extra rsbtbl lookup:
1066+ * A5. rsbtbl write lock
1067+ * A6. look up rsb in rsbtbl, not found
1068+ * A7. rsbtbl write unlock
1069+ * A8. create a new rsb
1070+ */
1071+
11041072 static int find_rsb(struct dlm_ls *ls, const void *name, int len,
11051073 int from_nodeid, unsigned int flags,
11061074 struct dlm_rsb **r_ret)
11071075 {
11081076 int dir_nodeid;
11091077 uint32_t hash;
1078+ int rv;
11101079
11111080 if (len > DLM_RESNAME_MAXLEN)
11121081 return -EINVAL;
@@ -1114,12 +1083,15 @@ static int find_rsb(struct dlm_ls *ls, const void *name, int len, · 1 unmodified line
11141083 hash = jhash(name, len, 0);
11151084 dir_nodeid = dlm_hash2nodeid(ls, hash);
11161085
1086+ rcu_read_lock();
11171087 if (dlm_no_directory(ls))
1118− return find_rsb_nodir(ls, name, len, hash, dir_nodeid,
1088+ rv = find_rsb_nodir(ls, name, len, hash, dir_nodeid,
11191089 from_nodeid, flags, r_ret);
11201090 else
1121− return find_rsb_dir(ls, name, len, hash, dir_nodeid,
1091+ rv = find_rsb_dir(ls, name, len, hash, dir_nodeid,
11221092 from_nodeid, flags, r_ret);
1093+ rcu_read_unlock();
1094+ return rv;
11231095 }
11241096
11251097 /* we have received a request and found that res_master_nodeid != our_nodeid,
@@ -1166,7 +1138,7 @@ static int validate_master_nodeid(struct dlm_ls *ls, struct dlm_rsb *r, · 40 unmodified lines
11661138 }
11671139
11681140 static void __dlm_master_lookup(struct dlm_ls *ls, struct dlm_rsb *r, int our_nodeid,
1169− int from_nodeid, bool toss_list, unsigned int flags,
1141+ int from_nodeid, bool is_inactive, unsigned int flags,
11701142 int *r_nodeid, int *result)
11711143 {
11721144 int fix_master = (flags & DLM_LU_RECOVER_MASTER);
@@ -1190,9 +1162,9 @@ static void __dlm_master_lookup(struct dlm_ls *ls, struct dlm_rsb *r, int our_no · 17 unmodified lines
11901162 r->res_nodeid = from_nodeid;
11911163 rsb_set_flag(r, RSB_NEW_MASTER);
11921164
1193− if (toss_list) {
1194− /* I don't think we should ever find it on toss list. */
1195− log_error(ls, "%s fix_master on toss", __func__);
1165+ if (is_inactive) {
1166+ /* I don't think we should ever find it inactive. */
1167+ log_error(ls, "%s fix_master inactive", __func__);
11961168 dlm_dump_rsb(r);
11971169 }
11981170 }
@@ -1232,7 +1204,7 @@ static void __dlm_master_lookup(struct dlm_ls *ls, struct dlm_rsb *r, int our_no · 33 unmodified lines
12321204 if (!from_master && !fix_master &&
12331205 (r->res_master_nodeid == from_nodeid)) {
12341206 /* this can happen when the master sends remove, the dir node
1235− * finds the rsb on the keep list and ignores the remove,
1207+ * finds the rsb on the active list and ignores the remove,
12361208 * and the former master sends a lookup
12371209 */
12381210
@@ -1276,8 +1248,8 @@ static void __dlm_master_lookup(struct dlm_ls *ls, struct dlm_rsb *r, int our_no · 37 unmodified lines
12761248 * . dlm_master_lookup RECOVER_MASTER (fix_master 1, from_master 0)
12771249 */
12781250
1279−int dlm_master_lookup(struct dlm_ls *ls, int from_nodeid, const char *name,
1280− int len, unsigned int flags, int *r_nodeid, int *result)
1251+static int _dlm_master_lookup(struct dlm_ls *ls, int from_nodeid, const char *name,
1252+ int len, unsigned int flags, int *r_nodeid, int *result)
12811253 {
12821254 struct dlm_rsb *r = NULL;
12831255 uint32_t hash;
@@ -1304,19 +1276,14 @@ int dlm_master_lookup(struct dlm_ls *ls, int from_nodeid, const char *name, · 20 unmodified lines
13041276 }
13051277
13061278 retry:
1307− error = pre_rsb_struct(ls);
1308− if (error < 0)
1309− return error;
1310−
1311− retry_lookup:
13121279
1313− /* check if the rsb is in keep state under read lock - likely path */
1280+ /* check if the rsb is active under read lock - likely path */
13141281 read_lock_bh(&ls->ls_rsbtbl_lock);
13151282 error = dlm_search_rsb_tree(&ls->ls_rsbtbl, name, len, &r);
13161283 if (!error) {
1317− if (rsb_flag(r, RSB_TOSS)) {
1284+ if (rsb_flag(r, RSB_INACTIVE)) {
13181285 read_unlock_bh(&ls->ls_rsbtbl_lock);
1319− goto do_toss;
1286+ goto do_inactive;
13201287 }
13211288
13221289 /* because the rsb is active, we need to lock_rsb before
@@ -1340,53 +1307,48 @@ int dlm_master_lookup(struct dlm_ls *ls, int from_nodeid, const char *name, · 17 unmodified lines
13401307 goto not_found;
13411308 }
13421309
1343− do_toss:
1310+ do_inactive:
13441311 /* unlikely path - relookup under write */
13451312 write_lock_bh(&ls->ls_rsbtbl_lock);
13461313
1347− /* rsb_mod_timer() requires to held ls_rsbtbl_lock in write lock
1348− * check if the rsb is still in toss state, if not relookup
1349− */
13501314 error = dlm_search_rsb_tree(&ls->ls_rsbtbl, name, len, &r);
13511315 if (!error) {
1352− if (!rsb_flag(r, RSB_TOSS)) {
1316+ if (!rsb_flag(r, RSB_INACTIVE)) {
13531317 write_unlock_bh(&ls->ls_rsbtbl_lock);
13541318 /* something as changed, very unlikely but
13551319 * try again
13561320 */
1357− goto retry_lookup;
1321+ goto retry;
13581322 }
13591323 } else {
13601324 write_unlock_bh(&ls->ls_rsbtbl_lock);
13611325 goto not_found;
13621326 }
13631327
1364− /* because the rsb is inactive (on toss list), it's not refcounted
1365− * and lock_rsb is not used, but is protected by the rsbtbl lock
1366− */
1328+ /* because the rsb is inactive, it's not refcounted and lock_rsb
1329+ is not used, but is protected by the rsbtbl lock */
13671330
13681331 __dlm_master_lookup(ls, r, our_nodeid, from_nodeid, true, flags,
13691332 r_nodeid, result);
13701333
1371− rsb_mod_timer(ls, r);
1372− /* the rsb was inactive (on toss list) */
1334+ /* A dir record rsb should never be on scan list. */
1335+ /* Try to fix this with del_scan? */
1336+ WARN_ON(!list_empty(&r->res_scan_list));
1337+
13731338 write_unlock_bh(&ls->ls_rsbtbl_lock);
13741339
13751340 return 0;
13761341
13771342 not_found:
13781343 error = get_rsb_struct(ls, name, len, &r);
1379− if (error == -EAGAIN)
1380− goto retry;
1381− if (error)
1344+ if (WARN_ON_ONCE(error))
13821345 goto out;
13831346
13841347 r->res_hash = hash;
13851348 r->res_dir_nodeid = our_nodeid;
13861349 r->res_master_nodeid = from_nodeid;
13871350 r->res_nodeid = from_nodeid;
1388− kref_init(&r->res_ref);
1389− rsb_set_flag(r, RSB_TOSS);
1351+ rsb_set_flag(r, RSB_INACTIVE);
13901352
13911353 write_lock_bh(&ls->ls_rsbtbl_lock);
13921354 error = rsb_insert(r, &ls->ls_rsbtbl);
@@ -1396,7 +1358,7 @@ int dlm_master_lookup(struct dlm_ls *ls, int from_nodeid, const char *name, · 3 unmodified lines
13961358 */
13971359 write_unlock_bh(&ls->ls_rsbtbl_lock);
13981360 dlm_free_rsb(r);
1399− goto retry_lookup;
1361+ goto retry;
14001362 } else if (error) {
14011363 write_unlock_bh(&ls->ls_rsbtbl_lock);
14021364 /* should never happen */
@@ -1404,8 +1366,7 @@ int dlm_master_lookup(struct dlm_ls *ls, int from_nodeid, const char *name, · 1 unmodified line
14041366 goto retry;
14051367 }
14061368
1407− list_add(&r->res_rsbs_list, &ls->ls_toss);
1408− rsb_mod_timer(ls, r);
1369+ list_add(&r->res_slow_list, &ls->ls_slow_inactive);
14091370 write_unlock_bh(&ls->ls_rsbtbl_lock);
14101371
14111372 if (result)
@@ -1415,12 +1376,22 @@ int dlm_master_lookup(struct dlm_ls *ls, int from_nodeid, const char *name, · 3 unmodified lines
14151376 return error;
14161377 }
14171378
1379+int dlm_master_lookup(struct dlm_ls *ls, int from_nodeid, const char *name,
1380+ int len, unsigned int flags, int *r_nodeid, int *result)
1381+{
1382+ int rv;
1383+ rcu_read_lock();
1384+ rv = _dlm_master_lookup(ls, from_nodeid, name, len, flags, r_nodeid, result);
1385+ rcu_read_unlock();
1386+ return rv;
1387+}
1388+
14181389 static void dlm_dump_rsb_hash(struct dlm_ls *ls, uint32_t hash)
14191390 {
14201391 struct dlm_rsb *r;
14211392
14221393 read_lock_bh(&ls->ls_rsbtbl_lock);
1423− list_for_each_entry(r, &ls->ls_keep, res_rsbs_list) {
1394+ list_for_each_entry(r, &ls->ls_slow_active, res_slow_list) {
14241395 if (r->res_hash == hash)
14251396 dlm_dump_rsb(r);
14261397 }
@@ -1442,15 +1413,28 @@ void dlm_dump_rsb_name(struct dlm_ls *ls, const char *name, int len) · 15 unmodified lines
14421413 read_unlock_bh(&ls->ls_rsbtbl_lock);
14431414 }
14441415
1445−static void toss_rsb(struct kref *kref)
1416+static void deactivate_rsb(struct kref *kref)
14461417 {
14471418 struct dlm_rsb *r = container_of(kref, struct dlm_rsb, res_ref);
14481419 struct dlm_ls *ls = r->res_ls;
1420+ int our_nodeid = dlm_our_nodeid();
14491421
14501422 DLM_ASSERT(list_empty(&r->res_root_list), dlm_print_rsb(r););
1451− rsb_set_flag(r, RSB_TOSS);
1452− list_move(&r->res_rsbs_list, &ls->ls_toss);
1453− rsb_mod_timer(ls, r);
1423+ rsb_set_flag(r, RSB_INACTIVE);
1424+ list_move(&r->res_slow_list, &ls->ls_slow_inactive);
1425+
1426+ /*
1427+ * When the rsb becomes unused:
1428+ * - If it's not a dir record for a remote master rsb,
1429+ * then it is put on the scan list to be freed.
1430+ * - If it's a dir record for a remote master rsb,
1431+ * then it is kept in the inactive state until
1432+ * receive_remove() from the master node.
1433+ */
1434+ if (!dlm_no_directory(ls) &&
1435+ (r->res_master_nodeid != our_nodeid) &&
1436+ (dlm_dir_nodeid(r) != our_nodeid))
1437+ add_scan(ls, r);
14541438
14551439 if (r->res_lvbptr) {
14561440 dlm_free_lvb(r->res_lvbptr);
@@ -1464,22 +1448,22 @@ static void unhold_rsb(struct dlm_rsb *r) · 7 unmodified lines
14641448 {
14651449 int rv;
14661450
1467− /* rsbs in toss state never get referenced */
1468− WARN_ON(rsb_flag(r, RSB_TOSS));
1469− rv = kref_put(&r->res_ref, toss_rsb);
1451+ /* inactive rsbs are not ref counted */
1452+ WARN_ON(rsb_flag(r, RSB_INACTIVE));
1453+ rv = kref_put(&r->res_ref, deactivate_rsb);
14701454 DLM_ASSERT(!rv, dlm_dump_rsb(r););
14711455 }
14721456
1473−void free_toss_rsb(struct dlm_rsb *r)
1457+void free_inactive_rsb(struct dlm_rsb *r)
14741458 {
1475− WARN_ON_ONCE(!rsb_flag(r, RSB_TOSS));
1459+ WARN_ON_ONCE(!rsb_flag(r, RSB_INACTIVE));
14761460
14771461 DLM_ASSERT(list_empty(&r->res_lookup), dlm_dump_rsb(r););
14781462 DLM_ASSERT(list_empty(&r->res_grantqueue), dlm_dump_rsb(r););
14791463 DLM_ASSERT(list_empty(&r->res_convertqueue), dlm_dump_rsb(r););
14801464 DLM_ASSERT(list_empty(&r->res_waitqueue), dlm_dump_rsb(r););
14811465 DLM_ASSERT(list_empty(&r->res_root_list), dlm_dump_rsb(r););
1482− DLM_ASSERT(list_empty(&r->res_toss_q_list), dlm_dump_rsb(r););
1466+ DLM_ASSERT(list_empty(&r->res_scan_list), dlm_dump_rsb(r););
14831467 DLM_ASSERT(list_empty(&r->res_recover_list), dlm_dump_rsb(r););
14841468 DLM_ASSERT(list_empty(&r->res_masters_list), dlm_dump_rsb(r););
14851469
@@ -1504,11 +1488,15 @@ static void detach_lkb(struct dlm_lkb *lkb) · 18 unmodified lines
15041488 }
15051489
15061490 static int _create_lkb(struct dlm_ls *ls, struct dlm_lkb **lkb_ret,
1507− int start, int end)
1491+ unsigned long start, unsigned long end)
15081492 {
1493+ struct xa_limit limit;
15091494 struct dlm_lkb *lkb;
15101495 int rv;
15111496
1497+ limit.max = end;
1498+ limit.min = start;
1499+
15121500 lkb = dlm_allocate_lkb(ls);
15131501 if (!lkb)
15141502 return -ENOMEM;
@@ -1522,14 +1510,12 @@ static int _create_lkb(struct dlm_ls *ls, struct dlm_lkb **lkb_ret, · 7 unmodified lines
15221510 INIT_LIST_HEAD(&lkb->lkb_ownqueue);
15231511 INIT_LIST_HEAD(&lkb->lkb_rsb_lookup);
15241512
1525− write_lock_bh(&ls->ls_lkbidr_lock);
1526− rv = idr_alloc(&ls->ls_lkbidr, lkb, start, end, GFP_NOWAIT);
1527− if (rv >= 0)
1528− lkb->lkb_id = rv;
1529− write_unlock_bh(&ls->ls_lkbidr_lock);
1513+ write_lock_bh(&ls->ls_lkbxa_lock);
1514+ rv = xa_alloc(&ls->ls_lkbxa, &lkb->lkb_id, lkb, limit, GFP_ATOMIC);
1515+ write_unlock_bh(&ls->ls_lkbxa_lock);
15301516
15311517 if (rv < 0) {
1532− log_error(ls, "create_lkb idr error %d", rv);
1518+ log_error(ls, "create_lkb xa error %d", rv);
15331519 dlm_free_lkb(lkb);
15341520 return rv;
15351521 }
@@ -1540,18 +1526,18 @@ static int _create_lkb(struct dlm_ls *ls, struct dlm_lkb **lkb_ret, · 4 unmodified lines
15401526
15411527 static int create_lkb(struct dlm_ls *ls, struct dlm_lkb **lkb_ret)
15421528 {
1543− return _create_lkb(ls, lkb_ret, 1, 0);
1529+ return _create_lkb(ls, lkb_ret, 1, ULONG_MAX);
15441530 }
15451531
15461532 static int find_lkb(struct dlm_ls *ls, uint32_t lkid, struct dlm_lkb **lkb_ret)
15471533 {
15481534 struct dlm_lkb *lkb;
15491535
1550− read_lock_bh(&ls->ls_lkbidr_lock);
1551− lkb = idr_find(&ls->ls_lkbidr, lkid);
1536+ read_lock_bh(&ls->ls_lkbxa_lock);
1537+ lkb = xa_load(&ls->ls_lkbxa, lkid);
15521538 if (lkb)
15531539 kref_get(&lkb->lkb_ref);
1554− read_unlock_bh(&ls->ls_lkbidr_lock);
1540+ read_unlock_bh(&ls->ls_lkbxa_lock);
15551541
15561542 *lkb_ret = lkb;
15571543 return lkb ? 0 : -ENOENT;
@@ -1576,10 +1562,10 @@ static int __put_lkb(struct dlm_ls *ls, struct dlm_lkb *lkb) · 18 unmodified lines
15761562 int rv;
15771563
15781564 rv = dlm_kref_put_write_lock_bh(&lkb->lkb_ref, kill_lkb,
1579− &ls->ls_lkbidr_lock);
1565+ &ls->ls_lkbxa_lock);
15801566 if (rv) {
1581− idr_remove(&ls->ls_lkbidr, lkid);
1582− write_unlock_bh(&ls->ls_lkbidr_lock);
1567+ xa_erase(&ls->ls_lkbxa, lkid);
1568+ write_unlock_bh(&ls->ls_lkbxa_lock);
15831569
15841570 detach_lkb(lkb);
15851571
@@ -4323,8 +4309,9 @@ static void receive_remove(struct dlm_ls *ls, const struct dlm_message *ms) · 2737 unmodified lines
43234309 return;
43244310 }
43254311
4326− /* Look for name in rsb toss state, if it's there, kill it.
4327− * If it's in non toss state, it's being used, and we should ignore this
4312+ /*
4313+ * Look for inactive rsb, if it's there, free it.
4314+ * If the rsb is active, it's being used, and we should ignore this
43284315 * message. This is an expected race between the dir node sending a
43294316 * request to the master node at the same time as the master node sends
43304317 * a remove to the dir node. The resolution to that race is for the
@@ -4347,16 +4334,18 @@ static void receive_remove(struct dlm_ls *ls, const struct dlm_message *ms) · 16 unmodified lines
43474334 return;
43484335 }
43494336
4350− if (!rsb_flag(r, RSB_TOSS)) {
4337+ if (!rsb_flag(r, RSB_INACTIVE)) {
43514338 if (r->res_master_nodeid != from_nodeid) {
43524339 /* should not happen */
4353− log_error(ls, "receive_remove keep from %d master %d",
4340+ log_error(ls, "receive_remove on active rsb from %d master %d",
43544341 from_nodeid, r->res_master_nodeid);
43554342 dlm_print_rsb(r);
43564343 write_unlock_bh(&ls->ls_rsbtbl_lock);
43574344 return;
43584345 }
43594346
4347+ /* Ignore the remove message, see race comment above. */
4348+
43604349 log_debug(ls, "receive_remove from %d master %d first %x %s",
43614350 from_nodeid, r->res_master_nodeid, r->res_first_lkid,
43624351 name);
@@ -4365,19 +4354,20 @@ static void receive_remove(struct dlm_ls *ls, const struct dlm_message *ms) · 2 unmodified lines
43654354 }
43664355
43674356 if (r->res_master_nodeid != from_nodeid) {
4368− log_error(ls, "receive_remove toss from %d master %d",
4357+ log_error(ls, "receive_remove inactive from %d master %d",
43694358 from_nodeid, r->res_master_nodeid);
43704359 dlm_print_rsb(r);
43714360 write_unlock_bh(&ls->ls_rsbtbl_lock);
43724361 return;
43734362 }
43744363
4375− list_del(&r->res_rsbs_list);
4364+ list_del(&r->res_slow_list);
43764365 rhashtable_remove_fast(&ls->ls_rsbtbl, &r->res_node,
43774366 dlm_rhash_rsb_params);
4367+ rsb_clear_flag(r, RSB_HASHED);
43784368 write_unlock_bh(&ls->ls_rsbtbl_lock);
43794369
4380− free_toss_rsb(r);
4370+ free_inactive_rsb(r);
43814371 }
43824372
43834373 static void receive_purge(struct dlm_ls *ls, const struct dlm_message *ms)
@@ -5444,7 +5434,7 @@ static struct dlm_rsb *find_grant_rsb(struct dlm_ls *ls) · 1060 unmodified lines
54445434 struct dlm_rsb *r;
54455435
54465436 read_lock_bh(&ls->ls_rsbtbl_lock);
5447− list_for_each_entry(r, &ls->ls_keep, res_rsbs_list) {
5437+ list_for_each_entry(r, &ls->ls_slow_active, res_slow_list) {
54485438 if (!rsb_flag(r, RSB_RECOVER_GRANT))
54495439 continue;
54505440 if (!is_master(r)) {
fs/dlm/lock.h+3−4
@@ -11,7 +11,6 @@ · 10 unmodified lines
1111 #ifndef __LOCK_DOT_H__
1212 #define __LOCK_DOT_H__
1313
14−void dlm_rsb_toss_timer(struct timer_list *timer);
1514 void dlm_dump_rsb(struct dlm_rsb *r);
1615 void dlm_dump_rsb_name(struct dlm_ls *ls, const char *name, int len);
1716 void dlm_print_lkb(struct dlm_lkb *lkb);
@@ -19,15 +18,15 @@ void dlm_receive_message_saved(struct dlm_ls *ls, const struct dlm_message *ms, · 1 unmodified line
1918 uint32_t saved_seq);
2019 void dlm_receive_buffer(const union dlm_packet *p, int nodeid);
2120 int dlm_modes_compat(int mode1, int mode2);
22−void free_toss_rsb(struct dlm_rsb *r);
21+void free_inactive_rsb(struct dlm_rsb *r);
2322 void dlm_put_rsb(struct dlm_rsb *r);
2423 void dlm_hold_rsb(struct dlm_rsb *r);
2524 int dlm_put_lkb(struct dlm_lkb *lkb);
26−void dlm_scan_rsbs(struct dlm_ls *ls);
2725 int dlm_lock_recovery_try(struct dlm_ls *ls);
2826 void dlm_lock_recovery(struct dlm_ls *ls);
2927 void dlm_unlock_recovery(struct dlm_ls *ls);
30−void dlm_timer_resume(struct dlm_ls *ls);
28+void dlm_rsb_scan(struct timer_list *timer);
29+void resume_scan_timer(struct dlm_ls *ls);
3130
3231 int dlm_master_lookup(struct dlm_ls *ls, int from_nodeid, const char *name,
3332 int len, unsigned int flags, int *r_nodeid, int *result);
fs/dlm/lockspace.c+58−73
@@ -38,7 +38,7 @@ static ssize_t dlm_control_store(struct dlm_ls *ls, const char *buf, size_t len) · 37 unmodified lines
3838
3939 if (rc)
4040 return rc;
41− ls = dlm_find_lockspace_local(ls->ls_local_handle);
41+ ls = dlm_find_lockspace_local(ls);
4242 if (!ls)
4343 return -EINVAL;
4444
@@ -265,18 +265,9 @@ struct dlm_ls *dlm_find_lockspace_global(uint32_t id) · 220 unmodified lines
265265
266266 struct dlm_ls *dlm_find_lockspace_local(dlm_lockspace_t *lockspace)
267267 {
268− struct dlm_ls *ls;
268+ struct dlm_ls *ls = lockspace;
269269
270− spin_lock_bh(&lslist_lock);
271− list_for_each_entry(ls, &lslist, ls_list) {
272− if (ls->ls_local_handle == lockspace) {
273− atomic_inc(&ls->ls_count);
274− goto out;
275− }
276− }
277− ls = NULL;
278− out:
279− spin_unlock_bh(&lslist_lock);
270+ atomic_inc(&ls->ls_count);
280271 return ls;
281272 }
282273
@@ -410,37 +401,37 @@ static int new_lockspace(const char *name, const char *cluster, · 127 unmodified lines
410401 atomic_set(&ls->ls_count, 0);
411402 init_waitqueue_head(&ls->ls_count_wait);
412403 ls->ls_flags = 0;
413− ls->ls_scan_time = jiffies;
414404
415405 if (ops && dlm_config.ci_recover_callbacks) {
416406 ls->ls_ops = ops;
417407 ls->ls_ops_arg = ops_arg;
418408 }
419409
410+ if (flags & DLM_LSFL_SOFTIRQ)
411+ set_bit(LSFL_SOFTIRQ, &ls->ls_flags);
412+
420413 /* ls_exflags are forced to match among nodes, and we don't
421414 * need to require all nodes to have some flags set
422415 */
423− ls->ls_exflags = (flags & ~(DLM_LSFL_FS | DLM_LSFL_NEWEXCL));
416+ ls->ls_exflags = (flags & ~(DLM_LSFL_FS | DLM_LSFL_NEWEXCL |
417+ DLM_LSFL_SOFTIRQ));
424418
425− INIT_LIST_HEAD(&ls->ls_toss);
426− INIT_LIST_HEAD(&ls->ls_keep);
419+ INIT_LIST_HEAD(&ls->ls_slow_inactive);
420+ INIT_LIST_HEAD(&ls->ls_slow_active);
427421 rwlock_init(&ls->ls_rsbtbl_lock);
428422
429423 error = rhashtable_init(&ls->ls_rsbtbl, &dlm_rhash_rsb_params);
430424 if (error)
431425 goto out_lsfree;
432426
433− idr_init(&ls->ls_lkbidr);
434− rwlock_init(&ls->ls_lkbidr_lock);
427+ xa_init_flags(&ls->ls_lkbxa, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_BH);
428+ rwlock_init(&ls->ls_lkbxa_lock);
435429
436430 INIT_LIST_HEAD(&ls->ls_waiters);
437431 spin_lock_init(&ls->ls_waiters_lock);
438432 INIT_LIST_HEAD(&ls->ls_orphans);
439433 spin_lock_init(&ls->ls_orphans_lock);
440434
441− INIT_LIST_HEAD(&ls->ls_new_rsb);
442− spin_lock_init(&ls->ls_new_rsb_spin);
443−
444435 INIT_LIST_HEAD(&ls->ls_nodes);
445436 INIT_LIST_HEAD(&ls->ls_nodes_gone);
446437 ls->ls_num_nodes = 0;
@@ -484,7 +475,7 @@ static int new_lockspace(const char *name, const char *cluster, · 37 unmodified lines
484475 ls->ls_recover_buf = kmalloc(DLM_MAX_SOCKET_BUFSIZE, GFP_NOFS);
485476 if (!ls->ls_recover_buf) {
486477 error = -ENOMEM;
487− goto out_lkbidr;
478+ goto out_lkbxa;
488479 }
489480
490481 ls->ls_slot = 0;
@@ -494,32 +485,31 @@ static int new_lockspace(const char *name, const char *cluster, · 3 unmodified lines
494485
495486 INIT_LIST_HEAD(&ls->ls_recover_list);
496487 spin_lock_init(&ls->ls_recover_list_lock);
497− idr_init(&ls->ls_recover_idr);
498− spin_lock_init(&ls->ls_recover_idr_lock);
488+ xa_init_flags(&ls->ls_recover_xa, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_BH);
489+ spin_lock_init(&ls->ls_recover_xa_lock);
499490 ls->ls_recover_list_count = 0;
500− ls->ls_local_handle = ls;
501491 init_waitqueue_head(&ls->ls_wait_general);
502492 INIT_LIST_HEAD(&ls->ls_masters_list);
503493 rwlock_init(&ls->ls_masters_lock);
504494 INIT_LIST_HEAD(&ls->ls_dir_dump_list);
505495 rwlock_init(&ls->ls_dir_dump_lock);
506496
507− INIT_LIST_HEAD(&ls->ls_toss_q);
508− spin_lock_init(&ls->ls_toss_q_lock);
509− timer_setup(&ls->ls_timer, dlm_rsb_toss_timer,
510− TIMER_DEFERRABLE);
497+ INIT_LIST_HEAD(&ls->ls_scan_list);
498+ spin_lock_init(&ls->ls_scan_lock);
499+ timer_setup(&ls->ls_scan_timer, dlm_rsb_scan, TIMER_DEFERRABLE);
511500
512501 spin_lock_bh(&lslist_lock);
513502 ls->ls_create_count = 1;
514503 list_add(&ls->ls_list, &lslist);
515504 spin_unlock_bh(&lslist_lock);
516505
517− if (flags & DLM_LSFL_FS) {
518− error = dlm_callback_start(ls);
519− if (error) {
520− log_error(ls, "can't start dlm_callback %d", error);
521− goto out_delist;
522− }
506+ if (flags & DLM_LSFL_FS)
507+ set_bit(LSFL_FS, &ls->ls_flags);
508+
509+ error = dlm_callback_start(ls);
510+ if (error) {
511+ log_error(ls, "can't start dlm_callback %d", error);
512+ goto out_delist;
523513 }
524514
525515 init_waitqueue_head(&ls->ls_recover_lock_wait);
@@ -584,10 +574,10 @@ static int new_lockspace(const char *name, const char *cluster, · 58 unmodified lines
584574 spin_lock_bh(&lslist_lock);
585575 list_del(&ls->ls_list);
586576 spin_unlock_bh(&lslist_lock);
587− idr_destroy(&ls->ls_recover_idr);
577+ xa_destroy(&ls->ls_recover_xa);
588578 kfree(ls->ls_recover_buf);
589− out_lkbidr:
590− idr_destroy(&ls->ls_lkbidr);
579+ out_lkbxa:
580+ xa_destroy(&ls->ls_lkbxa);
591581 rhashtable_destroy(&ls->ls_rsbtbl);
592582 out_lsfree:
593583 if (do_unreg)
@@ -643,26 +633,15 @@ int dlm_new_user_lockspace(const char *name, const char *cluster, · 49 unmodified lines
643633 void *ops_arg, int *ops_result,
644634 dlm_lockspace_t **lockspace)
645635 {
636+ if (flags & DLM_LSFL_SOFTIRQ)
637+ return -EINVAL;
638+
646639 return __dlm_new_lockspace(name, cluster, flags, lvblen, ops,
647640 ops_arg, ops_result, lockspace);
648641 }
649642
650−static int lkb_idr_is_local(int id, void *p, void *data)
651−{
652− struct dlm_lkb *lkb = p;
653−
654− return lkb->lkb_nodeid == 0 && lkb->lkb_grmode != DLM_LOCK_IV;
655−}
656−
657−static int lkb_idr_is_any(int id, void *p, void *data)
658−{
659− return 1;
660−}
661−
662−static int lkb_idr_free(int id, void *p, void *data)
643+static int lkb_idr_free(struct dlm_lkb *lkb)
663644 {
664− struct dlm_lkb *lkb = p;
665−
666645 if (lkb->lkb_lvbptr && test_bit(DLM_IFL_MSTCPY_BIT, &lkb->lkb_iflags))
667646 dlm_free_lvb(lkb->lkb_lvbptr);
668647
@@ -670,23 +649,34 @@ static int lkb_idr_free(int id, void *p, void *data) · 1 unmodified line
670649 return 0;
671650 }
672651
673−/* NOTE: We check the lkbidr here rather than the resource table.
652+/* NOTE: We check the lkbxa here rather than the resource table.
674653 This is because there may be LKBs queued as ASTs that have been unlinked
675654 from their RSBs and are pending deletion once the AST has been delivered */
676655
677656 static int lockspace_busy(struct dlm_ls *ls, int force)
678657 {
679− int rv;
658+ struct dlm_lkb *lkb;
659+ unsigned long id;
660+ int rv = 0;
680661
681− read_lock_bh(&ls->ls_lkbidr_lock);
662+ read_lock_bh(&ls->ls_lkbxa_lock);
682663 if (force == 0) {
683− rv = idr_for_each(&ls->ls_lkbidr, lkb_idr_is_any, ls);
664+ xa_for_each(&ls->ls_lkbxa, id, lkb) {
665+ rv = 1;
666+ break;
667+ }
684668 } else if (force == 1) {
685− rv = idr_for_each(&ls->ls_lkbidr, lkb_idr_is_local, ls);
669+ xa_for_each(&ls->ls_lkbxa, id, lkb) {
670+ if (lkb->lkb_nodeid == 0 &&
671+ lkb->lkb_grmode != DLM_LOCK_IV) {
672+ rv = 1;
673+ break;
674+ }
675+ }
686676 } else {
687677 rv = 0;
688678 }
689− read_unlock_bh(&ls->ls_lkbidr_lock);
679+ read_unlock_bh(&ls->ls_lkbxa_lock);
690680 return rv;
691681 }
692682
@@ -699,7 +689,8 @@ static void rhash_free_rsb(void *ptr, void *arg) · 6 unmodified lines
699689
700690 static int release_lockspace(struct dlm_ls *ls, int force)
701691 {
702− struct dlm_rsb *rsb;
692+ struct dlm_lkb *lkb;
693+ unsigned long id;
703694 int busy, rv;
704695
705696 busy = lockspace_busy(ls, force);
@@ -739,7 +730,7 @@ static int release_lockspace(struct dlm_ls *ls, int force) · 33 unmodified lines
739730 * time_shutdown_sync(), we don't care anymore
740731 */
741732 clear_bit(LSFL_RUNNING, &ls->ls_flags);
742− timer_shutdown_sync(&ls->ls_timer);
733+ timer_shutdown_sync(&ls->ls_scan_timer);
743734
744735 if (ls_count == 1) {
745736 dlm_clear_members(ls);
@@ -752,28 +743,22 @@ static int release_lockspace(struct dlm_ls *ls, int force) · 6 unmodified lines
752743
753744 dlm_delete_debug_file(ls);
754745
755− idr_destroy(&ls->ls_recover_idr);
746+ xa_destroy(&ls->ls_recover_xa);
756747 kfree(ls->ls_recover_buf);
757748
758749 /*
759− * Free all lkb's in idr
750+ * Free all lkb's in xa
760751 */
761−
762− idr_for_each(&ls->ls_lkbidr, lkb_idr_free, ls);
763− idr_destroy(&ls->ls_lkbidr);
752+ xa_for_each(&ls->ls_lkbxa, id, lkb) {
753+ lkb_idr_free(lkb);
754+ }
755+ xa_destroy(&ls->ls_lkbxa);
764756
765757 /*
766758 * Free all rsb's on rsbtbl
767759 */
768760 rhashtable_free_and_destroy(&ls->ls_rsbtbl, rhash_free_rsb, NULL);
769761
770− while (!list_empty(&ls->ls_new_rsb)) {
771− rsb = list_first_entry(&ls->ls_new_rsb, struct dlm_rsb,
772− res_hashchain);
773− list_del(&rsb->res_hashchain);
774− dlm_free_rsb(rsb);
775− }
776−
777762 /*
778763 * Free structures on any other lists
779764 */
fs/dlm/lowcomms.c+1−7
@@ -461,7 +461,7 @@ static bool dlm_lowcomms_con_has_addr(const struct connection *con, · 460 unmodified lines
461461 return false;
462462 }
463463
464−int dlm_lowcomms_addr(int nodeid, struct sockaddr_storage *addr, int len)
464+int dlm_lowcomms_addr(int nodeid, struct sockaddr_storage *addr)
465465 {
466466 struct connection *con;
467467 bool ret, idx;
@@ -858,12 +858,6 @@ static void free_processqueue_entry(struct processqueue_entry *pentry) · 390 unmodified lines
858858 kfree(pentry);
859859 }
860860
861−struct dlm_processed_nodes {
862− int nodeid;
863−
864− struct list_head list;
865−};
866−
867861 static void process_dlm_messages(struct work_struct *work)
868862 {
869863 struct processqueue_entry *pentry;
fs/dlm/lowcomms.h+1−1
@@ -46,7 +46,7 @@ void dlm_lowcomms_put_msg(struct dlm_msg *msg); · 45 unmodified lines
4646 int dlm_lowcomms_resend_msg(struct dlm_msg *msg);
4747 int dlm_lowcomms_connect_node(int nodeid);
4848 int dlm_lowcomms_nodes_set_mark(int nodeid, unsigned int mark);
49−int dlm_lowcomms_addr(int nodeid, struct sockaddr_storage *addr, int len);
49+int dlm_lowcomms_addr(int nodeid, struct sockaddr_storage *addr);
5050 void dlm_midcomms_receive_done(int nodeid);
5151 struct kmem_cache *dlm_lowcomms_writequeue_cache_create(void);
5252 struct kmem_cache *dlm_lowcomms_msg_cache_create(void);
fs/dlm/member.c+1−1
@@ -642,7 +642,7 @@ int dlm_ls_stop(struct dlm_ls *ls) · 641 unmodified lines
642642 set_bit(LSFL_RECOVER_STOP, &ls->ls_flags);
643643 new = test_and_clear_bit(LSFL_RUNNING, &ls->ls_flags);
644644 if (new)
645− timer_delete_sync(&ls->ls_timer);
645+ timer_delete_sync(&ls->ls_scan_timer);
646646 ls->ls_recover_seq++;
647647
648648 /* activate requestqueue and stop processing */
fs/dlm/memory.c+9−1
@@ -72,6 +72,8 @@ out: · 71 unmodified lines
7272
7373 void dlm_memory_exit(void)
7474 {
75+ rcu_barrier();
76+
7577 kmem_cache_destroy(writequeue_cache);
7678 kmem_cache_destroy(mhandle_cache);
7779 kmem_cache_destroy(msg_cache);
@@ -101,13 +103,19 @@ struct dlm_rsb *dlm_allocate_rsb(struct dlm_ls *ls) · 23 unmodified lines
101103 return r;
102104 }
103105
104−void dlm_free_rsb(struct dlm_rsb *r)
106+static void __free_rsb_rcu(struct rcu_head *rcu)
105107 {
108+ struct dlm_rsb *r = container_of(rcu, struct dlm_rsb, rcu);
106109 if (r->res_lvbptr)
107110 dlm_free_lvb(r->res_lvbptr);
108111 kmem_cache_free(rsb_cache, r);
109112 }
110113
114+void dlm_free_rsb(struct dlm_rsb *r)
115+{
116+ call_rcu(&r->rcu, __free_rsb_rcu);
117+}
118+
111119 struct dlm_lkb *dlm_allocate_lkb(struct dlm_ls *ls)
112120 {
113121 struct dlm_lkb *lkb;
fs/dlm/midcomms.c+2−2
@@ -334,12 +334,12 @@ static struct midcomms_node *nodeid2node(int nodeid) · 333 unmodified lines
334334 return __find_node(nodeid, nodeid_hash(nodeid));
335335 }
336336
337−int dlm_midcomms_addr(int nodeid, struct sockaddr_storage *addr, int len)
337+int dlm_midcomms_addr(int nodeid, struct sockaddr_storage *addr)
338338 {
339339 int ret, idx, r = nodeid_hash(nodeid);
340340 struct midcomms_node *node;
341341
342− ret = dlm_lowcomms_addr(nodeid, addr, len);
342+ ret = dlm_lowcomms_addr(nodeid, addr);
343343 if (ret)
344344 return ret;
345345
fs/dlm/midcomms.h+1−1
@@ -19,7 +19,7 @@ int dlm_process_incoming_buffer(int nodeid, unsigned char *buf, int buflen); · 18 unmodified lines
1919 struct dlm_mhandle *dlm_midcomms_get_mhandle(int nodeid, int len, char **ppc);
2020 void dlm_midcomms_commit_mhandle(struct dlm_mhandle *mh, const void *name,
2121 int namelen);
22−int dlm_midcomms_addr(int nodeid, struct sockaddr_storage *addr, int len);
22+int dlm_midcomms_addr(int nodeid, struct sockaddr_storage *addr);
2323 void dlm_midcomms_version_wait(void);
2424 int dlm_midcomms_close(int nodeid);
2525 int dlm_midcomms_start(void);
fs/dlm/recover.c+40−38
@@ -293,73 +293,78 @@ static void recover_list_clear(struct dlm_ls *ls) · 292 unmodified lines
293293 spin_unlock_bh(&ls->ls_recover_list_lock);
294294 }
295295
296−static int recover_idr_empty(struct dlm_ls *ls)
296+static int recover_xa_empty(struct dlm_ls *ls)
297297 {
298298 int empty = 1;
299299
300− spin_lock_bh(&ls->ls_recover_idr_lock);
300+ spin_lock_bh(&ls->ls_recover_xa_lock);
301301 if (ls->ls_recover_list_count)
302302 empty = 0;
303− spin_unlock_bh(&ls->ls_recover_idr_lock);
303+ spin_unlock_bh(&ls->ls_recover_xa_lock);
304304
305305 return empty;
306306 }
307307
308−static int recover_idr_add(struct dlm_rsb *r)
308+static int recover_xa_add(struct dlm_rsb *r)
309309 {
310310 struct dlm_ls *ls = r->res_ls;
311+ struct xa_limit limit = {
312+ .min = 1,
313+ .max = UINT_MAX,
314+ };
315+ uint32_t id;
311316 int rv;
312317
313− spin_lock_bh(&ls->ls_recover_idr_lock);
318+ spin_lock_bh(&ls->ls_recover_xa_lock);
314319 if (r->res_id) {
315320 rv = -1;
316321 goto out_unlock;
317322 }
318− rv = idr_alloc(&ls->ls_recover_idr, r, 1, 0, GFP_NOWAIT);
323+ rv = xa_alloc(&ls->ls_recover_xa, &id, r, limit, GFP_ATOMIC);
319324 if (rv < 0)
320325 goto out_unlock;
321326
322− r->res_id = rv;
327+ r->res_id = id;
323328 ls->ls_recover_list_count++;
324329 dlm_hold_rsb(r);
325330 rv = 0;
326331 out_unlock:
327− spin_unlock_bh(&ls->ls_recover_idr_lock);
332+ spin_unlock_bh(&ls->ls_recover_xa_lock);
328333 return rv;
329334 }
330335
331−static void recover_idr_del(struct dlm_rsb *r)
336+static void recover_xa_del(struct dlm_rsb *r)
332337 {
333338 struct dlm_ls *ls = r->res_ls;
334339
335− spin_lock_bh(&ls->ls_recover_idr_lock);
336− idr_remove(&ls->ls_recover_idr, r->res_id);
340+ spin_lock_bh(&ls->ls_recover_xa_lock);
341+ xa_erase_bh(&ls->ls_recover_xa, r->res_id);
337342 r->res_id = 0;
338343 ls->ls_recover_list_count--;
339− spin_unlock_bh(&ls->ls_recover_idr_lock);
344+ spin_unlock_bh(&ls->ls_recover_xa_lock);
340345
341346 dlm_put_rsb(r);
342347 }
343348
344−static struct dlm_rsb *recover_idr_find(struct dlm_ls *ls, uint64_t id)
349+static struct dlm_rsb *recover_xa_find(struct dlm_ls *ls, uint64_t id)
345350 {
346351 struct dlm_rsb *r;
347352
348− spin_lock_bh(&ls->ls_recover_idr_lock);
349− r = idr_find(&ls->ls_recover_idr, (int)id);
350− spin_unlock_bh(&ls->ls_recover_idr_lock);
353+ spin_lock_bh(&ls->ls_recover_xa_lock);
354+ r = xa_load(&ls->ls_recover_xa, (int)id);
355+ spin_unlock_bh(&ls->ls_recover_xa_lock);
351356 return r;
352357 }
353358
354−static void recover_idr_clear(struct dlm_ls *ls)
359+static void recover_xa_clear(struct dlm_ls *ls)
355360 {
356361 struct dlm_rsb *r;
357− int id;
362+ unsigned long id;
358363
359− spin_lock_bh(&ls->ls_recover_idr_lock);
364+ spin_lock_bh(&ls->ls_recover_xa_lock);
360365
361− idr_for_each_entry(&ls->ls_recover_idr, r, id) {
362− idr_remove(&ls->ls_recover_idr, id);
366+ xa_for_each(&ls->ls_recover_xa, id, r) {
367+ xa_erase_bh(&ls->ls_recover_xa, id);
363368 r->res_id = 0;
364369 r->res_recover_locks_count = 0;
365370 ls->ls_recover_list_count--;
@@ -372,7 +377,7 @@ static void recover_idr_clear(struct dlm_ls *ls) · 6 unmodified lines
372377 ls->ls_recover_list_count);
373378 ls->ls_recover_list_count = 0;
374379 }
375− spin_unlock_bh(&ls->ls_recover_idr_lock);
380+ spin_unlock_bh(&ls->ls_recover_xa_lock);
376381 }
377382
378383
@@ -470,7 +475,7 @@ static int recover_master(struct dlm_rsb *r, unsigned int *count, uint64_t seq) · 91 unmodified lines
470475 set_new_master(r);
471476 error = 0;
472477 } else {
473− recover_idr_add(r);
478+ recover_xa_add(r);
474479 error = dlm_send_rcom_lookup(r, dir_nodeid, seq);
475480 }
476481
@@ -551,10 +556,10 @@ int dlm_recover_masters(struct dlm_ls *ls, uint64_t seq, · 74 unmodified lines
551556
552557 log_rinfo(ls, "dlm_recover_masters %u of %u", count, total);
553558
554− error = dlm_wait_function(ls, &recover_idr_empty);
559+ error = dlm_wait_function(ls, &recover_xa_empty);
555560 out:
556561 if (error)
557− recover_idr_clear(ls);
562+ recover_xa_clear(ls);
558563 return error;
559564 }
560565
@@ -563,7 +568,7 @@ int dlm_recover_master_reply(struct dlm_ls *ls, const struct dlm_rcom *rc) · 2 unmodified lines
563568 struct dlm_rsb *r;
564569 int ret_nodeid, new_master;
565570
566− r = recover_idr_find(ls, le64_to_cpu(rc->rc_id));
571+ r = recover_xa_find(ls, le64_to_cpu(rc->rc_id));
567572 if (!r) {
568573 log_error(ls, "dlm_recover_master_reply no id %llx",
569574 (unsigned long long)le64_to_cpu(rc->rc_id));
@@ -582,9 +587,9 @@ int dlm_recover_master_reply(struct dlm_ls *ls, const struct dlm_rcom *rc) · 12 unmodified lines
582587 r->res_nodeid = new_master;
583588 set_new_master(r);
584589 unlock_rsb(r);
585− recover_idr_del(r);
590+ recover_xa_del(r);
586591
587− if (recover_idr_empty(ls))
592+ if (recover_xa_empty(ls))
588593 wake_up(&ls->ls_wait_general);
589594 out:
590595 return 0;
@@ -877,29 +882,26 @@ void dlm_recover_rsbs(struct dlm_ls *ls, const struct list_head *root_list) · 286 unmodified lines
877882 log_rinfo(ls, "dlm_recover_rsbs %d done", count);
878883 }
879884
880−/* Create a single list of all root rsb's to be used during recovery */
881−
882−void dlm_clear_toss(struct dlm_ls *ls)
885+void dlm_clear_inactive(struct dlm_ls *ls)
883886 {
884887 struct dlm_rsb *r, *safe;
885888 unsigned int count = 0;
886889
887890 write_lock_bh(&ls->ls_rsbtbl_lock);
888− list_for_each_entry_safe(r, safe, &ls->ls_toss, res_rsbs_list) {
889− list_del(&r->res_rsbs_list);
891+ list_for_each_entry_safe(r, safe, &ls->ls_slow_inactive, res_slow_list) {
892+ list_del(&r->res_slow_list);
890893 rhashtable_remove_fast(&ls->ls_rsbtbl, &r->res_node,
891894 dlm_rhash_rsb_params);
892895
893− /* remove it from the toss queue if its part of it */
894− if (!list_empty(&r->res_toss_q_list))
895− list_del_init(&r->res_toss_q_list);
896+ if (!list_empty(&r->res_scan_list))
897+ list_del_init(&r->res_scan_list);
896898
897− free_toss_rsb(r);
899+ free_inactive_rsb(r);
898900 count++;
899901 }
900902 write_unlock_bh(&ls->ls_rsbtbl_lock);
901903
902904 if (count)
903− log_rinfo(ls, "dlm_clear_toss %u done", count);
905+ log_rinfo(ls, "dlm_clear_inactive %u done", count);
904906 }
905907
fs/dlm/recover.h+1−1
@@ -25,7 +25,7 @@ int dlm_recover_master_reply(struct dlm_ls *ls, const struct dlm_rcom *rc); · 24 unmodified lines
2525 int dlm_recover_locks(struct dlm_ls *ls, uint64_t seq,
2626 const struct list_head *root_list);
2727 void dlm_recovered_lock(struct dlm_rsb *r);
28−void dlm_clear_toss(struct dlm_ls *ls);
28+void dlm_clear_inactive(struct dlm_ls *ls);
2929 void dlm_recover_rsbs(struct dlm_ls *ls, const struct list_head *root_list);
3030
3131 #endif /* __RECOVER_DOT_H__ */
fs/dlm/recoverd.c+7−7
@@ -33,7 +33,7 @@ static int dlm_create_masters_list(struct dlm_ls *ls) · 32 unmodified lines
3333 }
3434
3535 read_lock_bh(&ls->ls_rsbtbl_lock);
36− list_for_each_entry(r, &ls->ls_keep, res_rsbs_list) {
36+ list_for_each_entry(r, &ls->ls_slow_active, res_slow_list) {
3737 if (r->res_nodeid)
3838 continue;
3939
@@ -63,12 +63,12 @@ static void dlm_create_root_list(struct dlm_ls *ls, struct list_head *root_list) · 23 unmodified lines
6363 struct dlm_rsb *r;
6464
6565 read_lock_bh(&ls->ls_rsbtbl_lock);
66− list_for_each_entry(r, &ls->ls_keep, res_rsbs_list) {
66+ list_for_each_entry(r, &ls->ls_slow_active, res_slow_list) {
6767 list_add(&r->res_root_list, root_list);
6868 dlm_hold_rsb(r);
6969 }
7070
71− WARN_ON_ONCE(!list_empty(&ls->ls_toss));
71+ WARN_ON_ONCE(!list_empty(&ls->ls_slow_inactive));
7272 read_unlock_bh(&ls->ls_rsbtbl_lock);
7373 }
7474
@@ -98,16 +98,16 @@ static int enable_locking(struct dlm_ls *ls, uint64_t seq) · 23 unmodified lines
9898 spin_lock_bh(&ls->ls_recover_lock);
9999 if (ls->ls_recover_seq == seq) {
100100 set_bit(LSFL_RUNNING, &ls->ls_flags);
101− /* Schedule next timer if recovery put something on toss.
101+ /* Schedule next timer if recovery put something on inactive.
102102 *
103103 * The rsbs that was queued while recovery on toss hasn't
104104 * started yet because LSFL_RUNNING was set everything
105105 * else recovery hasn't started as well because ls_in_recovery
106106 * is still hold. So we should not run into the case that
107− * dlm_timer_resume() queues a timer that can occur in
107+ * resume_scan_timer() queues a timer that can occur in
108108 * a no op.
109109 */
110− dlm_timer_resume(ls);
110+ resume_scan_timer(ls);
111111 /* unblocks processes waiting to enter the dlm */
112112 up_write(&ls->ls_in_recovery);
113113 clear_bit(LSFL_RECOVER_LOCK, &ls->ls_flags);
@@ -131,7 +131,7 @@ static int ls_recover(struct dlm_ls *ls, struct dlm_recover *rv) · 17 unmodified lines
131131
132132 dlm_callback_suspend(ls);
133133
134− dlm_clear_toss(ls);
134+ dlm_clear_inactive(ls);
135135
136136 /*
137137 * This list of root rsb's will be the basis of most of the recovery
fs/dlm/user.c+18−24
@@ -182,7 +182,7 @@ void dlm_user_add_ast(struct dlm_lkb *lkb, uint32_t flags, int mode, · 181 unmodified lines
182182 struct dlm_user_args *ua;
183183 struct dlm_user_proc *proc;
184184 struct dlm_callback *cb;
185− int rv;
185+ int rv, copy_lvb;
186186
187187 if (test_bit(DLM_DFL_ORPHAN_BIT, &lkb->lkb_dflags) ||
188188 test_bit(DLM_IFL_DEAD_BIT, &lkb->lkb_iflags))
@@ -213,28 +213,22 @@ void dlm_user_add_ast(struct dlm_lkb *lkb, uint32_t flags, int mode, · 24 unmodified lines
213213
214214 spin_lock_bh(&proc->asts_spin);
215215
216− rv = dlm_queue_lkb_callback(lkb, flags, mode, status, sbflags, &cb);
217− switch (rv) {
218− case DLM_ENQUEUE_CALLBACK_NEED_SCHED:
219− cb->ua = *ua;
220− cb->lkb_lksb = &cb->ua.lksb;
221− if (cb->copy_lvb) {
222− memcpy(cb->lvbptr, ua->lksb.sb_lvbptr,
223− DLM_USER_LVB_LEN);
224− cb->lkb_lksb->sb_lvbptr = cb->lvbptr;
216+ if (!dlm_may_skip_callback(lkb, flags, mode, status, sbflags,
217+ &copy_lvb)) {
218+ rv = dlm_get_cb(lkb, flags, mode, status, sbflags, &cb);
219+ if (!rv) {
220+ cb->copy_lvb = copy_lvb;
221+ cb->ua = *ua;
222+ cb->lkb_lksb = &cb->ua.lksb;
223+ if (copy_lvb) {
224+ memcpy(cb->lvbptr, ua->lksb.sb_lvbptr,
225+ DLM_USER_LVB_LEN);
226+ cb->lkb_lksb->sb_lvbptr = cb->lvbptr;
227+ }
228+
229+ list_add_tail(&cb->list, &proc->asts);
230+ wake_up_interruptible(&proc->wait);
225231 }
226−
227− list_add_tail(&cb->list, &proc->asts);
228− wake_up_interruptible(&proc->wait);
229− break;
230− case DLM_ENQUEUE_CALLBACK_SUCCESS:
231− break;
232− case DLM_ENQUEUE_CALLBACK_FAILURE:
233− fallthrough;
234− default:
235− spin_unlock_bh(&proc->asts_spin);
236− WARN_ON_ONCE(1);
237− goto out;
238232 }
239233 spin_unlock_bh(&proc->asts_spin);
240234
@@ -454,7 +448,7 @@ static int device_remove_lockspace(struct dlm_lspace_params *params) · 213 unmodified lines
454448 if (params->flags & DLM_USER_LSFLG_FORCEFREE)
455449 force = 2;
456450
457− lockspace = ls->ls_local_handle;
451+ lockspace = ls;
458452 dlm_put_lockspace(ls);
459453
460454 /* The final dlm_release_lockspace waits for references to go to
@@ -657,7 +651,7 @@ static int device_open(struct inode *inode, struct file *file) · 196 unmodified lines
657651 return -ENOMEM;
658652 }
659653
660− proc->lockspace = ls->ls_local_handle;
654+ proc->lockspace = ls;
661655 INIT_LIST_HEAD(&proc->asts);
662656 INIT_LIST_HEAD(&proc->locks);
663657 INIT_LIST_HEAD(&proc->unlocking);
include/linux/dlm.h+16−1
@@ -35,6 +35,9 @@ struct dlm_lockspace_ops { · 34 unmodified lines
3535 int num_slots, int our_slot, uint32_t generation);
3636 };
3737
38+/* only relevant for kernel lockspaces, will be removed in future */
39+#define DLM_LSFL_SOFTIRQ __DLM_LSFL_RESERVED0
40+
3841 /*
3942 * dlm_new_lockspace
4043 *
@@ -55,6 +58,11 @@ struct dlm_lockspace_ops { · 14 unmodified lines
5558 * used to select the directory node. Must be the same on all nodes.
5659 * DLM_LSFL_NEWEXCL
5760 * dlm_new_lockspace() should return -EEXIST if the lockspace exists.
61+ * DLM_LSFL_SOFTIRQ
62+ * dlm request callbacks (ast, bast) are softirq safe. Flag should be
63+ * preferred by users. Will be default in some future. If set the
64+ * strongest context for ast, bast callback is softirq as it avoids
65+ * an additional context switch.
5866 *
5967 * lvblen: length of lvb in bytes. Must be multiple of 8.
6068 * dlm_new_lockspace() returns an error if this does not match
@@ -121,7 +129,14 @@ int dlm_release_lockspace(dlm_lockspace_t *lockspace, int force); · 60 unmodified lines
121129 * call.
122130 *
123131 * AST routines should not block (at least not for long), but may make
124− * any locking calls they please.
132+ * any locking calls they please. If DLM_LSFL_SOFTIRQ for kernel
133+ * users of dlm_new_lockspace() is passed the ast and bast callbacks
134+ * can be processed in softirq context. Also some of the callback
135+ * contexts are in the same context as the DLM lock request API, users
136+ * must not hold locks while calling dlm lock request API and trying
137+ * to acquire this lock in the callback again, this will end in a
138+ * lock recursion. For newer implementation the DLM_LSFL_SOFTIRQ
139+ * should be used.
125140 */
126141
127142 int dlm_lock(dlm_lockspace_t *lockspace,
include/uapi/linux/dlm.h+2−0
@@ -71,6 +71,8 @@ struct dlm_lksb { · 70 unmodified lines
7171 /* DLM_LSFL_TIMEWARN is deprecated and reserved. DO NOT USE! */
7272 #define DLM_LSFL_TIMEWARN 0x00000002
7373 #define DLM_LSFL_NEWEXCL 0x00000008
74+/* currently reserved due in-kernel use */
75+#define __DLM_LSFL_RESERVED0 0x00000010
7476
7577
7678 #endif /* _UAPI__DLM_DOT_H__ */

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t