History

sched/wait: Introduce wakeup boomark in wake_up_page_bit

Now that we have added breaks in the wait queue scan and allow bookmark on scan position, we put this logic in the wake_up_page_bit function. We can have very long page wait list in large system where multiple pages share the same wait list. We break the wake up walk here to allow other cpus a chance to access the list, and not to disable the interrupts when traversing the list for too long. This reduces the interrupt and rescheduling latency, and excessive page wait queue lock hold time. [ v2: Remove bookmark_wake_function ]

Tim Chen committed 9 years ago

Commit 11a19c71 parent 2554db93 files · +30 −1Browse files

Unified · Split

3 files
  1. include/linux/wait.h +2 −0
  2. kernel/sched/wait.c +7 −0
  3. mm/filemap.c +21 −1
include/linux/wait.h+2−0
@@ -185,6 +185,8 @@ __remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq · 184 unmodified lines
185 185
186void __wake_up(struct wait_queue_head *wq_head, unsigned int mode, int nr, void *key);186void __wake_up(struct wait_queue_head *wq_head, unsigned int mode, int nr, void *key);
187void __wake_up_locked_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);187void __wake_up_locked_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);
188void __wake_up_locked_key_bookmark(struct wait_queue_head *wq_head,
189 unsigned int mode, void *key, wait_queue_entry_t *bookmark);
188void __wake_up_sync_key(struct wait_queue_head *wq_head, unsigned int mode, int nr, void *key);190void __wake_up_sync_key(struct wait_queue_head *wq_head, unsigned int mode, int nr, void *key);
189void __wake_up_locked(struct wait_queue_head *wq_head, unsigned int mode, int nr);191void __wake_up_locked(struct wait_queue_head *wq_head, unsigned int mode, int nr);
190void __wake_up_sync(struct wait_queue_head *wq_head, unsigned int mode, int nr);192void __wake_up_sync(struct wait_queue_head *wq_head, unsigned int mode, int nr);
kernel/sched/wait.c+7−0
@@ -165,6 +165,13 @@ void __wake_up_locked_key(struct wait_queue_head *wq_head, unsigned int mode, vo · 164 unmodified lines
165}165}
166EXPORT_SYMBOL_GPL(__wake_up_locked_key);166EXPORT_SYMBOL_GPL(__wake_up_locked_key);
167 167
168void __wake_up_locked_key_bookmark(struct wait_queue_head *wq_head,
169 unsigned int mode, void *key, wait_queue_entry_t *bookmark)
170{
171 __wake_up_common(wq_head, mode, 1, 0, key, bookmark);
172}
173EXPORT_SYMBOL_GPL(__wake_up_locked_key_bookmark);
174
168/**175/**
169 * __wake_up_sync_key - wake up threads blocked on a waitqueue.176 * __wake_up_sync_key - wake up threads blocked on a waitqueue.
170 * @wq_head: the waitqueue177 * @wq_head: the waitqueue
mm/filemap.c+21−1
@@ -909,13 +909,33 @@ static void wake_up_page_bit(struct page *page, int bit_nr) · 908 unmodified lines
909 wait_queue_head_t *q = page_waitqueue(page);909 wait_queue_head_t *q = page_waitqueue(page);
910 struct wait_page_key key;910 struct wait_page_key key;
911 unsigned long flags;911 unsigned long flags;
912 wait_queue_entry_t bookmark;
912 913
913 key.page = page;914 key.page = page;
914 key.bit_nr = bit_nr;915 key.bit_nr = bit_nr;
915 key.page_match = 0;916 key.page_match = 0;
916 917
918 bookmark.flags = 0;
919 bookmark.private = NULL;
920 bookmark.func = NULL;
921 INIT_LIST_HEAD(&bookmark.entry);
922
917 spin_lock_irqsave(&q->lock, flags);923 spin_lock_irqsave(&q->lock, flags);
918 __wake_up_locked_key(q, TASK_NORMAL, &key);924 __wake_up_locked_key_bookmark(q, TASK_NORMAL, &key, &bookmark);
925
926 while (bookmark.flags & WQ_FLAG_BOOKMARK) {
927 /*
928 * Take a breather from holding the lock,
929 * allow pages that finish wake up asynchronously
930 * to acquire the lock and remove themselves
931 * from wait queue
932 */
933 spin_unlock_irqrestore(&q->lock, flags);
934 cpu_relax();
935 spin_lock_irqsave(&q->lock, flags);
936 __wake_up_locked_key_bookmark(q, TASK_NORMAL, &key, &bookmark);
937 }
938
919 /*939 /*
920 * It is possible for other pages to have collided on the waitqueue940 * It is possible for other pages to have collided on the waitqueue
921 * hash, so in that case check for a page match. That prevents a long-941 * hash, so in that case check for a page match. That prevents a long-

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t