History

Merge branch 'akpm' (patches from Andrew)

Mergr misc fixes from Andrew Morton: "11 fixes" Mostly VM fixes, one psi polling fix, and one parisc build fix. * emailed patches from Andrew Morton <akpm@linux-foundation.org>: mm/kasan: fix false positive invalid-free reports with CONFIG_KASAN_SW_TAGS=y mm/zsmalloc.c: fix race condition in zs_destroy_pool mm/zsmalloc.c: migration can leave pages in ZS_EMPTY indefinitely mm, page_owner: handle THP splits correctly userfaultfd_release: always remove uffd flags and clear vm_userfaultfd_ctx psi: get poll_work to run when calling poll syscall next time mm: memcontrol: flush percpu vmevents before releasing memcg mm: memcontrol: flush percpu vmstats before releasing memcg parisc: fix compilation errrors mm, page_alloc: move_freepages should not examine struct page of reserved memory mm/z3fold.c: fix race between migration and destruction

Linus Torvalds committed 7 years ago

Commit f47edb52 parents e67095f 00fb24a9 files · +260 −36Browse files

Unified · Split

9 files
  1. arch/parisc/include/asm/pgtable.h +1 −2
  2. fs/userfaultfd.c +13 −12
  3. kernel/sched/psi.c +8 −0
  4. mm/huge_memory.c +4 −0
  5. mm/kasan/common.c +8 −2
  6. mm/memcontrol.c +60 −0
  7. mm/page_alloc.c +4 −15
  8. mm/z3fold.c +89 −0
  9. mm/zsmalloc.c +73 −5
arch/parisc/include/asm/pgtable.h+1−2
@@ -2,6 +2,7 @@ · 1 unmodified line
2#ifndef _PARISC_PGTABLE_H2#ifndef _PARISC_PGTABLE_H
3#define _PARISC_PGTABLE_H3#define _PARISC_PGTABLE_H
4 4
5#include <asm/page.h>
5#include <asm-generic/4level-fixup.h>6#include <asm-generic/4level-fixup.h>
6 7
7#include <asm/fixmap.h>8#include <asm/fixmap.h>
@@ -98,8 +99,6 @@ static inline void purge_tlb_entries(struct mm_struct *mm, unsigned long addr) · 90 unmodified lines
98 99
99#endif /* !__ASSEMBLY__ */100#endif /* !__ASSEMBLY__ */
100 101
101#include <asm/page.h>
102
103#define pte_ERROR(e) \102#define pte_ERROR(e) \
104 printk("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e))103 printk("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e))
105#define pmd_ERROR(e) \104#define pmd_ERROR(e) \
fs/userfaultfd.c+13−12
@@ -880,6 +880,7 @@ static int userfaultfd_release(struct inode *inode, struct file *file) · 879 unmodified lines
880 /* len == 0 means wake all */880 /* len == 0 means wake all */
881 struct userfaultfd_wake_range range = { .len = 0, };881 struct userfaultfd_wake_range range = { .len = 0, };
882 unsigned long new_flags;882 unsigned long new_flags;
883 bool still_valid;
883 884
884 WRITE_ONCE(ctx->released, true);885 WRITE_ONCE(ctx->released, true);
885 886
@@ -895,8 +896,7 @@ static int userfaultfd_release(struct inode *inode, struct file *file) · 9 unmodified lines
895 * taking the mmap_sem for writing.896 * taking the mmap_sem for writing.
896 */897 */
897 down_write(&mm->mmap_sem);898 down_write(&mm->mmap_sem);
898 if (!mmget_still_valid(mm))899 still_valid = mmget_still_valid(mm);
899 goto skip_mm;
900 prev = NULL;900 prev = NULL;
901 for (vma = mm->mmap; vma; vma = vma->vm_next) {901 for (vma = mm->mmap; vma; vma = vma->vm_next) {
902 cond_resched();902 cond_resched();
@@ -907,19 +907,20 @@ static int userfaultfd_release(struct inode *inode, struct file *file) · 4 unmodified lines
907 continue;907 continue;
908 }908 }
909 new_flags = vma->vm_flags & ~(VM_UFFD_MISSING | VM_UFFD_WP);909 new_flags = vma->vm_flags & ~(VM_UFFD_MISSING | VM_UFFD_WP);
910 prev = vma_merge(mm, prev, vma->vm_start, vma->vm_end,910 if (still_valid) {
911 new_flags, vma->anon_vma,911 prev = vma_merge(mm, prev, vma->vm_start, vma->vm_end,
912 vma->vm_file, vma->vm_pgoff,912 new_flags, vma->anon_vma,
913 vma_policy(vma),913 vma->vm_file, vma->vm_pgoff,
914 NULL_VM_UFFD_CTX);914 vma_policy(vma),
915 if (prev)915 NULL_VM_UFFD_CTX);
916 vma = prev;916 if (prev)
917 else917 vma = prev;
918 prev = vma;918 else
919 prev = vma;
920 }
919 vma->vm_flags = new_flags;921 vma->vm_flags = new_flags;
920 vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;922 vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
921 }923 }
922skip_mm:
923 up_write(&mm->mmap_sem);924 up_write(&mm->mmap_sem);
924 mmput(mm);925 mmput(mm);
925wakeup:926wakeup:
kernel/sched/psi.c+8−0
@@ -1131,7 +1131,15 @@ static void psi_trigger_destroy(struct kref *ref) · 1130 unmodified lines
1131 * deadlock while waiting for psi_poll_work to acquire trigger_lock1131 * deadlock while waiting for psi_poll_work to acquire trigger_lock
1132 */1132 */
1133 if (kworker_to_destroy) {1133 if (kworker_to_destroy) {
1134 /*
1135 * After the RCU grace period has expired, the worker
1136 * can no longer be found through group->poll_kworker.
1137 * But it might have been already scheduled before
1138 * that - deschedule it cleanly before destroying it.
1139 */
1134 kthread_cancel_delayed_work_sync(&group->poll_work);1140 kthread_cancel_delayed_work_sync(&group->poll_work);
1141 atomic_set(&group->poll_scheduled, 0);
1142
1135 kthread_destroy_worker(kworker_to_destroy);1143 kthread_destroy_worker(kworker_to_destroy);
1136 }1144 }
1137 kfree(t);1145 kfree(t);
mm/huge_memory.c+4−0
@@ -32,6 +32,7 @@ · 31 unmodified lines
32#include <linux/shmem_fs.h>32#include <linux/shmem_fs.h>
33#include <linux/oom.h>33#include <linux/oom.h>
34#include <linux/numa.h>34#include <linux/numa.h>
35#include <linux/page_owner.h>
35 36
36#include <asm/tlb.h>37#include <asm/tlb.h>
37#include <asm/pgalloc.h>38#include <asm/pgalloc.h>
@@ -2516,6 +2517,9 @@ static void __split_huge_page(struct page *page, struct list_head *list, · 2478 unmodified lines
2516 }2517 }
2517 2518
2518 ClearPageCompound(head);2519 ClearPageCompound(head);
2520
2521 split_page_owner(head, HPAGE_PMD_ORDER);
2522
2519 /* See comment in __split_huge_page_tail() */2523 /* See comment in __split_huge_page_tail() */
2520 if (PageAnon(head)) {2524 if (PageAnon(head)) {
2521 /* Additional pin to swap cache */2525 /* Additional pin to swap cache */
mm/kasan/common.c+8−2
@@ -407,8 +407,14 @@ static inline bool shadow_invalid(u8 tag, s8 shadow_byte) · 406 unmodified lines
407 if (IS_ENABLED(CONFIG_KASAN_GENERIC))407 if (IS_ENABLED(CONFIG_KASAN_GENERIC))
408 return shadow_byte < 0 ||408 return shadow_byte < 0 ||
409 shadow_byte >= KASAN_SHADOW_SCALE_SIZE;409 shadow_byte >= KASAN_SHADOW_SCALE_SIZE;
410 else410
411 return tag != (u8)shadow_byte;411 /* else CONFIG_KASAN_SW_TAGS: */
412 if ((u8)shadow_byte == KASAN_TAG_INVALID)
413 return true;
414 if ((tag != KASAN_TAG_KERNEL) && (tag != (u8)shadow_byte))
415 return true;
416
417 return false;
412}418}
413 419
414static bool __kasan_slab_free(struct kmem_cache *cache, void *object,420static bool __kasan_slab_free(struct kmem_cache *cache, void *object,
mm/memcontrol.c+60−0
@@ -3260,6 +3260,60 @@ static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css, · 3259 unmodified lines
3260 }3260 }
3261}3261}
3262 3262
3263static void memcg_flush_percpu_vmstats(struct mem_cgroup *memcg)
3264{
3265 unsigned long stat[MEMCG_NR_STAT];
3266 struct mem_cgroup *mi;
3267 int node, cpu, i;
3268
3269 for (i = 0; i < MEMCG_NR_STAT; i++)
3270 stat[i] = 0;
3271
3272 for_each_online_cpu(cpu)
3273 for (i = 0; i < MEMCG_NR_STAT; i++)
3274 stat[i] += raw_cpu_read(memcg->vmstats_percpu->stat[i]);
3275
3276 for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
3277 for (i = 0; i < MEMCG_NR_STAT; i++)
3278 atomic_long_add(stat[i], &mi->vmstats[i]);
3279
3280 for_each_node(node) {
3281 struct mem_cgroup_per_node *pn = memcg->nodeinfo[node];
3282 struct mem_cgroup_per_node *pi;
3283
3284 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
3285 stat[i] = 0;
3286
3287 for_each_online_cpu(cpu)
3288 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
3289 stat[i] += raw_cpu_read(
3290 pn->lruvec_stat_cpu->count[i]);
3291
3292 for (pi = pn; pi; pi = parent_nodeinfo(pi, node))
3293 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
3294 atomic_long_add(stat[i], &pi->lruvec_stat[i]);
3295 }
3296}
3297
3298static void memcg_flush_percpu_vmevents(struct mem_cgroup *memcg)
3299{
3300 unsigned long events[NR_VM_EVENT_ITEMS];
3301 struct mem_cgroup *mi;
3302 int cpu, i;
3303
3304 for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
3305 events[i] = 0;
3306
3307 for_each_online_cpu(cpu)
3308 for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
3309 events[i] += raw_cpu_read(
3310 memcg->vmstats_percpu->events[i]);
3311
3312 for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
3313 for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
3314 atomic_long_add(events[i], &mi->vmevents[i]);
3315}
3316
3263#ifdef CONFIG_MEMCG_KMEM3317#ifdef CONFIG_MEMCG_KMEM
3264static int memcg_online_kmem(struct mem_cgroup *memcg)3318static int memcg_online_kmem(struct mem_cgroup *memcg)
3265{3319{
@@ -4682,6 +4736,12 @@ static void __mem_cgroup_free(struct mem_cgroup *memcg) · 1416 unmodified lines
4682{4736{
4683 int node;4737 int node;
4684 4738
4739 /*
4740 * Flush percpu vmstats and vmevents to guarantee the value correctness
4741 * on parent's and all ancestor levels.
4742 */
4743 memcg_flush_percpu_vmstats(memcg);
4744 memcg_flush_percpu_vmevents(memcg);
4685 for_each_node(node)4745 for_each_node(node)
4686 free_mem_cgroup_per_node_info(memcg, node);4746 free_mem_cgroup_per_node_info(memcg, node);
4687 free_percpu(memcg->vmstats_percpu);4747 free_percpu(memcg->vmstats_percpu);
mm/page_alloc.c+4−15
@@ -2238,27 +2238,12 @@ static int move_freepages(struct zone *zone, · 2237 unmodified lines
2238 unsigned int order;2238 unsigned int order;
2239 int pages_moved = 0;2239 int pages_moved = 0;
2240 2240
2241#ifndef CONFIG_HOLES_IN_ZONE
2242 /*
2243 * page_zone is not safe to call in this context when
2244 * CONFIG_HOLES_IN_ZONE is set. This bug check is probably redundant
2245 * anyway as we check zone boundaries in move_freepages_block().
2246 * Remove at a later date when no bug reports exist related to
2247 * grouping pages by mobility
2248 */
2249 VM_BUG_ON(pfn_valid(page_to_pfn(start_page)) &&
2250 pfn_valid(page_to_pfn(end_page)) &&
2251 page_zone(start_page) != page_zone(end_page));
2252#endif
2253 for (page = start_page; page <= end_page;) {2241 for (page = start_page; page <= end_page;) {
2254 if (!pfn_valid_within(page_to_pfn(page))) {2242 if (!pfn_valid_within(page_to_pfn(page))) {
2255 page++;2243 page++;
2256 continue;2244 continue;
2257 }2245 }
2258 2246
2259 /* Make sure we are not inadvertently changing nodes */
2260 VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page);
2261
2262 if (!PageBuddy(page)) {2247 if (!PageBuddy(page)) {
2263 /*2248 /*
2264 * We assume that pages that could be isolated for2249 * We assume that pages that could be isolated for
@@ -2273,6 +2258,10 @@ static int move_freepages(struct zone *zone, · 8 unmodified lines
2273 continue;2258 continue;
2274 }2259 }
2275 2260
2261 /* Make sure we are not inadvertently changing nodes */
2262 VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page);
2263 VM_BUG_ON_PAGE(page_zone(page) != zone, page);
2264
2276 order = page_order(page);2265 order = page_order(page);
2277 move_to_free_area(page, &zone->free_area[order], migratetype);2266 move_to_free_area(page, &zone->free_area[order], migratetype);
2278 page += 1 << order;2267 page += 1 << order;
mm/z3fold.c+89−0
@@ -41,6 +41,7 @@ · 40 unmodified lines
41#include <linux/workqueue.h>41#include <linux/workqueue.h>
42#include <linux/slab.h>42#include <linux/slab.h>
43#include <linux/spinlock.h>43#include <linux/spinlock.h>
44#include <linux/wait.h>
44#include <linux/zpool.h>45#include <linux/zpool.h>
45#include <linux/magic.h>46#include <linux/magic.h>
46 47
@@ -145,6 +146,8 @@ struct z3fold_header { · 98 unmodified lines
145 * @release_wq: workqueue for safe page release146 * @release_wq: workqueue for safe page release
146 * @work: work_struct for safe page release147 * @work: work_struct for safe page release
147 * @inode: inode for z3fold pseudo filesystem148 * @inode: inode for z3fold pseudo filesystem
149 * @destroying: bool to stop migration once we start destruction
150 * @isolated: int to count the number of pages currently in isolation
148 *151 *
149 * This structure is allocated at pool creation time and maintains metadata152 * This structure is allocated at pool creation time and maintains metadata
150 * pertaining to a particular z3fold pool.153 * pertaining to a particular z3fold pool.
@@ -163,8 +166,11 @@ struct z3fold_pool { · 12 unmodified lines
163 const struct zpool_ops *zpool_ops;166 const struct zpool_ops *zpool_ops;
164 struct workqueue_struct *compact_wq;167 struct workqueue_struct *compact_wq;
165 struct workqueue_struct *release_wq;168 struct workqueue_struct *release_wq;
169 struct wait_queue_head isolate_wait;
166 struct work_struct work;170 struct work_struct work;
167 struct inode *inode;171 struct inode *inode;
172 bool destroying;
173 int isolated;
168};174};
169 175
170/*176/*
@@ -769,6 +775,7 @@ static struct z3fold_pool *z3fold_create_pool(const char *name, gfp_t gfp, · 598 unmodified lines
769 goto out_c;775 goto out_c;
770 spin_lock_init(&pool->lock);776 spin_lock_init(&pool->lock);
771 spin_lock_init(&pool->stale_lock);777 spin_lock_init(&pool->stale_lock);
778 init_waitqueue_head(&pool->isolate_wait);
772 pool->unbuddied = __alloc_percpu(sizeof(struct list_head)*NCHUNKS, 2);779 pool->unbuddied = __alloc_percpu(sizeof(struct list_head)*NCHUNKS, 2);
773 if (!pool->unbuddied)780 if (!pool->unbuddied)
774 goto out_pool;781 goto out_pool;
@@ -808,6 +815,15 @@ out: · 33 unmodified lines
808 return NULL;815 return NULL;
809}816}
810 817
818static bool pool_isolated_are_drained(struct z3fold_pool *pool)
819{
820 bool ret;
821
822 spin_lock(&pool->lock);
823 ret = pool->isolated == 0;
824 spin_unlock(&pool->lock);
825 return ret;
826}
811/**827/**
812 * z3fold_destroy_pool() - destroys an existing z3fold pool828 * z3fold_destroy_pool() - destroys an existing z3fold pool
813 * @pool: the z3fold pool to be destroyed829 * @pool: the z3fold pool to be destroyed
@@ -817,6 +833,22 @@ out: · 3 unmodified lines
817static void z3fold_destroy_pool(struct z3fold_pool *pool)833static void z3fold_destroy_pool(struct z3fold_pool *pool)
818{834{
819 kmem_cache_destroy(pool->c_handle);835 kmem_cache_destroy(pool->c_handle);
836 /*
837 * We set pool-> destroying under lock to ensure that
838 * z3fold_page_isolate() sees any changes to destroying. This way we
839 * avoid the need for any memory barriers.
840 */
841
842 spin_lock(&pool->lock);
843 pool->destroying = true;
844 spin_unlock(&pool->lock);
845
846 /*
847 * We need to ensure that no pages are being migrated while we destroy
848 * these workqueues, as migration can queue work on either of the
849 * workqueues.
850 */
851 wait_event(pool->isolate_wait, !pool_isolated_are_drained(pool));
820 852
821 /*853 /*
822 * We need to destroy pool->compact_wq before pool->release_wq,854 * We need to destroy pool->compact_wq before pool->release_wq,
@@ -1307,6 +1339,28 @@ static u64 z3fold_get_pool_size(struct z3fold_pool *pool) · 484 unmodified lines
1307 return atomic64_read(&pool->pages_nr);1339 return atomic64_read(&pool->pages_nr);
1308}1340}
1309 1341
1342/*
1343 * z3fold_dec_isolated() expects to be called while pool->lock is held.
1344 */
1345static void z3fold_dec_isolated(struct z3fold_pool *pool)
1346{
1347 assert_spin_locked(&pool->lock);
1348 VM_BUG_ON(pool->isolated <= 0);
1349 pool->isolated--;
1350
1351 /*
1352 * If we have no more isolated pages, we have to see if
1353 * z3fold_destroy_pool() is waiting for a signal.
1354 */
1355 if (pool->isolated == 0 && waitqueue_active(&pool->isolate_wait))
1356 wake_up_all(&pool->isolate_wait);
1357}
1358
1359static void z3fold_inc_isolated(struct z3fold_pool *pool)
1360{
1361 pool->isolated++;
1362}
1363
1310static bool z3fold_page_isolate(struct page *page, isolate_mode_t mode)1364static bool z3fold_page_isolate(struct page *page, isolate_mode_t mode)
1311{1365{
1312 struct z3fold_header *zhdr;1366 struct z3fold_header *zhdr;
@@ -1333,6 +1387,33 @@ static bool z3fold_page_isolate(struct page *page, isolate_mode_t mode) · 20 unmodified lines
1333 spin_lock(&pool->lock);1387 spin_lock(&pool->lock);
1334 if (!list_empty(&page->lru))1388 if (!list_empty(&page->lru))
1335 list_del(&page->lru);1389 list_del(&page->lru);
1390 /*
1391 * We need to check for destruction while holding pool->lock, as
1392 * otherwise destruction could see 0 isolated pages, and
1393 * proceed.
1394 */
1395 if (unlikely(pool->destroying)) {
1396 spin_unlock(&pool->lock);
1397 /*
1398 * If this page isn't stale, somebody else holds a
1399 * reference to it. Let't drop our refcount so that they
1400 * can call the release logic.
1401 */
1402 if (unlikely(kref_put(&zhdr->refcount,
1403 release_z3fold_page_locked))) {
1404 /*
1405 * If we get here we have kref problems, so we
1406 * should freak out.
1407 */
1408 WARN(1, "Z3fold is experiencing kref problems\n");
1409 return false;
1410 }
1411 z3fold_page_unlock(zhdr);
1412 return false;
1413 }
1414
1415
1416 z3fold_inc_isolated(pool);
1336 spin_unlock(&pool->lock);1417 spin_unlock(&pool->lock);
1337 z3fold_page_unlock(zhdr);1418 z3fold_page_unlock(zhdr);
1338 return true;1419 return true;
@@ -1401,6 +1482,10 @@ static int z3fold_page_migrate(struct address_space *mapping, struct page *newpa · 62 unmodified lines
1401 1482
1402 queue_work_on(new_zhdr->cpu, pool->compact_wq, &new_zhdr->work);1483 queue_work_on(new_zhdr->cpu, pool->compact_wq, &new_zhdr->work);
1403 1484
1485 spin_lock(&pool->lock);
1486 z3fold_dec_isolated(pool);
1487 spin_unlock(&pool->lock);
1488
1404 page_mapcount_reset(page);1489 page_mapcount_reset(page);
1405 put_page(page);1490 put_page(page);
1406 return 0;1491 return 0;
@@ -1420,10 +1505,14 @@ static void z3fold_page_putback(struct page *page) · 13 unmodified lines
1420 INIT_LIST_HEAD(&page->lru);1505 INIT_LIST_HEAD(&page->lru);
1421 if (kref_put(&zhdr->refcount, release_z3fold_page_locked)) {1506 if (kref_put(&zhdr->refcount, release_z3fold_page_locked)) {
1422 atomic64_dec(&pool->pages_nr);1507 atomic64_dec(&pool->pages_nr);
1508 spin_lock(&pool->lock);
1509 z3fold_dec_isolated(pool);
1510 spin_unlock(&pool->lock);
1423 return;1511 return;
1424 }1512 }
1425 spin_lock(&pool->lock);1513 spin_lock(&pool->lock);
1426 list_add(&page->lru, &pool->lru);1514 list_add(&page->lru, &pool->lru);
1515 z3fold_dec_isolated(pool);
1427 spin_unlock(&pool->lock);1516 spin_unlock(&pool->lock);
1428 z3fold_page_unlock(zhdr);1517 z3fold_page_unlock(zhdr);
1429}1518}
mm/zsmalloc.c+73−5
@@ -54,6 +54,7 @@ · 53 unmodified lines
54#include <linux/mount.h>54#include <linux/mount.h>
55#include <linux/pseudo_fs.h>55#include <linux/pseudo_fs.h>
56#include <linux/migrate.h>56#include <linux/migrate.h>
57#include <linux/wait.h>
57#include <linux/pagemap.h>58#include <linux/pagemap.h>
58#include <linux/fs.h>59#include <linux/fs.h>
59 60
@@ -268,6 +269,10 @@ struct zs_pool { · 208 unmodified lines
268#ifdef CONFIG_COMPACTION269#ifdef CONFIG_COMPACTION
269 struct inode *inode;270 struct inode *inode;
270 struct work_struct free_work;271 struct work_struct free_work;
272 /* A wait queue for when migration races with async_free_zspage() */
273 struct wait_queue_head migration_wait;
274 atomic_long_t isolated_pages;
275 bool destroying;
271#endif276#endif
272};277};
273 278
@@ -1862,6 +1867,31 @@ static void dec_zspage_isolation(struct zspage *zspage) · 1588 unmodified lines
1862 zspage->isolated--;1867 zspage->isolated--;
1863}1868}
1864 1869
1870static void putback_zspage_deferred(struct zs_pool *pool,
1871 struct size_class *class,
1872 struct zspage *zspage)
1873{
1874 enum fullness_group fg;
1875
1876 fg = putback_zspage(class, zspage);
1877 if (fg == ZS_EMPTY)
1878 schedule_work(&pool->free_work);
1879
1880}
1881
1882static inline void zs_pool_dec_isolated(struct zs_pool *pool)
1883{
1884 VM_BUG_ON(atomic_long_read(&pool->isolated_pages) <= 0);
1885 atomic_long_dec(&pool->isolated_pages);
1886 /*
1887 * There's no possibility of racing, since wait_for_isolated_drain()
1888 * checks the isolated count under &class->lock after enqueuing
1889 * on migration_wait.
1890 */
1891 if (atomic_long_read(&pool->isolated_pages) == 0 && pool->destroying)
1892 wake_up_all(&pool->migration_wait);
1893}
1894
1865static void replace_sub_page(struct size_class *class, struct zspage *zspage,1895static void replace_sub_page(struct size_class *class, struct zspage *zspage,
1866 struct page *newpage, struct page *oldpage)1896 struct page *newpage, struct page *oldpage)
1867{1897{
@@ -1931,6 +1961,7 @@ static bool zs_page_isolate(struct page *page, isolate_mode_t mode) · 63 unmodified lines
1931 */1961 */
1932 if (!list_empty(&zspage->list) && !is_zspage_isolated(zspage)) {1962 if (!list_empty(&zspage->list) && !is_zspage_isolated(zspage)) {
1933 get_zspage_mapping(zspage, &class_idx, &fullness);1963 get_zspage_mapping(zspage, &class_idx, &fullness);
1964 atomic_long_inc(&pool->isolated_pages);
1934 remove_zspage(class, zspage, fullness);1965 remove_zspage(class, zspage, fullness);
1935 }1966 }
1936 1967
@@ -2030,8 +2061,16 @@ static int zs_page_migrate(struct address_space *mapping, struct page *newpage, · 93 unmodified lines
2030 * Page migration is done so let's putback isolated zspage to2061 * Page migration is done so let's putback isolated zspage to
2031 * the list if @page is final isolated subpage in the zspage.2062 * the list if @page is final isolated subpage in the zspage.
2032 */2063 */
2033 if (!is_zspage_isolated(zspage))2064 if (!is_zspage_isolated(zspage)) {
2034 putback_zspage(class, zspage);2065 /*
2066 * We cannot race with zs_destroy_pool() here because we wait
2067 * for isolation to hit zero before we start destroying.
2068 * Also, we ensure that everyone can see pool->destroying before
2069 * we start waiting.
2070 */
2071 putback_zspage_deferred(pool, class, zspage);
2072 zs_pool_dec_isolated(pool);
2073 }
2035 2074
2036 reset_page(page);2075 reset_page(page);
2037 put_page(page);2076 put_page(page);
@@ -2077,13 +2116,12 @@ static void zs_page_putback(struct page *page) · 39 unmodified lines
2077 spin_lock(&class->lock);2116 spin_lock(&class->lock);
2078 dec_zspage_isolation(zspage);2117 dec_zspage_isolation(zspage);
2079 if (!is_zspage_isolated(zspage)) {2118 if (!is_zspage_isolated(zspage)) {
2080 fg = putback_zspage(class, zspage);
2081 /*2119 /*
2082 * Due to page_lock, we cannot free zspage immediately2120 * Due to page_lock, we cannot free zspage immediately
2083 * so let's defer.2121 * so let's defer.
2084 */2122 */
2085 if (fg == ZS_EMPTY)2123 putback_zspage_deferred(pool, class, zspage);
2086 schedule_work(&pool->free_work);2124 zs_pool_dec_isolated(pool);
2087 }2125 }
2088 spin_unlock(&class->lock);2126 spin_unlock(&class->lock);
2089}2127}
@@ -2107,8 +2145,36 @@ static int zs_register_migration(struct zs_pool *pool) · 17 unmodified lines
2107 return 0;2145 return 0;
2108}2146}
2109 2147
2148static bool pool_isolated_are_drained(struct zs_pool *pool)
2149{
2150 return atomic_long_read(&pool->isolated_pages) == 0;
2151}
2152
2153/* Function for resolving migration */
2154static void wait_for_isolated_drain(struct zs_pool *pool)
2155{
2156
2157 /*
2158 * We're in the process of destroying the pool, so there are no
2159 * active allocations. zs_page_isolate() fails for completely free
2160 * zspages, so we need only wait for the zs_pool's isolated
2161 * count to hit zero.
2162 */
2163 wait_event(pool->migration_wait,
2164 pool_isolated_are_drained(pool));
2165}
2166
2110static void zs_unregister_migration(struct zs_pool *pool)2167static void zs_unregister_migration(struct zs_pool *pool)
2111{2168{
2169 pool->destroying = true;
2170 /*
2171 * We need a memory barrier here to ensure global visibility of
2172 * pool->destroying. Thus pool->isolated pages will either be 0 in which
2173 * case we don't care, or it will be > 0 and pool->destroying will
2174 * ensure that we wake up once isolation hits 0.
2175 */
2176 smp_mb();
2177 wait_for_isolated_drain(pool); /* This can block */
2112 flush_work(&pool->free_work);2178 flush_work(&pool->free_work);
2113 iput(pool->inode);2179 iput(pool->inode);
2114}2180}
@@ -2346,6 +2412,8 @@ struct zs_pool *zs_create_pool(const char *name) · 231 unmodified lines
2346 if (!pool->name)2412 if (!pool->name)
2347 goto err;2413 goto err;
2348 2414
2415 init_waitqueue_head(&pool->migration_wait);
2416
2349 if (create_cache(pool))2417 if (create_cache(pool))
2350 goto err;2418 goto err;
2351 2419