mm, futex: fix shared futex pgoff on shmem huge page

If more than one futex is placed on a shmem huge page, it can happen that waking the second wakes the first instead, and leaves the second waiting: the key's shared.pgoff is wrong. When 3.11 commit 13d60f4b6ab5 ("futex: Take hugepages into account when generating futex_key"), the only shared huge pages came from hugetlbfs, and the code added to deal with its exceptional page->index was put into hugetlb source. Then that was missed when 4.8 added shmem huge pages. page_to_pgoff() is what others use for this nowadays: except that, as currently written, it gives the right answer on hugetlbfs head, but nonsense on hugetlbfs tails. Fix that by calling hugetlbfs-specific hugetlb_basepage_index() on PageHuge tails as well as on head. Yes, it's unconventional to declare hugetlb_basepage_index() there in pagemap.h, rather than in hugetlb.h; but I do not expect anything but page_to_pgoff() ever to need it. [akpm@linux-foundation.org: give hugetlb_basepage_index() prototype the correct scope]

Hugh Dickins committed 5 years ago

Commit fe19bd31 parent 5fa54344 files · +9 −28Browse files

Unified · Split

4 files

include/linux/hugetlb.h +0 −16
include/linux/pagemap.h +7 −6
kernel/futex.c +1 −2
mm/hugetlb.c +1 −4

include/linux/hugetlb.h+0−16

@@ -741,17 +741,6 @@ static inline int hstate_index(struct hstate *h) · 740 unmodified lines
741	741	return h - hstates;
742	742	}
743	743
744		−pgoff_t __basepage_index(struct page *page);
745		−
746		−/* Return page->index in PAGE_SIZE units */
747		−static inline pgoff_t basepage_index(struct page *page)
748		−{
749		− if (!PageCompound(page))
750		− return page->index;
751		−
752		− return __basepage_index(page);
753		−}
754		−
755	744	extern int dissolve_free_huge_page(struct page *page);
756	745	extern int dissolve_free_huge_pages(unsigned long start_pfn,
757	746	unsigned long end_pfn);
@@ -988,11 +977,6 @@ static inline int hstate_index(struct hstate *h) · 230 unmodified lines
988	977	return 0;
989	978	}
990	979
991		−static inline pgoff_t basepage_index(struct page *page)
992		−{
993		− return page->index;
994		−}
995		−
996	980	static inline int dissolve_free_huge_page(struct page *page)
997	981	{
998	982	return 0;

include/linux/pagemap.h+7−6

@@ -516,7 +516,7 @@ static inline struct page read_mapping_page(struct address_space mapping, · 515 unmodified lines
516	516	}
517	517
518	518	/*
519		− * Get index of the page with in radix-tree
	519	+ * Get index of the page within radix-tree (but not for hugetlb pages).
520	520	* (TODO: remove once hugetlb pages will have ->index in PAGE_SIZE)
521	521	*/
522	522	static inline pgoff_t page_to_index(struct page *page)
@@ -535,15 +535,16 @@ static inline pgoff_t page_to_index(struct page *page) · 12 unmodified lines
535	535	return pgoff;
536	536	}
537	537
	538	+extern pgoff_t hugetlb_basepage_index(struct page *page);
	539	+
538	540	/*
539		− * Get the offset in PAGE_SIZE.
540		− * (TODO: hugepage should have ->index in PAGE_SIZE)
	541	+ * Get the offset in PAGE_SIZE (even for hugetlb pages).
	542	+ * (TODO: hugetlb pages should have ->index in PAGE_SIZE)
541	543	*/
542	544	static inline pgoff_t page_to_pgoff(struct page *page)
543	545	{
544		− if (unlikely(PageHeadHuge(page)))
545		− return page->index << compound_order(page);
546		−
	546	+ if (unlikely(PageHuge(page)))
	547	+ return hugetlb_basepage_index(page);
547	548	return page_to_index(page);
548	549	}
549	550

kernel/futex.c+1−2

@@ -35,7 +35,6 @@ · 34 unmodified lines
35	35	#include <linux/jhash.h>
36	36	#include <linux/pagemap.h>
37	37	#include <linux/syscalls.h>
38		−#include <linux/hugetlb.h>
39	38	#include <linux/freezer.h>
40	39	#include <linux/memblock.h>
41	40	#include <linux/fault-inject.h>
@@ -650,7 +649,7 @@ again: · 608 unmodified lines
650	649
651	650	key->both.offset \|= FUT_OFF_INODE; /* inode-based key */
652	651	key->shared.i_seq = get_inode_sequence_number(inode);
653		− key->shared.pgoff = basepage_index(tail);
	652	+ key->shared.pgoff = page_to_pgoff(tail);
654	653	rcu_read_unlock();
655	654	}
656	655

mm/hugetlb.c+1−4

@@ -1588,15 +1588,12 @@ struct address_space hugetlb_page_mapping_lock_write(struct page hpage) · 1587 unmodified lines
1588	1588	return NULL;
1589	1589	}
1590	1590
1591		−pgoff_t __basepage_index(struct page *page)
	1591	+pgoff_t hugetlb_basepage_index(struct page *page)
1592	1592	{
1593	1593	struct page *page_head = compound_head(page);
1594	1594	pgoff_t index = page_index(page_head);
1595	1595	unsigned long compound_idx;
1596	1596
1597		− if (!PageHuge(page_head))
1598		− return page_index(page);
1599		−
1600	1597	if (compound_order(page_head) >= MAX_ORDER)
1601	1598	compound_idx = page_to_pfn(page) - page_to_pfn(page_head);
1602	1599	else