When freeing a page with an order >= shuffle_page_order randomly select the front or back of the list for insertion. While the mm tries to defragment physical pages into huge pages this can tend to make the page allocator more predictable over time. Inject the front-back randomness to preserve the initial randomness established by shuffle_free_memory() when the kernel was booted. The overhead of this manipulation is constrained by only being applied for MAX_ORDER sized pages by default. [akpm@linux-foundation.org: coding-style fixes] Link: http://lkml.kernel.org/r/154899812788.3165233.9066631950746578517.stgit@dwillia2-desk3.amr.corp.intel.com Signed-off-by: Dan Williams <dan.j.williams@intel.com> Reviewed-by: Kees Cook <keescook@chromium.org> Cc: Michal Hocko <mhocko@suse.com> Cc: Dave Hansen <dave.hansen@linux.intel.com> Cc: Keith Busch <keith.busch@intel.com> Cc: Robert Elliott <elliott@hpe.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Unified · Split
| @@ -116,6 +116,18 @@ static inline void add_to_free_area_tail(struct page *page, struct free_area *ar · 115 unmodified lines | |||
| 116 | area->nr_free++; | 116 | area->nr_free++; |
| 117 | } | 117 | } |
| 118 | 118 | ||
| 119 | #ifdef CONFIG_SHUFFLE_PAGE_ALLOCATOR | ||
| 120 | /* Used to preserve page allocation order entropy */ | ||
| 121 | void add_to_free_area_random(struct page *page, struct free_area *area, | ||
| 122 | int migratetype); | ||
| 123 | #else | ||
| 124 | static inline void add_to_free_area_random(struct page *page, | ||
| 125 | struct free_area *area, int migratetype) | ||
| 126 | { | ||
| 127 | add_to_free_area(page, area, migratetype); | ||
| 128 | } | ||
| 129 | #endif | ||
| 130 | |||
| 119 | /* Used for pages which are on another list */ | 131 | /* Used for pages which are on another list */ |
| 120 | static inline void move_to_free_area(struct page *page, struct free_area *area, | 132 | static inline void move_to_free_area(struct page *page, struct free_area *area, |
| 121 | int migratetype) | 133 | int migratetype) |
| @@ -43,6 +43,7 @@ · 42 unmodified lines | |||
| 43 | #include <linux/mempolicy.h> | 43 | #include <linux/mempolicy.h> |
| 44 | #include <linux/memremap.h> | 44 | #include <linux/memremap.h> |
| 45 | #include <linux/stop_machine.h> | 45 | #include <linux/stop_machine.h> |
| 46 | #include <linux/random.h> | ||
| 46 | #include <linux/sort.h> | 47 | #include <linux/sort.h> |
| 47 | #include <linux/pfn.h> | 48 | #include <linux/pfn.h> |
| 48 | #include <linux/backing-dev.h> | 49 | #include <linux/backing-dev.h> |
| @@ -958,7 +959,8 @@ done_merging: · 909 unmodified lines | |||
| 958 | * so it's less likely to be used soon and more likely to be merged | 959 | * so it's less likely to be used soon and more likely to be merged |
| 959 | * as a higher order page | 960 | * as a higher order page |
| 960 | */ | 961 | */ |
| 961 | if ((order < MAX_ORDER-2) && pfn_valid_within(buddy_pfn)) { | 962 | if ((order < MAX_ORDER-2) && pfn_valid_within(buddy_pfn) |
| 963 | && !is_shuffle_order(order)) { | ||
| 962 | struct page *higher_page, *higher_buddy; | 964 | struct page *higher_page, *higher_buddy; |
| 963 | combined_pfn = buddy_pfn & pfn; | 965 | combined_pfn = buddy_pfn & pfn; |
| 964 | higher_page = page + (combined_pfn - pfn); | 966 | higher_page = page + (combined_pfn - pfn); |
| @@ -972,7 +974,12 @@ done_merging: · 7 unmodified lines | |||
| 972 | } | 974 | } |
| 973 | } | 975 | } |
| 974 | 976 | ||
| 975 | add_to_free_area(page, &zone->free_area[order], migratetype); | 977 | if (is_shuffle_order(order)) |
| 978 | add_to_free_area_random(page, &zone->free_area[order], | ||
| 979 | migratetype); | ||
| 980 | else | ||
| 981 | add_to_free_area(page, &zone->free_area[order], migratetype); | ||
| 982 | |||
| 976 | } | 983 | } |
| 977 | 984 | ||
| 978 | /* | 985 | /* |
| @@ -182,3 +182,26 @@ void __meminit __shuffle_free_memory(pg_data_t *pgdat) · 181 unmodified lines | |||
| 182 | for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++) | 182 | for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++) |
| 183 | shuffle_zone(z); | 183 | shuffle_zone(z); |
| 184 | } | 184 | } |
| 185 | |||
| 186 | void add_to_free_area_random(struct page *page, struct free_area *area, | ||
| 187 | int migratetype) | ||
| 188 | { | ||
| 189 | static u64 rand; | ||
| 190 | static u8 rand_bits; | ||
| 191 | |||
| 192 | /* | ||
| 193 | * The lack of locking is deliberate. If 2 threads race to | ||
| 194 | * update the rand state it just adds to the entropy. | ||
| 195 | */ | ||
| 196 | if (rand_bits == 0) { | ||
| 197 | rand_bits = 64; | ||
| 198 | rand = get_random_u64(); | ||
| 199 | } | ||
| 200 | |||
| 201 | if (rand & 1) | ||
| 202 | add_to_free_area(page, area, migratetype); | ||
| 203 | else | ||
| 204 | add_to_free_area_tail(page, area, migratetype); | ||
| 205 | rand_bits--; | ||
| 206 | rand >>= 1; | ||
| 207 | } | ||
| @@ -36,6 +36,13 @@ static inline void shuffle_zone(struct zone *z) · 35 unmodified lines | |||
| 36 | return; | 36 | return; |
| 37 | __shuffle_zone(z); | 37 | __shuffle_zone(z); |
| 38 | } | 38 | } |
| 39 | |||
| 40 | static inline bool is_shuffle_order(int order) | ||
| 41 | { | ||
| 42 | if (!static_branch_unlikely(&page_alloc_shuffle_key)) | ||
| 43 | return false; | ||
| 44 | return order >= SHUFFLE_ORDER; | ||
| 45 | } | ||
| 39 | #else | 46 | #else |
| 40 | static inline void shuffle_free_memory(pg_data_t *pgdat) | 47 | static inline void shuffle_free_memory(pg_data_t *pgdat) |
| 41 | { | 48 | { |
| @@ -48,5 +55,10 @@ static inline void shuffle_zone(struct zone *z) · 6 unmodified lines | |||
| 48 | static inline void page_alloc_shuffle(enum mm_shuffle_ctl ctl) | 55 | static inline void page_alloc_shuffle(enum mm_shuffle_ctl ctl) |
| 49 | { | 56 | { |
| 50 | } | 57 | } |
| 58 | |||
| 59 | static inline bool is_shuffle_order(int order) | ||
| 60 | { | ||
| 61 | return false; | ||
| 62 | } | ||
| 51 | #endif | 63 | #endif |
| 52 | #endif /* _MM_SHUFFLE_H */ | 64 | #endif /* _MM_SHUFFLE_H */ |