History

percpu: set PCPU_BITMAP_BLOCK_SIZE to PAGE_SIZE

Previously, block size was flexible based on the constraint that the GCD(PCPU_BITMAP_BLOCK_SIZE, PAGE_SIZE) > 1. However, this carried the overhead that keeping a floating number of populated free pages required scanning over the free regions of a chunk. Setting the block size to be fixed at PAGE_SIZE lets us know when an empty page becomes used as we will break a full contig_hint of a block. This means we no longer have to scan the whole chunk upon breaking a contig_hint which empty page management piggybacked off. A later patch takes advantage of this to optimize the allocation path by only scanning forward using the scan_hint introduced later too.

Dennis Zhou committed 7 years ago

Commit b239f7d1 parent 8744d853 files · +48 −76Browse files

Unified · Split

3 files
  1. include/linux/percpu.h +3 −9
  2. mm/percpu-km.c +1 −1
  3. mm/percpu.c +44 −66
include/linux/percpu.h+3−9
@@ -26,16 +26,10 @@ · 25 unmodified lines
2626 #define PCPU_MIN_ALLOC_SHIFT 2
2727 #define PCPU_MIN_ALLOC_SIZE (1 << PCPU_MIN_ALLOC_SHIFT)
2828
29−/* number of bits per page, used to trigger a scan if blocks are > PAGE_SIZE */
30−#define PCPU_BITS_PER_PAGE (PAGE_SIZE >> PCPU_MIN_ALLOC_SHIFT)
31−
3229 /*
33− * This determines the size of each metadata block. There are several subtle
34− * constraints around this constant. The reserved region must be a multiple of
35− * PCPU_BITMAP_BLOCK_SIZE. Additionally, PCPU_BITMAP_BLOCK_SIZE must be a
36− * multiple of PAGE_SIZE or PAGE_SIZE must be a multiple of
37− * PCPU_BITMAP_BLOCK_SIZE to align with the populated page map. The unit_size
38− * also has to be a multiple of PCPU_BITMAP_BLOCK_SIZE to ensure full blocks.
30+ * The PCPU_BITMAP_BLOCK_SIZE must be the same size as PAGE_SIZE as the
31+ * updating of hints is used to manage the nr_empty_pop_pages in both
32+ * the chunk and globally.
3933 */
4034 #define PCPU_BITMAP_BLOCK_SIZE PAGE_SIZE
4135 #define PCPU_BITMAP_BLOCK_BITS (PCPU_BITMAP_BLOCK_SIZE >> \
mm/percpu-km.c+1−1
@@ -70,7 +70,7 @@ static struct pcpu_chunk *pcpu_create_chunk(gfp_t gfp) · 69 unmodified lines
7070 chunk->base_addr = page_address(pages);
7171
7272 spin_lock_irqsave(&pcpu_lock, flags);
73− pcpu_chunk_populated(chunk, 0, nr_pages, false);
73+ pcpu_chunk_populated(chunk, 0, nr_pages);
7474 spin_unlock_irqrestore(&pcpu_lock, flags);
7575
7676 pcpu_stats_chunk_alloc();
mm/percpu.c+44−66
@@ -527,37 +527,20 @@ static void pcpu_chunk_relocate(struct pcpu_chunk *chunk, int oslot) · 526 unmodified lines
527527 __pcpu_chunk_move(chunk, nslot, oslot < nslot);
528528 }
529529
530−/**
531− * pcpu_cnt_pop_pages- counts populated backing pages in range
530+/*
531+ * pcpu_update_empty_pages - update empty page counters
532532 * @chunk: chunk of interest
533− * @bit_off: start offset
534− * @bits: size of area to check
533+ * @nr: nr of empty pages
535534 *
536− * Calculates the number of populated pages in the region
537− * [page_start, page_end). This keeps track of how many empty populated
538− * pages are available and decide if async work should be scheduled.
539− *
540− * RETURNS:
541− * The nr of populated pages.
535+ * This is used to keep track of the empty pages now based on the premise
536+ * a md_block covers a page. The hint update functions recognize if a block
537+ * is made full or broken to calculate deltas for keeping track of free pages.
542538 */
543−static inline int pcpu_cnt_pop_pages(struct pcpu_chunk *chunk, int bit_off,
544− int bits)
539+static inline void pcpu_update_empty_pages(struct pcpu_chunk *chunk, int nr)
545540 {
546− int page_start = PFN_UP(bit_off * PCPU_MIN_ALLOC_SIZE);
547− int page_end = PFN_DOWN((bit_off + bits) * PCPU_MIN_ALLOC_SIZE);
548−
549− if (page_start >= page_end)
550− return 0;
551−
552− /*
553− * bitmap_weight counts the number of bits set in a bitmap up to
554− * the specified number of bits. This is counting the populated
555− * pages up to page_end and then subtracting the populated pages
556− * up to page_start to count the populated pages in
557− * [page_start, page_end).
558− */
559− return bitmap_weight(chunk->populated, page_end) -
560− bitmap_weight(chunk->populated, page_start);
541+ chunk->nr_empty_pop_pages += nr;
542+ if (chunk != pcpu_reserved_chunk)
543+ pcpu_nr_empty_pop_pages += nr;
561544 }
562545
563546 /*
@@ -608,36 +591,19 @@ static void pcpu_chunk_update(struct pcpu_chunk *chunk, int bit_off, int bits) · 44 unmodified lines
608591 * Updates:
609592 * chunk->contig_bits
610593 * chunk->contig_bits_start
611− * nr_empty_pop_pages (chunk and global)
612594 */
613595 static void pcpu_chunk_refresh_hint(struct pcpu_chunk *chunk)
614596 {
615− int bit_off, bits, nr_empty_pop_pages;
597+ int bit_off, bits;
616598
617599 /* clear metadata */
618600 chunk->contig_bits = 0;
619601
620602 bit_off = chunk->first_bit;
621− bits = nr_empty_pop_pages = 0;
603+ bits = 0;
622604 pcpu_for_each_md_free_region(chunk, bit_off, bits) {
623605 pcpu_chunk_update(chunk, bit_off, bits);
624−
625− nr_empty_pop_pages += pcpu_cnt_pop_pages(chunk, bit_off, bits);
626606 }
627−
628− /*
629− * Keep track of nr_empty_pop_pages.
630− *
631− * The chunk maintains the previous number of free pages it held,
632− * so the delta is used to update the global counter. The reserved
633− * chunk is not part of the free page count as they are populated
634− * at init and are special to serving reserved allocations.
635− */
636− if (chunk != pcpu_reserved_chunk)
637− pcpu_nr_empty_pop_pages +=
638− (nr_empty_pop_pages - chunk->nr_empty_pop_pages);
639−
640− chunk->nr_empty_pop_pages = nr_empty_pop_pages;
641607 }
642608
643609 /**
@@ -709,6 +675,7 @@ static void pcpu_block_refresh_hint(struct pcpu_chunk *chunk, int index) · 65 unmodified lines
709675 static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off,
710676 int bits)
711677 {
678+ int nr_empty_pages = 0;
712679 struct pcpu_block_md *s_block, *e_block, *block;
713680 int s_index, e_index; /* block indexes of the freed allocation */
714681 int s_off, e_off; /* block offsets of the freed allocation */
@@ -733,6 +700,9 @@ static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off, · 18 unmodified lines
733700 * If the allocation breaks the contig_hint, a scan is required to
734701 * restore this hint.
735702 */
703+ if (s_block->contig_hint == PCPU_BITMAP_BLOCK_BITS)
704+ nr_empty_pages++;
705+
736706 if (s_off == s_block->first_free)
737707 s_block->first_free = find_next_zero_bit(
738708 pcpu_index_alloc_map(chunk, s_index),
@@ -760,6 +730,9 @@ static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off, · 21 unmodified lines
760730 * Update e_block.
761731 */
762732 if (s_index != e_index) {
733+ if (e_block->contig_hint == PCPU_BITMAP_BLOCK_BITS)
734+ nr_empty_pages++;
735+
763736 /*
764737 * When the allocation is across blocks, the end is along
765738 * the left part of the e_block.
@@ -784,6 +757,7 @@ static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off, · 18 unmodified lines
784757 }
785758
786759 /* update in-between md_blocks */
760+ nr_empty_pages += (e_index - s_index - 1);
787761 for (block = s_block + 1; block < e_block; block++) {
788762 block->contig_hint = 0;
789763 block->left_free = 0;
@@ -791,6 +765,9 @@ static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off, · 1 unmodified line
791765 }
792766 }
793767
768+ if (nr_empty_pages)
769+ pcpu_update_empty_pages(chunk, -nr_empty_pages);
770+
794771 /*
795772 * The only time a full chunk scan is required is if the chunk
796773 * contig hint is broken. Otherwise, it means a smaller space
@@ -823,6 +800,7 @@ static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off, · 26 unmodified lines
823800 static void pcpu_block_update_hint_free(struct pcpu_chunk *chunk, int bit_off,
824801 int bits)
825802 {
803+ int nr_empty_pages = 0;
826804 struct pcpu_block_md *s_block, *e_block, *block;
827805 int s_index, e_index; /* block indexes of the freed allocation */
828806 int s_off, e_off; /* block offsets of the freed allocation */
@@ -876,14 +854,19 @@ static void pcpu_block_update_hint_free(struct pcpu_chunk *chunk, int bit_off, · 47 unmodified lines
876854
877855 /* update s_block */
878856 e_off = (s_index == e_index) ? end : PCPU_BITMAP_BLOCK_BITS;
857+ if (!start && e_off == PCPU_BITMAP_BLOCK_BITS)
858+ nr_empty_pages++;
879859 pcpu_block_update(s_block, start, e_off);
880860
881861 /* freeing in the same block */
882862 if (s_index != e_index) {
883863 /* update e_block */
864+ if (end == PCPU_BITMAP_BLOCK_BITS)
865+ nr_empty_pages++;
884866 pcpu_block_update(e_block, 0, end);
885867
886868 /* reset md_blocks in the middle */
869+ nr_empty_pages += (e_index - s_index - 1);
887870 for (block = s_block + 1; block < e_block; block++) {
888871 block->first_free = 0;
889872 block->contig_hint_start = 0;
@@ -893,15 +876,16 @@ static void pcpu_block_update_hint_free(struct pcpu_chunk *chunk, int bit_off, · 3 unmodified lines
893876 }
894877 }
895878
879+ if (nr_empty_pages)
880+ pcpu_update_empty_pages(chunk, nr_empty_pages);
881+
896882 /*
897− * Refresh chunk metadata when the free makes a page free, a block
898− * free, or spans across blocks. The contig hint may be off by up to
899− * a page, but if the hint is contained in a block, it will be accurate
900− * with the else condition below.
883+ * Refresh chunk metadata when the free makes a block free or spans
884+ * across blocks. The contig_hint may be off by up to a page, but if
885+ * the contig_hint is contained in a block, it will be accurate with
886+ * the else condition below.
901887 */
902− if ((ALIGN_DOWN(end, min(PCPU_BITS_PER_PAGE, PCPU_BITMAP_BLOCK_BITS)) >
903− ALIGN(start, min(PCPU_BITS_PER_PAGE, PCPU_BITMAP_BLOCK_BITS))) ||
904− s_index != e_index)
888+ if (((end - start) >= PCPU_BITMAP_BLOCK_BITS) || s_index != e_index)
905889 pcpu_chunk_refresh_hint(chunk);
906890 else
907891 pcpu_chunk_update(chunk, pcpu_block_off_to_off(s_index, start),
@@ -1178,9 +1162,7 @@ static struct pcpu_chunk * __init pcpu_alloc_first_chunk(unsigned long tmp_addr, · 270 unmodified lines
11781162 chunk->immutable = true;
11791163 bitmap_fill(chunk->populated, chunk->nr_pages);
11801164 chunk->nr_populated = chunk->nr_pages;
1181− chunk->nr_empty_pop_pages =
1182− pcpu_cnt_pop_pages(chunk, start_offset / PCPU_MIN_ALLOC_SIZE,
1183− map_size / PCPU_MIN_ALLOC_SIZE);
1165+ chunk->nr_empty_pop_pages = chunk->nr_pages;
11841166
11851167 chunk->contig_bits = map_size / PCPU_MIN_ALLOC_SIZE;
11861168 chunk->free_bytes = map_size;
@@ -1275,7 +1257,6 @@ static void pcpu_free_chunk(struct pcpu_chunk *chunk) · 88 unmodified lines
12751257 * @chunk: pcpu_chunk which got populated
12761258 * @page_start: the start page
12771259 * @page_end: the end page
1278− * @for_alloc: if this is to populate for allocation
12791260 *
12801261 * Pages in [@page_start,@page_end) have been populated to @chunk. Update
12811262 * the bookkeeping information accordingly. Must be called after each
@@ -1285,7 +1266,7 @@ static void pcpu_free_chunk(struct pcpu_chunk *chunk) · 3 unmodified lines
12851266 * is to serve an allocation in that area.
12861267 */
12871268 static void pcpu_chunk_populated(struct pcpu_chunk *chunk, int page_start,
1288− int page_end, bool for_alloc)
1269+ int page_end)
12891270 {
12901271 int nr = page_end - page_start;
12911272
@@ -1295,10 +1276,7 @@ static void pcpu_chunk_populated(struct pcpu_chunk *chunk, int page_start, · 3 unmodified lines
12951276 chunk->nr_populated += nr;
12961277 pcpu_nr_populated += nr;
12971278
1298− if (!for_alloc) {
1299− chunk->nr_empty_pop_pages += nr;
1300− pcpu_nr_empty_pop_pages += nr;
1301− }
1279+ pcpu_update_empty_pages(chunk, nr);
13021280 }
13031281
13041282 /**
@@ -1320,9 +1298,9 @@ static void pcpu_chunk_depopulated(struct pcpu_chunk *chunk, · 15 unmodified lines
13201298
13211299 bitmap_clear(chunk->populated, page_start, nr);
13221300 chunk->nr_populated -= nr;
1323− chunk->nr_empty_pop_pages -= nr;
1324− pcpu_nr_empty_pop_pages -= nr;
13251301 pcpu_nr_populated -= nr;
1302+
1303+ pcpu_update_empty_pages(chunk, -nr);
13261304 }
13271305
13281306 /*
@@ -1537,7 +1515,7 @@ area_found: · 208 unmodified lines
15371515 err = "failed to populate";
15381516 goto fail_unlock;
15391517 }
1540− pcpu_chunk_populated(chunk, rs, re, true);
1518+ pcpu_chunk_populated(chunk, rs, re);
15411519 spin_unlock_irqrestore(&pcpu_lock, flags);
15421520 }
15431521
@@ -1736,7 +1714,7 @@ retry_pop: · 192 unmodified lines
17361714 if (!ret) {
17371715 nr_to_pop -= nr;
17381716 spin_lock_irq(&pcpu_lock);
1739− pcpu_chunk_populated(chunk, rs, rs + nr, false);
1717+ pcpu_chunk_populated(chunk, rs, rs + nr);
17401718 spin_unlock_irq(&pcpu_lock);
17411719 } else {
17421720 nr_to_pop = 0;

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t