History

percpu: set PCPU_BITMAP_BLOCK_SIZE to PAGE_SIZE

Previously, block size was flexible based on the constraint that the GCD(PCPU_BITMAP_BLOCK_SIZE, PAGE_SIZE) > 1. However, this carried the overhead that keeping a floating number of populated free pages required scanning over the free regions of a chunk. Setting the block size to be fixed at PAGE_SIZE lets us know when an empty page becomes used as we will break a full contig_hint of a block. This means we no longer have to scan the whole chunk upon breaking a contig_hint which empty page management piggybacked off. A later patch takes advantage of this to optimize the allocation path by only scanning forward using the scan_hint introduced later too.

Dennis Zhou committed 7 years ago

Commit b239f7d1 parent 8744d853 files · +48 −76Browse files

Unified · Split

3 files
  1. include/linux/percpu.h +3 −9
  2. mm/percpu-km.c +1 −1
  3. mm/percpu.c +44 −66
include/linux/percpu.h+3−9
@@ -26,16 +26,10 @@ · 25 unmodified lines
26#define PCPU_MIN_ALLOC_SHIFT 226#define PCPU_MIN_ALLOC_SHIFT 2
27#define PCPU_MIN_ALLOC_SIZE (1 << PCPU_MIN_ALLOC_SHIFT)27#define PCPU_MIN_ALLOC_SIZE (1 << PCPU_MIN_ALLOC_SHIFT)
28 28
29/* number of bits per page, used to trigger a scan if blocks are > PAGE_SIZE */
30#define PCPU_BITS_PER_PAGE (PAGE_SIZE >> PCPU_MIN_ALLOC_SHIFT)
31
32/*29/*
33 * This determines the size of each metadata block. There are several subtle30 * The PCPU_BITMAP_BLOCK_SIZE must be the same size as PAGE_SIZE as the
34 * constraints around this constant. The reserved region must be a multiple of31 * updating of hints is used to manage the nr_empty_pop_pages in both
35 * PCPU_BITMAP_BLOCK_SIZE. Additionally, PCPU_BITMAP_BLOCK_SIZE must be a32 * the chunk and globally.
36 * multiple of PAGE_SIZE or PAGE_SIZE must be a multiple of
37 * PCPU_BITMAP_BLOCK_SIZE to align with the populated page map. The unit_size
38 * also has to be a multiple of PCPU_BITMAP_BLOCK_SIZE to ensure full blocks.
39 */33 */
40#define PCPU_BITMAP_BLOCK_SIZE PAGE_SIZE34#define PCPU_BITMAP_BLOCK_SIZE PAGE_SIZE
41#define PCPU_BITMAP_BLOCK_BITS (PCPU_BITMAP_BLOCK_SIZE >> \35#define PCPU_BITMAP_BLOCK_BITS (PCPU_BITMAP_BLOCK_SIZE >> \
mm/percpu-km.c+1−1
@@ -70,7 +70,7 @@ static struct pcpu_chunk *pcpu_create_chunk(gfp_t gfp) · 69 unmodified lines
70 chunk->base_addr = page_address(pages);70 chunk->base_addr = page_address(pages);
71 71
72 spin_lock_irqsave(&pcpu_lock, flags);72 spin_lock_irqsave(&pcpu_lock, flags);
73 pcpu_chunk_populated(chunk, 0, nr_pages, false);73 pcpu_chunk_populated(chunk, 0, nr_pages);
74 spin_unlock_irqrestore(&pcpu_lock, flags);74 spin_unlock_irqrestore(&pcpu_lock, flags);
75 75
76 pcpu_stats_chunk_alloc();76 pcpu_stats_chunk_alloc();
mm/percpu.c+44−66
@@ -527,37 +527,20 @@ static void pcpu_chunk_relocate(struct pcpu_chunk *chunk, int oslot) · 526 unmodified lines
527 __pcpu_chunk_move(chunk, nslot, oslot < nslot);527 __pcpu_chunk_move(chunk, nslot, oslot < nslot);
528}528}
529 529
530/**530/*
531 * pcpu_cnt_pop_pages- counts populated backing pages in range531 * pcpu_update_empty_pages - update empty page counters
532 * @chunk: chunk of interest532 * @chunk: chunk of interest
533 * @bit_off: start offset533 * @nr: nr of empty pages
534 * @bits: size of area to check
535 *534 *
536 * Calculates the number of populated pages in the region535 * This is used to keep track of the empty pages now based on the premise
537 * [page_start, page_end). This keeps track of how many empty populated536 * a md_block covers a page. The hint update functions recognize if a block
538 * pages are available and decide if async work should be scheduled.537 * is made full or broken to calculate deltas for keeping track of free pages.
539 *
540 * RETURNS:
541 * The nr of populated pages.
542 */538 */
543static inline int pcpu_cnt_pop_pages(struct pcpu_chunk *chunk, int bit_off,539static inline void pcpu_update_empty_pages(struct pcpu_chunk *chunk, int nr)
544 int bits)
545{540{
546 int page_start = PFN_UP(bit_off * PCPU_MIN_ALLOC_SIZE);541 chunk->nr_empty_pop_pages += nr;
547 int page_end = PFN_DOWN((bit_off + bits) * PCPU_MIN_ALLOC_SIZE);542 if (chunk != pcpu_reserved_chunk)
548 543 pcpu_nr_empty_pop_pages += nr;
549 if (page_start >= page_end)
550 return 0;
551
552 /*
553 * bitmap_weight counts the number of bits set in a bitmap up to
554 * the specified number of bits. This is counting the populated
555 * pages up to page_end and then subtracting the populated pages
556 * up to page_start to count the populated pages in
557 * [page_start, page_end).
558 */
559 return bitmap_weight(chunk->populated, page_end) -
560 bitmap_weight(chunk->populated, page_start);
561}544}
562 545
563/*546/*
@@ -608,36 +591,19 @@ static void pcpu_chunk_update(struct pcpu_chunk *chunk, int bit_off, int bits) · 44 unmodified lines
608 * Updates:591 * Updates:
609 * chunk->contig_bits592 * chunk->contig_bits
610 * chunk->contig_bits_start593 * chunk->contig_bits_start
611 * nr_empty_pop_pages (chunk and global)
612 */594 */
613static void pcpu_chunk_refresh_hint(struct pcpu_chunk *chunk)595static void pcpu_chunk_refresh_hint(struct pcpu_chunk *chunk)
614{596{
615 int bit_off, bits, nr_empty_pop_pages;597 int bit_off, bits;
616 598
617 /* clear metadata */599 /* clear metadata */
618 chunk->contig_bits = 0;600 chunk->contig_bits = 0;
619 601
620 bit_off = chunk->first_bit;602 bit_off = chunk->first_bit;
621 bits = nr_empty_pop_pages = 0;603 bits = 0;
622 pcpu_for_each_md_free_region(chunk, bit_off, bits) {604 pcpu_for_each_md_free_region(chunk, bit_off, bits) {
623 pcpu_chunk_update(chunk, bit_off, bits);605 pcpu_chunk_update(chunk, bit_off, bits);
624
625 nr_empty_pop_pages += pcpu_cnt_pop_pages(chunk, bit_off, bits);
626 }606 }
627
628 /*
629 * Keep track of nr_empty_pop_pages.
630 *
631 * The chunk maintains the previous number of free pages it held,
632 * so the delta is used to update the global counter. The reserved
633 * chunk is not part of the free page count as they are populated
634 * at init and are special to serving reserved allocations.
635 */
636 if (chunk != pcpu_reserved_chunk)
637 pcpu_nr_empty_pop_pages +=
638 (nr_empty_pop_pages - chunk->nr_empty_pop_pages);
639
640 chunk->nr_empty_pop_pages = nr_empty_pop_pages;
641}607}
642 608
643/**609/**
@@ -709,6 +675,7 @@ static void pcpu_block_refresh_hint(struct pcpu_chunk *chunk, int index) · 65 unmodified lines
709static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off,675static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off,
710 int bits)676 int bits)
711{677{
678 int nr_empty_pages = 0;
712 struct pcpu_block_md *s_block, *e_block, *block;679 struct pcpu_block_md *s_block, *e_block, *block;
713 int s_index, e_index; /* block indexes of the freed allocation */680 int s_index, e_index; /* block indexes of the freed allocation */
714 int s_off, e_off; /* block offsets of the freed allocation */681 int s_off, e_off; /* block offsets of the freed allocation */
@@ -733,6 +700,9 @@ static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off, · 18 unmodified lines
733 * If the allocation breaks the contig_hint, a scan is required to700 * If the allocation breaks the contig_hint, a scan is required to
734 * restore this hint.701 * restore this hint.
735 */702 */
703 if (s_block->contig_hint == PCPU_BITMAP_BLOCK_BITS)
704 nr_empty_pages++;
705
736 if (s_off == s_block->first_free)706 if (s_off == s_block->first_free)
737 s_block->first_free = find_next_zero_bit(707 s_block->first_free = find_next_zero_bit(
738 pcpu_index_alloc_map(chunk, s_index),708 pcpu_index_alloc_map(chunk, s_index),
@@ -760,6 +730,9 @@ static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off, · 21 unmodified lines
760 * Update e_block.730 * Update e_block.
761 */731 */
762 if (s_index != e_index) {732 if (s_index != e_index) {
733 if (e_block->contig_hint == PCPU_BITMAP_BLOCK_BITS)
734 nr_empty_pages++;
735
763 /*736 /*
764 * When the allocation is across blocks, the end is along737 * When the allocation is across blocks, the end is along
765 * the left part of the e_block.738 * the left part of the e_block.
@@ -784,6 +757,7 @@ static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off, · 18 unmodified lines
784 }757 }
785 758
786 /* update in-between md_blocks */759 /* update in-between md_blocks */
760 nr_empty_pages += (e_index - s_index - 1);
787 for (block = s_block + 1; block < e_block; block++) {761 for (block = s_block + 1; block < e_block; block++) {
788 block->contig_hint = 0;762 block->contig_hint = 0;
789 block->left_free = 0;763 block->left_free = 0;
@@ -791,6 +765,9 @@ static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off, · 1 unmodified line
791 }765 }
792 }766 }
793 767
768 if (nr_empty_pages)
769 pcpu_update_empty_pages(chunk, -nr_empty_pages);
770
794 /*771 /*
795 * The only time a full chunk scan is required is if the chunk772 * The only time a full chunk scan is required is if the chunk
796 * contig hint is broken. Otherwise, it means a smaller space773 * contig hint is broken. Otherwise, it means a smaller space
@@ -823,6 +800,7 @@ static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off, · 26 unmodified lines
823static void pcpu_block_update_hint_free(struct pcpu_chunk *chunk, int bit_off,800static void pcpu_block_update_hint_free(struct pcpu_chunk *chunk, int bit_off,
824 int bits)801 int bits)
825{802{
803 int nr_empty_pages = 0;
826 struct pcpu_block_md *s_block, *e_block, *block;804 struct pcpu_block_md *s_block, *e_block, *block;
827 int s_index, e_index; /* block indexes of the freed allocation */805 int s_index, e_index; /* block indexes of the freed allocation */
828 int s_off, e_off; /* block offsets of the freed allocation */806 int s_off, e_off; /* block offsets of the freed allocation */
@@ -876,14 +854,19 @@ static void pcpu_block_update_hint_free(struct pcpu_chunk *chunk, int bit_off, · 47 unmodified lines
876 854
877 /* update s_block */855 /* update s_block */
878 e_off = (s_index == e_index) ? end : PCPU_BITMAP_BLOCK_BITS;856 e_off = (s_index == e_index) ? end : PCPU_BITMAP_BLOCK_BITS;
857 if (!start && e_off == PCPU_BITMAP_BLOCK_BITS)
858 nr_empty_pages++;
879 pcpu_block_update(s_block, start, e_off);859 pcpu_block_update(s_block, start, e_off);
880 860
881 /* freeing in the same block */861 /* freeing in the same block */
882 if (s_index != e_index) {862 if (s_index != e_index) {
883 /* update e_block */863 /* update e_block */
864 if (end == PCPU_BITMAP_BLOCK_BITS)
865 nr_empty_pages++;
884 pcpu_block_update(e_block, 0, end);866 pcpu_block_update(e_block, 0, end);
885 867
886 /* reset md_blocks in the middle */868 /* reset md_blocks in the middle */
869 nr_empty_pages += (e_index - s_index - 1);
887 for (block = s_block + 1; block < e_block; block++) {870 for (block = s_block + 1; block < e_block; block++) {
888 block->first_free = 0;871 block->first_free = 0;
889 block->contig_hint_start = 0;872 block->contig_hint_start = 0;
@@ -893,15 +876,16 @@ static void pcpu_block_update_hint_free(struct pcpu_chunk *chunk, int bit_off, · 3 unmodified lines
893 }876 }
894 }877 }
895 878
879 if (nr_empty_pages)
880 pcpu_update_empty_pages(chunk, nr_empty_pages);
881
896 /*882 /*
897 * Refresh chunk metadata when the free makes a page free, a block883 * Refresh chunk metadata when the free makes a block free or spans
898 * free, or spans across blocks. The contig hint may be off by up to884 * across blocks. The contig_hint may be off by up to a page, but if
899 * a page, but if the hint is contained in a block, it will be accurate885 * the contig_hint is contained in a block, it will be accurate with
900 * with the else condition below.886 * the else condition below.
901 */887 */
902 if ((ALIGN_DOWN(end, min(PCPU_BITS_PER_PAGE, PCPU_BITMAP_BLOCK_BITS)) >888 if (((end - start) >= PCPU_BITMAP_BLOCK_BITS) || s_index != e_index)
903 ALIGN(start, min(PCPU_BITS_PER_PAGE, PCPU_BITMAP_BLOCK_BITS))) ||
904 s_index != e_index)
905 pcpu_chunk_refresh_hint(chunk);889 pcpu_chunk_refresh_hint(chunk);
906 else890 else
907 pcpu_chunk_update(chunk, pcpu_block_off_to_off(s_index, start),891 pcpu_chunk_update(chunk, pcpu_block_off_to_off(s_index, start),
@@ -1178,9 +1162,7 @@ static struct pcpu_chunk * __init pcpu_alloc_first_chunk(unsigned long tmp_addr, · 270 unmodified lines
1178 chunk->immutable = true;1162 chunk->immutable = true;
1179 bitmap_fill(chunk->populated, chunk->nr_pages);1163 bitmap_fill(chunk->populated, chunk->nr_pages);
1180 chunk->nr_populated = chunk->nr_pages;1164 chunk->nr_populated = chunk->nr_pages;
1181 chunk->nr_empty_pop_pages =1165 chunk->nr_empty_pop_pages = chunk->nr_pages;
1182 pcpu_cnt_pop_pages(chunk, start_offset / PCPU_MIN_ALLOC_SIZE,
1183 map_size / PCPU_MIN_ALLOC_SIZE);
1184 1166
1185 chunk->contig_bits = map_size / PCPU_MIN_ALLOC_SIZE;1167 chunk->contig_bits = map_size / PCPU_MIN_ALLOC_SIZE;
1186 chunk->free_bytes = map_size;1168 chunk->free_bytes = map_size;
@@ -1275,7 +1257,6 @@ static void pcpu_free_chunk(struct pcpu_chunk *chunk) · 88 unmodified lines
1275 * @chunk: pcpu_chunk which got populated1257 * @chunk: pcpu_chunk which got populated
1276 * @page_start: the start page1258 * @page_start: the start page
1277 * @page_end: the end page1259 * @page_end: the end page
1278 * @for_alloc: if this is to populate for allocation
1279 *1260 *
1280 * Pages in [@page_start,@page_end) have been populated to @chunk. Update1261 * Pages in [@page_start,@page_end) have been populated to @chunk. Update
1281 * the bookkeeping information accordingly. Must be called after each1262 * the bookkeeping information accordingly. Must be called after each
@@ -1285,7 +1266,7 @@ static void pcpu_free_chunk(struct pcpu_chunk *chunk) · 3 unmodified lines
1285 * is to serve an allocation in that area.1266 * is to serve an allocation in that area.
1286 */1267 */
1287static void pcpu_chunk_populated(struct pcpu_chunk *chunk, int page_start,1268static void pcpu_chunk_populated(struct pcpu_chunk *chunk, int page_start,
1288 int page_end, bool for_alloc)1269 int page_end)
1289{1270{
1290 int nr = page_end - page_start;1271 int nr = page_end - page_start;
1291 1272
@@ -1295,10 +1276,7 @@ static void pcpu_chunk_populated(struct pcpu_chunk *chunk, int page_start, · 3 unmodified lines
1295 chunk->nr_populated += nr;1276 chunk->nr_populated += nr;
1296 pcpu_nr_populated += nr;1277 pcpu_nr_populated += nr;
1297 1278
1298 if (!for_alloc) {1279 pcpu_update_empty_pages(chunk, nr);
1299 chunk->nr_empty_pop_pages += nr;
1300 pcpu_nr_empty_pop_pages += nr;
1301 }
1302}1280}
1303 1281
1304/**1282/**
@@ -1320,9 +1298,9 @@ static void pcpu_chunk_depopulated(struct pcpu_chunk *chunk, · 15 unmodified lines
1320 1298
1321 bitmap_clear(chunk->populated, page_start, nr);1299 bitmap_clear(chunk->populated, page_start, nr);
1322 chunk->nr_populated -= nr;1300 chunk->nr_populated -= nr;
1323 chunk->nr_empty_pop_pages -= nr;
1324 pcpu_nr_empty_pop_pages -= nr;
1325 pcpu_nr_populated -= nr;1301 pcpu_nr_populated -= nr;
1302
1303 pcpu_update_empty_pages(chunk, -nr);
1326}1304}
1327 1305
1328/*1306/*
@@ -1537,7 +1515,7 @@ area_found: · 208 unmodified lines
1537 err = "failed to populate";1515 err = "failed to populate";
1538 goto fail_unlock;1516 goto fail_unlock;
1539 }1517 }
1540 pcpu_chunk_populated(chunk, rs, re, true);1518 pcpu_chunk_populated(chunk, rs, re);
1541 spin_unlock_irqrestore(&pcpu_lock, flags);1519 spin_unlock_irqrestore(&pcpu_lock, flags);
1542 }1520 }
1543 1521
@@ -1736,7 +1714,7 @@ retry_pop: · 192 unmodified lines
1736 if (!ret) {1714 if (!ret) {
1737 nr_to_pop -= nr;1715 nr_to_pop -= nr;
1738 spin_lock_irq(&pcpu_lock);1716 spin_lock_irq(&pcpu_lock);
1739 pcpu_chunk_populated(chunk, rs, rs + nr, false);1717 pcpu_chunk_populated(chunk, rs, rs + nr);
1740 spin_unlock_irq(&pcpu_lock);1718 spin_unlock_irq(&pcpu_lock);
1741 } else {1719 } else {
1742 nr_to_pop = 0;1720 nr_to_pop = 0;

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t