| @@ -527,37 +527,20 @@ static void pcpu_chunk_relocate(struct pcpu_chunk *chunk, int oslot) · 526 unmodified lines |
| 527 | __pcpu_chunk_move(chunk, nslot, oslot < nslot); | 527 | __pcpu_chunk_move(chunk, nslot, oslot < nslot); |
| 528 | } | 528 | } |
| 529 | | 529 | |
| 530 | /** | 530 | /* |
| 531 | * pcpu_cnt_pop_pages- counts populated backing pages in range | 531 | * pcpu_update_empty_pages - update empty page counters |
| 532 | * @chunk: chunk of interest | 532 | * @chunk: chunk of interest |
| 533 | * @bit_off: start offset | 533 | * @nr: nr of empty pages |
| 534 | * @bits: size of area to check | | |
| 535 | * | 534 | * |
| 536 | * Calculates the number of populated pages in the region | 535 | * This is used to keep track of the empty pages now based on the premise |
| 537 | * [page_start, page_end). This keeps track of how many empty populated | 536 | * a md_block covers a page. The hint update functions recognize if a block |
| 538 | * pages are available and decide if async work should be scheduled. | 537 | * is made full or broken to calculate deltas for keeping track of free pages. |
| 539 | * | | |
| 540 | * RETURNS: | | |
| 541 | * The nr of populated pages. | | |
| 542 | */ | 538 | */ |
| 543 | static inline int pcpu_cnt_pop_pages(struct pcpu_chunk *chunk, int bit_off, | 539 | static inline void pcpu_update_empty_pages(struct pcpu_chunk *chunk, int nr) |
| 544 | int bits) | | |
| 545 | { | 540 | { |
| 546 | int page_start = PFN_UP(bit_off * PCPU_MIN_ALLOC_SIZE); | 541 | chunk->nr_empty_pop_pages += nr; |
| 547 | int page_end = PFN_DOWN((bit_off + bits) * PCPU_MIN_ALLOC_SIZE); | 542 | if (chunk != pcpu_reserved_chunk) |
| 548 | | 543 | pcpu_nr_empty_pop_pages += nr; |
| 549 | if (page_start >= page_end) | | |
| 550 | return 0; | | |
| 551 | | | |
| 552 | /* | | |
| 553 | * bitmap_weight counts the number of bits set in a bitmap up to | | |
| 554 | * the specified number of bits. This is counting the populated | | |
| 555 | * pages up to page_end and then subtracting the populated pages | | |
| 556 | * up to page_start to count the populated pages in | | |
| 557 | * [page_start, page_end). | | |
| 558 | */ | | |
| 559 | return bitmap_weight(chunk->populated, page_end) - | | |
| 560 | bitmap_weight(chunk->populated, page_start); | | |
| 561 | } | 544 | } |
| 562 | | 545 | |
| 563 | /* | 546 | /* |
| @@ -608,36 +591,19 @@ static void pcpu_chunk_update(struct pcpu_chunk *chunk, int bit_off, int bits) · 44 unmodified lines |
| 608 | * Updates: | 591 | * Updates: |
| 609 | * chunk->contig_bits | 592 | * chunk->contig_bits |
| 610 | * chunk->contig_bits_start | 593 | * chunk->contig_bits_start |
| 611 | * nr_empty_pop_pages (chunk and global) | | |
| 612 | */ | 594 | */ |
| 613 | static void pcpu_chunk_refresh_hint(struct pcpu_chunk *chunk) | 595 | static void pcpu_chunk_refresh_hint(struct pcpu_chunk *chunk) |
| 614 | { | 596 | { |
| 615 | int bit_off, bits, nr_empty_pop_pages; | 597 | int bit_off, bits; |
| 616 | | 598 | |
| 617 | /* clear metadata */ | 599 | /* clear metadata */ |
| 618 | chunk->contig_bits = 0; | 600 | chunk->contig_bits = 0; |
| 619 | | 601 | |
| 620 | bit_off = chunk->first_bit; | 602 | bit_off = chunk->first_bit; |
| 621 | bits = nr_empty_pop_pages = 0; | 603 | bits = 0; |
| 622 | pcpu_for_each_md_free_region(chunk, bit_off, bits) { | 604 | pcpu_for_each_md_free_region(chunk, bit_off, bits) { |
| 623 | pcpu_chunk_update(chunk, bit_off, bits); | 605 | pcpu_chunk_update(chunk, bit_off, bits); |
| 624 | | | |
| 625 | nr_empty_pop_pages += pcpu_cnt_pop_pages(chunk, bit_off, bits); | | |
| 626 | } | 606 | } |
| 627 | | | |
| 628 | /* | | |
| 629 | * Keep track of nr_empty_pop_pages. | | |
| 630 | * | | |
| 631 | * The chunk maintains the previous number of free pages it held, | | |
| 632 | * so the delta is used to update the global counter. The reserved | | |
| 633 | * chunk is not part of the free page count as they are populated | | |
| 634 | * at init and are special to serving reserved allocations. | | |
| 635 | */ | | |
| 636 | if (chunk != pcpu_reserved_chunk) | | |
| 637 | pcpu_nr_empty_pop_pages += | | |
| 638 | (nr_empty_pop_pages - chunk->nr_empty_pop_pages); | | |
| 639 | | | |
| 640 | chunk->nr_empty_pop_pages = nr_empty_pop_pages; | | |
| 641 | } | 607 | } |
| 642 | | 608 | |
| 643 | /** | 609 | /** |
| @@ -709,6 +675,7 @@ static void pcpu_block_refresh_hint(struct pcpu_chunk *chunk, int index) · 65 unmodified lines |
| 709 | static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off, | 675 | static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off, |
| 710 | int bits) | 676 | int bits) |
| 711 | { | 677 | { |
| | 678 | int nr_empty_pages = 0; |
| 712 | struct pcpu_block_md *s_block, *e_block, *block; | 679 | struct pcpu_block_md *s_block, *e_block, *block; |
| 713 | int s_index, e_index; /* block indexes of the freed allocation */ | 680 | int s_index, e_index; /* block indexes of the freed allocation */ |
| 714 | int s_off, e_off; /* block offsets of the freed allocation */ | 681 | int s_off, e_off; /* block offsets of the freed allocation */ |
| @@ -733,6 +700,9 @@ static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off, · 18 unmodified lines |
| 733 | * If the allocation breaks the contig_hint, a scan is required to | 700 | * If the allocation breaks the contig_hint, a scan is required to |
| 734 | * restore this hint. | 701 | * restore this hint. |
| 735 | */ | 702 | */ |
| | 703 | if (s_block->contig_hint == PCPU_BITMAP_BLOCK_BITS) |
| | 704 | nr_empty_pages++; |
| | 705 | |
| 736 | if (s_off == s_block->first_free) | 706 | if (s_off == s_block->first_free) |
| 737 | s_block->first_free = find_next_zero_bit( | 707 | s_block->first_free = find_next_zero_bit( |
| 738 | pcpu_index_alloc_map(chunk, s_index), | 708 | pcpu_index_alloc_map(chunk, s_index), |
| @@ -760,6 +730,9 @@ static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off, · 21 unmodified lines |
| 760 | * Update e_block. | 730 | * Update e_block. |
| 761 | */ | 731 | */ |
| 762 | if (s_index != e_index) { | 732 | if (s_index != e_index) { |
| | 733 | if (e_block->contig_hint == PCPU_BITMAP_BLOCK_BITS) |
| | 734 | nr_empty_pages++; |
| | 735 | |
| 763 | /* | 736 | /* |
| 764 | * When the allocation is across blocks, the end is along | 737 | * When the allocation is across blocks, the end is along |
| 765 | * the left part of the e_block. | 738 | * the left part of the e_block. |
| @@ -784,6 +757,7 @@ static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off, · 18 unmodified lines |
| 784 | } | 757 | } |
| 785 | | 758 | |
| 786 | /* update in-between md_blocks */ | 759 | /* update in-between md_blocks */ |
| | 760 | nr_empty_pages += (e_index - s_index - 1); |
| 787 | for (block = s_block + 1; block < e_block; block++) { | 761 | for (block = s_block + 1; block < e_block; block++) { |
| 788 | block->contig_hint = 0; | 762 | block->contig_hint = 0; |
| 789 | block->left_free = 0; | 763 | block->left_free = 0; |
| @@ -791,6 +765,9 @@ static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off, · 1 unmodified line |
| 791 | } | 765 | } |
| 792 | } | 766 | } |
| 793 | | 767 | |
| | 768 | if (nr_empty_pages) |
| | 769 | pcpu_update_empty_pages(chunk, -nr_empty_pages); |
| | 770 | |
| 794 | /* | 771 | /* |
| 795 | * The only time a full chunk scan is required is if the chunk | 772 | * The only time a full chunk scan is required is if the chunk |
| 796 | * contig hint is broken. Otherwise, it means a smaller space | 773 | * contig hint is broken. Otherwise, it means a smaller space |
| @@ -823,6 +800,7 @@ static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off, · 26 unmodified lines |
| 823 | static void pcpu_block_update_hint_free(struct pcpu_chunk *chunk, int bit_off, | 800 | static void pcpu_block_update_hint_free(struct pcpu_chunk *chunk, int bit_off, |
| 824 | int bits) | 801 | int bits) |
| 825 | { | 802 | { |
| | 803 | int nr_empty_pages = 0; |
| 826 | struct pcpu_block_md *s_block, *e_block, *block; | 804 | struct pcpu_block_md *s_block, *e_block, *block; |
| 827 | int s_index, e_index; /* block indexes of the freed allocation */ | 805 | int s_index, e_index; /* block indexes of the freed allocation */ |
| 828 | int s_off, e_off; /* block offsets of the freed allocation */ | 806 | int s_off, e_off; /* block offsets of the freed allocation */ |
| @@ -876,14 +854,19 @@ static void pcpu_block_update_hint_free(struct pcpu_chunk *chunk, int bit_off, · 47 unmodified lines |
| 876 | | 854 | |
| 877 | /* update s_block */ | 855 | /* update s_block */ |
| 878 | e_off = (s_index == e_index) ? end : PCPU_BITMAP_BLOCK_BITS; | 856 | e_off = (s_index == e_index) ? end : PCPU_BITMAP_BLOCK_BITS; |
| | 857 | if (!start && e_off == PCPU_BITMAP_BLOCK_BITS) |
| | 858 | nr_empty_pages++; |
| 879 | pcpu_block_update(s_block, start, e_off); | 859 | pcpu_block_update(s_block, start, e_off); |
| 880 | | 860 | |
| 881 | /* freeing in the same block */ | 861 | /* freeing in the same block */ |
| 882 | if (s_index != e_index) { | 862 | if (s_index != e_index) { |
| 883 | /* update e_block */ | 863 | /* update e_block */ |
| | 864 | if (end == PCPU_BITMAP_BLOCK_BITS) |
| | 865 | nr_empty_pages++; |
| 884 | pcpu_block_update(e_block, 0, end); | 866 | pcpu_block_update(e_block, 0, end); |
| 885 | | 867 | |
| 886 | /* reset md_blocks in the middle */ | 868 | /* reset md_blocks in the middle */ |
| | 869 | nr_empty_pages += (e_index - s_index - 1); |
| 887 | for (block = s_block + 1; block < e_block; block++) { | 870 | for (block = s_block + 1; block < e_block; block++) { |
| 888 | block->first_free = 0; | 871 | block->first_free = 0; |
| 889 | block->contig_hint_start = 0; | 872 | block->contig_hint_start = 0; |
| @@ -893,15 +876,16 @@ static void pcpu_block_update_hint_free(struct pcpu_chunk *chunk, int bit_off, · 3 unmodified lines |
| 893 | } | 876 | } |
| 894 | } | 877 | } |
| 895 | | 878 | |
| | 879 | if (nr_empty_pages) |
| | 880 | pcpu_update_empty_pages(chunk, nr_empty_pages); |
| | 881 | |
| 896 | /* | 882 | /* |
| 897 | * Refresh chunk metadata when the free makes a page free, a block | 883 | * Refresh chunk metadata when the free makes a block free or spans |
| 898 | * free, or spans across blocks. The contig hint may be off by up to | 884 | * across blocks. The contig_hint may be off by up to a page, but if |
| 899 | * a page, but if the hint is contained in a block, it will be accurate | 885 | * the contig_hint is contained in a block, it will be accurate with |
| 900 | * with the else condition below. | 886 | * the else condition below. |
| 901 | */ | 887 | */ |
| 902 | if ((ALIGN_DOWN(end, min(PCPU_BITS_PER_PAGE, PCPU_BITMAP_BLOCK_BITS)) > | 888 | if (((end - start) >= PCPU_BITMAP_BLOCK_BITS) || s_index != e_index) |
| 903 | ALIGN(start, min(PCPU_BITS_PER_PAGE, PCPU_BITMAP_BLOCK_BITS))) || | | |
| 904 | s_index != e_index) | | |
| 905 | pcpu_chunk_refresh_hint(chunk); | 889 | pcpu_chunk_refresh_hint(chunk); |
| 906 | else | 890 | else |
| 907 | pcpu_chunk_update(chunk, pcpu_block_off_to_off(s_index, start), | 891 | pcpu_chunk_update(chunk, pcpu_block_off_to_off(s_index, start), |
| @@ -1178,9 +1162,7 @@ static struct pcpu_chunk * __init pcpu_alloc_first_chunk(unsigned long tmp_addr, · 270 unmodified lines |
| 1178 | chunk->immutable = true; | 1162 | chunk->immutable = true; |
| 1179 | bitmap_fill(chunk->populated, chunk->nr_pages); | 1163 | bitmap_fill(chunk->populated, chunk->nr_pages); |
| 1180 | chunk->nr_populated = chunk->nr_pages; | 1164 | chunk->nr_populated = chunk->nr_pages; |
| 1181 | chunk->nr_empty_pop_pages = | 1165 | chunk->nr_empty_pop_pages = chunk->nr_pages; |
| 1182 | pcpu_cnt_pop_pages(chunk, start_offset / PCPU_MIN_ALLOC_SIZE, | | |
| 1183 | map_size / PCPU_MIN_ALLOC_SIZE); | | |
| 1184 | | 1166 | |
| 1185 | chunk->contig_bits = map_size / PCPU_MIN_ALLOC_SIZE; | 1167 | chunk->contig_bits = map_size / PCPU_MIN_ALLOC_SIZE; |
| 1186 | chunk->free_bytes = map_size; | 1168 | chunk->free_bytes = map_size; |
| @@ -1275,7 +1257,6 @@ static void pcpu_free_chunk(struct pcpu_chunk *chunk) · 88 unmodified lines |
| 1275 | * @chunk: pcpu_chunk which got populated | 1257 | * @chunk: pcpu_chunk which got populated |
| 1276 | * @page_start: the start page | 1258 | * @page_start: the start page |
| 1277 | * @page_end: the end page | 1259 | * @page_end: the end page |
| 1278 | * @for_alloc: if this is to populate for allocation | | |
| 1279 | * | 1260 | * |
| 1280 | * Pages in [@page_start,@page_end) have been populated to @chunk. Update | 1261 | * Pages in [@page_start,@page_end) have been populated to @chunk. Update |
| 1281 | * the bookkeeping information accordingly. Must be called after each | 1262 | * the bookkeeping information accordingly. Must be called after each |
| @@ -1285,7 +1266,7 @@ static void pcpu_free_chunk(struct pcpu_chunk *chunk) · 3 unmodified lines |
| 1285 | * is to serve an allocation in that area. | 1266 | * is to serve an allocation in that area. |
| 1286 | */ | 1267 | */ |
| 1287 | static void pcpu_chunk_populated(struct pcpu_chunk *chunk, int page_start, | 1268 | static void pcpu_chunk_populated(struct pcpu_chunk *chunk, int page_start, |
| 1288 | int page_end, bool for_alloc) | 1269 | int page_end) |
| 1289 | { | 1270 | { |
| 1290 | int nr = page_end - page_start; | 1271 | int nr = page_end - page_start; |
| 1291 | | 1272 | |
| @@ -1295,10 +1276,7 @@ static void pcpu_chunk_populated(struct pcpu_chunk *chunk, int page_start, · 3 unmodified lines |
| 1295 | chunk->nr_populated += nr; | 1276 | chunk->nr_populated += nr; |
| 1296 | pcpu_nr_populated += nr; | 1277 | pcpu_nr_populated += nr; |
| 1297 | | 1278 | |
| 1298 | if (!for_alloc) { | 1279 | pcpu_update_empty_pages(chunk, nr); |
| 1299 | chunk->nr_empty_pop_pages += nr; | | |
| 1300 | pcpu_nr_empty_pop_pages += nr; | | |
| 1301 | } | | |
| 1302 | } | 1280 | } |
| 1303 | | 1281 | |
| 1304 | /** | 1282 | /** |
| @@ -1320,9 +1298,9 @@ static void pcpu_chunk_depopulated(struct pcpu_chunk *chunk, · 15 unmodified lines |
| 1320 | | 1298 | |
| 1321 | bitmap_clear(chunk->populated, page_start, nr); | 1299 | bitmap_clear(chunk->populated, page_start, nr); |
| 1322 | chunk->nr_populated -= nr; | 1300 | chunk->nr_populated -= nr; |
| 1323 | chunk->nr_empty_pop_pages -= nr; | | |
| 1324 | pcpu_nr_empty_pop_pages -= nr; | | |
| 1325 | pcpu_nr_populated -= nr; | 1301 | pcpu_nr_populated -= nr; |
| | 1302 | |
| | 1303 | pcpu_update_empty_pages(chunk, -nr); |
| 1326 | } | 1304 | } |
| 1327 | | 1305 | |
| 1328 | /* | 1306 | /* |
| @@ -1537,7 +1515,7 @@ area_found: · 208 unmodified lines |
| 1537 | err = "failed to populate"; | 1515 | err = "failed to populate"; |
| 1538 | goto fail_unlock; | 1516 | goto fail_unlock; |
| 1539 | } | 1517 | } |
| 1540 | pcpu_chunk_populated(chunk, rs, re, true); | 1518 | pcpu_chunk_populated(chunk, rs, re); |
| 1541 | spin_unlock_irqrestore(&pcpu_lock, flags); | 1519 | spin_unlock_irqrestore(&pcpu_lock, flags); |
| 1542 | } | 1520 | } |
| 1543 | | 1521 | |
| @@ -1736,7 +1714,7 @@ retry_pop: · 192 unmodified lines |
| 1736 | if (!ret) { | 1714 | if (!ret) { |
| 1737 | nr_to_pop -= nr; | 1715 | nr_to_pop -= nr; |
| 1738 | spin_lock_irq(&pcpu_lock); | 1716 | spin_lock_irq(&pcpu_lock); |
| 1739 | pcpu_chunk_populated(chunk, rs, rs + nr, false); | 1717 | pcpu_chunk_populated(chunk, rs, rs + nr); |
| 1740 | spin_unlock_irq(&pcpu_lock); | 1718 | spin_unlock_irq(&pcpu_lock); |
| 1741 | } else { | 1719 | } else { |
| 1742 | nr_to_pop = 0; | 1720 | nr_to_pop = 0; |