| @@ -527,37 +527,20 @@ static void pcpu_chunk_relocate(struct pcpu_chunk *chunk, int oslot) · 526 unmodified lines |
| 527 | 527 | __pcpu_chunk_move(chunk, nslot, oslot < nslot); |
| 528 | 528 | } |
| 529 | 529 | |
| 530 | | −/** |
| 531 | | − * pcpu_cnt_pop_pages- counts populated backing pages in range |
| 530 | +/* |
| 531 | + * pcpu_update_empty_pages - update empty page counters |
| 532 | 532 | * @chunk: chunk of interest |
| 533 | | − * @bit_off: start offset |
| 534 | | − * @bits: size of area to check |
| 533 | + * @nr: nr of empty pages |
| 535 | 534 | * |
| 536 | | − * Calculates the number of populated pages in the region |
| 537 | | − * [page_start, page_end). This keeps track of how many empty populated |
| 538 | | − * pages are available and decide if async work should be scheduled. |
| 539 | | − * |
| 540 | | − * RETURNS: |
| 541 | | − * The nr of populated pages. |
| 535 | + * This is used to keep track of the empty pages now based on the premise |
| 536 | + * a md_block covers a page. The hint update functions recognize if a block |
| 537 | + * is made full or broken to calculate deltas for keeping track of free pages. |
| 542 | 538 | */ |
| 543 | | −static inline int pcpu_cnt_pop_pages(struct pcpu_chunk *chunk, int bit_off, |
| 544 | | − int bits) |
| 539 | +static inline void pcpu_update_empty_pages(struct pcpu_chunk *chunk, int nr) |
| 545 | 540 | { |
| 546 | | − int page_start = PFN_UP(bit_off * PCPU_MIN_ALLOC_SIZE); |
| 547 | | − int page_end = PFN_DOWN((bit_off + bits) * PCPU_MIN_ALLOC_SIZE); |
| 548 | | − |
| 549 | | − if (page_start >= page_end) |
| 550 | | − return 0; |
| 551 | | − |
| 552 | | − /* |
| 553 | | − * bitmap_weight counts the number of bits set in a bitmap up to |
| 554 | | − * the specified number of bits. This is counting the populated |
| 555 | | − * pages up to page_end and then subtracting the populated pages |
| 556 | | − * up to page_start to count the populated pages in |
| 557 | | − * [page_start, page_end). |
| 558 | | − */ |
| 559 | | − return bitmap_weight(chunk->populated, page_end) - |
| 560 | | − bitmap_weight(chunk->populated, page_start); |
| 541 | + chunk->nr_empty_pop_pages += nr; |
| 542 | + if (chunk != pcpu_reserved_chunk) |
| 543 | + pcpu_nr_empty_pop_pages += nr; |
| 561 | 544 | } |
| 562 | 545 | |
| 563 | 546 | /* |
| @@ -608,36 +591,19 @@ static void pcpu_chunk_update(struct pcpu_chunk *chunk, int bit_off, int bits) · 44 unmodified lines |
| 608 | 591 | * Updates: |
| 609 | 592 | * chunk->contig_bits |
| 610 | 593 | * chunk->contig_bits_start |
| 611 | | − * nr_empty_pop_pages (chunk and global) |
| 612 | 594 | */ |
| 613 | 595 | static void pcpu_chunk_refresh_hint(struct pcpu_chunk *chunk) |
| 614 | 596 | { |
| 615 | | − int bit_off, bits, nr_empty_pop_pages; |
| 597 | + int bit_off, bits; |
| 616 | 598 | |
| 617 | 599 | /* clear metadata */ |
| 618 | 600 | chunk->contig_bits = 0; |
| 619 | 601 | |
| 620 | 602 | bit_off = chunk->first_bit; |
| 621 | | − bits = nr_empty_pop_pages = 0; |
| 603 | + bits = 0; |
| 622 | 604 | pcpu_for_each_md_free_region(chunk, bit_off, bits) { |
| 623 | 605 | pcpu_chunk_update(chunk, bit_off, bits); |
| 624 | | − |
| 625 | | − nr_empty_pop_pages += pcpu_cnt_pop_pages(chunk, bit_off, bits); |
| 626 | 606 | } |
| 627 | | − |
| 628 | | − /* |
| 629 | | − * Keep track of nr_empty_pop_pages. |
| 630 | | − * |
| 631 | | − * The chunk maintains the previous number of free pages it held, |
| 632 | | − * so the delta is used to update the global counter. The reserved |
| 633 | | − * chunk is not part of the free page count as they are populated |
| 634 | | − * at init and are special to serving reserved allocations. |
| 635 | | − */ |
| 636 | | − if (chunk != pcpu_reserved_chunk) |
| 637 | | − pcpu_nr_empty_pop_pages += |
| 638 | | − (nr_empty_pop_pages - chunk->nr_empty_pop_pages); |
| 639 | | − |
| 640 | | − chunk->nr_empty_pop_pages = nr_empty_pop_pages; |
| 641 | 607 | } |
| 642 | 608 | |
| 643 | 609 | /** |
| @@ -709,6 +675,7 @@ static void pcpu_block_refresh_hint(struct pcpu_chunk *chunk, int index) · 65 unmodified lines |
| 709 | 675 | static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off, |
| 710 | 676 | int bits) |
| 711 | 677 | { |
| 678 | + int nr_empty_pages = 0; |
| 712 | 679 | struct pcpu_block_md *s_block, *e_block, *block; |
| 713 | 680 | int s_index, e_index; /* block indexes of the freed allocation */ |
| 714 | 681 | int s_off, e_off; /* block offsets of the freed allocation */ |
| @@ -733,6 +700,9 @@ static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off, · 18 unmodified lines |
| 733 | 700 | * If the allocation breaks the contig_hint, a scan is required to |
| 734 | 701 | * restore this hint. |
| 735 | 702 | */ |
| 703 | + if (s_block->contig_hint == PCPU_BITMAP_BLOCK_BITS) |
| 704 | + nr_empty_pages++; |
| 705 | + |
| 736 | 706 | if (s_off == s_block->first_free) |
| 737 | 707 | s_block->first_free = find_next_zero_bit( |
| 738 | 708 | pcpu_index_alloc_map(chunk, s_index), |
| @@ -760,6 +730,9 @@ static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off, · 21 unmodified lines |
| 760 | 730 | * Update e_block. |
| 761 | 731 | */ |
| 762 | 732 | if (s_index != e_index) { |
| 733 | + if (e_block->contig_hint == PCPU_BITMAP_BLOCK_BITS) |
| 734 | + nr_empty_pages++; |
| 735 | + |
| 763 | 736 | /* |
| 764 | 737 | * When the allocation is across blocks, the end is along |
| 765 | 738 | * the left part of the e_block. |
| @@ -784,6 +757,7 @@ static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off, · 18 unmodified lines |
| 784 | 757 | } |
| 785 | 758 | |
| 786 | 759 | /* update in-between md_blocks */ |
| 760 | + nr_empty_pages += (e_index - s_index - 1); |
| 787 | 761 | for (block = s_block + 1; block < e_block; block++) { |
| 788 | 762 | block->contig_hint = 0; |
| 789 | 763 | block->left_free = 0; |
| @@ -791,6 +765,9 @@ static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off, · 1 unmodified line |
| 791 | 765 | } |
| 792 | 766 | } |
| 793 | 767 | |
| 768 | + if (nr_empty_pages) |
| 769 | + pcpu_update_empty_pages(chunk, -nr_empty_pages); |
| 770 | + |
| 794 | 771 | /* |
| 795 | 772 | * The only time a full chunk scan is required is if the chunk |
| 796 | 773 | * contig hint is broken. Otherwise, it means a smaller space |
| @@ -823,6 +800,7 @@ static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off, · 26 unmodified lines |
| 823 | 800 | static void pcpu_block_update_hint_free(struct pcpu_chunk *chunk, int bit_off, |
| 824 | 801 | int bits) |
| 825 | 802 | { |
| 803 | + int nr_empty_pages = 0; |
| 826 | 804 | struct pcpu_block_md *s_block, *e_block, *block; |
| 827 | 805 | int s_index, e_index; /* block indexes of the freed allocation */ |
| 828 | 806 | int s_off, e_off; /* block offsets of the freed allocation */ |
| @@ -876,14 +854,19 @@ static void pcpu_block_update_hint_free(struct pcpu_chunk *chunk, int bit_off, · 47 unmodified lines |
| 876 | 854 | |
| 877 | 855 | /* update s_block */ |
| 878 | 856 | e_off = (s_index == e_index) ? end : PCPU_BITMAP_BLOCK_BITS; |
| 857 | + if (!start && e_off == PCPU_BITMAP_BLOCK_BITS) |
| 858 | + nr_empty_pages++; |
| 879 | 859 | pcpu_block_update(s_block, start, e_off); |
| 880 | 860 | |
| 881 | 861 | /* freeing in the same block */ |
| 882 | 862 | if (s_index != e_index) { |
| 883 | 863 | /* update e_block */ |
| 864 | + if (end == PCPU_BITMAP_BLOCK_BITS) |
| 865 | + nr_empty_pages++; |
| 884 | 866 | pcpu_block_update(e_block, 0, end); |
| 885 | 867 | |
| 886 | 868 | /* reset md_blocks in the middle */ |
| 869 | + nr_empty_pages += (e_index - s_index - 1); |
| 887 | 870 | for (block = s_block + 1; block < e_block; block++) { |
| 888 | 871 | block->first_free = 0; |
| 889 | 872 | block->contig_hint_start = 0; |
| @@ -893,15 +876,16 @@ static void pcpu_block_update_hint_free(struct pcpu_chunk *chunk, int bit_off, · 3 unmodified lines |
| 893 | 876 | } |
| 894 | 877 | } |
| 895 | 878 | |
| 879 | + if (nr_empty_pages) |
| 880 | + pcpu_update_empty_pages(chunk, nr_empty_pages); |
| 881 | + |
| 896 | 882 | /* |
| 897 | | − * Refresh chunk metadata when the free makes a page free, a block |
| 898 | | − * free, or spans across blocks. The contig hint may be off by up to |
| 899 | | − * a page, but if the hint is contained in a block, it will be accurate |
| 900 | | − * with the else condition below. |
| 883 | + * Refresh chunk metadata when the free makes a block free or spans |
| 884 | + * across blocks. The contig_hint may be off by up to a page, but if |
| 885 | + * the contig_hint is contained in a block, it will be accurate with |
| 886 | + * the else condition below. |
| 901 | 887 | */ |
| 902 | | − if ((ALIGN_DOWN(end, min(PCPU_BITS_PER_PAGE, PCPU_BITMAP_BLOCK_BITS)) > |
| 903 | | − ALIGN(start, min(PCPU_BITS_PER_PAGE, PCPU_BITMAP_BLOCK_BITS))) || |
| 904 | | − s_index != e_index) |
| 888 | + if (((end - start) >= PCPU_BITMAP_BLOCK_BITS) || s_index != e_index) |
| 905 | 889 | pcpu_chunk_refresh_hint(chunk); |
| 906 | 890 | else |
| 907 | 891 | pcpu_chunk_update(chunk, pcpu_block_off_to_off(s_index, start), |
| @@ -1178,9 +1162,7 @@ static struct pcpu_chunk * __init pcpu_alloc_first_chunk(unsigned long tmp_addr, · 270 unmodified lines |
| 1178 | 1162 | chunk->immutable = true; |
| 1179 | 1163 | bitmap_fill(chunk->populated, chunk->nr_pages); |
| 1180 | 1164 | chunk->nr_populated = chunk->nr_pages; |
| 1181 | | − chunk->nr_empty_pop_pages = |
| 1182 | | − pcpu_cnt_pop_pages(chunk, start_offset / PCPU_MIN_ALLOC_SIZE, |
| 1183 | | − map_size / PCPU_MIN_ALLOC_SIZE); |
| 1165 | + chunk->nr_empty_pop_pages = chunk->nr_pages; |
| 1184 | 1166 | |
| 1185 | 1167 | chunk->contig_bits = map_size / PCPU_MIN_ALLOC_SIZE; |
| 1186 | 1168 | chunk->free_bytes = map_size; |
| @@ -1275,7 +1257,6 @@ static void pcpu_free_chunk(struct pcpu_chunk *chunk) · 88 unmodified lines |
| 1275 | 1257 | * @chunk: pcpu_chunk which got populated |
| 1276 | 1258 | * @page_start: the start page |
| 1277 | 1259 | * @page_end: the end page |
| 1278 | | − * @for_alloc: if this is to populate for allocation |
| 1279 | 1260 | * |
| 1280 | 1261 | * Pages in [@page_start,@page_end) have been populated to @chunk. Update |
| 1281 | 1262 | * the bookkeeping information accordingly. Must be called after each |
| @@ -1285,7 +1266,7 @@ static void pcpu_free_chunk(struct pcpu_chunk *chunk) · 3 unmodified lines |
| 1285 | 1266 | * is to serve an allocation in that area. |
| 1286 | 1267 | */ |
| 1287 | 1268 | static void pcpu_chunk_populated(struct pcpu_chunk *chunk, int page_start, |
| 1288 | | − int page_end, bool for_alloc) |
| 1269 | + int page_end) |
| 1289 | 1270 | { |
| 1290 | 1271 | int nr = page_end - page_start; |
| 1291 | 1272 | |
| @@ -1295,10 +1276,7 @@ static void pcpu_chunk_populated(struct pcpu_chunk *chunk, int page_start, · 3 unmodified lines |
| 1295 | 1276 | chunk->nr_populated += nr; |
| 1296 | 1277 | pcpu_nr_populated += nr; |
| 1297 | 1278 | |
| 1298 | | − if (!for_alloc) { |
| 1299 | | − chunk->nr_empty_pop_pages += nr; |
| 1300 | | − pcpu_nr_empty_pop_pages += nr; |
| 1301 | | − } |
| 1279 | + pcpu_update_empty_pages(chunk, nr); |
| 1302 | 1280 | } |
| 1303 | 1281 | |
| 1304 | 1282 | /** |
| @@ -1320,9 +1298,9 @@ static void pcpu_chunk_depopulated(struct pcpu_chunk *chunk, · 15 unmodified lines |
| 1320 | 1298 | |
| 1321 | 1299 | bitmap_clear(chunk->populated, page_start, nr); |
| 1322 | 1300 | chunk->nr_populated -= nr; |
| 1323 | | − chunk->nr_empty_pop_pages -= nr; |
| 1324 | | − pcpu_nr_empty_pop_pages -= nr; |
| 1325 | 1301 | pcpu_nr_populated -= nr; |
| 1302 | + |
| 1303 | + pcpu_update_empty_pages(chunk, -nr); |
| 1326 | 1304 | } |
| 1327 | 1305 | |
| 1328 | 1306 | /* |
| @@ -1537,7 +1515,7 @@ area_found: · 208 unmodified lines |
| 1537 | 1515 | err = "failed to populate"; |
| 1538 | 1516 | goto fail_unlock; |
| 1539 | 1517 | } |
| 1540 | | − pcpu_chunk_populated(chunk, rs, re, true); |
| 1518 | + pcpu_chunk_populated(chunk, rs, re); |
| 1541 | 1519 | spin_unlock_irqrestore(&pcpu_lock, flags); |
| 1542 | 1520 | } |
| 1543 | 1521 | |
| @@ -1736,7 +1714,7 @@ retry_pop: · 192 unmodified lines |
| 1736 | 1714 | if (!ret) { |
| 1737 | 1715 | nr_to_pop -= nr; |
| 1738 | 1716 | spin_lock_irq(&pcpu_lock); |
| 1739 | | − pcpu_chunk_populated(chunk, rs, rs + nr, false); |
| 1717 | + pcpu_chunk_populated(chunk, rs, rs + nr); |
| 1740 | 1718 | spin_unlock_irq(&pcpu_lock); |
| 1741 | 1719 | } else { |
| 1742 | 1720 | nr_to_pop = 0; |