Bitmaps are fairly popular for their space efficiency, but we don't have generic iterators available. Make percpu's bitmap region iterators available to everyone.
Unified · Split
| @@ -456,6 +456,41 @@ static inline int bitmap_parse(const char *buf, unsigned int buflen, · 455 unmodified lines | ||
| 456 | 456 | return __bitmap_parse(buf, buflen, 0, maskp, nmaskbits); |
| 457 | 457 | } |
| 458 | 458 | |
| 459 | +static inline void bitmap_next_clear_region(unsigned long *bitmap, | |
| 460 | + unsigned int *rs, unsigned int *re, | |
| 461 | + unsigned int end) | |
| 462 | +{ | |
| 463 | + *rs = find_next_zero_bit(bitmap, end, *rs); | |
| 464 | + *re = find_next_bit(bitmap, end, *rs + 1); | |
| 465 | +} | |
| 466 | + | |
| 467 | +static inline void bitmap_next_set_region(unsigned long *bitmap, | |
| 468 | + unsigned int *rs, unsigned int *re, | |
| 469 | + unsigned int end) | |
| 470 | +{ | |
| 471 | + *rs = find_next_bit(bitmap, end, *rs); | |
| 472 | + *re = find_next_zero_bit(bitmap, end, *rs + 1); | |
| 473 | +} | |
| 474 | + | |
| 475 | +/* | |
| 476 | + * Bitmap region iterators. Iterates over the bitmap between [@start, @end). | |
| 477 | + * @rs and @re should be integer variables and will be set to start and end | |
| 478 | + * index of the current clear or set region. | |
| 479 | + */ | |
| 480 | +#define bitmap_for_each_clear_region(bitmap, rs, re, start, end) \ | |
| 481 | + for ((rs) = (start), \ | |
| 482 | + bitmap_next_clear_region((bitmap), &(rs), &(re), (end)); \ | |
| 483 | + (rs) < (re); \ | |
| 484 | + (rs) = (re) + 1, \ | |
| 485 | + bitmap_next_clear_region((bitmap), &(rs), &(re), (end))) | |
| 486 | + | |
| 487 | +#define bitmap_for_each_set_region(bitmap, rs, re, start, end) \ | |
| 488 | + for ((rs) = (start), \ | |
| 489 | + bitmap_next_set_region((bitmap), &(rs), &(re), (end)); \ | |
| 490 | + (rs) < (re); \ | |
| 491 | + (rs) = (re) + 1, \ | |
| 492 | + bitmap_next_set_region((bitmap), &(rs), &(re), (end))) | |
| 493 | + | |
| 459 | 494 | /** |
| 460 | 495 | * BITMAP_FROM_U64() - Represent u64 value in the format suitable for bitmap. |
| 461 | 496 | * @n: u64 value |
| @@ -270,33 +270,6 @@ static unsigned long pcpu_chunk_addr(struct pcpu_chunk *chunk, · 269 unmodified lines | ||
| 270 | 270 | pcpu_unit_page_offset(cpu, page_idx); |
| 271 | 271 | } |
| 272 | 272 | |
| 273 | −static void pcpu_next_unpop(unsigned long *bitmap, int *rs, int *re, int end) | |
| 274 | −{ | |
| 275 | − *rs = find_next_zero_bit(bitmap, end, *rs); | |
| 276 | − *re = find_next_bit(bitmap, end, *rs + 1); | |
| 277 | −} | |
| 278 | − | |
| 279 | −static void pcpu_next_pop(unsigned long *bitmap, int *rs, int *re, int end) | |
| 280 | −{ | |
| 281 | − *rs = find_next_bit(bitmap, end, *rs); | |
| 282 | − *re = find_next_zero_bit(bitmap, end, *rs + 1); | |
| 283 | −} | |
| 284 | − | |
| 285 | −/* | |
| 286 | − * Bitmap region iterators. Iterates over the bitmap between | |
| 287 | − * [@start, @end) in @chunk. @rs and @re should be integer variables | |
| 288 | − * and will be set to start and end index of the current free region. | |
| 289 | − */ | |
| 290 | −#define pcpu_for_each_unpop_region(bitmap, rs, re, start, end) \ | |
| 291 | − for ((rs) = (start), pcpu_next_unpop((bitmap), &(rs), &(re), (end)); \ | |
| 292 | − (rs) < (re); \ | |
| 293 | − (rs) = (re) + 1, pcpu_next_unpop((bitmap), &(rs), &(re), (end))) | |
| 294 | − | |
| 295 | −#define pcpu_for_each_pop_region(bitmap, rs, re, start, end) \ | |
| 296 | − for ((rs) = (start), pcpu_next_pop((bitmap), &(rs), &(re), (end)); \ | |
| 297 | − (rs) < (re); \ | |
| 298 | − (rs) = (re) + 1, pcpu_next_pop((bitmap), &(rs), &(re), (end))) | |
| 299 | − | |
| 300 | 273 | /* |
| 301 | 274 | * The following are helper functions to help access bitmaps and convert |
| 302 | 275 | * between bitmap offsets to address offsets. |
| @@ -732,9 +705,8 @@ static void pcpu_chunk_refresh_hint(struct pcpu_chunk *chunk, bool full_scan) · 429 unmodified lines | ||
| 732 | 705 | } |
| 733 | 706 | |
| 734 | 707 | bits = 0; |
| 735 | − pcpu_for_each_md_free_region(chunk, bit_off, bits) { | |
| 708 | + pcpu_for_each_md_free_region(chunk, bit_off, bits) | |
| 736 | 709 | pcpu_block_update(chunk_md, bit_off, bit_off + bits); |
| 737 | − } | |
| 738 | 710 | } |
| 739 | 711 | |
| 740 | 712 | /** |
| @@ -749,7 +721,7 @@ static void pcpu_block_refresh_hint(struct pcpu_chunk *chunk, int index) · 8 unmodified lines | ||
| 749 | 721 | { |
| 750 | 722 | struct pcpu_block_md *block = chunk->md_blocks + index; |
| 751 | 723 | unsigned long *alloc_map = pcpu_index_alloc_map(chunk, index); |
| 752 | − int rs, re, start; /* region start, region end */ | |
| 724 | + unsigned int rs, re, start; /* region start, region end */ | |
| 753 | 725 | |
| 754 | 726 | /* promote scan_hint to contig_hint */ |
| 755 | 727 | if (block->scan_hint) { |
| @@ -765,10 +737,9 @@ static void pcpu_block_refresh_hint(struct pcpu_chunk *chunk, int index) · 9 unmodified lines | ||
| 765 | 737 | block->right_free = 0; |
| 766 | 738 | |
| 767 | 739 | /* iterate over free areas and update the contig hints */ |
| 768 | − pcpu_for_each_unpop_region(alloc_map, rs, re, start, | |
| 769 | − PCPU_BITMAP_BLOCK_BITS) { | |
| 740 | + bitmap_for_each_clear_region(alloc_map, rs, re, start, | |
| 741 | + PCPU_BITMAP_BLOCK_BITS) | |
| 770 | 742 | pcpu_block_update(block, rs, re); |
| 771 | − } | |
| 772 | 743 | } |
| 773 | 744 | |
| 774 | 745 | /** |
| @@ -1041,13 +1012,13 @@ static void pcpu_block_update_hint_free(struct pcpu_chunk *chunk, int bit_off, · 266 unmodified lines | ||
| 1041 | 1012 | static bool pcpu_is_populated(struct pcpu_chunk *chunk, int bit_off, int bits, |
| 1042 | 1013 | int *next_off) |
| 1043 | 1014 | { |
| 1044 | − int page_start, page_end, rs, re; | |
| 1015 | + unsigned int page_start, page_end, rs, re; | |
| 1045 | 1016 | |
| 1046 | 1017 | page_start = PFN_DOWN(bit_off * PCPU_MIN_ALLOC_SIZE); |
| 1047 | 1018 | page_end = PFN_UP((bit_off + bits) * PCPU_MIN_ALLOC_SIZE); |
| 1048 | 1019 | |
| 1049 | 1020 | rs = page_start; |
| 1050 | − pcpu_next_unpop(chunk->populated, &rs, &re, page_end); | |
| 1021 | + bitmap_next_clear_region(chunk->populated, &rs, &re, page_end); | |
| 1051 | 1022 | if (rs >= page_end) |
| 1052 | 1023 | return true; |
| 1053 | 1024 | |
| @@ -1702,13 +1673,13 @@ area_found: · 648 unmodified lines | ||
| 1702 | 1673 | |
| 1703 | 1674 | /* populate if not all pages are already there */ |
| 1704 | 1675 | if (!is_atomic) { |
| 1705 | − int page_start, page_end, rs, re; | |
| 1676 | + unsigned int page_start, page_end, rs, re; | |
| 1706 | 1677 | |
| 1707 | 1678 | page_start = PFN_DOWN(off); |
| 1708 | 1679 | page_end = PFN_UP(off + size); |
| 1709 | 1680 | |
| 1710 | − pcpu_for_each_unpop_region(chunk->populated, rs, re, | |
| 1711 | − page_start, page_end) { | |
| 1681 | + bitmap_for_each_clear_region(chunk->populated, rs, re, | |
| 1682 | + page_start, page_end) { | |
| 1712 | 1683 | WARN_ON(chunk->immutable); |
| 1713 | 1684 | |
| 1714 | 1685 | ret = pcpu_populate_chunk(chunk, rs, re, pcpu_gfp); |
| @@ -1858,10 +1829,10 @@ static void pcpu_balance_workfn(struct work_struct *work) · 143 unmodified lines | ||
| 1858 | 1829 | spin_unlock_irq(&pcpu_lock); |
| 1859 | 1830 | |
| 1860 | 1831 | list_for_each_entry_safe(chunk, next, &to_free, list) { |
| 1861 | − int rs, re; | |
| 1832 | + unsigned int rs, re; | |
| 1862 | 1833 | |
| 1863 | − pcpu_for_each_pop_region(chunk->populated, rs, re, 0, | |
| 1864 | − chunk->nr_pages) { | |
| 1834 | + bitmap_for_each_set_region(chunk->populated, rs, re, 0, | |
| 1835 | + chunk->nr_pages) { | |
| 1865 | 1836 | pcpu_depopulate_chunk(chunk, rs, re); |
| 1866 | 1837 | spin_lock_irq(&pcpu_lock); |
| 1867 | 1838 | pcpu_chunk_depopulated(chunk, rs, re); |
| @@ -1893,7 +1864,7 @@ retry_pop: · 25 unmodified lines | ||
| 1893 | 1864 | } |
| 1894 | 1865 | |
| 1895 | 1866 | for (slot = pcpu_size_to_slot(PAGE_SIZE); slot < pcpu_nr_slots; slot++) { |
| 1896 | − int nr_unpop = 0, rs, re; | |
| 1867 | + unsigned int nr_unpop = 0, rs, re; | |
| 1897 | 1868 | |
| 1898 | 1869 | if (!nr_to_pop) |
| 1899 | 1870 | break; |
| @@ -1910,9 +1881,9 @@ retry_pop: · 10 unmodified lines | ||
| 1910 | 1881 | continue; |
| 1911 | 1882 | |
| 1912 | 1883 | /* @chunk can't go away while pcpu_alloc_mutex is held */ |
| 1913 | − pcpu_for_each_unpop_region(chunk->populated, rs, re, 0, | |
| 1914 | − chunk->nr_pages) { | |
| 1915 | − int nr = min(re - rs, nr_to_pop); | |
| 1884 | + bitmap_for_each_clear_region(chunk->populated, rs, re, 0, | |
| 1885 | + chunk->nr_pages) { | |
| 1886 | + int nr = min_t(int, re - rs, nr_to_pop); | |
| 1916 | 1887 | |
| 1917 | 1888 | ret = pcpu_populate_chunk(chunk, rs, rs + nr, gfp); |
| 1918 | 1889 | if (!ret) { |