Bitmaps are fairly popular for their space efficiency, but we don't have generic iterators available. Make percpu's bitmap region iterators available to everyone.
Unified · Split
| @@ -456,6 +456,41 @@ static inline int bitmap_parse(const char *buf, unsigned int buflen, · 455 unmodified lines | |||
| 456 | return __bitmap_parse(buf, buflen, 0, maskp, nmaskbits); | 456 | return __bitmap_parse(buf, buflen, 0, maskp, nmaskbits); |
| 457 | } | 457 | } |
| 458 | 458 | ||
| 459 | static inline void bitmap_next_clear_region(unsigned long *bitmap, | ||
| 460 | unsigned int *rs, unsigned int *re, | ||
| 461 | unsigned int end) | ||
| 462 | { | ||
| 463 | *rs = find_next_zero_bit(bitmap, end, *rs); | ||
| 464 | *re = find_next_bit(bitmap, end, *rs + 1); | ||
| 465 | } | ||
| 466 | |||
| 467 | static inline void bitmap_next_set_region(unsigned long *bitmap, | ||
| 468 | unsigned int *rs, unsigned int *re, | ||
| 469 | unsigned int end) | ||
| 470 | { | ||
| 471 | *rs = find_next_bit(bitmap, end, *rs); | ||
| 472 | *re = find_next_zero_bit(bitmap, end, *rs + 1); | ||
| 473 | } | ||
| 474 | |||
| 475 | /* | ||
| 476 | * Bitmap region iterators. Iterates over the bitmap between [@start, @end). | ||
| 477 | * @rs and @re should be integer variables and will be set to start and end | ||
| 478 | * index of the current clear or set region. | ||
| 479 | */ | ||
| 480 | #define bitmap_for_each_clear_region(bitmap, rs, re, start, end) \ | ||
| 481 | for ((rs) = (start), \ | ||
| 482 | bitmap_next_clear_region((bitmap), &(rs), &(re), (end)); \ | ||
| 483 | (rs) < (re); \ | ||
| 484 | (rs) = (re) + 1, \ | ||
| 485 | bitmap_next_clear_region((bitmap), &(rs), &(re), (end))) | ||
| 486 | |||
| 487 | #define bitmap_for_each_set_region(bitmap, rs, re, start, end) \ | ||
| 488 | for ((rs) = (start), \ | ||
| 489 | bitmap_next_set_region((bitmap), &(rs), &(re), (end)); \ | ||
| 490 | (rs) < (re); \ | ||
| 491 | (rs) = (re) + 1, \ | ||
| 492 | bitmap_next_set_region((bitmap), &(rs), &(re), (end))) | ||
| 493 | |||
| 459 | /** | 494 | /** |
| 460 | * BITMAP_FROM_U64() - Represent u64 value in the format suitable for bitmap. | 495 | * BITMAP_FROM_U64() - Represent u64 value in the format suitable for bitmap. |
| 461 | * @n: u64 value | 496 | * @n: u64 value |
| @@ -270,33 +270,6 @@ static unsigned long pcpu_chunk_addr(struct pcpu_chunk *chunk, · 269 unmodified lines | |||
| 270 | pcpu_unit_page_offset(cpu, page_idx); | 270 | pcpu_unit_page_offset(cpu, page_idx); |
| 271 | } | 271 | } |
| 272 | 272 | ||
| 273 | static void pcpu_next_unpop(unsigned long *bitmap, int *rs, int *re, int end) | ||
| 274 | { | ||
| 275 | *rs = find_next_zero_bit(bitmap, end, *rs); | ||
| 276 | *re = find_next_bit(bitmap, end, *rs + 1); | ||
| 277 | } | ||
| 278 | |||
| 279 | static void pcpu_next_pop(unsigned long *bitmap, int *rs, int *re, int end) | ||
| 280 | { | ||
| 281 | *rs = find_next_bit(bitmap, end, *rs); | ||
| 282 | *re = find_next_zero_bit(bitmap, end, *rs + 1); | ||
| 283 | } | ||
| 284 | |||
| 285 | /* | ||
| 286 | * Bitmap region iterators. Iterates over the bitmap between | ||
| 287 | * [@start, @end) in @chunk. @rs and @re should be integer variables | ||
| 288 | * and will be set to start and end index of the current free region. | ||
| 289 | */ | ||
| 290 | #define pcpu_for_each_unpop_region(bitmap, rs, re, start, end) \ | ||
| 291 | for ((rs) = (start), pcpu_next_unpop((bitmap), &(rs), &(re), (end)); \ | ||
| 292 | (rs) < (re); \ | ||
| 293 | (rs) = (re) + 1, pcpu_next_unpop((bitmap), &(rs), &(re), (end))) | ||
| 294 | |||
| 295 | #define pcpu_for_each_pop_region(bitmap, rs, re, start, end) \ | ||
| 296 | for ((rs) = (start), pcpu_next_pop((bitmap), &(rs), &(re), (end)); \ | ||
| 297 | (rs) < (re); \ | ||
| 298 | (rs) = (re) + 1, pcpu_next_pop((bitmap), &(rs), &(re), (end))) | ||
| 299 | |||
| 300 | /* | 273 | /* |
| 301 | * The following are helper functions to help access bitmaps and convert | 274 | * The following are helper functions to help access bitmaps and convert |
| 302 | * between bitmap offsets to address offsets. | 275 | * between bitmap offsets to address offsets. |
| @@ -732,9 +705,8 @@ static void pcpu_chunk_refresh_hint(struct pcpu_chunk *chunk, bool full_scan) · 429 unmodified lines | |||
| 732 | } | 705 | } |
| 733 | 706 | ||
| 734 | bits = 0; | 707 | bits = 0; |
| 735 | pcpu_for_each_md_free_region(chunk, bit_off, bits) { | 708 | pcpu_for_each_md_free_region(chunk, bit_off, bits) |
| 736 | pcpu_block_update(chunk_md, bit_off, bit_off + bits); | 709 | pcpu_block_update(chunk_md, bit_off, bit_off + bits); |
| 737 | } | ||
| 738 | } | 710 | } |
| 739 | 711 | ||
| 740 | /** | 712 | /** |
| @@ -749,7 +721,7 @@ static void pcpu_block_refresh_hint(struct pcpu_chunk *chunk, int index) · 8 unmodified lines | |||
| 749 | { | 721 | { |
| 750 | struct pcpu_block_md *block = chunk->md_blocks + index; | 722 | struct pcpu_block_md *block = chunk->md_blocks + index; |
| 751 | unsigned long *alloc_map = pcpu_index_alloc_map(chunk, index); | 723 | unsigned long *alloc_map = pcpu_index_alloc_map(chunk, index); |
| 752 | int rs, re, start; /* region start, region end */ | 724 | unsigned int rs, re, start; /* region start, region end */ |
| 753 | 725 | ||
| 754 | /* promote scan_hint to contig_hint */ | 726 | /* promote scan_hint to contig_hint */ |
| 755 | if (block->scan_hint) { | 727 | if (block->scan_hint) { |
| @@ -765,10 +737,9 @@ static void pcpu_block_refresh_hint(struct pcpu_chunk *chunk, int index) · 9 unmodified lines | |||
| 765 | block->right_free = 0; | 737 | block->right_free = 0; |
| 766 | 738 | ||
| 767 | /* iterate over free areas and update the contig hints */ | 739 | /* iterate over free areas and update the contig hints */ |
| 768 | pcpu_for_each_unpop_region(alloc_map, rs, re, start, | 740 | bitmap_for_each_clear_region(alloc_map, rs, re, start, |
| 769 | PCPU_BITMAP_BLOCK_BITS) { | 741 | PCPU_BITMAP_BLOCK_BITS) |
| 770 | pcpu_block_update(block, rs, re); | 742 | pcpu_block_update(block, rs, re); |
| 771 | } | ||
| 772 | } | 743 | } |
| 773 | 744 | ||
| 774 | /** | 745 | /** |
| @@ -1041,13 +1012,13 @@ static void pcpu_block_update_hint_free(struct pcpu_chunk *chunk, int bit_off, · 266 unmodified lines | |||
| 1041 | static bool pcpu_is_populated(struct pcpu_chunk *chunk, int bit_off, int bits, | 1012 | static bool pcpu_is_populated(struct pcpu_chunk *chunk, int bit_off, int bits, |
| 1042 | int *next_off) | 1013 | int *next_off) |
| 1043 | { | 1014 | { |
| 1044 | int page_start, page_end, rs, re; | 1015 | unsigned int page_start, page_end, rs, re; |
| 1045 | 1016 | ||
| 1046 | page_start = PFN_DOWN(bit_off * PCPU_MIN_ALLOC_SIZE); | 1017 | page_start = PFN_DOWN(bit_off * PCPU_MIN_ALLOC_SIZE); |
| 1047 | page_end = PFN_UP((bit_off + bits) * PCPU_MIN_ALLOC_SIZE); | 1018 | page_end = PFN_UP((bit_off + bits) * PCPU_MIN_ALLOC_SIZE); |
| 1048 | 1019 | ||
| 1049 | rs = page_start; | 1020 | rs = page_start; |
| 1050 | pcpu_next_unpop(chunk->populated, &rs, &re, page_end); | 1021 | bitmap_next_clear_region(chunk->populated, &rs, &re, page_end); |
| 1051 | if (rs >= page_end) | 1022 | if (rs >= page_end) |
| 1052 | return true; | 1023 | return true; |
| 1053 | 1024 | ||
| @@ -1702,13 +1673,13 @@ area_found: · 648 unmodified lines | |||
| 1702 | 1673 | ||
| 1703 | /* populate if not all pages are already there */ | 1674 | /* populate if not all pages are already there */ |
| 1704 | if (!is_atomic) { | 1675 | if (!is_atomic) { |
| 1705 | int page_start, page_end, rs, re; | 1676 | unsigned int page_start, page_end, rs, re; |
| 1706 | 1677 | ||
| 1707 | page_start = PFN_DOWN(off); | 1678 | page_start = PFN_DOWN(off); |
| 1708 | page_end = PFN_UP(off + size); | 1679 | page_end = PFN_UP(off + size); |
| 1709 | 1680 | ||
| 1710 | pcpu_for_each_unpop_region(chunk->populated, rs, re, | 1681 | bitmap_for_each_clear_region(chunk->populated, rs, re, |
| 1711 | page_start, page_end) { | 1682 | page_start, page_end) { |
| 1712 | WARN_ON(chunk->immutable); | 1683 | WARN_ON(chunk->immutable); |
| 1713 | 1684 | ||
| 1714 | ret = pcpu_populate_chunk(chunk, rs, re, pcpu_gfp); | 1685 | ret = pcpu_populate_chunk(chunk, rs, re, pcpu_gfp); |
| @@ -1858,10 +1829,10 @@ static void pcpu_balance_workfn(struct work_struct *work) · 143 unmodified lines | |||
| 1858 | spin_unlock_irq(&pcpu_lock); | 1829 | spin_unlock_irq(&pcpu_lock); |
| 1859 | 1830 | ||
| 1860 | list_for_each_entry_safe(chunk, next, &to_free, list) { | 1831 | list_for_each_entry_safe(chunk, next, &to_free, list) { |
| 1861 | int rs, re; | 1832 | unsigned int rs, re; |
| 1862 | 1833 | ||
| 1863 | pcpu_for_each_pop_region(chunk->populated, rs, re, 0, | 1834 | bitmap_for_each_set_region(chunk->populated, rs, re, 0, |
| 1864 | chunk->nr_pages) { | 1835 | chunk->nr_pages) { |
| 1865 | pcpu_depopulate_chunk(chunk, rs, re); | 1836 | pcpu_depopulate_chunk(chunk, rs, re); |
| 1866 | spin_lock_irq(&pcpu_lock); | 1837 | spin_lock_irq(&pcpu_lock); |
| 1867 | pcpu_chunk_depopulated(chunk, rs, re); | 1838 | pcpu_chunk_depopulated(chunk, rs, re); |
| @@ -1893,7 +1864,7 @@ retry_pop: · 25 unmodified lines | |||
| 1893 | } | 1864 | } |
| 1894 | 1865 | ||
| 1895 | for (slot = pcpu_size_to_slot(PAGE_SIZE); slot < pcpu_nr_slots; slot++) { | 1866 | for (slot = pcpu_size_to_slot(PAGE_SIZE); slot < pcpu_nr_slots; slot++) { |
| 1896 | int nr_unpop = 0, rs, re; | 1867 | unsigned int nr_unpop = 0, rs, re; |
| 1897 | 1868 | ||
| 1898 | if (!nr_to_pop) | 1869 | if (!nr_to_pop) |
| 1899 | break; | 1870 | break; |
| @@ -1910,9 +1881,9 @@ retry_pop: · 10 unmodified lines | |||
| 1910 | continue; | 1881 | continue; |
| 1911 | 1882 | ||
| 1912 | /* @chunk can't go away while pcpu_alloc_mutex is held */ | 1883 | /* @chunk can't go away while pcpu_alloc_mutex is held */ |
| 1913 | pcpu_for_each_unpop_region(chunk->populated, rs, re, 0, | 1884 | bitmap_for_each_clear_region(chunk->populated, rs, re, 0, |
| 1914 | chunk->nr_pages) { | 1885 | chunk->nr_pages) { |
| 1915 | int nr = min(re - rs, nr_to_pop); | 1886 | int nr = min_t(int, re - rs, nr_to_pop); |
| 1916 | 1887 | ||
| 1917 | ret = pcpu_populate_chunk(chunk, rs, rs + nr, gfp); | 1888 | ret = pcpu_populate_chunk(chunk, rs, rs + nr, gfp); |
| 1918 | if (!ret) { | 1889 | if (!ret) { |