Pull bitmap updates from Yury Norov: - switch all bitmamp APIs from inline to __always_inline (Brian Norris) The __always_inline series improves on code generation, and now with the latest compiler versions is required to avoid compilation warnings. It spent enough in my backlog, and I'm thankful to Brian Norris for taking over and moving it forward. - introduce GENMASK_U128() macro (Anshuman Khandual) GENMASK_U128() is a prerequisite needed for arm64 development * tag 'bitmap-for-6.12' of https://github.com/norov/linux: lib/test_bits.c: Add tests for GENMASK_U128() uapi: Define GENMASK_U128 nodemask: Switch from inline to __always_inline cpumask: Switch from inline to __always_inline bitmap: Switch from inline to __always_inline find: Switch from inline to __always_inline
Showing the changes to include/linux/bitmap.h (1 of 8 files). Show all
Unified · Split
| @@ -203,12 +203,12 @@ unsigned long bitmap_find_next_zero_area_off(unsigned long *map, · 202 unmodified lines | ||
| 203 | 203 | * the bit offset of all zero areas this function finds is multiples of that |
| 204 | 204 | * power of 2. A @align_mask of 0 means no alignment is required. |
| 205 | 205 | */ |
| 206 | −static inline unsigned long | |
| 207 | −bitmap_find_next_zero_area(unsigned long *map, | |
| 208 | − unsigned long size, | |
| 209 | − unsigned long start, | |
| 210 | − unsigned int nr, | |
| 211 | − unsigned long align_mask) | |
| 206 | +static __always_inline | |
| 207 | +unsigned long bitmap_find_next_zero_area(unsigned long *map, | |
| 208 | + unsigned long size, | |
| 209 | + unsigned long start, | |
| 210 | + unsigned int nr, | |
| 211 | + unsigned long align_mask) | |
| 212 | 212 | { |
| 213 | 213 | return bitmap_find_next_zero_area_off(map, size, start, nr, |
| 214 | 214 | align_mask, 0); |
| @@ -228,7 +228,7 @@ void bitmap_fold(unsigned long *dst, const unsigned long *orig, · 13 unmodified lines | ||
| 228 | 228 | |
| 229 | 229 | #define bitmap_size(nbits) (ALIGN(nbits, BITS_PER_LONG) / BITS_PER_BYTE) |
| 230 | 230 | |
| 231 | −static inline void bitmap_zero(unsigned long *dst, unsigned int nbits) | |
| 231 | +static __always_inline void bitmap_zero(unsigned long *dst, unsigned int nbits) | |
| 232 | 232 | { |
| 233 | 233 | unsigned int len = bitmap_size(nbits); |
| 234 | 234 | |
| @@ -238,7 +238,7 @@ static inline void bitmap_zero(unsigned long *dst, unsigned int nbits) · 3 unmodified lines | ||
| 238 | 238 | memset(dst, 0, len); |
| 239 | 239 | } |
| 240 | 240 | |
| 241 | −static inline void bitmap_fill(unsigned long *dst, unsigned int nbits) | |
| 241 | +static __always_inline void bitmap_fill(unsigned long *dst, unsigned int nbits) | |
| 242 | 242 | { |
| 243 | 243 | unsigned int len = bitmap_size(nbits); |
| 244 | 244 | |
| @@ -248,8 +248,8 @@ static inline void bitmap_fill(unsigned long *dst, unsigned int nbits) · 3 unmodified lines | ||
| 248 | 248 | memset(dst, 0xff, len); |
| 249 | 249 | } |
| 250 | 250 | |
| 251 | −static inline void bitmap_copy(unsigned long *dst, const unsigned long *src, | |
| 252 | − unsigned int nbits) | |
| 251 | +static __always_inline | |
| 252 | +void bitmap_copy(unsigned long *dst, const unsigned long *src, unsigned int nbits) | |
| 253 | 253 | { |
| 254 | 254 | unsigned int len = bitmap_size(nbits); |
| 255 | 255 | |
| @@ -262,8 +262,8 @@ static inline void bitmap_copy(unsigned long *dst, const unsigned long *src, · 6 unmodified lines | ||
| 262 | 262 | /* |
| 263 | 263 | * Copy bitmap and clear tail bits in last word. |
| 264 | 264 | */ |
| 265 | −static inline void bitmap_copy_clear_tail(unsigned long *dst, | |
| 266 | − const unsigned long *src, unsigned int nbits) | |
| 265 | +static __always_inline | |
| 266 | +void bitmap_copy_clear_tail(unsigned long *dst, const unsigned long *src, unsigned int nbits) | |
| 267 | 267 | { |
| 268 | 268 | bitmap_copy(dst, src, nbits); |
| 269 | 269 | if (nbits % BITS_PER_LONG) |
| @@ -318,16 +318,18 @@ void bitmap_to_arr64(u64 *buf, const unsigned long *bitmap, unsigned int nbits); · 48 unmodified lines | ||
| 318 | 318 | bitmap_copy_clear_tail((unsigned long *)(buf), (const unsigned long *)(bitmap), (nbits)) |
| 319 | 319 | #endif |
| 320 | 320 | |
| 321 | −static inline bool bitmap_and(unsigned long *dst, const unsigned long *src1, | |
| 322 | − const unsigned long *src2, unsigned int nbits) | |
| 321 | +static __always_inline | |
| 322 | +bool bitmap_and(unsigned long *dst, const unsigned long *src1, | |
| 323 | + const unsigned long *src2, unsigned int nbits) | |
| 323 | 324 | { |
| 324 | 325 | if (small_const_nbits(nbits)) |
| 325 | 326 | return (*dst = *src1 & *src2 & BITMAP_LAST_WORD_MASK(nbits)) != 0; |
| 326 | 327 | return __bitmap_and(dst, src1, src2, nbits); |
| 327 | 328 | } |
| 328 | 329 | |
| 329 | −static inline void bitmap_or(unsigned long *dst, const unsigned long *src1, | |
| 330 | − const unsigned long *src2, unsigned int nbits) | |
| 330 | +static __always_inline | |
| 331 | +void bitmap_or(unsigned long *dst, const unsigned long *src1, | |
| 332 | + const unsigned long *src2, unsigned int nbits) | |
| 331 | 333 | { |
| 332 | 334 | if (small_const_nbits(nbits)) |
| 333 | 335 | *dst = *src1 | *src2; |
| @@ -335,8 +337,9 @@ static inline void bitmap_or(unsigned long *dst, const unsigned long *src1, · 1 unmodified line | ||
| 335 | 337 | __bitmap_or(dst, src1, src2, nbits); |
| 336 | 338 | } |
| 337 | 339 | |
| 338 | −static inline void bitmap_xor(unsigned long *dst, const unsigned long *src1, | |
| 339 | − const unsigned long *src2, unsigned int nbits) | |
| 340 | +static __always_inline | |
| 341 | +void bitmap_xor(unsigned long *dst, const unsigned long *src1, | |
| 342 | + const unsigned long *src2, unsigned int nbits) | |
| 340 | 343 | { |
| 341 | 344 | if (small_const_nbits(nbits)) |
| 342 | 345 | *dst = *src1 ^ *src2; |
| @@ -344,16 +347,17 @@ static inline void bitmap_xor(unsigned long *dst, const unsigned long *src1, · 1 unmodified line | ||
| 344 | 347 | __bitmap_xor(dst, src1, src2, nbits); |
| 345 | 348 | } |
| 346 | 349 | |
| 347 | −static inline bool bitmap_andnot(unsigned long *dst, const unsigned long *src1, | |
| 348 | − const unsigned long *src2, unsigned int nbits) | |
| 350 | +static __always_inline | |
| 351 | +bool bitmap_andnot(unsigned long *dst, const unsigned long *src1, | |
| 352 | + const unsigned long *src2, unsigned int nbits) | |
| 349 | 353 | { |
| 350 | 354 | if (small_const_nbits(nbits)) |
| 351 | 355 | return (*dst = *src1 & ~(*src2) & BITMAP_LAST_WORD_MASK(nbits)) != 0; |
| 352 | 356 | return __bitmap_andnot(dst, src1, src2, nbits); |
| 353 | 357 | } |
| 354 | 358 | |
| 355 | −static inline void bitmap_complement(unsigned long *dst, const unsigned long *src, | |
| 356 | − unsigned int nbits) | |
| 359 | +static __always_inline | |
| 360 | +void bitmap_complement(unsigned long *dst, const unsigned long *src, unsigned int nbits) | |
| 357 | 361 | { |
| 358 | 362 | if (small_const_nbits(nbits)) |
| 359 | 363 | *dst = ~(*src); |
| @@ -368,8 +372,8 @@ static inline void bitmap_complement(unsigned long *dst, const unsigned long *sr · 8 unmodified lines | ||
| 368 | 372 | #endif |
| 369 | 373 | #define BITMAP_MEM_MASK (BITMAP_MEM_ALIGNMENT - 1) |
| 370 | 374 | |
| 371 | −static inline bool bitmap_equal(const unsigned long *src1, | |
| 372 | − const unsigned long *src2, unsigned int nbits) | |
| 375 | +static __always_inline | |
| 376 | +bool bitmap_equal(const unsigned long *src1, const unsigned long *src2, unsigned int nbits) | |
| 373 | 377 | { |
| 374 | 378 | if (small_const_nbits(nbits)) |
| 375 | 379 | return !((*src1 ^ *src2) & BITMAP_LAST_WORD_MASK(nbits)); |
| @@ -388,10 +392,9 @@ static inline bool bitmap_equal(const unsigned long *src1, · 12 unmodified lines | ||
| 388 | 392 | * |
| 389 | 393 | * Returns: True if (*@src1 | *@src2) == *@src3, false otherwise |
| 390 | 394 | */ |
| 391 | −static inline bool bitmap_or_equal(const unsigned long *src1, | |
| 392 | − const unsigned long *src2, | |
| 393 | − const unsigned long *src3, | |
| 394 | − unsigned int nbits) | |
| 395 | +static __always_inline | |
| 396 | +bool bitmap_or_equal(const unsigned long *src1, const unsigned long *src2, | |
| 397 | + const unsigned long *src3, unsigned int nbits) | |
| 395 | 398 | { |
| 396 | 399 | if (!small_const_nbits(nbits)) |
| 397 | 400 | return __bitmap_or_equal(src1, src2, src3, nbits); |
| @@ -399,9 +402,8 @@ static inline bool bitmap_or_equal(const unsigned long *src1, · 1 unmodified line | ||
| 399 | 402 | return !(((*src1 | *src2) ^ *src3) & BITMAP_LAST_WORD_MASK(nbits)); |
| 400 | 403 | } |
| 401 | 404 | |
| 402 | −static inline bool bitmap_intersects(const unsigned long *src1, | |
| 403 | − const unsigned long *src2, | |
| 404 | − unsigned int nbits) | |
| 405 | +static __always_inline | |
| 406 | +bool bitmap_intersects(const unsigned long *src1, const unsigned long *src2, unsigned int nbits) | |
| 405 | 407 | { |
| 406 | 408 | if (small_const_nbits(nbits)) |
| 407 | 409 | return ((*src1 & *src2) & BITMAP_LAST_WORD_MASK(nbits)) != 0; |
| @@ -409,8 +411,8 @@ static inline bool bitmap_intersects(const unsigned long *src1, · 1 unmodified line | ||
| 409 | 411 | return __bitmap_intersects(src1, src2, nbits); |
| 410 | 412 | } |
| 411 | 413 | |
| 412 | −static inline bool bitmap_subset(const unsigned long *src1, | |
| 413 | − const unsigned long *src2, unsigned int nbits) | |
| 414 | +static __always_inline | |
| 415 | +bool bitmap_subset(const unsigned long *src1, const unsigned long *src2, unsigned int nbits) | |
| 414 | 416 | { |
| 415 | 417 | if (small_const_nbits(nbits)) |
| 416 | 418 | return ! ((*src1 & ~(*src2)) & BITMAP_LAST_WORD_MASK(nbits)); |
| @@ -418,7 +420,8 @@ static inline bool bitmap_subset(const unsigned long *src1, · 1 unmodified line | ||
| 418 | 420 | return __bitmap_subset(src1, src2, nbits); |
| 419 | 421 | } |
| 420 | 422 | |
| 421 | −static inline bool bitmap_empty(const unsigned long *src, unsigned nbits) | |
| 423 | +static __always_inline | |
| 424 | +bool bitmap_empty(const unsigned long *src, unsigned nbits) | |
| 422 | 425 | { |
| 423 | 426 | if (small_const_nbits(nbits)) |
| 424 | 427 | return ! (*src & BITMAP_LAST_WORD_MASK(nbits)); |
| @@ -426,7 +429,8 @@ static inline bool bitmap_empty(const unsigned long *src, unsigned nbits) · 1 unmodified line | ||
| 426 | 429 | return find_first_bit(src, nbits) == nbits; |
| 427 | 430 | } |
| 428 | 431 | |
| 429 | −static inline bool bitmap_full(const unsigned long *src, unsigned int nbits) | |
| 432 | +static __always_inline | |
| 433 | +bool bitmap_full(const unsigned long *src, unsigned int nbits) | |
| 430 | 434 | { |
| 431 | 435 | if (small_const_nbits(nbits)) |
| 432 | 436 | return ! (~(*src) & BITMAP_LAST_WORD_MASK(nbits)); |
| @@ -460,8 +464,8 @@ unsigned long bitmap_weight_andnot(const unsigned long *src1, · 27 unmodified lines | ||
| 460 | 464 | return __bitmap_weight_andnot(src1, src2, nbits); |
| 461 | 465 | } |
| 462 | 466 | |
| 463 | −static __always_inline void bitmap_set(unsigned long *map, unsigned int start, | |
| 464 | − unsigned int nbits) | |
| 467 | +static __always_inline | |
| 468 | +void bitmap_set(unsigned long *map, unsigned int start, unsigned int nbits) | |
| 465 | 469 | { |
| 466 | 470 | if (__builtin_constant_p(nbits) && nbits == 1) |
| 467 | 471 | __set_bit(start, map); |
| @@ -476,8 +480,8 @@ static __always_inline void bitmap_set(unsigned long *map, unsigned int start, · 8 unmodified lines | ||
| 476 | 480 | __bitmap_set(map, start, nbits); |
| 477 | 481 | } |
| 478 | 482 | |
| 479 | −static __always_inline void bitmap_clear(unsigned long *map, unsigned int start, | |
| 480 | − unsigned int nbits) | |
| 483 | +static __always_inline | |
| 484 | +void bitmap_clear(unsigned long *map, unsigned int start, unsigned int nbits) | |
| 481 | 485 | { |
| 482 | 486 | if (__builtin_constant_p(nbits) && nbits == 1) |
| 483 | 487 | __clear_bit(start, map); |
| @@ -492,8 +496,9 @@ static __always_inline void bitmap_clear(unsigned long *map, unsigned int start, · 8 unmodified lines | ||
| 492 | 496 | __bitmap_clear(map, start, nbits); |
| 493 | 497 | } |
| 494 | 498 | |
| 495 | −static inline void bitmap_shift_right(unsigned long *dst, const unsigned long *src, | |
| 496 | − unsigned int shift, unsigned int nbits) | |
| 499 | +static __always_inline | |
| 500 | +void bitmap_shift_right(unsigned long *dst, const unsigned long *src, | |
| 501 | + unsigned int shift, unsigned int nbits) | |
| 497 | 502 | { |
| 498 | 503 | if (small_const_nbits(nbits)) |
| 499 | 504 | *dst = (*src & BITMAP_LAST_WORD_MASK(nbits)) >> shift; |
| @@ -501,8 +506,9 @@ static inline void bitmap_shift_right(unsigned long *dst, const unsigned long *s · 1 unmodified line | ||
| 501 | 506 | __bitmap_shift_right(dst, src, shift, nbits); |
| 502 | 507 | } |
| 503 | 508 | |
| 504 | −static inline void bitmap_shift_left(unsigned long *dst, const unsigned long *src, | |
| 505 | − unsigned int shift, unsigned int nbits) | |
| 509 | +static __always_inline | |
| 510 | +void bitmap_shift_left(unsigned long *dst, const unsigned long *src, | |
| 511 | + unsigned int shift, unsigned int nbits) | |
| 506 | 512 | { |
| 507 | 513 | if (small_const_nbits(nbits)) |
| 508 | 514 | *dst = (*src << shift) & BITMAP_LAST_WORD_MASK(nbits); |
| @@ -510,11 +516,12 @@ static inline void bitmap_shift_left(unsigned long *dst, const unsigned long *sr · 1 unmodified line | ||
| 510 | 516 | __bitmap_shift_left(dst, src, shift, nbits); |
| 511 | 517 | } |
| 512 | 518 | |
| 513 | −static inline void bitmap_replace(unsigned long *dst, | |
| 514 | − const unsigned long *old, | |
| 515 | − const unsigned long *new, | |
| 516 | − const unsigned long *mask, | |
| 517 | − unsigned int nbits) | |
| 519 | +static __always_inline | |
| 520 | +void bitmap_replace(unsigned long *dst, | |
| 521 | + const unsigned long *old, | |
| 522 | + const unsigned long *new, | |
| 523 | + const unsigned long *mask, | |
| 524 | + unsigned int nbits) | |
| 518 | 525 | { |
| 519 | 526 | if (small_const_nbits(nbits)) |
| 520 | 527 | *dst = (*old & ~(*mask)) | (*new & *mask); |
| @@ -557,8 +564,9 @@ static inline void bitmap_replace(unsigned long *dst, · 36 unmodified lines | ||
| 557 | 564 | * bitmap_gather() can be seen as the 'reverse' bitmap_scatter() operation. |
| 558 | 565 | * See bitmap_scatter() for details related to this relationship. |
| 559 | 566 | */ |
| 560 | −static inline void bitmap_scatter(unsigned long *dst, const unsigned long *src, | |
| 561 | − const unsigned long *mask, unsigned int nbits) | |
| 567 | +static __always_inline | |
| 568 | +void bitmap_scatter(unsigned long *dst, const unsigned long *src, | |
| 569 | + const unsigned long *mask, unsigned int nbits) | |
| 562 | 570 | { |
| 563 | 571 | unsigned int n = 0; |
| 564 | 572 | unsigned int bit; |
| @@ -611,8 +619,9 @@ static inline void bitmap_scatter(unsigned long *dst, const unsigned long *src, · 46 unmodified lines | ||
| 611 | 619 | * bitmap_scatter(res, src, mask, n) and a call to |
| 612 | 620 | * bitmap_scatter(res, result, mask, n) will lead to the same res value. |
| 613 | 621 | */ |
| 614 | −static inline void bitmap_gather(unsigned long *dst, const unsigned long *src, | |
| 615 | − const unsigned long *mask, unsigned int nbits) | |
| 622 | +static __always_inline | |
| 623 | +void bitmap_gather(unsigned long *dst, const unsigned long *src, | |
| 624 | + const unsigned long *mask, unsigned int nbits) | |
| 616 | 625 | { |
| 617 | 626 | unsigned int n = 0; |
| 618 | 627 | unsigned int bit; |
| @@ -623,9 +632,9 @@ static inline void bitmap_gather(unsigned long *dst, const unsigned long *src, · 4 unmodified lines | ||
| 623 | 632 | __assign_bit(n++, dst, test_bit(bit, src)); |
| 624 | 633 | } |
| 625 | 634 | |
| 626 | −static inline void bitmap_next_set_region(unsigned long *bitmap, | |
| 627 | − unsigned int *rs, unsigned int *re, | |
| 628 | − unsigned int end) | |
| 635 | +static __always_inline | |
| 636 | +void bitmap_next_set_region(unsigned long *bitmap, unsigned int *rs, | |
| 637 | + unsigned int *re, unsigned int end) | |
| 629 | 638 | { |
| 630 | 639 | *rs = find_next_bit(bitmap, end, *rs); |
| 631 | 640 | *re = find_next_zero_bit(bitmap, end, *rs + 1); |
| @@ -640,7 +649,8 @@ static inline void bitmap_next_set_region(unsigned long *bitmap, · 8 unmodified lines | ||
| 640 | 649 | * This is the complement to __bitmap_find_free_region() and releases |
| 641 | 650 | * the found region (by clearing it in the bitmap). |
| 642 | 651 | */ |
| 643 | −static inline void bitmap_release_region(unsigned long *bitmap, unsigned int pos, int order) | |
| 652 | +static __always_inline | |
| 653 | +void bitmap_release_region(unsigned long *bitmap, unsigned int pos, int order) | |
| 644 | 654 | { |
| 645 | 655 | bitmap_clear(bitmap, pos, BIT(order)); |
| 646 | 656 | } |
| @@ -656,7 +666,8 @@ static inline void bitmap_release_region(unsigned long *bitmap, unsigned int pos · 9 unmodified lines | ||
| 656 | 666 | * Returns: 0 on success, or %-EBUSY if specified region wasn't |
| 657 | 667 | * free (not all bits were zero). |
| 658 | 668 | */ |
| 659 | −static inline int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos, int order) | |
| 669 | +static __always_inline | |
| 670 | +int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos, int order) | |
| 660 | 671 | { |
| 661 | 672 | unsigned int len = BIT(order); |
| 662 | 673 | |
| @@ -680,7 +691,8 @@ static inline int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos · 17 unmodified lines | ||
| 680 | 691 | * Returns: the bit offset in bitmap of the allocated region, |
| 681 | 692 | * or -errno on failure. |
| 682 | 693 | */ |
| 683 | −static inline int bitmap_find_free_region(unsigned long *bitmap, unsigned int bits, int order) | |
| 694 | +static __always_inline | |
| 695 | +int bitmap_find_free_region(unsigned long *bitmap, unsigned int bits, int order) | |
| 684 | 696 | { |
| 685 | 697 | unsigned int pos, end; /* scans bitmap by regions of size order */ |
| 686 | 698 | |
| @@ -734,7 +746,7 @@ static inline int bitmap_find_free_region(unsigned long *bitmap, unsigned int bi · 47 unmodified lines | ||
| 734 | 746 | * That is ``(u32 *)(&val)[0]`` gets the upper 32 bits, |
| 735 | 747 | * but we expect the lower 32-bits of u64. |
| 736 | 748 | */ |
| 737 | −static inline void bitmap_from_u64(unsigned long *dst, u64 mask) | |
| 749 | +static __always_inline void bitmap_from_u64(unsigned long *dst, u64 mask) | |
| 738 | 750 | { |
| 739 | 751 | bitmap_from_arr64(dst, &mask, 64); |
| 740 | 752 | } |
| @@ -749,9 +761,8 @@ static inline void bitmap_from_u64(unsigned long *dst, u64 mask) · 8 unmodified lines | ||
| 749 | 761 | * @map memory region. For @nbits = 0 and @nbits > BITS_PER_LONG the return |
| 750 | 762 | * value is undefined. |
| 751 | 763 | */ |
| 752 | −static inline unsigned long bitmap_read(const unsigned long *map, | |
| 753 | − unsigned long start, | |
| 754 | − unsigned long nbits) | |
| 764 | +static __always_inline | |
| 765 | +unsigned long bitmap_read(const unsigned long *map, unsigned long start, unsigned long nbits) | |
| 755 | 766 | { |
| 756 | 767 | size_t index = BIT_WORD(start); |
| 757 | 768 | unsigned long offset = start % BITS_PER_LONG; |
| @@ -784,8 +795,9 @@ static inline unsigned long bitmap_read(const unsigned long *map, · 26 unmodified lines | ||
| 784 | 795 | * |
| 785 | 796 | * For @nbits == 0 and @nbits > BITS_PER_LONG no writes are performed. |
| 786 | 797 | */ |
| 787 | −static inline void bitmap_write(unsigned long *map, unsigned long value, | |
| 788 | − unsigned long start, unsigned long nbits) | |
| 798 | +static __always_inline | |
| 799 | +void bitmap_write(unsigned long *map, unsigned long value, | |
| 800 | + unsigned long start, unsigned long nbits) | |
| 789 | 801 | { |
| 790 | 802 | size_t index; |
| 791 | 803 | unsigned long offset; |