History

Merge tag 'bitmap-for-6.12' of https://github.com/norov/linux

Pull bitmap updates from Yury Norov: - switch all bitmamp APIs from inline to __always_inline (Brian Norris) The __always_inline series improves on code generation, and now with the latest compiler versions is required to avoid compilation warnings. It spent enough in my backlog, and I'm thankful to Brian Norris for taking over and moving it forward. - introduce GENMASK_U128() macro (Anshuman Khandual) GENMASK_U128() is a prerequisite needed for arm64 development * tag 'bitmap-for-6.12' of https://github.com/norov/linux: lib/test_bits.c: Add tests for GENMASK_U128() uapi: Define GENMASK_U128 nodemask: Switch from inline to __always_inline cpumask: Switch from inline to __always_inline bitmap: Switch from inline to __always_inline find: Switch from inline to __always_inline

Linus Torvalds committed 2 years ago

Commit 9c445752 parents ba33a49 d7bcc378 files · +325 −232Browse files

Showing the changes to include/linux/bitmap.h (1 of 8 files). Show all

Unified · Split

include/linux/bitmap.h+76−64
@@ -203,12 +203,12 @@ unsigned long bitmap_find_next_zero_area_off(unsigned long *map, · 202 unmodified lines
203 * the bit offset of all zero areas this function finds is multiples of that203 * the bit offset of all zero areas this function finds is multiples of that
204 * power of 2. A @align_mask of 0 means no alignment is required.204 * power of 2. A @align_mask of 0 means no alignment is required.
205 */205 */
206static inline unsigned long206static __always_inline
207bitmap_find_next_zero_area(unsigned long *map,207unsigned long bitmap_find_next_zero_area(unsigned long *map,
208 unsigned long size,208 unsigned long size,
209 unsigned long start,209 unsigned long start,
210 unsigned int nr,210 unsigned int nr,
211 unsigned long align_mask)211 unsigned long align_mask)
212{212{
213 return bitmap_find_next_zero_area_off(map, size, start, nr,213 return bitmap_find_next_zero_area_off(map, size, start, nr,
214 align_mask, 0);214 align_mask, 0);
@@ -228,7 +228,7 @@ void bitmap_fold(unsigned long *dst, const unsigned long *orig, · 13 unmodified lines
228 228
229#define bitmap_size(nbits) (ALIGN(nbits, BITS_PER_LONG) / BITS_PER_BYTE)229#define bitmap_size(nbits) (ALIGN(nbits, BITS_PER_LONG) / BITS_PER_BYTE)
230 230
231static inline void bitmap_zero(unsigned long *dst, unsigned int nbits)231static __always_inline void bitmap_zero(unsigned long *dst, unsigned int nbits)
232{232{
233 unsigned int len = bitmap_size(nbits);233 unsigned int len = bitmap_size(nbits);
234 234
@@ -238,7 +238,7 @@ static inline void bitmap_zero(unsigned long *dst, unsigned int nbits) · 3 unmodified lines
238 memset(dst, 0, len);238 memset(dst, 0, len);
239}239}
240 240
241static inline void bitmap_fill(unsigned long *dst, unsigned int nbits)241static __always_inline void bitmap_fill(unsigned long *dst, unsigned int nbits)
242{242{
243 unsigned int len = bitmap_size(nbits);243 unsigned int len = bitmap_size(nbits);
244 244
@@ -248,8 +248,8 @@ static inline void bitmap_fill(unsigned long *dst, unsigned int nbits) · 3 unmodified lines
248 memset(dst, 0xff, len);248 memset(dst, 0xff, len);
249}249}
250 250
251static inline void bitmap_copy(unsigned long *dst, const unsigned long *src,251static __always_inline
252 unsigned int nbits)252void bitmap_copy(unsigned long *dst, const unsigned long *src, unsigned int nbits)
253{253{
254 unsigned int len = bitmap_size(nbits);254 unsigned int len = bitmap_size(nbits);
255 255
@@ -262,8 +262,8 @@ static inline void bitmap_copy(unsigned long *dst, const unsigned long *src, · 6 unmodified lines
262/*262/*
263 * Copy bitmap and clear tail bits in last word.263 * Copy bitmap and clear tail bits in last word.
264 */264 */
265static inline void bitmap_copy_clear_tail(unsigned long *dst,265static __always_inline
266 const unsigned long *src, unsigned int nbits)266void bitmap_copy_clear_tail(unsigned long *dst, const unsigned long *src, unsigned int nbits)
267{267{
268 bitmap_copy(dst, src, nbits);268 bitmap_copy(dst, src, nbits);
269 if (nbits % BITS_PER_LONG)269 if (nbits % BITS_PER_LONG)
@@ -318,16 +318,18 @@ void bitmap_to_arr64(u64 *buf, const unsigned long *bitmap, unsigned int nbits); · 48 unmodified lines
318 bitmap_copy_clear_tail((unsigned long *)(buf), (const unsigned long *)(bitmap), (nbits))318 bitmap_copy_clear_tail((unsigned long *)(buf), (const unsigned long *)(bitmap), (nbits))
319#endif319#endif
320 320
321static inline bool bitmap_and(unsigned long *dst, const unsigned long *src1,321static __always_inline
322 const unsigned long *src2, unsigned int nbits)322bool bitmap_and(unsigned long *dst, const unsigned long *src1,
323 const unsigned long *src2, unsigned int nbits)
323{324{
324 if (small_const_nbits(nbits))325 if (small_const_nbits(nbits))
325 return (*dst = *src1 & *src2 & BITMAP_LAST_WORD_MASK(nbits)) != 0;326 return (*dst = *src1 & *src2 & BITMAP_LAST_WORD_MASK(nbits)) != 0;
326 return __bitmap_and(dst, src1, src2, nbits);327 return __bitmap_and(dst, src1, src2, nbits);
327}328}
328 329
329static inline void bitmap_or(unsigned long *dst, const unsigned long *src1,330static __always_inline
330 const unsigned long *src2, unsigned int nbits)331void bitmap_or(unsigned long *dst, const unsigned long *src1,
332 const unsigned long *src2, unsigned int nbits)
331{333{
332 if (small_const_nbits(nbits))334 if (small_const_nbits(nbits))
333 *dst = *src1 | *src2;335 *dst = *src1 | *src2;
@@ -335,8 +337,9 @@ static inline void bitmap_or(unsigned long *dst, const unsigned long *src1, · 1 unmodified line
335 __bitmap_or(dst, src1, src2, nbits);337 __bitmap_or(dst, src1, src2, nbits);
336}338}
337 339
338static inline void bitmap_xor(unsigned long *dst, const unsigned long *src1,340static __always_inline
339 const unsigned long *src2, unsigned int nbits)341void bitmap_xor(unsigned long *dst, const unsigned long *src1,
342 const unsigned long *src2, unsigned int nbits)
340{343{
341 if (small_const_nbits(nbits))344 if (small_const_nbits(nbits))
342 *dst = *src1 ^ *src2;345 *dst = *src1 ^ *src2;
@@ -344,16 +347,17 @@ static inline void bitmap_xor(unsigned long *dst, const unsigned long *src1, · 1 unmodified line
344 __bitmap_xor(dst, src1, src2, nbits);347 __bitmap_xor(dst, src1, src2, nbits);
345}348}
346 349
347static inline bool bitmap_andnot(unsigned long *dst, const unsigned long *src1,350static __always_inline
348 const unsigned long *src2, unsigned int nbits)351bool bitmap_andnot(unsigned long *dst, const unsigned long *src1,
352 const unsigned long *src2, unsigned int nbits)
349{353{
350 if (small_const_nbits(nbits))354 if (small_const_nbits(nbits))
351 return (*dst = *src1 & ~(*src2) & BITMAP_LAST_WORD_MASK(nbits)) != 0;355 return (*dst = *src1 & ~(*src2) & BITMAP_LAST_WORD_MASK(nbits)) != 0;
352 return __bitmap_andnot(dst, src1, src2, nbits);356 return __bitmap_andnot(dst, src1, src2, nbits);
353}357}
354 358
355static inline void bitmap_complement(unsigned long *dst, const unsigned long *src,359static __always_inline
356 unsigned int nbits)360void bitmap_complement(unsigned long *dst, const unsigned long *src, unsigned int nbits)
357{361{
358 if (small_const_nbits(nbits))362 if (small_const_nbits(nbits))
359 *dst = ~(*src);363 *dst = ~(*src);
@@ -368,8 +372,8 @@ static inline void bitmap_complement(unsigned long *dst, const unsigned long *sr · 8 unmodified lines
368#endif372#endif
369#define BITMAP_MEM_MASK (BITMAP_MEM_ALIGNMENT - 1)373#define BITMAP_MEM_MASK (BITMAP_MEM_ALIGNMENT - 1)
370 374
371static inline bool bitmap_equal(const unsigned long *src1,375static __always_inline
372 const unsigned long *src2, unsigned int nbits)376bool bitmap_equal(const unsigned long *src1, const unsigned long *src2, unsigned int nbits)
373{377{
374 if (small_const_nbits(nbits))378 if (small_const_nbits(nbits))
375 return !((*src1 ^ *src2) & BITMAP_LAST_WORD_MASK(nbits));379 return !((*src1 ^ *src2) & BITMAP_LAST_WORD_MASK(nbits));
@@ -388,10 +392,9 @@ static inline bool bitmap_equal(const unsigned long *src1, · 12 unmodified lines
388 *392 *
389 * Returns: True if (*@src1 | *@src2) == *@src3, false otherwise393 * Returns: True if (*@src1 | *@src2) == *@src3, false otherwise
390 */394 */
391static inline bool bitmap_or_equal(const unsigned long *src1,395static __always_inline
392 const unsigned long *src2,396bool bitmap_or_equal(const unsigned long *src1, const unsigned long *src2,
393 const unsigned long *src3,397 const unsigned long *src3, unsigned int nbits)
394 unsigned int nbits)
395{398{
396 if (!small_const_nbits(nbits))399 if (!small_const_nbits(nbits))
397 return __bitmap_or_equal(src1, src2, src3, nbits);400 return __bitmap_or_equal(src1, src2, src3, nbits);
@@ -399,9 +402,8 @@ static inline bool bitmap_or_equal(const unsigned long *src1, · 1 unmodified line
399 return !(((*src1 | *src2) ^ *src3) & BITMAP_LAST_WORD_MASK(nbits));402 return !(((*src1 | *src2) ^ *src3) & BITMAP_LAST_WORD_MASK(nbits));
400}403}
401 404
402static inline bool bitmap_intersects(const unsigned long *src1,405static __always_inline
403 const unsigned long *src2,406bool bitmap_intersects(const unsigned long *src1, const unsigned long *src2, unsigned int nbits)
404 unsigned int nbits)
405{407{
406 if (small_const_nbits(nbits))408 if (small_const_nbits(nbits))
407 return ((*src1 & *src2) & BITMAP_LAST_WORD_MASK(nbits)) != 0;409 return ((*src1 & *src2) & BITMAP_LAST_WORD_MASK(nbits)) != 0;
@@ -409,8 +411,8 @@ static inline bool bitmap_intersects(const unsigned long *src1, · 1 unmodified line
409 return __bitmap_intersects(src1, src2, nbits);411 return __bitmap_intersects(src1, src2, nbits);
410}412}
411 413
412static inline bool bitmap_subset(const unsigned long *src1,414static __always_inline
413 const unsigned long *src2, unsigned int nbits)415bool bitmap_subset(const unsigned long *src1, const unsigned long *src2, unsigned int nbits)
414{416{
415 if (small_const_nbits(nbits))417 if (small_const_nbits(nbits))
416 return ! ((*src1 & ~(*src2)) & BITMAP_LAST_WORD_MASK(nbits));418 return ! ((*src1 & ~(*src2)) & BITMAP_LAST_WORD_MASK(nbits));
@@ -418,7 +420,8 @@ static inline bool bitmap_subset(const unsigned long *src1, · 1 unmodified line
418 return __bitmap_subset(src1, src2, nbits);420 return __bitmap_subset(src1, src2, nbits);
419}421}
420 422
421static inline bool bitmap_empty(const unsigned long *src, unsigned nbits)423static __always_inline
424bool bitmap_empty(const unsigned long *src, unsigned nbits)
422{425{
423 if (small_const_nbits(nbits))426 if (small_const_nbits(nbits))
424 return ! (*src & BITMAP_LAST_WORD_MASK(nbits));427 return ! (*src & BITMAP_LAST_WORD_MASK(nbits));
@@ -426,7 +429,8 @@ static inline bool bitmap_empty(const unsigned long *src, unsigned nbits) · 1 unmodified line
426 return find_first_bit(src, nbits) == nbits;429 return find_first_bit(src, nbits) == nbits;
427}430}
428 431
429static inline bool bitmap_full(const unsigned long *src, unsigned int nbits)432static __always_inline
433bool bitmap_full(const unsigned long *src, unsigned int nbits)
430{434{
431 if (small_const_nbits(nbits))435 if (small_const_nbits(nbits))
432 return ! (~(*src) & BITMAP_LAST_WORD_MASK(nbits));436 return ! (~(*src) & BITMAP_LAST_WORD_MASK(nbits));
@@ -460,8 +464,8 @@ unsigned long bitmap_weight_andnot(const unsigned long *src1, · 27 unmodified lines
460 return __bitmap_weight_andnot(src1, src2, nbits);464 return __bitmap_weight_andnot(src1, src2, nbits);
461}465}
462 466
463static __always_inline void bitmap_set(unsigned long *map, unsigned int start,467static __always_inline
464 unsigned int nbits)468void bitmap_set(unsigned long *map, unsigned int start, unsigned int nbits)
465{469{
466 if (__builtin_constant_p(nbits) && nbits == 1)470 if (__builtin_constant_p(nbits) && nbits == 1)
467 __set_bit(start, map);471 __set_bit(start, map);
@@ -476,8 +480,8 @@ static __always_inline void bitmap_set(unsigned long *map, unsigned int start, · 8 unmodified lines
476 __bitmap_set(map, start, nbits);480 __bitmap_set(map, start, nbits);
477}481}
478 482
479static __always_inline void bitmap_clear(unsigned long *map, unsigned int start,483static __always_inline
480 unsigned int nbits)484void bitmap_clear(unsigned long *map, unsigned int start, unsigned int nbits)
481{485{
482 if (__builtin_constant_p(nbits) && nbits == 1)486 if (__builtin_constant_p(nbits) && nbits == 1)
483 __clear_bit(start, map);487 __clear_bit(start, map);
@@ -492,8 +496,9 @@ static __always_inline void bitmap_clear(unsigned long *map, unsigned int start, · 8 unmodified lines
492 __bitmap_clear(map, start, nbits);496 __bitmap_clear(map, start, nbits);
493}497}
494 498
495static inline void bitmap_shift_right(unsigned long *dst, const unsigned long *src,499static __always_inline
496 unsigned int shift, unsigned int nbits)500void bitmap_shift_right(unsigned long *dst, const unsigned long *src,
501 unsigned int shift, unsigned int nbits)
497{502{
498 if (small_const_nbits(nbits))503 if (small_const_nbits(nbits))
499 *dst = (*src & BITMAP_LAST_WORD_MASK(nbits)) >> shift;504 *dst = (*src & BITMAP_LAST_WORD_MASK(nbits)) >> shift;
@@ -501,8 +506,9 @@ static inline void bitmap_shift_right(unsigned long *dst, const unsigned long *s · 1 unmodified line
501 __bitmap_shift_right(dst, src, shift, nbits);506 __bitmap_shift_right(dst, src, shift, nbits);
502}507}
503 508
504static inline void bitmap_shift_left(unsigned long *dst, const unsigned long *src,509static __always_inline
505 unsigned int shift, unsigned int nbits)510void bitmap_shift_left(unsigned long *dst, const unsigned long *src,
511 unsigned int shift, unsigned int nbits)
506{512{
507 if (small_const_nbits(nbits))513 if (small_const_nbits(nbits))
508 *dst = (*src << shift) & BITMAP_LAST_WORD_MASK(nbits);514 *dst = (*src << shift) & BITMAP_LAST_WORD_MASK(nbits);
@@ -510,11 +516,12 @@ static inline void bitmap_shift_left(unsigned long *dst, const unsigned long *sr · 1 unmodified line
510 __bitmap_shift_left(dst, src, shift, nbits);516 __bitmap_shift_left(dst, src, shift, nbits);
511}517}
512 518
513static inline void bitmap_replace(unsigned long *dst,519static __always_inline
514 const unsigned long *old,520void bitmap_replace(unsigned long *dst,
515 const unsigned long *new,521 const unsigned long *old,
516 const unsigned long *mask,522 const unsigned long *new,
517 unsigned int nbits)523 const unsigned long *mask,
524 unsigned int nbits)
518{525{
519 if (small_const_nbits(nbits))526 if (small_const_nbits(nbits))
520 *dst = (*old & ~(*mask)) | (*new & *mask);527 *dst = (*old & ~(*mask)) | (*new & *mask);
@@ -557,8 +564,9 @@ static inline void bitmap_replace(unsigned long *dst, · 36 unmodified lines
557 * bitmap_gather() can be seen as the 'reverse' bitmap_scatter() operation.564 * bitmap_gather() can be seen as the 'reverse' bitmap_scatter() operation.
558 * See bitmap_scatter() for details related to this relationship.565 * See bitmap_scatter() for details related to this relationship.
559 */566 */
560static inline void bitmap_scatter(unsigned long *dst, const unsigned long *src,567static __always_inline
561 const unsigned long *mask, unsigned int nbits)568void bitmap_scatter(unsigned long *dst, const unsigned long *src,
569 const unsigned long *mask, unsigned int nbits)
562{570{
563 unsigned int n = 0;571 unsigned int n = 0;
564 unsigned int bit;572 unsigned int bit;
@@ -611,8 +619,9 @@ static inline void bitmap_scatter(unsigned long *dst, const unsigned long *src, · 46 unmodified lines
611 * bitmap_scatter(res, src, mask, n) and a call to619 * bitmap_scatter(res, src, mask, n) and a call to
612 * bitmap_scatter(res, result, mask, n) will lead to the same res value.620 * bitmap_scatter(res, result, mask, n) will lead to the same res value.
613 */621 */
614static inline void bitmap_gather(unsigned long *dst, const unsigned long *src,622static __always_inline
615 const unsigned long *mask, unsigned int nbits)623void bitmap_gather(unsigned long *dst, const unsigned long *src,
624 const unsigned long *mask, unsigned int nbits)
616{625{
617 unsigned int n = 0;626 unsigned int n = 0;
618 unsigned int bit;627 unsigned int bit;
@@ -623,9 +632,9 @@ static inline void bitmap_gather(unsigned long *dst, const unsigned long *src, · 4 unmodified lines
623 __assign_bit(n++, dst, test_bit(bit, src));632 __assign_bit(n++, dst, test_bit(bit, src));
624}633}
625 634
626static inline void bitmap_next_set_region(unsigned long *bitmap,635static __always_inline
627 unsigned int *rs, unsigned int *re,636void bitmap_next_set_region(unsigned long *bitmap, unsigned int *rs,
628 unsigned int end)637 unsigned int *re, unsigned int end)
629{638{
630 *rs = find_next_bit(bitmap, end, *rs);639 *rs = find_next_bit(bitmap, end, *rs);
631 *re = find_next_zero_bit(bitmap, end, *rs + 1);640 *re = find_next_zero_bit(bitmap, end, *rs + 1);
@@ -640,7 +649,8 @@ static inline void bitmap_next_set_region(unsigned long *bitmap, · 8 unmodified lines
640 * This is the complement to __bitmap_find_free_region() and releases649 * This is the complement to __bitmap_find_free_region() and releases
641 * the found region (by clearing it in the bitmap).650 * the found region (by clearing it in the bitmap).
642 */651 */
643static inline void bitmap_release_region(unsigned long *bitmap, unsigned int pos, int order)652static __always_inline
653void bitmap_release_region(unsigned long *bitmap, unsigned int pos, int order)
644{654{
645 bitmap_clear(bitmap, pos, BIT(order));655 bitmap_clear(bitmap, pos, BIT(order));
646}656}
@@ -656,7 +666,8 @@ static inline void bitmap_release_region(unsigned long *bitmap, unsigned int pos · 9 unmodified lines
656 * Returns: 0 on success, or %-EBUSY if specified region wasn't666 * Returns: 0 on success, or %-EBUSY if specified region wasn't
657 * free (not all bits were zero).667 * free (not all bits were zero).
658 */668 */
659static inline int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos, int order)669static __always_inline
670int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos, int order)
660{671{
661 unsigned int len = BIT(order);672 unsigned int len = BIT(order);
662 673
@@ -680,7 +691,8 @@ static inline int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos · 17 unmodified lines
680 * Returns: the bit offset in bitmap of the allocated region,691 * Returns: the bit offset in bitmap of the allocated region,
681 * or -errno on failure.692 * or -errno on failure.
682 */693 */
683static inline int bitmap_find_free_region(unsigned long *bitmap, unsigned int bits, int order)694static __always_inline
695int bitmap_find_free_region(unsigned long *bitmap, unsigned int bits, int order)
684{696{
685 unsigned int pos, end; /* scans bitmap by regions of size order */697 unsigned int pos, end; /* scans bitmap by regions of size order */
686 698
@@ -734,7 +746,7 @@ static inline int bitmap_find_free_region(unsigned long *bitmap, unsigned int bi · 47 unmodified lines
734 * That is ``(u32 *)(&val)[0]`` gets the upper 32 bits,746 * That is ``(u32 *)(&val)[0]`` gets the upper 32 bits,
735 * but we expect the lower 32-bits of u64.747 * but we expect the lower 32-bits of u64.
736 */748 */
737static inline void bitmap_from_u64(unsigned long *dst, u64 mask)749static __always_inline void bitmap_from_u64(unsigned long *dst, u64 mask)
738{750{
739 bitmap_from_arr64(dst, &mask, 64);751 bitmap_from_arr64(dst, &mask, 64);
740}752}
@@ -749,9 +761,8 @@ static inline void bitmap_from_u64(unsigned long *dst, u64 mask) · 8 unmodified lines
749 * @map memory region. For @nbits = 0 and @nbits > BITS_PER_LONG the return761 * @map memory region. For @nbits = 0 and @nbits > BITS_PER_LONG the return
750 * value is undefined.762 * value is undefined.
751 */763 */
752static inline unsigned long bitmap_read(const unsigned long *map,764static __always_inline
753 unsigned long start,765unsigned long bitmap_read(const unsigned long *map, unsigned long start, unsigned long nbits)
754 unsigned long nbits)
755{766{
756 size_t index = BIT_WORD(start);767 size_t index = BIT_WORD(start);
757 unsigned long offset = start % BITS_PER_LONG;768 unsigned long offset = start % BITS_PER_LONG;
@@ -784,8 +795,9 @@ static inline unsigned long bitmap_read(const unsigned long *map, · 26 unmodified lines
784 *795 *
785 * For @nbits == 0 and @nbits > BITS_PER_LONG no writes are performed.796 * For @nbits == 0 and @nbits > BITS_PER_LONG no writes are performed.
786 */797 */
787static inline void bitmap_write(unsigned long *map, unsigned long value,798static __always_inline
788 unsigned long start, unsigned long nbits)799void bitmap_write(unsigned long *map, unsigned long value,
800 unsigned long start, unsigned long nbits)
789{801{
790 size_t index;802 size_t index;
791 unsigned long offset;803 unsigned long offset;

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t