History

Merge tag 'bitmap-for-6.12' of https://github.com/norov/linux

Pull bitmap updates from Yury Norov: - switch all bitmamp APIs from inline to __always_inline (Brian Norris) The __always_inline series improves on code generation, and now with the latest compiler versions is required to avoid compilation warnings. It spent enough in my backlog, and I'm thankful to Brian Norris for taking over and moving it forward. - introduce GENMASK_U128() macro (Anshuman Khandual) GENMASK_U128() is a prerequisite needed for arm64 development * tag 'bitmap-for-6.12' of https://github.com/norov/linux: lib/test_bits.c: Add tests for GENMASK_U128() uapi: Define GENMASK_U128 nodemask: Switch from inline to __always_inline cpumask: Switch from inline to __always_inline bitmap: Switch from inline to __always_inline find: Switch from inline to __always_inline

Linus Torvalds committed 2 years ago

Commit 9c445752 parents ba33a49 d7bcc378 files · +325 −232Browse files

Unified · Split

8 files
  1. include/linux/bitmap.h +76 −64
  2. include/linux/bits.h +15 −0
  3. include/linux/cpumask.h +112 −100
  4. include/linux/find.h +25 −25
  5. include/linux/nodemask.h +43 −43
  6. include/uapi/linux/bits.h +3 −0
  7. include/uapi/linux/const.h +17 −0
  8. lib/test_bits.c +34 −0
include/linux/bitmap.h+76−64
@@ -203,12 +203,12 @@ unsigned long bitmap_find_next_zero_area_off(unsigned long *map, · 202 unmodified lines
203 * the bit offset of all zero areas this function finds is multiples of that203 * the bit offset of all zero areas this function finds is multiples of that
204 * power of 2. A @align_mask of 0 means no alignment is required.204 * power of 2. A @align_mask of 0 means no alignment is required.
205 */205 */
206static inline unsigned long206static __always_inline
207bitmap_find_next_zero_area(unsigned long *map,207unsigned long bitmap_find_next_zero_area(unsigned long *map,
208 unsigned long size,208 unsigned long size,
209 unsigned long start,209 unsigned long start,
210 unsigned int nr,210 unsigned int nr,
211 unsigned long align_mask)211 unsigned long align_mask)
212{212{
213 return bitmap_find_next_zero_area_off(map, size, start, nr,213 return bitmap_find_next_zero_area_off(map, size, start, nr,
214 align_mask, 0);214 align_mask, 0);
@@ -228,7 +228,7 @@ void bitmap_fold(unsigned long *dst, const unsigned long *orig, · 13 unmodified lines
228 228
229#define bitmap_size(nbits) (ALIGN(nbits, BITS_PER_LONG) / BITS_PER_BYTE)229#define bitmap_size(nbits) (ALIGN(nbits, BITS_PER_LONG) / BITS_PER_BYTE)
230 230
231static inline void bitmap_zero(unsigned long *dst, unsigned int nbits)231static __always_inline void bitmap_zero(unsigned long *dst, unsigned int nbits)
232{232{
233 unsigned int len = bitmap_size(nbits);233 unsigned int len = bitmap_size(nbits);
234 234
@@ -238,7 +238,7 @@ static inline void bitmap_zero(unsigned long *dst, unsigned int nbits) · 3 unmodified lines
238 memset(dst, 0, len);238 memset(dst, 0, len);
239}239}
240 240
241static inline void bitmap_fill(unsigned long *dst, unsigned int nbits)241static __always_inline void bitmap_fill(unsigned long *dst, unsigned int nbits)
242{242{
243 unsigned int len = bitmap_size(nbits);243 unsigned int len = bitmap_size(nbits);
244 244
@@ -248,8 +248,8 @@ static inline void bitmap_fill(unsigned long *dst, unsigned int nbits) · 3 unmodified lines
248 memset(dst, 0xff, len);248 memset(dst, 0xff, len);
249}249}
250 250
251static inline void bitmap_copy(unsigned long *dst, const unsigned long *src,251static __always_inline
252 unsigned int nbits)252void bitmap_copy(unsigned long *dst, const unsigned long *src, unsigned int nbits)
253{253{
254 unsigned int len = bitmap_size(nbits);254 unsigned int len = bitmap_size(nbits);
255 255
@@ -262,8 +262,8 @@ static inline void bitmap_copy(unsigned long *dst, const unsigned long *src, · 6 unmodified lines
262/*262/*
263 * Copy bitmap and clear tail bits in last word.263 * Copy bitmap and clear tail bits in last word.
264 */264 */
265static inline void bitmap_copy_clear_tail(unsigned long *dst,265static __always_inline
266 const unsigned long *src, unsigned int nbits)266void bitmap_copy_clear_tail(unsigned long *dst, const unsigned long *src, unsigned int nbits)
267{267{
268 bitmap_copy(dst, src, nbits);268 bitmap_copy(dst, src, nbits);
269 if (nbits % BITS_PER_LONG)269 if (nbits % BITS_PER_LONG)
@@ -318,16 +318,18 @@ void bitmap_to_arr64(u64 *buf, const unsigned long *bitmap, unsigned int nbits); · 48 unmodified lines
318 bitmap_copy_clear_tail((unsigned long *)(buf), (const unsigned long *)(bitmap), (nbits))318 bitmap_copy_clear_tail((unsigned long *)(buf), (const unsigned long *)(bitmap), (nbits))
319#endif319#endif
320 320
321static inline bool bitmap_and(unsigned long *dst, const unsigned long *src1,321static __always_inline
322 const unsigned long *src2, unsigned int nbits)322bool bitmap_and(unsigned long *dst, const unsigned long *src1,
323 const unsigned long *src2, unsigned int nbits)
323{324{
324 if (small_const_nbits(nbits))325 if (small_const_nbits(nbits))
325 return (*dst = *src1 & *src2 & BITMAP_LAST_WORD_MASK(nbits)) != 0;326 return (*dst = *src1 & *src2 & BITMAP_LAST_WORD_MASK(nbits)) != 0;
326 return __bitmap_and(dst, src1, src2, nbits);327 return __bitmap_and(dst, src1, src2, nbits);
327}328}
328 329
329static inline void bitmap_or(unsigned long *dst, const unsigned long *src1,330static __always_inline
330 const unsigned long *src2, unsigned int nbits)331void bitmap_or(unsigned long *dst, const unsigned long *src1,
332 const unsigned long *src2, unsigned int nbits)
331{333{
332 if (small_const_nbits(nbits))334 if (small_const_nbits(nbits))
333 *dst = *src1 | *src2;335 *dst = *src1 | *src2;
@@ -335,8 +337,9 @@ static inline void bitmap_or(unsigned long *dst, const unsigned long *src1, · 1 unmodified line
335 __bitmap_or(dst, src1, src2, nbits);337 __bitmap_or(dst, src1, src2, nbits);
336}338}
337 339
338static inline void bitmap_xor(unsigned long *dst, const unsigned long *src1,340static __always_inline
339 const unsigned long *src2, unsigned int nbits)341void bitmap_xor(unsigned long *dst, const unsigned long *src1,
342 const unsigned long *src2, unsigned int nbits)
340{343{
341 if (small_const_nbits(nbits))344 if (small_const_nbits(nbits))
342 *dst = *src1 ^ *src2;345 *dst = *src1 ^ *src2;
@@ -344,16 +347,17 @@ static inline void bitmap_xor(unsigned long *dst, const unsigned long *src1, · 1 unmodified line
344 __bitmap_xor(dst, src1, src2, nbits);347 __bitmap_xor(dst, src1, src2, nbits);
345}348}
346 349
347static inline bool bitmap_andnot(unsigned long *dst, const unsigned long *src1,350static __always_inline
348 const unsigned long *src2, unsigned int nbits)351bool bitmap_andnot(unsigned long *dst, const unsigned long *src1,
352 const unsigned long *src2, unsigned int nbits)
349{353{
350 if (small_const_nbits(nbits))354 if (small_const_nbits(nbits))
351 return (*dst = *src1 & ~(*src2) & BITMAP_LAST_WORD_MASK(nbits)) != 0;355 return (*dst = *src1 & ~(*src2) & BITMAP_LAST_WORD_MASK(nbits)) != 0;
352 return __bitmap_andnot(dst, src1, src2, nbits);356 return __bitmap_andnot(dst, src1, src2, nbits);
353}357}
354 358
355static inline void bitmap_complement(unsigned long *dst, const unsigned long *src,359static __always_inline
356 unsigned int nbits)360void bitmap_complement(unsigned long *dst, const unsigned long *src, unsigned int nbits)
357{361{
358 if (small_const_nbits(nbits))362 if (small_const_nbits(nbits))
359 *dst = ~(*src);363 *dst = ~(*src);
@@ -368,8 +372,8 @@ static inline void bitmap_complement(unsigned long *dst, const unsigned long *sr · 8 unmodified lines
368#endif372#endif
369#define BITMAP_MEM_MASK (BITMAP_MEM_ALIGNMENT - 1)373#define BITMAP_MEM_MASK (BITMAP_MEM_ALIGNMENT - 1)
370 374
371static inline bool bitmap_equal(const unsigned long *src1,375static __always_inline
372 const unsigned long *src2, unsigned int nbits)376bool bitmap_equal(const unsigned long *src1, const unsigned long *src2, unsigned int nbits)
373{377{
374 if (small_const_nbits(nbits))378 if (small_const_nbits(nbits))
375 return !((*src1 ^ *src2) & BITMAP_LAST_WORD_MASK(nbits));379 return !((*src1 ^ *src2) & BITMAP_LAST_WORD_MASK(nbits));
@@ -388,10 +392,9 @@ static inline bool bitmap_equal(const unsigned long *src1, · 12 unmodified lines
388 *392 *
389 * Returns: True if (*@src1 | *@src2) == *@src3, false otherwise393 * Returns: True if (*@src1 | *@src2) == *@src3, false otherwise
390 */394 */
391static inline bool bitmap_or_equal(const unsigned long *src1,395static __always_inline
392 const unsigned long *src2,396bool bitmap_or_equal(const unsigned long *src1, const unsigned long *src2,
393 const unsigned long *src3,397 const unsigned long *src3, unsigned int nbits)
394 unsigned int nbits)
395{398{
396 if (!small_const_nbits(nbits))399 if (!small_const_nbits(nbits))
397 return __bitmap_or_equal(src1, src2, src3, nbits);400 return __bitmap_or_equal(src1, src2, src3, nbits);
@@ -399,9 +402,8 @@ static inline bool bitmap_or_equal(const unsigned long *src1, · 1 unmodified line
399 return !(((*src1 | *src2) ^ *src3) & BITMAP_LAST_WORD_MASK(nbits));402 return !(((*src1 | *src2) ^ *src3) & BITMAP_LAST_WORD_MASK(nbits));
400}403}
401 404
402static inline bool bitmap_intersects(const unsigned long *src1,405static __always_inline
403 const unsigned long *src2,406bool bitmap_intersects(const unsigned long *src1, const unsigned long *src2, unsigned int nbits)
404 unsigned int nbits)
405{407{
406 if (small_const_nbits(nbits))408 if (small_const_nbits(nbits))
407 return ((*src1 & *src2) & BITMAP_LAST_WORD_MASK(nbits)) != 0;409 return ((*src1 & *src2) & BITMAP_LAST_WORD_MASK(nbits)) != 0;
@@ -409,8 +411,8 @@ static inline bool bitmap_intersects(const unsigned long *src1, · 1 unmodified line
409 return __bitmap_intersects(src1, src2, nbits);411 return __bitmap_intersects(src1, src2, nbits);
410}412}
411 413
412static inline bool bitmap_subset(const unsigned long *src1,414static __always_inline
413 const unsigned long *src2, unsigned int nbits)415bool bitmap_subset(const unsigned long *src1, const unsigned long *src2, unsigned int nbits)
414{416{
415 if (small_const_nbits(nbits))417 if (small_const_nbits(nbits))
416 return ! ((*src1 & ~(*src2)) & BITMAP_LAST_WORD_MASK(nbits));418 return ! ((*src1 & ~(*src2)) & BITMAP_LAST_WORD_MASK(nbits));
@@ -418,7 +420,8 @@ static inline bool bitmap_subset(const unsigned long *src1, · 1 unmodified line
418 return __bitmap_subset(src1, src2, nbits);420 return __bitmap_subset(src1, src2, nbits);
419}421}
420 422
421static inline bool bitmap_empty(const unsigned long *src, unsigned nbits)423static __always_inline
424bool bitmap_empty(const unsigned long *src, unsigned nbits)
422{425{
423 if (small_const_nbits(nbits))426 if (small_const_nbits(nbits))
424 return ! (*src & BITMAP_LAST_WORD_MASK(nbits));427 return ! (*src & BITMAP_LAST_WORD_MASK(nbits));
@@ -426,7 +429,8 @@ static inline bool bitmap_empty(const unsigned long *src, unsigned nbits) · 1 unmodified line
426 return find_first_bit(src, nbits) == nbits;429 return find_first_bit(src, nbits) == nbits;
427}430}
428 431
429static inline bool bitmap_full(const unsigned long *src, unsigned int nbits)432static __always_inline
433bool bitmap_full(const unsigned long *src, unsigned int nbits)
430{434{
431 if (small_const_nbits(nbits))435 if (small_const_nbits(nbits))
432 return ! (~(*src) & BITMAP_LAST_WORD_MASK(nbits));436 return ! (~(*src) & BITMAP_LAST_WORD_MASK(nbits));
@@ -460,8 +464,8 @@ unsigned long bitmap_weight_andnot(const unsigned long *src1, · 27 unmodified lines
460 return __bitmap_weight_andnot(src1, src2, nbits);464 return __bitmap_weight_andnot(src1, src2, nbits);
461}465}
462 466
463static __always_inline void bitmap_set(unsigned long *map, unsigned int start,467static __always_inline
464 unsigned int nbits)468void bitmap_set(unsigned long *map, unsigned int start, unsigned int nbits)
465{469{
466 if (__builtin_constant_p(nbits) && nbits == 1)470 if (__builtin_constant_p(nbits) && nbits == 1)
467 __set_bit(start, map);471 __set_bit(start, map);
@@ -476,8 +480,8 @@ static __always_inline void bitmap_set(unsigned long *map, unsigned int start, · 8 unmodified lines
476 __bitmap_set(map, start, nbits);480 __bitmap_set(map, start, nbits);
477}481}
478 482
479static __always_inline void bitmap_clear(unsigned long *map, unsigned int start,483static __always_inline
480 unsigned int nbits)484void bitmap_clear(unsigned long *map, unsigned int start, unsigned int nbits)
481{485{
482 if (__builtin_constant_p(nbits) && nbits == 1)486 if (__builtin_constant_p(nbits) && nbits == 1)
483 __clear_bit(start, map);487 __clear_bit(start, map);
@@ -492,8 +496,9 @@ static __always_inline void bitmap_clear(unsigned long *map, unsigned int start, · 8 unmodified lines
492 __bitmap_clear(map, start, nbits);496 __bitmap_clear(map, start, nbits);
493}497}
494 498
495static inline void bitmap_shift_right(unsigned long *dst, const unsigned long *src,499static __always_inline
496 unsigned int shift, unsigned int nbits)500void bitmap_shift_right(unsigned long *dst, const unsigned long *src,
501 unsigned int shift, unsigned int nbits)
497{502{
498 if (small_const_nbits(nbits))503 if (small_const_nbits(nbits))
499 *dst = (*src & BITMAP_LAST_WORD_MASK(nbits)) >> shift;504 *dst = (*src & BITMAP_LAST_WORD_MASK(nbits)) >> shift;
@@ -501,8 +506,9 @@ static inline void bitmap_shift_right(unsigned long *dst, const unsigned long *s · 1 unmodified line
501 __bitmap_shift_right(dst, src, shift, nbits);506 __bitmap_shift_right(dst, src, shift, nbits);
502}507}
503 508
504static inline void bitmap_shift_left(unsigned long *dst, const unsigned long *src,509static __always_inline
505 unsigned int shift, unsigned int nbits)510void bitmap_shift_left(unsigned long *dst, const unsigned long *src,
511 unsigned int shift, unsigned int nbits)
506{512{
507 if (small_const_nbits(nbits))513 if (small_const_nbits(nbits))
508 *dst = (*src << shift) & BITMAP_LAST_WORD_MASK(nbits);514 *dst = (*src << shift) & BITMAP_LAST_WORD_MASK(nbits);
@@ -510,11 +516,12 @@ static inline void bitmap_shift_left(unsigned long *dst, const unsigned long *sr · 1 unmodified line
510 __bitmap_shift_left(dst, src, shift, nbits);516 __bitmap_shift_left(dst, src, shift, nbits);
511}517}
512 518
513static inline void bitmap_replace(unsigned long *dst,519static __always_inline
514 const unsigned long *old,520void bitmap_replace(unsigned long *dst,
515 const unsigned long *new,521 const unsigned long *old,
516 const unsigned long *mask,522 const unsigned long *new,
517 unsigned int nbits)523 const unsigned long *mask,
524 unsigned int nbits)
518{525{
519 if (small_const_nbits(nbits))526 if (small_const_nbits(nbits))
520 *dst = (*old & ~(*mask)) | (*new & *mask);527 *dst = (*old & ~(*mask)) | (*new & *mask);
@@ -557,8 +564,9 @@ static inline void bitmap_replace(unsigned long *dst, · 36 unmodified lines
557 * bitmap_gather() can be seen as the 'reverse' bitmap_scatter() operation.564 * bitmap_gather() can be seen as the 'reverse' bitmap_scatter() operation.
558 * See bitmap_scatter() for details related to this relationship.565 * See bitmap_scatter() for details related to this relationship.
559 */566 */
560static inline void bitmap_scatter(unsigned long *dst, const unsigned long *src,567static __always_inline
561 const unsigned long *mask, unsigned int nbits)568void bitmap_scatter(unsigned long *dst, const unsigned long *src,
569 const unsigned long *mask, unsigned int nbits)
562{570{
563 unsigned int n = 0;571 unsigned int n = 0;
564 unsigned int bit;572 unsigned int bit;
@@ -611,8 +619,9 @@ static inline void bitmap_scatter(unsigned long *dst, const unsigned long *src, · 46 unmodified lines
611 * bitmap_scatter(res, src, mask, n) and a call to619 * bitmap_scatter(res, src, mask, n) and a call to
612 * bitmap_scatter(res, result, mask, n) will lead to the same res value.620 * bitmap_scatter(res, result, mask, n) will lead to the same res value.
613 */621 */
614static inline void bitmap_gather(unsigned long *dst, const unsigned long *src,622static __always_inline
615 const unsigned long *mask, unsigned int nbits)623void bitmap_gather(unsigned long *dst, const unsigned long *src,
624 const unsigned long *mask, unsigned int nbits)
616{625{
617 unsigned int n = 0;626 unsigned int n = 0;
618 unsigned int bit;627 unsigned int bit;
@@ -623,9 +632,9 @@ static inline void bitmap_gather(unsigned long *dst, const unsigned long *src, · 4 unmodified lines
623 __assign_bit(n++, dst, test_bit(bit, src));632 __assign_bit(n++, dst, test_bit(bit, src));
624}633}
625 634
626static inline void bitmap_next_set_region(unsigned long *bitmap,635static __always_inline
627 unsigned int *rs, unsigned int *re,636void bitmap_next_set_region(unsigned long *bitmap, unsigned int *rs,
628 unsigned int end)637 unsigned int *re, unsigned int end)
629{638{
630 *rs = find_next_bit(bitmap, end, *rs);639 *rs = find_next_bit(bitmap, end, *rs);
631 *re = find_next_zero_bit(bitmap, end, *rs + 1);640 *re = find_next_zero_bit(bitmap, end, *rs + 1);
@@ -640,7 +649,8 @@ static inline void bitmap_next_set_region(unsigned long *bitmap, · 8 unmodified lines
640 * This is the complement to __bitmap_find_free_region() and releases649 * This is the complement to __bitmap_find_free_region() and releases
641 * the found region (by clearing it in the bitmap).650 * the found region (by clearing it in the bitmap).
642 */651 */
643static inline void bitmap_release_region(unsigned long *bitmap, unsigned int pos, int order)652static __always_inline
653void bitmap_release_region(unsigned long *bitmap, unsigned int pos, int order)
644{654{
645 bitmap_clear(bitmap, pos, BIT(order));655 bitmap_clear(bitmap, pos, BIT(order));
646}656}
@@ -656,7 +666,8 @@ static inline void bitmap_release_region(unsigned long *bitmap, unsigned int pos · 9 unmodified lines
656 * Returns: 0 on success, or %-EBUSY if specified region wasn't666 * Returns: 0 on success, or %-EBUSY if specified region wasn't
657 * free (not all bits were zero).667 * free (not all bits were zero).
658 */668 */
659static inline int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos, int order)669static __always_inline
670int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos, int order)
660{671{
661 unsigned int len = BIT(order);672 unsigned int len = BIT(order);
662 673
@@ -680,7 +691,8 @@ static inline int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos · 17 unmodified lines
680 * Returns: the bit offset in bitmap of the allocated region,691 * Returns: the bit offset in bitmap of the allocated region,
681 * or -errno on failure.692 * or -errno on failure.
682 */693 */
683static inline int bitmap_find_free_region(unsigned long *bitmap, unsigned int bits, int order)694static __always_inline
695int bitmap_find_free_region(unsigned long *bitmap, unsigned int bits, int order)
684{696{
685 unsigned int pos, end; /* scans bitmap by regions of size order */697 unsigned int pos, end; /* scans bitmap by regions of size order */
686 698
@@ -734,7 +746,7 @@ static inline int bitmap_find_free_region(unsigned long *bitmap, unsigned int bi · 47 unmodified lines
734 * That is ``(u32 *)(&val)[0]`` gets the upper 32 bits,746 * That is ``(u32 *)(&val)[0]`` gets the upper 32 bits,
735 * but we expect the lower 32-bits of u64.747 * but we expect the lower 32-bits of u64.
736 */748 */
737static inline void bitmap_from_u64(unsigned long *dst, u64 mask)749static __always_inline void bitmap_from_u64(unsigned long *dst, u64 mask)
738{750{
739 bitmap_from_arr64(dst, &mask, 64);751 bitmap_from_arr64(dst, &mask, 64);
740}752}
@@ -749,9 +761,8 @@ static inline void bitmap_from_u64(unsigned long *dst, u64 mask) · 8 unmodified lines
749 * @map memory region. For @nbits = 0 and @nbits > BITS_PER_LONG the return761 * @map memory region. For @nbits = 0 and @nbits > BITS_PER_LONG the return
750 * value is undefined.762 * value is undefined.
751 */763 */
752static inline unsigned long bitmap_read(const unsigned long *map,764static __always_inline
753 unsigned long start,765unsigned long bitmap_read(const unsigned long *map, unsigned long start, unsigned long nbits)
754 unsigned long nbits)
755{766{
756 size_t index = BIT_WORD(start);767 size_t index = BIT_WORD(start);
757 unsigned long offset = start % BITS_PER_LONG;768 unsigned long offset = start % BITS_PER_LONG;
@@ -784,8 +795,9 @@ static inline unsigned long bitmap_read(const unsigned long *map, · 26 unmodified lines
784 *795 *
785 * For @nbits == 0 and @nbits > BITS_PER_LONG no writes are performed.796 * For @nbits == 0 and @nbits > BITS_PER_LONG no writes are performed.
786 */797 */
787static inline void bitmap_write(unsigned long *map, unsigned long value,798static __always_inline
788 unsigned long start, unsigned long nbits)799void bitmap_write(unsigned long *map, unsigned long value,
800 unsigned long start, unsigned long nbits)
789{801{
790 size_t index;802 size_t index;
791 unsigned long offset;803 unsigned long offset;
include/linux/bits.h+15−0
@@ -36,4 +36,19 @@ · 35 unmodified lines
36#define GENMASK_ULL(h, l) \36#define GENMASK_ULL(h, l) \
37 (GENMASK_INPUT_CHECK(h, l) + __GENMASK_ULL(h, l))37 (GENMASK_INPUT_CHECK(h, l) + __GENMASK_ULL(h, l))
38 38
39#if !defined(__ASSEMBLY__)
40/*
41 * Missing asm support
42 *
43 * __GENMASK_U128() depends on _BIT128() which would not work
44 * in the asm code, as it shifts an 'unsigned __init128' data
45 * type instead of direct representation of 128 bit constants
46 * such as long and unsigned long. The fundamental problem is
47 * that a 128 bit constant will get silently truncated by the
48 * gcc compiler.
49 */
50#define GENMASK_U128(h, l) \
51 (GENMASK_INPUT_CHECK(h, l) + __GENMASK_U128(h, l))
52#endif
53
39#endif /* __LINUX_BITS_H */54#endif /* __LINUX_BITS_H */
include/linux/cpumask.h+112−100
@@ -30,7 +30,7 @@ · 29 unmodified lines
30extern unsigned int nr_cpu_ids;30extern unsigned int nr_cpu_ids;
31#endif31#endif
32 32
33static inline void set_nr_cpu_ids(unsigned int nr)33static __always_inline void set_nr_cpu_ids(unsigned int nr)
34{34{
35#if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS)35#if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS)
36 WARN_ON(nr != nr_cpu_ids);36 WARN_ON(nr != nr_cpu_ids);
@@ -149,7 +149,7 @@ static __always_inline unsigned int cpumask_check(unsigned int cpu) · 112 unmodified lines
149 *149 *
150 * Return: >= nr_cpu_ids if no cpus set.150 * Return: >= nr_cpu_ids if no cpus set.
151 */151 */
152static inline unsigned int cpumask_first(const struct cpumask *srcp)152static __always_inline unsigned int cpumask_first(const struct cpumask *srcp)
153{153{
154 return find_first_bit(cpumask_bits(srcp), small_cpumask_bits);154 return find_first_bit(cpumask_bits(srcp), small_cpumask_bits);
155}155}
@@ -160,7 +160,7 @@ static inline unsigned int cpumask_first(const struct cpumask *srcp) · 4 unmodified lines
160 *160 *
161 * Return: >= nr_cpu_ids if all cpus are set.161 * Return: >= nr_cpu_ids if all cpus are set.
162 */162 */
163static inline unsigned int cpumask_first_zero(const struct cpumask *srcp)163static __always_inline unsigned int cpumask_first_zero(const struct cpumask *srcp)
164{164{
165 return find_first_zero_bit(cpumask_bits(srcp), small_cpumask_bits);165 return find_first_zero_bit(cpumask_bits(srcp), small_cpumask_bits);
166}166}
@@ -172,7 +172,7 @@ static inline unsigned int cpumask_first_zero(const struct cpumask *srcp) · 5 unmodified lines
172 *172 *
173 * Return: >= nr_cpu_ids if no cpus set in both. See also cpumask_next_and().173 * Return: >= nr_cpu_ids if no cpus set in both. See also cpumask_next_and().
174 */174 */
175static inline175static __always_inline
176unsigned int cpumask_first_and(const struct cpumask *srcp1, const struct cpumask *srcp2)176unsigned int cpumask_first_and(const struct cpumask *srcp1, const struct cpumask *srcp2)
177{177{
178 return find_first_and_bit(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits);178 return find_first_and_bit(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits);
@@ -186,7 +186,7 @@ unsigned int cpumask_first_and(const struct cpumask *srcp1, const struct cpumask · 7 unmodified lines
186 *186 *
187 * Return: >= nr_cpu_ids if no cpus set in all.187 * Return: >= nr_cpu_ids if no cpus set in all.
188 */188 */
189static inline189static __always_inline
190unsigned int cpumask_first_and_and(const struct cpumask *srcp1,190unsigned int cpumask_first_and_and(const struct cpumask *srcp1,
191 const struct cpumask *srcp2,191 const struct cpumask *srcp2,
192 const struct cpumask *srcp3)192 const struct cpumask *srcp3)
@@ -201,7 +201,7 @@ unsigned int cpumask_first_and_and(const struct cpumask *srcp1, · 8 unmodified lines
201 *201 *
202 * Return: >= nr_cpumask_bits if no CPUs set.202 * Return: >= nr_cpumask_bits if no CPUs set.
203 */203 */
204static inline unsigned int cpumask_last(const struct cpumask *srcp)204static __always_inline unsigned int cpumask_last(const struct cpumask *srcp)
205{205{
206 return find_last_bit(cpumask_bits(srcp), small_cpumask_bits);206 return find_last_bit(cpumask_bits(srcp), small_cpumask_bits);
207}207}
@@ -213,7 +213,7 @@ static inline unsigned int cpumask_last(const struct cpumask *srcp) · 5 unmodified lines
213 *213 *
214 * Return: >= nr_cpu_ids if no further cpus set.214 * Return: >= nr_cpu_ids if no further cpus set.
215 */215 */
216static inline216static __always_inline
217unsigned int cpumask_next(int n, const struct cpumask *srcp)217unsigned int cpumask_next(int n, const struct cpumask *srcp)
218{218{
219 /* -1 is a legal arg here. */219 /* -1 is a legal arg here. */
@@ -229,7 +229,8 @@ unsigned int cpumask_next(int n, const struct cpumask *srcp) · 9 unmodified lines
229 *229 *
230 * Return: >= nr_cpu_ids if no further cpus unset.230 * Return: >= nr_cpu_ids if no further cpus unset.
231 */231 */
232static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)232static __always_inline
233unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
233{234{
234 /* -1 is a legal arg here. */235 /* -1 is a legal arg here. */
235 if (n != -1)236 if (n != -1)
@@ -239,18 +240,21 @@ static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp) · 3 unmodified lines
239 240
240#if NR_CPUS == 1241#if NR_CPUS == 1
241/* Uniprocessor: there is only one valid CPU */242/* Uniprocessor: there is only one valid CPU */
242static inline unsigned int cpumask_local_spread(unsigned int i, int node)243static __always_inline
244unsigned int cpumask_local_spread(unsigned int i, int node)
243{245{
244 return 0;246 return 0;
245}247}
246 248
247static inline unsigned int cpumask_any_and_distribute(const struct cpumask *src1p,249static __always_inline
248 const struct cpumask *src2p)250unsigned int cpumask_any_and_distribute(const struct cpumask *src1p,
251 const struct cpumask *src2p)
249{252{
250 return cpumask_first_and(src1p, src2p);253 return cpumask_first_and(src1p, src2p);
251}254}
252 255
253static inline unsigned int cpumask_any_distribute(const struct cpumask *srcp)256static __always_inline
257unsigned int cpumask_any_distribute(const struct cpumask *srcp)
254{258{
255 return cpumask_first(srcp);259 return cpumask_first(srcp);
256}260}
@@ -269,9 +273,9 @@ unsigned int cpumask_any_distribute(const struct cpumask *srcp); · 12 unmodified lines
269 *273 *
270 * Return: >= nr_cpu_ids if no further cpus set in both.274 * Return: >= nr_cpu_ids if no further cpus set in both.
271 */275 */
272static inline276static __always_inline
273unsigned int cpumask_next_and(int n, const struct cpumask *src1p,277unsigned int cpumask_next_and(int n, const struct cpumask *src1p,
274 const struct cpumask *src2p)278 const struct cpumask *src2p)
275{279{
276 /* -1 is a legal arg here. */280 /* -1 is a legal arg here. */
277 if (n != -1)281 if (n != -1)
@@ -291,7 +295,7 @@ unsigned int cpumask_next_and(int n, const struct cpumask *src1p, · 13 unmodified lines
291 for_each_set_bit(cpu, cpumask_bits(mask), small_cpumask_bits)295 for_each_set_bit(cpu, cpumask_bits(mask), small_cpumask_bits)
292 296
293#if NR_CPUS == 1297#if NR_CPUS == 1
294static inline298static __always_inline
295unsigned int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap)299unsigned int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap)
296{300{
297 cpumask_check(start);301 cpumask_check(start);
@@ -394,7 +398,7 @@ unsigned int __pure cpumask_next_wrap(int n, const struct cpumask *mask, int sta · 96 unmodified lines
394 * Often used to find any cpu but smp_processor_id() in a mask.398 * Often used to find any cpu but smp_processor_id() in a mask.
395 * Return: >= nr_cpu_ids if no cpus set.399 * Return: >= nr_cpu_ids if no cpus set.
396 */400 */
397static inline401static __always_inline
398unsigned int cpumask_any_but(const struct cpumask *mask, unsigned int cpu)402unsigned int cpumask_any_but(const struct cpumask *mask, unsigned int cpu)
399{403{
400 unsigned int i;404 unsigned int i;
@@ -414,7 +418,7 @@ unsigned int cpumask_any_but(const struct cpumask *mask, unsigned int cpu) · 13 unmodified lines
414 *418 *
415 * Returns >= nr_cpu_ids if no cpus set.419 * Returns >= nr_cpu_ids if no cpus set.
416 */420 */
417static inline421static __always_inline
418unsigned int cpumask_any_and_but(const struct cpumask *mask1,422unsigned int cpumask_any_and_but(const struct cpumask *mask1,
419 const struct cpumask *mask2,423 const struct cpumask *mask2,
420 unsigned int cpu)424 unsigned int cpu)
@@ -436,7 +440,8 @@ unsigned int cpumask_any_and_but(const struct cpumask *mask1, · 15 unmodified lines
436 *440 *
437 * Return: >= nr_cpu_ids if such cpu doesn't exist.441 * Return: >= nr_cpu_ids if such cpu doesn't exist.
438 */442 */
439static inline unsigned int cpumask_nth(unsigned int cpu, const struct cpumask *srcp)443static __always_inline
444unsigned int cpumask_nth(unsigned int cpu, const struct cpumask *srcp)
440{445{
441 return find_nth_bit(cpumask_bits(srcp), small_cpumask_bits, cpumask_check(cpu));446 return find_nth_bit(cpumask_bits(srcp), small_cpumask_bits, cpumask_check(cpu));
442}447}
@@ -449,7 +454,7 @@ static inline unsigned int cpumask_nth(unsigned int cpu, const struct cpumask *s · 6 unmodified lines
449 *454 *
450 * Return: >= nr_cpu_ids if such cpu doesn't exist.455 * Return: >= nr_cpu_ids if such cpu doesn't exist.
451 */456 */
452static inline457static __always_inline
453unsigned int cpumask_nth_and(unsigned int cpu, const struct cpumask *srcp1,458unsigned int cpumask_nth_and(unsigned int cpu, const struct cpumask *srcp1,
454 const struct cpumask *srcp2)459 const struct cpumask *srcp2)
455{460{
@@ -465,7 +470,7 @@ unsigned int cpumask_nth_and(unsigned int cpu, const struct cpumask *srcp1, · 9 unmodified lines
465 *470 *
466 * Return: >= nr_cpu_ids if such cpu doesn't exist.471 * Return: >= nr_cpu_ids if such cpu doesn't exist.
467 */472 */
468static inline473static __always_inline
469unsigned int cpumask_nth_andnot(unsigned int cpu, const struct cpumask *srcp1,474unsigned int cpumask_nth_andnot(unsigned int cpu, const struct cpumask *srcp1,
470 const struct cpumask *srcp2)475 const struct cpumask *srcp2)
471{476{
@@ -508,12 +513,14 @@ unsigned int cpumask_nth_and_andnot(unsigned int cpu, const struct cpumask *srcp · 36 unmodified lines
508 * @cpu: cpu number (< nr_cpu_ids)513 * @cpu: cpu number (< nr_cpu_ids)
509 * @dstp: the cpumask pointer514 * @dstp: the cpumask pointer
510 */515 */
511static __always_inline void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)516static __always_inline
517void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
512{518{
513 set_bit(cpumask_check(cpu), cpumask_bits(dstp));519 set_bit(cpumask_check(cpu), cpumask_bits(dstp));
514}520}
515 521
516static __always_inline void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)522static __always_inline
523void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
517{524{
518 __set_bit(cpumask_check(cpu), cpumask_bits(dstp));525 __set_bit(cpumask_check(cpu), cpumask_bits(dstp));
519}526}
@@ -557,7 +564,8 @@ static __always_inline void __cpumask_assign_cpu(int cpu, struct cpumask *dstp, · 37 unmodified lines
557 *564 *
558 * Return: true if @cpu is set in @cpumask, else returns false565 * Return: true if @cpu is set in @cpumask, else returns false
559 */566 */
560static __always_inline bool cpumask_test_cpu(int cpu, const struct cpumask *cpumask)567static __always_inline
568bool cpumask_test_cpu(int cpu, const struct cpumask *cpumask)
561{569{
562 return test_bit(cpumask_check(cpu), cpumask_bits((cpumask)));570 return test_bit(cpumask_check(cpu), cpumask_bits((cpumask)));
563}571}
@@ -571,7 +579,8 @@ static __always_inline bool cpumask_test_cpu(int cpu, const struct cpumask *cpum · 7 unmodified lines
571 *579 *
572 * Return: true if @cpu is set in old bitmap of @cpumask, else returns false580 * Return: true if @cpu is set in old bitmap of @cpumask, else returns false
573 */581 */
574static __always_inline bool cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)582static __always_inline
583bool cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)
575{584{
576 return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask));585 return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask));
577}586}
@@ -585,7 +594,8 @@ static __always_inline bool cpumask_test_and_set_cpu(int cpu, struct cpumask *cp · 7 unmodified lines
585 *594 *
586 * Return: true if @cpu is set in old bitmap of @cpumask, else returns false595 * Return: true if @cpu is set in old bitmap of @cpumask, else returns false
587 */596 */
588static __always_inline bool cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask)597static __always_inline
598bool cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask)
589{599{
590 return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask));600 return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask));
591}601}
@@ -594,7 +604,7 @@ static __always_inline bool cpumask_test_and_clear_cpu(int cpu, struct cpumask * · 2 unmodified lines
594 * cpumask_setall - set all cpus (< nr_cpu_ids) in a cpumask604 * cpumask_setall - set all cpus (< nr_cpu_ids) in a cpumask
595 * @dstp: the cpumask pointer605 * @dstp: the cpumask pointer
596 */606 */
597static inline void cpumask_setall(struct cpumask *dstp)607static __always_inline void cpumask_setall(struct cpumask *dstp)
598{608{
599 if (small_const_nbits(small_cpumask_bits)) {609 if (small_const_nbits(small_cpumask_bits)) {
600 cpumask_bits(dstp)[0] = BITMAP_LAST_WORD_MASK(nr_cpumask_bits);610 cpumask_bits(dstp)[0] = BITMAP_LAST_WORD_MASK(nr_cpumask_bits);
@@ -607,7 +617,7 @@ static inline void cpumask_setall(struct cpumask *dstp) · 6 unmodified lines
607 * cpumask_clear - clear all cpus (< nr_cpu_ids) in a cpumask617 * cpumask_clear - clear all cpus (< nr_cpu_ids) in a cpumask
608 * @dstp: the cpumask pointer618 * @dstp: the cpumask pointer
609 */619 */
610static inline void cpumask_clear(struct cpumask *dstp)620static __always_inline void cpumask_clear(struct cpumask *dstp)
611{621{
612 bitmap_zero(cpumask_bits(dstp), large_cpumask_bits);622 bitmap_zero(cpumask_bits(dstp), large_cpumask_bits);
613}623}
@@ -620,9 +630,9 @@ static inline void cpumask_clear(struct cpumask *dstp) · 6 unmodified lines
620 *630 *
621 * Return: false if *@dstp is empty, else returns true631 * Return: false if *@dstp is empty, else returns true
622 */632 */
623static inline bool cpumask_and(struct cpumask *dstp,633static __always_inline
624 const struct cpumask *src1p,634bool cpumask_and(struct cpumask *dstp, const struct cpumask *src1p,
625 const struct cpumask *src2p)635 const struct cpumask *src2p)
626{636{
627 return bitmap_and(cpumask_bits(dstp), cpumask_bits(src1p),637 return bitmap_and(cpumask_bits(dstp), cpumask_bits(src1p),
628 cpumask_bits(src2p), small_cpumask_bits);638 cpumask_bits(src2p), small_cpumask_bits);
@@ -634,8 +644,9 @@ static inline bool cpumask_and(struct cpumask *dstp, · 5 unmodified lines
634 * @src1p: the first input644 * @src1p: the first input
635 * @src2p: the second input645 * @src2p: the second input
636 */646 */
637static inline void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p,647static __always_inline
638 const struct cpumask *src2p)648void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p,
649 const struct cpumask *src2p)
639{650{
640 bitmap_or(cpumask_bits(dstp), cpumask_bits(src1p),651 bitmap_or(cpumask_bits(dstp), cpumask_bits(src1p),
641 cpumask_bits(src2p), small_cpumask_bits);652 cpumask_bits(src2p), small_cpumask_bits);
@@ -647,9 +658,9 @@ static inline void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p, · 5 unmodified lines
647 * @src1p: the first input658 * @src1p: the first input
648 * @src2p: the second input659 * @src2p: the second input
649 */660 */
650static inline void cpumask_xor(struct cpumask *dstp,661static __always_inline
651 const struct cpumask *src1p,662void cpumask_xor(struct cpumask *dstp, const struct cpumask *src1p,
652 const struct cpumask *src2p)663 const struct cpumask *src2p)
653{664{
654 bitmap_xor(cpumask_bits(dstp), cpumask_bits(src1p),665 bitmap_xor(cpumask_bits(dstp), cpumask_bits(src1p),
655 cpumask_bits(src2p), small_cpumask_bits);666 cpumask_bits(src2p), small_cpumask_bits);
@@ -663,9 +674,9 @@ static inline void cpumask_xor(struct cpumask *dstp, · 7 unmodified lines
663 *674 *
664 * Return: false if *@dstp is empty, else returns true675 * Return: false if *@dstp is empty, else returns true
665 */676 */
666static inline bool cpumask_andnot(struct cpumask *dstp,677static __always_inline
667 const struct cpumask *src1p,678bool cpumask_andnot(struct cpumask *dstp, const struct cpumask *src1p,
668 const struct cpumask *src2p)679 const struct cpumask *src2p)
669{680{
670 return bitmap_andnot(cpumask_bits(dstp), cpumask_bits(src1p),681 return bitmap_andnot(cpumask_bits(dstp), cpumask_bits(src1p),
671 cpumask_bits(src2p), small_cpumask_bits);682 cpumask_bits(src2p), small_cpumask_bits);
@@ -678,8 +689,8 @@ static inline bool cpumask_andnot(struct cpumask *dstp, · 6 unmodified lines
678 *689 *
679 * Return: true if the cpumasks are equal, false if not690 * Return: true if the cpumasks are equal, false if not
680 */691 */
681static inline bool cpumask_equal(const struct cpumask *src1p,692static __always_inline
682 const struct cpumask *src2p)693bool cpumask_equal(const struct cpumask *src1p, const struct cpumask *src2p)
683{694{
684 return bitmap_equal(cpumask_bits(src1p), cpumask_bits(src2p),695 return bitmap_equal(cpumask_bits(src1p), cpumask_bits(src2p),
685 small_cpumask_bits);696 small_cpumask_bits);
@@ -694,9 +705,9 @@ static inline bool cpumask_equal(const struct cpumask *src1p, · 8 unmodified lines
694 * Return: true if first cpumask ORed with second cpumask == third cpumask,705 * Return: true if first cpumask ORed with second cpumask == third cpumask,
695 * otherwise false706 * otherwise false
696 */707 */
697static inline bool cpumask_or_equal(const struct cpumask *src1p,708static __always_inline
698 const struct cpumask *src2p,709bool cpumask_or_equal(const struct cpumask *src1p, const struct cpumask *src2p,
699 const struct cpumask *src3p)710 const struct cpumask *src3p)
700{711{
701 return bitmap_or_equal(cpumask_bits(src1p), cpumask_bits(src2p),712 return bitmap_or_equal(cpumask_bits(src1p), cpumask_bits(src2p),
702 cpumask_bits(src3p), small_cpumask_bits);713 cpumask_bits(src3p), small_cpumask_bits);
@@ -710,8 +721,8 @@ static inline bool cpumask_or_equal(const struct cpumask *src1p, · 7 unmodified lines
710 * Return: true if first cpumask ANDed with second cpumask is non-empty,721 * Return: true if first cpumask ANDed with second cpumask is non-empty,
711 * otherwise false722 * otherwise false
712 */723 */
713static inline bool cpumask_intersects(const struct cpumask *src1p,724static __always_inline
714 const struct cpumask *src2p)725bool cpumask_intersects(const struct cpumask *src1p, const struct cpumask *src2p)
715{726{
716 return bitmap_intersects(cpumask_bits(src1p), cpumask_bits(src2p),727 return bitmap_intersects(cpumask_bits(src1p), cpumask_bits(src2p),
717 small_cpumask_bits);728 small_cpumask_bits);
@@ -724,8 +735,8 @@ static inline bool cpumask_intersects(const struct cpumask *src1p, · 6 unmodified lines
724 *735 *
725 * Return: true if *@src1p is a subset of *@src2p, else returns false736 * Return: true if *@src1p is a subset of *@src2p, else returns false
726 */737 */
727static inline bool cpumask_subset(const struct cpumask *src1p,738static __always_inline
728 const struct cpumask *src2p)739bool cpumask_subset(const struct cpumask *src1p, const struct cpumask *src2p)
729{740{
730 return bitmap_subset(cpumask_bits(src1p), cpumask_bits(src2p),741 return bitmap_subset(cpumask_bits(src1p), cpumask_bits(src2p),
731 small_cpumask_bits);742 small_cpumask_bits);
@@ -737,7 +748,7 @@ static inline bool cpumask_subset(const struct cpumask *src1p, · 5 unmodified lines
737 *748 *
738 * Return: true if srcp is empty (has no bits set), else false749 * Return: true if srcp is empty (has no bits set), else false
739 */750 */
740static inline bool cpumask_empty(const struct cpumask *srcp)751static __always_inline bool cpumask_empty(const struct cpumask *srcp)
741{752{
742 return bitmap_empty(cpumask_bits(srcp), small_cpumask_bits);753 return bitmap_empty(cpumask_bits(srcp), small_cpumask_bits);
743}754}
@@ -748,7 +759,7 @@ static inline bool cpumask_empty(const struct cpumask *srcp) · 4 unmodified lines
748 *759 *
749 * Return: true if srcp is full (has all bits set), else false760 * Return: true if srcp is full (has all bits set), else false
750 */761 */
751static inline bool cpumask_full(const struct cpumask *srcp)762static __always_inline bool cpumask_full(const struct cpumask *srcp)
752{763{
753 return bitmap_full(cpumask_bits(srcp), nr_cpumask_bits);764 return bitmap_full(cpumask_bits(srcp), nr_cpumask_bits);
754}765}
@@ -759,7 +770,7 @@ static inline bool cpumask_full(const struct cpumask *srcp) · 4 unmodified lines
759 *770 *
760 * Return: count of bits set in *srcp771 * Return: count of bits set in *srcp
761 */772 */
762static inline unsigned int cpumask_weight(const struct cpumask *srcp)773static __always_inline unsigned int cpumask_weight(const struct cpumask *srcp)
763{774{
764 return bitmap_weight(cpumask_bits(srcp), small_cpumask_bits);775 return bitmap_weight(cpumask_bits(srcp), small_cpumask_bits);
765}776}
@@ -771,8 +782,8 @@ static inline unsigned int cpumask_weight(const struct cpumask *srcp) · 5 unmodified lines
771 *782 *
772 * Return: count of bits set in both *srcp1 and *srcp2783 * Return: count of bits set in both *srcp1 and *srcp2
773 */784 */
774static inline unsigned int cpumask_weight_and(const struct cpumask *srcp1,785static __always_inline
775 const struct cpumask *srcp2)786unsigned int cpumask_weight_and(const struct cpumask *srcp1, const struct cpumask *srcp2)
776{787{
777 return bitmap_weight_and(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits);788 return bitmap_weight_and(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits);
778}789}
@@ -784,8 +795,9 @@ static inline unsigned int cpumask_weight_and(const struct cpumask *srcp1, · 5 unmodified lines
784 *795 *
785 * Return: count of bits set in both *srcp1 and *srcp2796 * Return: count of bits set in both *srcp1 and *srcp2
786 */797 */
787static inline unsigned int cpumask_weight_andnot(const struct cpumask *srcp1,798static __always_inline
788 const struct cpumask *srcp2)799unsigned int cpumask_weight_andnot(const struct cpumask *srcp1,
800 const struct cpumask *srcp2)
789{801{
790 return bitmap_weight_andnot(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits);802 return bitmap_weight_andnot(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits);
791}803}
@@ -796,8 +808,8 @@ static inline unsigned int cpumask_weight_andnot(const struct cpumask *srcp1, · 4 unmodified lines
796 * @srcp: the input to shift808 * @srcp: the input to shift
797 * @n: the number of bits to shift by809 * @n: the number of bits to shift by
798 */810 */
799static inline void cpumask_shift_right(struct cpumask *dstp,811static __always_inline
800 const struct cpumask *srcp, int n)812void cpumask_shift_right(struct cpumask *dstp, const struct cpumask *srcp, int n)
801{813{
802 bitmap_shift_right(cpumask_bits(dstp), cpumask_bits(srcp), n,814 bitmap_shift_right(cpumask_bits(dstp), cpumask_bits(srcp), n,
803 small_cpumask_bits);815 small_cpumask_bits);
@@ -809,8 +821,8 @@ static inline void cpumask_shift_right(struct cpumask *dstp, · 5 unmodified lines
809 * @srcp: the input to shift821 * @srcp: the input to shift
810 * @n: the number of bits to shift by822 * @n: the number of bits to shift by
811 */823 */
812static inline void cpumask_shift_left(struct cpumask *dstp,824static __always_inline
813 const struct cpumask *srcp, int n)825void cpumask_shift_left(struct cpumask *dstp, const struct cpumask *srcp, int n)
814{826{
815 bitmap_shift_left(cpumask_bits(dstp), cpumask_bits(srcp), n,827 bitmap_shift_left(cpumask_bits(dstp), cpumask_bits(srcp), n,
816 nr_cpumask_bits);828 nr_cpumask_bits);
@@ -821,8 +833,8 @@ static inline void cpumask_shift_left(struct cpumask *dstp, · 4 unmodified lines
821 * @dstp: the result833 * @dstp: the result
822 * @srcp: the input cpumask834 * @srcp: the input cpumask
823 */835 */
824static inline void cpumask_copy(struct cpumask *dstp,836static __always_inline
825 const struct cpumask *srcp)837void cpumask_copy(struct cpumask *dstp, const struct cpumask *srcp)
826{838{
827 bitmap_copy(cpumask_bits(dstp), cpumask_bits(srcp), large_cpumask_bits);839 bitmap_copy(cpumask_bits(dstp), cpumask_bits(srcp), large_cpumask_bits);
828}840}
@@ -858,8 +870,8 @@ static inline void cpumask_copy(struct cpumask *dstp, · 29 unmodified lines
858 *870 *
859 * Return: -errno, or 0 for success.871 * Return: -errno, or 0 for success.
860 */872 */
861static inline int cpumask_parse_user(const char __user *buf, int len,873static __always_inline
862 struct cpumask *dstp)874int cpumask_parse_user(const char __user *buf, int len, struct cpumask *dstp)
863{875{
864 return bitmap_parse_user(buf, len, cpumask_bits(dstp), nr_cpumask_bits);876 return bitmap_parse_user(buf, len, cpumask_bits(dstp), nr_cpumask_bits);
865}877}
@@ -872,8 +884,8 @@ static inline int cpumask_parse_user(const char __user *buf, int len, · 6 unmodified lines
872 *884 *
873 * Return: -errno, or 0 for success.885 * Return: -errno, or 0 for success.
874 */886 */
875static inline int cpumask_parselist_user(const char __user *buf, int len,887static __always_inline
876 struct cpumask *dstp)888int cpumask_parselist_user(const char __user *buf, int len, struct cpumask *dstp)
877{889{
878 return bitmap_parselist_user(buf, len, cpumask_bits(dstp),890 return bitmap_parselist_user(buf, len, cpumask_bits(dstp),
879 nr_cpumask_bits);891 nr_cpumask_bits);
@@ -886,7 +898,7 @@ static inline int cpumask_parselist_user(const char __user *buf, int len, · 6 unmodified lines
886 *898 *
887 * Return: -errno, or 0 for success.899 * Return: -errno, or 0 for success.
888 */900 */
889static inline int cpumask_parse(const char *buf, struct cpumask *dstp)901static __always_inline int cpumask_parse(const char *buf, struct cpumask *dstp)
890{902{
891 return bitmap_parse(buf, UINT_MAX, cpumask_bits(dstp), nr_cpumask_bits);903 return bitmap_parse(buf, UINT_MAX, cpumask_bits(dstp), nr_cpumask_bits);
892}904}
@@ -898,7 +910,7 @@ static inline int cpumask_parse(const char *buf, struct cpumask *dstp) · 5 unmodified lines
898 *910 *
899 * Return: -errno, or 0 for success.911 * Return: -errno, or 0 for success.
900 */912 */
901static inline int cpulist_parse(const char *buf, struct cpumask *dstp)913static __always_inline int cpulist_parse(const char *buf, struct cpumask *dstp)
902{914{
903 return bitmap_parselist(buf, cpumask_bits(dstp), nr_cpumask_bits);915 return bitmap_parselist(buf, cpumask_bits(dstp), nr_cpumask_bits);
904}916}
@@ -908,7 +920,7 @@ static inline int cpulist_parse(const char *buf, struct cpumask *dstp) · 3 unmodified lines
908 *920 *
909 * Return: size to allocate for a &struct cpumask in bytes921 * Return: size to allocate for a &struct cpumask in bytes
910 */922 */
911static inline unsigned int cpumask_size(void)923static __always_inline unsigned int cpumask_size(void)
912{924{
913 return bitmap_size(large_cpumask_bits);925 return bitmap_size(large_cpumask_bits);
914}926}
@@ -920,7 +932,7 @@ static inline unsigned int cpumask_size(void) · 5 unmodified lines
920 932
921bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);933bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
922 934
923static inline935static __always_inline
924bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node)936bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node)
925{937{
926 return alloc_cpumask_var_node(mask, flags | __GFP_ZERO, node);938 return alloc_cpumask_var_node(mask, flags | __GFP_ZERO, node);
@@ -938,13 +950,13 @@ bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node) · 11 unmodified lines
938 *950 *
939 * Return: %true if allocation succeeded, %false if not951 * Return: %true if allocation succeeded, %false if not
940 */952 */
941static inline953static __always_inline
942bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)954bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
943{955{
944 return alloc_cpumask_var_node(mask, flags, NUMA_NO_NODE);956 return alloc_cpumask_var_node(mask, flags, NUMA_NO_NODE);
945}957}
946 958
947static inline959static __always_inline
948bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)960bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
949{961{
950 return alloc_cpumask_var(mask, flags | __GFP_ZERO);962 return alloc_cpumask_var(mask, flags | __GFP_ZERO);
@@ -954,7 +966,7 @@ void alloc_bootmem_cpumask_var(cpumask_var_t *mask); · 3 unmodified lines
954void free_cpumask_var(cpumask_var_t mask);966void free_cpumask_var(cpumask_var_t mask);
955void free_bootmem_cpumask_var(cpumask_var_t mask);967void free_bootmem_cpumask_var(cpumask_var_t mask);
956 968
957static inline bool cpumask_available(cpumask_var_t mask)969static __always_inline bool cpumask_available(cpumask_var_t mask)
958{970{
959 return mask != NULL;971 return mask != NULL;
960}972}
@@ -964,43 +976,43 @@ static inline bool cpumask_available(cpumask_var_t mask) · 3 unmodified lines
964#define this_cpu_cpumask_var_ptr(x) this_cpu_ptr(x)976#define this_cpu_cpumask_var_ptr(x) this_cpu_ptr(x)
965#define __cpumask_var_read_mostly977#define __cpumask_var_read_mostly
966 978
967static inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)979static __always_inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
968{980{
969 return true;981 return true;
970}982}
971 983
972static inline bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,984static __always_inline bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
973 int node)985 int node)
974{986{
975 return true;987 return true;
976}988}
977 989
978static inline bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)990static __always_inline bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
979{991{
980 cpumask_clear(*mask);992 cpumask_clear(*mask);
981 return true;993 return true;
982}994}
983 995
984static inline bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,996static __always_inline bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
985 int node)997 int node)
986{998{
987 cpumask_clear(*mask);999 cpumask_clear(*mask);
988 return true;1000 return true;
989}1001}
990 1002
991static inline void alloc_bootmem_cpumask_var(cpumask_var_t *mask)1003static __always_inline void alloc_bootmem_cpumask_var(cpumask_var_t *mask)
992{1004{
993}1005}
994 1006
995static inline void free_cpumask_var(cpumask_var_t mask)1007static __always_inline void free_cpumask_var(cpumask_var_t mask)
996{1008{
997}1009}
998 1010
999static inline void free_bootmem_cpumask_var(cpumask_var_t mask)1011static __always_inline void free_bootmem_cpumask_var(cpumask_var_t mask)
1000{1012{
1001}1013}
1002 1014
1003static inline bool cpumask_available(cpumask_var_t mask)1015static __always_inline bool cpumask_available(cpumask_var_t mask)
1004{1016{
1005 return true;1017 return true;
1006}1018}
@@ -1058,7 +1070,7 @@ void set_cpu_online(unsigned int cpu, bool online); · 51 unmodified lines
1058 ((struct cpumask *)(1 ? (bitmap) \1070 ((struct cpumask *)(1 ? (bitmap) \
1059 : (void *)sizeof(__check_is_bitmap(bitmap))))1071 : (void *)sizeof(__check_is_bitmap(bitmap))))
1060 1072
1061static inline int __check_is_bitmap(const unsigned long *bitmap)1073static __always_inline int __check_is_bitmap(const unsigned long *bitmap)
1062{1074{
1063 return 1;1075 return 1;
1064}1076}
@@ -1073,7 +1085,7 @@ static inline int __check_is_bitmap(const unsigned long *bitmap) · 8 unmodified lines
1073extern const unsigned long1085extern const unsigned long
1074 cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)];1086 cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)];
1075 1087
1076static inline const struct cpumask *get_cpu_mask(unsigned int cpu)1088static __always_inline const struct cpumask *get_cpu_mask(unsigned int cpu)
1077{1089{
1078 const unsigned long *p = cpu_bit_bitmap[1 + cpu % BITS_PER_LONG];1090 const unsigned long *p = cpu_bit_bitmap[1 + cpu % BITS_PER_LONG];
1079 p -= cpu / BITS_PER_LONG;1091 p -= cpu / BITS_PER_LONG;
@@ -1100,32 +1112,32 @@ static __always_inline unsigned int num_online_cpus(void) · 20 unmodified lines
1100#define num_present_cpus() cpumask_weight(cpu_present_mask)1112#define num_present_cpus() cpumask_weight(cpu_present_mask)
1101#define num_active_cpus() cpumask_weight(cpu_active_mask)1113#define num_active_cpus() cpumask_weight(cpu_active_mask)
1102 1114
1103static inline bool cpu_online(unsigned int cpu)1115static __always_inline bool cpu_online(unsigned int cpu)
1104{1116{
1105 return cpumask_test_cpu(cpu, cpu_online_mask);1117 return cpumask_test_cpu(cpu, cpu_online_mask);
1106}1118}
1107 1119
1108static inline bool cpu_enabled(unsigned int cpu)1120static __always_inline bool cpu_enabled(unsigned int cpu)
1109{1121{
1110 return cpumask_test_cpu(cpu, cpu_enabled_mask);1122 return cpumask_test_cpu(cpu, cpu_enabled_mask);
1111}1123}
1112 1124
1113static inline bool cpu_possible(unsigned int cpu)1125static __always_inline bool cpu_possible(unsigned int cpu)
1114{1126{
1115 return cpumask_test_cpu(cpu, cpu_possible_mask);1127 return cpumask_test_cpu(cpu, cpu_possible_mask);
1116}1128}
1117 1129
1118static inline bool cpu_present(unsigned int cpu)1130static __always_inline bool cpu_present(unsigned int cpu)
1119{1131{
1120 return cpumask_test_cpu(cpu, cpu_present_mask);1132 return cpumask_test_cpu(cpu, cpu_present_mask);
1121}1133}
1122 1134
1123static inline bool cpu_active(unsigned int cpu)1135static __always_inline bool cpu_active(unsigned int cpu)
1124{1136{
1125 return cpumask_test_cpu(cpu, cpu_active_mask);1137 return cpumask_test_cpu(cpu, cpu_active_mask);
1126}1138}
1127 1139
1128static inline bool cpu_dying(unsigned int cpu)1140static __always_inline bool cpu_dying(unsigned int cpu)
1129{1141{
1130 return cpumask_test_cpu(cpu, cpu_dying_mask);1142 return cpumask_test_cpu(cpu, cpu_dying_mask);
1131}1143}
@@ -1138,32 +1150,32 @@ static inline bool cpu_dying(unsigned int cpu) · 6 unmodified lines
1138#define num_present_cpus() 1U1150#define num_present_cpus() 1U
1139#define num_active_cpus() 1U1151#define num_active_cpus() 1U
1140 1152
1141static inline bool cpu_online(unsigned int cpu)1153static __always_inline bool cpu_online(unsigned int cpu)
1142{1154{
1143 return cpu == 0;1155 return cpu == 0;
1144}1156}
1145 1157
1146static inline bool cpu_possible(unsigned int cpu)1158static __always_inline bool cpu_possible(unsigned int cpu)
1147{1159{
1148 return cpu == 0;1160 return cpu == 0;
1149}1161}
1150 1162
1151static inline bool cpu_enabled(unsigned int cpu)1163static __always_inline bool cpu_enabled(unsigned int cpu)
1152{1164{
1153 return cpu == 0;1165 return cpu == 0;
1154}1166}
1155 1167
1156static inline bool cpu_present(unsigned int cpu)1168static __always_inline bool cpu_present(unsigned int cpu)
1157{1169{
1158 return cpu == 0;1170 return cpu == 0;
1159}1171}
1160 1172
1161static inline bool cpu_active(unsigned int cpu)1173static __always_inline bool cpu_active(unsigned int cpu)
1162{1174{
1163 return cpu == 0;1175 return cpu == 0;
1164}1176}
1165 1177
1166static inline bool cpu_dying(unsigned int cpu)1178static __always_inline bool cpu_dying(unsigned int cpu)
1167{1179{
1168 return false;1180 return false;
1169}1181}
@@ -1197,7 +1209,7 @@ static inline bool cpu_dying(unsigned int cpu) · 27 unmodified lines
1197 * Return: the length of the (null-terminated) @buf string, zero if1209 * Return: the length of the (null-terminated) @buf string, zero if
1198 * nothing is copied.1210 * nothing is copied.
1199 */1211 */
1200static inline ssize_t1212static __always_inline ssize_t
1201cpumap_print_to_pagebuf(bool list, char *buf, const struct cpumask *mask)1213cpumap_print_to_pagebuf(bool list, char *buf, const struct cpumask *mask)
1202{1214{
1203 return bitmap_print_to_pagebuf(list, buf, cpumask_bits(mask),1215 return bitmap_print_to_pagebuf(list, buf, cpumask_bits(mask),
@@ -1220,9 +1232,9 @@ cpumap_print_to_pagebuf(bool list, char *buf, const struct cpumask *mask) · 16 unmodified lines
1220 * Return: the length of how many bytes have been copied, excluding1232 * Return: the length of how many bytes have been copied, excluding
1221 * terminating '\0'.1233 * terminating '\0'.
1222 */1234 */
1223static inline ssize_t1235static __always_inline
1224cpumap_print_bitmask_to_buf(char *buf, const struct cpumask *mask,1236ssize_t cpumap_print_bitmask_to_buf(char *buf, const struct cpumask *mask,
1225 loff_t off, size_t count)1237 loff_t off, size_t count)
1226{1238{
1227 return bitmap_print_bitmask_to_buf(buf, cpumask_bits(mask),1239 return bitmap_print_bitmask_to_buf(buf, cpumask_bits(mask),
1228 nr_cpu_ids, off, count) - 1;1240 nr_cpu_ids, off, count) - 1;
@@ -1242,9 +1254,9 @@ cpumap_print_bitmask_to_buf(char *buf, const struct cpumask *mask, · 13 unmodified lines
1242 * Return: the length of how many bytes have been copied, excluding1254 * Return: the length of how many bytes have been copied, excluding
1243 * terminating '\0'.1255 * terminating '\0'.
1244 */1256 */
1245static inline ssize_t1257static __always_inline
1246cpumap_print_list_to_buf(char *buf, const struct cpumask *mask,1258ssize_t cpumap_print_list_to_buf(char *buf, const struct cpumask *mask,
1247 loff_t off, size_t count)1259 loff_t off, size_t count)
1248{1260{
1249 return bitmap_print_list_to_buf(buf, cpumask_bits(mask),1261 return bitmap_print_list_to_buf(buf, cpumask_bits(mask),
1250 nr_cpu_ids, off, count) - 1;1262 nr_cpu_ids, off, count) - 1;
include/linux/find.h+25−25
@@ -52,7 +52,7 @@ unsigned long _find_next_bit_le(const unsigned long *addr, unsigned · 51 unmodified lines
52 * Returns the bit number for the next set bit52 * Returns the bit number for the next set bit
53 * If no bits are set, returns @size.53 * If no bits are set, returns @size.
54 */54 */
55static inline55static __always_inline
56unsigned long find_next_bit(const unsigned long *addr, unsigned long size,56unsigned long find_next_bit(const unsigned long *addr, unsigned long size,
57 unsigned long offset)57 unsigned long offset)
58{58{
@@ -81,7 +81,7 @@ unsigned long find_next_bit(const unsigned long *addr, unsigned long size, · 22 unmodified lines
81 * Returns the bit number for the next set bit81 * Returns the bit number for the next set bit
82 * If no bits are set, returns @size.82 * If no bits are set, returns @size.
83 */83 */
84static inline84static __always_inline
85unsigned long find_next_and_bit(const unsigned long *addr1,85unsigned long find_next_and_bit(const unsigned long *addr1,
86 const unsigned long *addr2, unsigned long size,86 const unsigned long *addr2, unsigned long size,
87 unsigned long offset)87 unsigned long offset)
@@ -112,7 +112,7 @@ unsigned long find_next_and_bit(const unsigned long *addr1, · 24 unmodified lines
112 * Returns the bit number for the next set bit112 * Returns the bit number for the next set bit
113 * If no bits are set, returns @size.113 * If no bits are set, returns @size.
114 */114 */
115static inline115static __always_inline
116unsigned long find_next_andnot_bit(const unsigned long *addr1,116unsigned long find_next_andnot_bit(const unsigned long *addr1,
117 const unsigned long *addr2, unsigned long size,117 const unsigned long *addr2, unsigned long size,
118 unsigned long offset)118 unsigned long offset)
@@ -142,7 +142,7 @@ unsigned long find_next_andnot_bit(const unsigned long *addr1, · 23 unmodified lines
142 * Returns the bit number for the next set bit142 * Returns the bit number for the next set bit
143 * If no bits are set, returns @size.143 * If no bits are set, returns @size.
144 */144 */
145static inline145static __always_inline
146unsigned long find_next_or_bit(const unsigned long *addr1,146unsigned long find_next_or_bit(const unsigned long *addr1,
147 const unsigned long *addr2, unsigned long size,147 const unsigned long *addr2, unsigned long size,
148 unsigned long offset)148 unsigned long offset)
@@ -171,7 +171,7 @@ unsigned long find_next_or_bit(const unsigned long *addr1, · 22 unmodified lines
171 * Returns the bit number of the next zero bit171 * Returns the bit number of the next zero bit
172 * If no bits are zero, returns @size.172 * If no bits are zero, returns @size.
173 */173 */
174static inline174static __always_inline
175unsigned long find_next_zero_bit(const unsigned long *addr, unsigned long size,175unsigned long find_next_zero_bit(const unsigned long *addr, unsigned long size,
176 unsigned long offset)176 unsigned long offset)
177{177{
@@ -198,7 +198,7 @@ unsigned long find_next_zero_bit(const unsigned long *addr, unsigned long size, · 20 unmodified lines
198 * Returns the bit number of the first set bit.198 * Returns the bit number of the first set bit.
199 * If no bits are set, returns @size.199 * If no bits are set, returns @size.
200 */200 */
201static inline201static __always_inline
202unsigned long find_first_bit(const unsigned long *addr, unsigned long size)202unsigned long find_first_bit(const unsigned long *addr, unsigned long size)
203{203{
204 if (small_const_nbits(size)) {204 if (small_const_nbits(size)) {
@@ -224,7 +224,7 @@ unsigned long find_first_bit(const unsigned long *addr, unsigned long size) · 19 unmodified lines
224 * Returns the bit number of the N'th set bit.224 * Returns the bit number of the N'th set bit.
225 * If no such, returns >= @size.225 * If no such, returns >= @size.
226 */226 */
227static inline227static __always_inline
228unsigned long find_nth_bit(const unsigned long *addr, unsigned long size, unsigned long n)228unsigned long find_nth_bit(const unsigned long *addr, unsigned long size, unsigned long n)
229{229{
230 if (n >= size)230 if (n >= size)
@@ -249,7 +249,7 @@ unsigned long find_nth_bit(const unsigned long *addr, unsigned long size, unsign · 18 unmodified lines
249 * Returns the bit number of the N'th set bit.249 * Returns the bit number of the N'th set bit.
250 * If no such, returns @size.250 * If no such, returns @size.
251 */251 */
252static inline252static __always_inline
253unsigned long find_nth_and_bit(const unsigned long *addr1, const unsigned long *addr2,253unsigned long find_nth_and_bit(const unsigned long *addr1, const unsigned long *addr2,
254 unsigned long size, unsigned long n)254 unsigned long size, unsigned long n)
255{255{
@@ -276,7 +276,7 @@ unsigned long find_nth_and_bit(const unsigned long *addr1, const unsigned long * · 20 unmodified lines
276 * Returns the bit number of the N'th set bit.276 * Returns the bit number of the N'th set bit.
277 * If no such, returns @size.277 * If no such, returns @size.
278 */278 */
279static inline279static __always_inline
280unsigned long find_nth_andnot_bit(const unsigned long *addr1, const unsigned long *addr2,280unsigned long find_nth_andnot_bit(const unsigned long *addr1, const unsigned long *addr2,
281 unsigned long size, unsigned long n)281 unsigned long size, unsigned long n)
282{282{
@@ -332,7 +332,7 @@ unsigned long find_nth_and_andnot_bit(const unsigned long *addr1, · 49 unmodified lines
332 * Returns the bit number for the next set bit332 * Returns the bit number for the next set bit
333 * If no bits are set, returns @size.333 * If no bits are set, returns @size.
334 */334 */
335static inline335static __always_inline
336unsigned long find_first_and_bit(const unsigned long *addr1,336unsigned long find_first_and_bit(const unsigned long *addr1,
337 const unsigned long *addr2,337 const unsigned long *addr2,
338 unsigned long size)338 unsigned long size)
@@ -357,7 +357,7 @@ unsigned long find_first_and_bit(const unsigned long *addr1, · 18 unmodified lines
357 * Returns the bit number for the first set bit357 * Returns the bit number for the first set bit
358 * If no bits are set, returns @size.358 * If no bits are set, returns @size.
359 */359 */
360static inline360static __always_inline
361unsigned long find_first_and_and_bit(const unsigned long *addr1,361unsigned long find_first_and_and_bit(const unsigned long *addr1,
362 const unsigned long *addr2,362 const unsigned long *addr2,
363 const unsigned long *addr3,363 const unsigned long *addr3,
@@ -381,7 +381,7 @@ unsigned long find_first_and_and_bit(const unsigned long *addr1, · 17 unmodified lines
381 * Returns the bit number of the first cleared bit.381 * Returns the bit number of the first cleared bit.
382 * If no bits are zero, returns @size.382 * If no bits are zero, returns @size.
383 */383 */
384static inline384static __always_inline
385unsigned long find_first_zero_bit(const unsigned long *addr, unsigned long size)385unsigned long find_first_zero_bit(const unsigned long *addr, unsigned long size)
386{386{
387 if (small_const_nbits(size)) {387 if (small_const_nbits(size)) {
@@ -402,7 +402,7 @@ unsigned long find_first_zero_bit(const unsigned long *addr, unsigned long size) · 14 unmodified lines
402 *402 *
403 * Returns the bit number of the last set bit, or size.403 * Returns the bit number of the last set bit, or size.
404 */404 */
405static inline405static __always_inline
406unsigned long find_last_bit(const unsigned long *addr, unsigned long size)406unsigned long find_last_bit(const unsigned long *addr, unsigned long size)
407{407{
408 if (small_const_nbits(size)) {408 if (small_const_nbits(size)) {
@@ -425,7 +425,7 @@ unsigned long find_last_bit(const unsigned long *addr, unsigned long size) · 16 unmodified lines
425 * Returns the bit number for the next set bit, or first set bit up to @offset425 * Returns the bit number for the next set bit, or first set bit up to @offset
426 * If no bits are set, returns @size.426 * If no bits are set, returns @size.
427 */427 */
428static inline428static __always_inline
429unsigned long find_next_and_bit_wrap(const unsigned long *addr1,429unsigned long find_next_and_bit_wrap(const unsigned long *addr1,
430 const unsigned long *addr2,430 const unsigned long *addr2,
431 unsigned long size, unsigned long offset)431 unsigned long size, unsigned long offset)
@@ -448,7 +448,7 @@ unsigned long find_next_and_bit_wrap(const unsigned long *addr1, · 16 unmodified lines
448 * Returns the bit number for the next set bit, or first set bit up to @offset448 * Returns the bit number for the next set bit, or first set bit up to @offset
449 * If no bits are set, returns @size.449 * If no bits are set, returns @size.
450 */450 */
451static inline451static __always_inline
452unsigned long find_next_bit_wrap(const unsigned long *addr,452unsigned long find_next_bit_wrap(const unsigned long *addr,
453 unsigned long size, unsigned long offset)453 unsigned long size, unsigned long offset)
454{454{
@@ -465,7 +465,7 @@ unsigned long find_next_bit_wrap(const unsigned long *addr, · 10 unmodified lines
465 * Helper for for_each_set_bit_wrap(). Make sure you're doing right thing465 * Helper for for_each_set_bit_wrap(). Make sure you're doing right thing
466 * before using it alone.466 * before using it alone.
467 */467 */
468static inline468static __always_inline
469unsigned long __for_each_wrap(const unsigned long *bitmap, unsigned long size,469unsigned long __for_each_wrap(const unsigned long *bitmap, unsigned long size,
470 unsigned long start, unsigned long n)470 unsigned long start, unsigned long n)
471{471{
@@ -506,20 +506,20 @@ extern unsigned long find_next_clump8(unsigned long *clump, · 34 unmodified lines
506 506
507#if defined(__LITTLE_ENDIAN)507#if defined(__LITTLE_ENDIAN)
508 508
509static inline unsigned long find_next_zero_bit_le(const void *addr,509static __always_inline
510 unsigned long size, unsigned long offset)510unsigned long find_next_zero_bit_le(const void *addr, unsigned long size, unsigned long offset)
511{511{
512 return find_next_zero_bit(addr, size, offset);512 return find_next_zero_bit(addr, size, offset);
513}513}
514 514
515static inline unsigned long find_next_bit_le(const void *addr,515static __always_inline
516 unsigned long size, unsigned long offset)516unsigned long find_next_bit_le(const void *addr, unsigned long size, unsigned long offset)
517{517{
518 return find_next_bit(addr, size, offset);518 return find_next_bit(addr, size, offset);
519}519}
520 520
521static inline unsigned long find_first_zero_bit_le(const void *addr,521static __always_inline
522 unsigned long size)522unsigned long find_first_zero_bit_le(const void *addr, unsigned long size)
523{523{
524 return find_first_zero_bit(addr, size);524 return find_first_zero_bit(addr, size);
525}525}
@@ -527,7 +527,7 @@ static inline unsigned long find_first_zero_bit_le(const void *addr, · 1 unmodified line
527#elif defined(__BIG_ENDIAN)527#elif defined(__BIG_ENDIAN)
528 528
529#ifndef find_next_zero_bit_le529#ifndef find_next_zero_bit_le
530static inline530static __always_inline
531unsigned long find_next_zero_bit_le(const void *addr, unsigned531unsigned long find_next_zero_bit_le(const void *addr, unsigned
532 long size, unsigned long offset)532 long size, unsigned long offset)
533{533{
@@ -546,7 +546,7 @@ unsigned long find_next_zero_bit_le(const void *addr, unsigned · 12 unmodified lines
546#endif546#endif
547 547
548#ifndef find_first_zero_bit_le548#ifndef find_first_zero_bit_le
549static inline549static __always_inline
550unsigned long find_first_zero_bit_le(const void *addr, unsigned long size)550unsigned long find_first_zero_bit_le(const void *addr, unsigned long size)
551{551{
552 if (small_const_nbits(size)) {552 if (small_const_nbits(size)) {
@@ -560,7 +560,7 @@ unsigned long find_first_zero_bit_le(const void *addr, unsigned long size) · 7 unmodified lines
560#endif560#endif
561 561
562#ifndef find_next_bit_le562#ifndef find_next_bit_le
563static inline563static __always_inline
564unsigned long find_next_bit_le(const void *addr, unsigned564unsigned long find_next_bit_le(const void *addr, unsigned
565 long size, unsigned long offset)565 long size, unsigned long offset)
566{566{
include/linux/nodemask.h+43−43
@@ -107,11 +107,11 @@ extern nodemask_t _unused_nodemask_arg_; · 106 unmodified lines
107 */107 */
108#define nodemask_pr_args(maskp) __nodemask_pr_numnodes(maskp), \108#define nodemask_pr_args(maskp) __nodemask_pr_numnodes(maskp), \
109 __nodemask_pr_bits(maskp)109 __nodemask_pr_bits(maskp)
110static inline unsigned int __nodemask_pr_numnodes(const nodemask_t *m)110static __always_inline unsigned int __nodemask_pr_numnodes(const nodemask_t *m)
111{111{
112 return m ? MAX_NUMNODES : 0;112 return m ? MAX_NUMNODES : 0;
113}113}
114static inline const unsigned long *__nodemask_pr_bits(const nodemask_t *m)114static __always_inline const unsigned long *__nodemask_pr_bits(const nodemask_t *m)
115{115{
116 return m ? m->bits : NULL;116 return m ? m->bits : NULL;
117}117}
@@ -132,19 +132,19 @@ static __always_inline void __node_set(int node, volatile nodemask_t *dstp) · 14 unmodified lines
132}132}
133 133
134#define node_clear(node, dst) __node_clear((node), &(dst))134#define node_clear(node, dst) __node_clear((node), &(dst))
135static inline void __node_clear(int node, volatile nodemask_t *dstp)135static __always_inline void __node_clear(int node, volatile nodemask_t *dstp)
136{136{
137 clear_bit(node, dstp->bits);137 clear_bit(node, dstp->bits);
138}138}
139 139
140#define nodes_setall(dst) __nodes_setall(&(dst), MAX_NUMNODES)140#define nodes_setall(dst) __nodes_setall(&(dst), MAX_NUMNODES)
141static inline void __nodes_setall(nodemask_t *dstp, unsigned int nbits)141static __always_inline void __nodes_setall(nodemask_t *dstp, unsigned int nbits)
142{142{
143 bitmap_fill(dstp->bits, nbits);143 bitmap_fill(dstp->bits, nbits);
144}144}
145 145
146#define nodes_clear(dst) __nodes_clear(&(dst), MAX_NUMNODES)146#define nodes_clear(dst) __nodes_clear(&(dst), MAX_NUMNODES)
147static inline void __nodes_clear(nodemask_t *dstp, unsigned int nbits)147static __always_inline void __nodes_clear(nodemask_t *dstp, unsigned int nbits)
148{148{
149 bitmap_zero(dstp->bits, nbits);149 bitmap_zero(dstp->bits, nbits);
150}150}
@@ -154,14 +154,14 @@ static inline void __nodes_clear(nodemask_t *dstp, unsigned int nbits) · 3 unmodified lines
154 154
155#define node_test_and_set(node, nodemask) \155#define node_test_and_set(node, nodemask) \
156 __node_test_and_set((node), &(nodemask))156 __node_test_and_set((node), &(nodemask))
157static inline bool __node_test_and_set(int node, nodemask_t *addr)157static __always_inline bool __node_test_and_set(int node, nodemask_t *addr)
158{158{
159 return test_and_set_bit(node, addr->bits);159 return test_and_set_bit(node, addr->bits);
160}160}
161 161
162#define nodes_and(dst, src1, src2) \162#define nodes_and(dst, src1, src2) \
163 __nodes_and(&(dst), &(src1), &(src2), MAX_NUMNODES)163 __nodes_and(&(dst), &(src1), &(src2), MAX_NUMNODES)
164static inline void __nodes_and(nodemask_t *dstp, const nodemask_t *src1p,164static __always_inline void __nodes_and(nodemask_t *dstp, const nodemask_t *src1p,
165 const nodemask_t *src2p, unsigned int nbits)165 const nodemask_t *src2p, unsigned int nbits)
166{166{
167 bitmap_and(dstp->bits, src1p->bits, src2p->bits, nbits);167 bitmap_and(dstp->bits, src1p->bits, src2p->bits, nbits);
@@ -169,7 +169,7 @@ static inline void __nodes_and(nodemask_t *dstp, const nodemask_t *src1p, · 1 unmodified line
169 169
170#define nodes_or(dst, src1, src2) \170#define nodes_or(dst, src1, src2) \
171 __nodes_or(&(dst), &(src1), &(src2), MAX_NUMNODES)171 __nodes_or(&(dst), &(src1), &(src2), MAX_NUMNODES)
172static inline void __nodes_or(nodemask_t *dstp, const nodemask_t *src1p,172static __always_inline void __nodes_or(nodemask_t *dstp, const nodemask_t *src1p,
173 const nodemask_t *src2p, unsigned int nbits)173 const nodemask_t *src2p, unsigned int nbits)
174{174{
175 bitmap_or(dstp->bits, src1p->bits, src2p->bits, nbits);175 bitmap_or(dstp->bits, src1p->bits, src2p->bits, nbits);
@@ -177,7 +177,7 @@ static inline void __nodes_or(nodemask_t *dstp, const nodemask_t *src1p, · 1 unmodified line
177 177
178#define nodes_xor(dst, src1, src2) \178#define nodes_xor(dst, src1, src2) \
179 __nodes_xor(&(dst), &(src1), &(src2), MAX_NUMNODES)179 __nodes_xor(&(dst), &(src1), &(src2), MAX_NUMNODES)
180static inline void __nodes_xor(nodemask_t *dstp, const nodemask_t *src1p,180static __always_inline void __nodes_xor(nodemask_t *dstp, const nodemask_t *src1p,
181 const nodemask_t *src2p, unsigned int nbits)181 const nodemask_t *src2p, unsigned int nbits)
182{182{
183 bitmap_xor(dstp->bits, src1p->bits, src2p->bits, nbits);183 bitmap_xor(dstp->bits, src1p->bits, src2p->bits, nbits);
@@ -185,7 +185,7 @@ static inline void __nodes_xor(nodemask_t *dstp, const nodemask_t *src1p, · 1 unmodified line
185 185
186#define nodes_andnot(dst, src1, src2) \186#define nodes_andnot(dst, src1, src2) \
187 __nodes_andnot(&(dst), &(src1), &(src2), MAX_NUMNODES)187 __nodes_andnot(&(dst), &(src1), &(src2), MAX_NUMNODES)
188static inline void __nodes_andnot(nodemask_t *dstp, const nodemask_t *src1p,188static __always_inline void __nodes_andnot(nodemask_t *dstp, const nodemask_t *src1p,
189 const nodemask_t *src2p, unsigned int nbits)189 const nodemask_t *src2p, unsigned int nbits)
190{190{
191 bitmap_andnot(dstp->bits, src1p->bits, src2p->bits, nbits);191 bitmap_andnot(dstp->bits, src1p->bits, src2p->bits, nbits);
@@ -193,7 +193,7 @@ static inline void __nodes_andnot(nodemask_t *dstp, const nodemask_t *src1p, · 1 unmodified line
193 193
194#define nodes_complement(dst, src) \194#define nodes_complement(dst, src) \
195 __nodes_complement(&(dst), &(src), MAX_NUMNODES)195 __nodes_complement(&(dst), &(src), MAX_NUMNODES)
196static inline void __nodes_complement(nodemask_t *dstp,196static __always_inline void __nodes_complement(nodemask_t *dstp,
197 const nodemask_t *srcp, unsigned int nbits)197 const nodemask_t *srcp, unsigned int nbits)
198{198{
199 bitmap_complement(dstp->bits, srcp->bits, nbits);199 bitmap_complement(dstp->bits, srcp->bits, nbits);
@@ -201,7 +201,7 @@ static inline void __nodes_complement(nodemask_t *dstp, · 1 unmodified line
201 201
202#define nodes_equal(src1, src2) \202#define nodes_equal(src1, src2) \
203 __nodes_equal(&(src1), &(src2), MAX_NUMNODES)203 __nodes_equal(&(src1), &(src2), MAX_NUMNODES)
204static inline bool __nodes_equal(const nodemask_t *src1p,204static __always_inline bool __nodes_equal(const nodemask_t *src1p,
205 const nodemask_t *src2p, unsigned int nbits)205 const nodemask_t *src2p, unsigned int nbits)
206{206{
207 return bitmap_equal(src1p->bits, src2p->bits, nbits);207 return bitmap_equal(src1p->bits, src2p->bits, nbits);
@@ -209,7 +209,7 @@ static inline bool __nodes_equal(const nodemask_t *src1p, · 1 unmodified line
209 209
210#define nodes_intersects(src1, src2) \210#define nodes_intersects(src1, src2) \
211 __nodes_intersects(&(src1), &(src2), MAX_NUMNODES)211 __nodes_intersects(&(src1), &(src2), MAX_NUMNODES)
212static inline bool __nodes_intersects(const nodemask_t *src1p,212static __always_inline bool __nodes_intersects(const nodemask_t *src1p,
213 const nodemask_t *src2p, unsigned int nbits)213 const nodemask_t *src2p, unsigned int nbits)
214{214{
215 return bitmap_intersects(src1p->bits, src2p->bits, nbits);215 return bitmap_intersects(src1p->bits, src2p->bits, nbits);
@@ -217,33 +217,33 @@ static inline bool __nodes_intersects(const nodemask_t *src1p, · 1 unmodified line
217 217
218#define nodes_subset(src1, src2) \218#define nodes_subset(src1, src2) \
219 __nodes_subset(&(src1), &(src2), MAX_NUMNODES)219 __nodes_subset(&(src1), &(src2), MAX_NUMNODES)
220static inline bool __nodes_subset(const nodemask_t *src1p,220static __always_inline bool __nodes_subset(const nodemask_t *src1p,
221 const nodemask_t *src2p, unsigned int nbits)221 const nodemask_t *src2p, unsigned int nbits)
222{222{
223 return bitmap_subset(src1p->bits, src2p->bits, nbits);223 return bitmap_subset(src1p->bits, src2p->bits, nbits);
224}224}
225 225
226#define nodes_empty(src) __nodes_empty(&(src), MAX_NUMNODES)226#define nodes_empty(src) __nodes_empty(&(src), MAX_NUMNODES)
227static inline bool __nodes_empty(const nodemask_t *srcp, unsigned int nbits)227static __always_inline bool __nodes_empty(const nodemask_t *srcp, unsigned int nbits)
228{228{
229 return bitmap_empty(srcp->bits, nbits);229 return bitmap_empty(srcp->bits, nbits);
230}230}
231 231
232#define nodes_full(nodemask) __nodes_full(&(nodemask), MAX_NUMNODES)232#define nodes_full(nodemask) __nodes_full(&(nodemask), MAX_NUMNODES)
233static inline bool __nodes_full(const nodemask_t *srcp, unsigned int nbits)233static __always_inline bool __nodes_full(const nodemask_t *srcp, unsigned int nbits)
234{234{
235 return bitmap_full(srcp->bits, nbits);235 return bitmap_full(srcp->bits, nbits);
236}236}
237 237
238#define nodes_weight(nodemask) __nodes_weight(&(nodemask), MAX_NUMNODES)238#define nodes_weight(nodemask) __nodes_weight(&(nodemask), MAX_NUMNODES)
239static inline int __nodes_weight(const nodemask_t *srcp, unsigned int nbits)239static __always_inline int __nodes_weight(const nodemask_t *srcp, unsigned int nbits)
240{240{
241 return bitmap_weight(srcp->bits, nbits);241 return bitmap_weight(srcp->bits, nbits);
242}242}
243 243
244#define nodes_shift_right(dst, src, n) \244#define nodes_shift_right(dst, src, n) \
245 __nodes_shift_right(&(dst), &(src), (n), MAX_NUMNODES)245 __nodes_shift_right(&(dst), &(src), (n), MAX_NUMNODES)
246static inline void __nodes_shift_right(nodemask_t *dstp,246static __always_inline void __nodes_shift_right(nodemask_t *dstp,
247 const nodemask_t *srcp, int n, int nbits)247 const nodemask_t *srcp, int n, int nbits)
248{248{
249 bitmap_shift_right(dstp->bits, srcp->bits, n, nbits);249 bitmap_shift_right(dstp->bits, srcp->bits, n, nbits);
@@ -251,7 +251,7 @@ static inline void __nodes_shift_right(nodemask_t *dstp, · 1 unmodified line
251 251
252#define nodes_shift_left(dst, src, n) \252#define nodes_shift_left(dst, src, n) \
253 __nodes_shift_left(&(dst), &(src), (n), MAX_NUMNODES)253 __nodes_shift_left(&(dst), &(src), (n), MAX_NUMNODES)
254static inline void __nodes_shift_left(nodemask_t *dstp,254static __always_inline void __nodes_shift_left(nodemask_t *dstp,
255 const nodemask_t *srcp, int n, int nbits)255 const nodemask_t *srcp, int n, int nbits)
256{256{
257 bitmap_shift_left(dstp->bits, srcp->bits, n, nbits);257 bitmap_shift_left(dstp->bits, srcp->bits, n, nbits);
@@ -261,13 +261,13 @@ static inline void __nodes_shift_left(nodemask_t *dstp, · 3 unmodified lines
261 > MAX_NUMNODES, then the silly min_ts could be dropped. */261 > MAX_NUMNODES, then the silly min_ts could be dropped. */
262 262
263#define first_node(src) __first_node(&(src))263#define first_node(src) __first_node(&(src))
264static inline unsigned int __first_node(const nodemask_t *srcp)264static __always_inline unsigned int __first_node(const nodemask_t *srcp)
265{265{
266 return min_t(unsigned int, MAX_NUMNODES, find_first_bit(srcp->bits, MAX_NUMNODES));266 return min_t(unsigned int, MAX_NUMNODES, find_first_bit(srcp->bits, MAX_NUMNODES));
267}267}
268 268
269#define next_node(n, src) __next_node((n), &(src))269#define next_node(n, src) __next_node((n), &(src))
270static inline unsigned int __next_node(int n, const nodemask_t *srcp)270static __always_inline unsigned int __next_node(int n, const nodemask_t *srcp)
271{271{
272 return min_t(unsigned int, MAX_NUMNODES, find_next_bit(srcp->bits, MAX_NUMNODES, n+1));272 return min_t(unsigned int, MAX_NUMNODES, find_next_bit(srcp->bits, MAX_NUMNODES, n+1));
273}273}
@@ -277,7 +277,7 @@ static inline unsigned int __next_node(int n, const nodemask_t *srcp) · 3 unmodified lines
277 * the first node in src if needed. Returns MAX_NUMNODES if src is empty.277 * the first node in src if needed. Returns MAX_NUMNODES if src is empty.
278 */278 */
279#define next_node_in(n, src) __next_node_in((n), &(src))279#define next_node_in(n, src) __next_node_in((n), &(src))
280static inline unsigned int __next_node_in(int node, const nodemask_t *srcp)280static __always_inline unsigned int __next_node_in(int node, const nodemask_t *srcp)
281{281{
282 unsigned int ret = __next_node(node, srcp);282 unsigned int ret = __next_node(node, srcp);
283 283
@@ -286,7 +286,7 @@ static inline unsigned int __next_node_in(int node, const nodemask_t *srcp) · 2 unmodified lines
286 return ret;286 return ret;
287}287}
288 288
289static inline void init_nodemask_of_node(nodemask_t *mask, int node)289static __always_inline void init_nodemask_of_node(nodemask_t *mask, int node)
290{290{
291 nodes_clear(*mask);291 nodes_clear(*mask);
292 node_set(node, *mask);292 node_set(node, *mask);
@@ -304,7 +304,7 @@ static inline void init_nodemask_of_node(nodemask_t *mask, int node) · 11 unmodified lines
304})304})
305 305
306#define first_unset_node(mask) __first_unset_node(&(mask))306#define first_unset_node(mask) __first_unset_node(&(mask))
307static inline unsigned int __first_unset_node(const nodemask_t *maskp)307static __always_inline unsigned int __first_unset_node(const nodemask_t *maskp)
308{308{
309 return min_t(unsigned int, MAX_NUMNODES,309 return min_t(unsigned int, MAX_NUMNODES,
310 find_first_zero_bit(maskp->bits, MAX_NUMNODES));310 find_first_zero_bit(maskp->bits, MAX_NUMNODES));
@@ -338,21 +338,21 @@ static inline unsigned int __first_unset_node(const nodemask_t *maskp) · 27 unmodified lines
338 338
339#define nodemask_parse_user(ubuf, ulen, dst) \339#define nodemask_parse_user(ubuf, ulen, dst) \
340 __nodemask_parse_user((ubuf), (ulen), &(dst), MAX_NUMNODES)340 __nodemask_parse_user((ubuf), (ulen), &(dst), MAX_NUMNODES)
341static inline int __nodemask_parse_user(const char __user *buf, int len,341static __always_inline int __nodemask_parse_user(const char __user *buf, int len,
342 nodemask_t *dstp, int nbits)342 nodemask_t *dstp, int nbits)
343{343{
344 return bitmap_parse_user(buf, len, dstp->bits, nbits);344 return bitmap_parse_user(buf, len, dstp->bits, nbits);
345}345}
346 346
347#define nodelist_parse(buf, dst) __nodelist_parse((buf), &(dst), MAX_NUMNODES)347#define nodelist_parse(buf, dst) __nodelist_parse((buf), &(dst), MAX_NUMNODES)
348static inline int __nodelist_parse(const char *buf, nodemask_t *dstp, int nbits)348static __always_inline int __nodelist_parse(const char *buf, nodemask_t *dstp, int nbits)
349{349{
350 return bitmap_parselist(buf, dstp->bits, nbits);350 return bitmap_parselist(buf, dstp->bits, nbits);
351}351}
352 352
353#define node_remap(oldbit, old, new) \353#define node_remap(oldbit, old, new) \
354 __node_remap((oldbit), &(old), &(new), MAX_NUMNODES)354 __node_remap((oldbit), &(old), &(new), MAX_NUMNODES)
355static inline int __node_remap(int oldbit,355static __always_inline int __node_remap(int oldbit,
356 const nodemask_t *oldp, const nodemask_t *newp, int nbits)356 const nodemask_t *oldp, const nodemask_t *newp, int nbits)
357{357{
358 return bitmap_bitremap(oldbit, oldp->bits, newp->bits, nbits);358 return bitmap_bitremap(oldbit, oldp->bits, newp->bits, nbits);
@@ -360,7 +360,7 @@ static inline int __node_remap(int oldbit, · 1 unmodified line
360 360
361#define nodes_remap(dst, src, old, new) \361#define nodes_remap(dst, src, old, new) \
362 __nodes_remap(&(dst), &(src), &(old), &(new), MAX_NUMNODES)362 __nodes_remap(&(dst), &(src), &(old), &(new), MAX_NUMNODES)
363static inline void __nodes_remap(nodemask_t *dstp, const nodemask_t *srcp,363static __always_inline void __nodes_remap(nodemask_t *dstp, const nodemask_t *srcp,
364 const nodemask_t *oldp, const nodemask_t *newp, int nbits)364 const nodemask_t *oldp, const nodemask_t *newp, int nbits)
365{365{
366 bitmap_remap(dstp->bits, srcp->bits, oldp->bits, newp->bits, nbits);366 bitmap_remap(dstp->bits, srcp->bits, oldp->bits, newp->bits, nbits);
@@ -368,7 +368,7 @@ static inline void __nodes_remap(nodemask_t *dstp, const nodemask_t *srcp, · 1 unmodified line
368 368
369#define nodes_onto(dst, orig, relmap) \369#define nodes_onto(dst, orig, relmap) \
370 __nodes_onto(&(dst), &(orig), &(relmap), MAX_NUMNODES)370 __nodes_onto(&(dst), &(orig), &(relmap), MAX_NUMNODES)
371static inline void __nodes_onto(nodemask_t *dstp, const nodemask_t *origp,371static __always_inline void __nodes_onto(nodemask_t *dstp, const nodemask_t *origp,
372 const nodemask_t *relmapp, int nbits)372 const nodemask_t *relmapp, int nbits)
373{373{
374 bitmap_onto(dstp->bits, origp->bits, relmapp->bits, nbits);374 bitmap_onto(dstp->bits, origp->bits, relmapp->bits, nbits);
@@ -376,7 +376,7 @@ static inline void __nodes_onto(nodemask_t *dstp, const nodemask_t *origp, · 1 unmodified line
376 376
377#define nodes_fold(dst, orig, sz) \377#define nodes_fold(dst, orig, sz) \
378 __nodes_fold(&(dst), &(orig), sz, MAX_NUMNODES)378 __nodes_fold(&(dst), &(orig), sz, MAX_NUMNODES)
379static inline void __nodes_fold(nodemask_t *dstp, const nodemask_t *origp,379static __always_inline void __nodes_fold(nodemask_t *dstp, const nodemask_t *origp,
380 int sz, int nbits)380 int sz, int nbits)
381{381{
382 bitmap_fold(dstp->bits, origp->bits, sz, nbits);382 bitmap_fold(dstp->bits, origp->bits, sz, nbits);
@@ -418,22 +418,22 @@ enum node_states { · 35 unmodified lines
418extern nodemask_t node_states[NR_NODE_STATES];418extern nodemask_t node_states[NR_NODE_STATES];
419 419
420#if MAX_NUMNODES > 1420#if MAX_NUMNODES > 1
421static inline int node_state(int node, enum node_states state)421static __always_inline int node_state(int node, enum node_states state)
422{422{
423 return node_isset(node, node_states[state]);423 return node_isset(node, node_states[state]);
424}424}
425 425
426static inline void node_set_state(int node, enum node_states state)426static __always_inline void node_set_state(int node, enum node_states state)
427{427{
428 __node_set(node, &node_states[state]);428 __node_set(node, &node_states[state]);
429}429}
430 430
431static inline void node_clear_state(int node, enum node_states state)431static __always_inline void node_clear_state(int node, enum node_states state)
432{432{
433 __node_clear(node, &node_states[state]);433 __node_clear(node, &node_states[state]);
434}434}
435 435
436static inline int num_node_state(enum node_states state)436static __always_inline int num_node_state(enum node_states state)
437{437{
438 return nodes_weight(node_states[state]);438 return nodes_weight(node_states[state]);
439}439}
@@ -443,11 +443,11 @@ static inline int num_node_state(enum node_states state) · 3 unmodified lines
443 443
444#define first_online_node first_node(node_states[N_ONLINE])444#define first_online_node first_node(node_states[N_ONLINE])
445#define first_memory_node first_node(node_states[N_MEMORY])445#define first_memory_node first_node(node_states[N_MEMORY])
446static inline unsigned int next_online_node(int nid)446static __always_inline unsigned int next_online_node(int nid)
447{447{
448 return next_node(nid, node_states[N_ONLINE]);448 return next_node(nid, node_states[N_ONLINE]);
449}449}
450static inline unsigned int next_memory_node(int nid)450static __always_inline unsigned int next_memory_node(int nid)
451{451{
452 return next_node(nid, node_states[N_MEMORY]);452 return next_node(nid, node_states[N_MEMORY]);
453}453}
@@ -455,13 +455,13 @@ static inline unsigned int next_memory_node(int nid) · 1 unmodified line
455extern unsigned int nr_node_ids;455extern unsigned int nr_node_ids;
456extern unsigned int nr_online_nodes;456extern unsigned int nr_online_nodes;
457 457
458static inline void node_set_online(int nid)458static __always_inline void node_set_online(int nid)
459{459{
460 node_set_state(nid, N_ONLINE);460 node_set_state(nid, N_ONLINE);
461 nr_online_nodes = num_node_state(N_ONLINE);461 nr_online_nodes = num_node_state(N_ONLINE);
462}462}
463 463
464static inline void node_set_offline(int nid)464static __always_inline void node_set_offline(int nid)
465{465{
466 node_clear_state(nid, N_ONLINE);466 node_clear_state(nid, N_ONLINE);
467 nr_online_nodes = num_node_state(N_ONLINE);467 nr_online_nodes = num_node_state(N_ONLINE);
@@ -469,20 +469,20 @@ static inline void node_set_offline(int nid) · 1 unmodified line
469 469
470#else470#else
471 471
472static inline int node_state(int node, enum node_states state)472static __always_inline int node_state(int node, enum node_states state)
473{473{
474 return node == 0;474 return node == 0;
475}475}
476 476
477static inline void node_set_state(int node, enum node_states state)477static __always_inline void node_set_state(int node, enum node_states state)
478{478{
479}479}
480 480
481static inline void node_clear_state(int node, enum node_states state)481static __always_inline void node_clear_state(int node, enum node_states state)
482{482{
483}483}
484 484
485static inline int num_node_state(enum node_states state)485static __always_inline int num_node_state(enum node_states state)
486{486{
487 return 1;487 return 1;
488}488}
@@ -502,7 +502,7 @@ static inline int num_node_state(enum node_states state) · 13 unmodified lines
502 502
503#endif503#endif
504 504
505static inline int node_random(const nodemask_t *maskp)505static __always_inline int node_random(const nodemask_t *maskp)
506{506{
507#if defined(CONFIG_NUMA) && (MAX_NUMNODES > 1)507#if defined(CONFIG_NUMA) && (MAX_NUMNODES > 1)
508 int w, bit;508 int w, bit;
include/uapi/linux/bits.h+3−0
@@ -12,4 +12,7 @@ · 11 unmodified lines
12 (((~_ULL(0)) - (_ULL(1) << (l)) + 1) & \12 (((~_ULL(0)) - (_ULL(1) << (l)) + 1) & \
13 (~_ULL(0) >> (__BITS_PER_LONG_LONG - 1 - (h))))13 (~_ULL(0) >> (__BITS_PER_LONG_LONG - 1 - (h))))
14 14
15#define __GENMASK_U128(h, l) \
16 ((_BIT128((h)) << 1) - (_BIT128(l)))
17
15#endif /* _UAPI_LINUX_BITS_H */18#endif /* _UAPI_LINUX_BITS_H */
include/uapi/linux/const.h+17−0
@@ -28,6 +28,23 @@ · 27 unmodified lines
28#define _BITUL(x) (_UL(1) << (x))28#define _BITUL(x) (_UL(1) << (x))
29#define _BITULL(x) (_ULL(1) << (x))29#define _BITULL(x) (_ULL(1) << (x))
30 30
31#if !defined(__ASSEMBLY__)
32/*
33 * Missing asm support
34 *
35 * __BIT128() would not work in the asm code, as it shifts an
36 * 'unsigned __init128' data type as direct representation of
37 * 128 bit constants is not supported in the gcc compiler, as
38 * they get silently truncated.
39 *
40 * TODO: Please revisit this implementation when gcc compiler
41 * starts representing 128 bit constants directly like long
42 * and unsigned long etc. Subsequently drop the comment for
43 * GENMASK_U128() which would then start supporting asm code.
44 */
45#define _BIT128(x) ((unsigned __int128)(1) << (x))
46#endif
47
31#define __ALIGN_KERNEL(x, a) __ALIGN_KERNEL_MASK(x, (__typeof__(x))(a) - 1)48#define __ALIGN_KERNEL(x, a) __ALIGN_KERNEL_MASK(x, (__typeof__(x))(a) - 1)
32#define __ALIGN_KERNEL_MASK(x, mask) (((x) + (mask)) & ~(mask))49#define __ALIGN_KERNEL_MASK(x, mask) (((x) + (mask)) & ~(mask))
33 50
lib/test_bits.c+34−0
@@ -39,6 +39,36 @@ static void genmask_ull_test(struct kunit *test) · 38 unmodified lines
39#endif39#endif
40}40}
41 41
42static void genmask_u128_test(struct kunit *test)
43{
44#ifdef CONFIG_ARCH_SUPPORTS_INT128
45 /* Below 64 bit masks */
46 KUNIT_EXPECT_EQ(test, 0x0000000000000001ull, GENMASK_U128(0, 0));
47 KUNIT_EXPECT_EQ(test, 0x0000000000000003ull, GENMASK_U128(1, 0));
48 KUNIT_EXPECT_EQ(test, 0x0000000000000006ull, GENMASK_U128(2, 1));
49 KUNIT_EXPECT_EQ(test, 0x00000000ffffffffull, GENMASK_U128(31, 0));
50 KUNIT_EXPECT_EQ(test, 0x000000ffffe00000ull, GENMASK_U128(39, 21));
51 KUNIT_EXPECT_EQ(test, 0xffffffffffffffffull, GENMASK_U128(63, 0));
52
53 /* Above 64 bit masks - only 64 bit portion can be validated once */
54 KUNIT_EXPECT_EQ(test, 0xffffffffffffffffull, GENMASK_U128(64, 0) >> 1);
55 KUNIT_EXPECT_EQ(test, 0x00000000ffffffffull, GENMASK_U128(81, 50) >> 50);
56 KUNIT_EXPECT_EQ(test, 0x0000000000ffffffull, GENMASK_U128(87, 64) >> 64);
57 KUNIT_EXPECT_EQ(test, 0x0000000000ff0000ull, GENMASK_U128(87, 80) >> 64);
58
59 KUNIT_EXPECT_EQ(test, 0xffffffffffffffffull, GENMASK_U128(127, 0) >> 64);
60 KUNIT_EXPECT_EQ(test, 0xffffffffffffffffull, (u64)GENMASK_U128(127, 0));
61 KUNIT_EXPECT_EQ(test, 0x0000000000000003ull, GENMASK_U128(127, 126) >> 126);
62 KUNIT_EXPECT_EQ(test, 0x0000000000000001ull, GENMASK_U128(127, 127) >> 127);
63#ifdef TEST_GENMASK_FAILURES
64 /* these should fail compilation */
65 GENMASK_U128(0, 1);
66 GENMASK_U128(0, 10);
67 GENMASK_U128(9, 10);
68#endif /* TEST_GENMASK_FAILURES */
69#endif /* CONFIG_ARCH_SUPPORTS_INT128 */
70}
71
42static void genmask_input_check_test(struct kunit *test)72static void genmask_input_check_test(struct kunit *test)
43{73{
44 unsigned int x, y;74 unsigned int x, y;
@@ -56,12 +86,16 @@ static void genmask_input_check_test(struct kunit *test) · 11 unmodified lines
56 /* Valid input */86 /* Valid input */
57 KUNIT_EXPECT_EQ(test, 0, GENMASK_INPUT_CHECK(1, 1));87 KUNIT_EXPECT_EQ(test, 0, GENMASK_INPUT_CHECK(1, 1));
58 KUNIT_EXPECT_EQ(test, 0, GENMASK_INPUT_CHECK(39, 21));88 KUNIT_EXPECT_EQ(test, 0, GENMASK_INPUT_CHECK(39, 21));
89 KUNIT_EXPECT_EQ(test, 0, GENMASK_INPUT_CHECK(100, 80));
90 KUNIT_EXPECT_EQ(test, 0, GENMASK_INPUT_CHECK(110, 65));
91 KUNIT_EXPECT_EQ(test, 0, GENMASK_INPUT_CHECK(127, 0));
59}92}
60 93
61 94
62static struct kunit_case bits_test_cases[] = {95static struct kunit_case bits_test_cases[] = {
63 KUNIT_CASE(genmask_test),96 KUNIT_CASE(genmask_test),
64 KUNIT_CASE(genmask_ull_test),97 KUNIT_CASE(genmask_ull_test),
98 KUNIT_CASE(genmask_u128_test),
65 KUNIT_CASE(genmask_input_check_test),99 KUNIT_CASE(genmask_input_check_test),
66 {}100 {}
67};101};