Merge tag 'bitmap-for-6.12' of https://github.com/norov/linux

Pull bitmap updates from Yury Norov: - switch all bitmamp APIs from inline to __always_inline (Brian Norris) The __always_inline series improves on code generation, and now with the latest compiler versions is required to avoid compilation warnings. It spent enough in my backlog, and I'm thankful to Brian Norris for taking over and moving it forward. - introduce GENMASK_U128() macro (Anshuman Khandual) GENMASK_U128() is a prerequisite needed for arm64 development * tag 'bitmap-for-6.12' of https://github.com/norov/linux: lib/test_bits.c: Add tests for GENMASK_U128() uapi: Define GENMASK_U128 nodemask: Switch from inline to __always_inline cpumask: Switch from inline to __always_inline bitmap: Switch from inline to __always_inline find: Switch from inline to __always_inline

Linus Torvalds committed 2 years ago

Commit 9c445752 parents ba33a49 d7bcc378 files · +325 −232Browse files

Unified · Split

8 files

include/linux/bitmap.h +76 −64
include/linux/bits.h +15 −0
include/linux/cpumask.h +112 −100
include/linux/find.h +25 −25
include/linux/nodemask.h +43 −43
include/uapi/linux/bits.h +3 −0
include/uapi/linux/const.h +17 −0
lib/test_bits.c +34 −0

include/linux/bitmap.h+76−64

@@ -203,12 +203,12 @@ unsigned long bitmap_find_next_zero_area_off(unsigned long *map, · 202 unmodified lines
203	203	* the bit offset of all zero areas this function finds is multiples of that
204	204	* power of 2. A @align_mask of 0 means no alignment is required.
205	205	*/
206		−static inline unsigned long
207		−bitmap_find_next_zero_area(unsigned long *map,
208		− unsigned long size,
209		− unsigned long start,
210		− unsigned int nr,
211		− unsigned long align_mask)
	206	+static __always_inline
	207	+unsigned long bitmap_find_next_zero_area(unsigned long *map,
	208	+ unsigned long size,
	209	+ unsigned long start,
	210	+ unsigned int nr,
	211	+ unsigned long align_mask)
212	212	{
213	213	return bitmap_find_next_zero_area_off(map, size, start, nr,
214	214	align_mask, 0);
@@ -228,7 +228,7 @@ void bitmap_fold(unsigned long dst, const unsigned long orig, · 13 unmodified lines
228	228
229	229	#define bitmap_size(nbits) (ALIGN(nbits, BITS_PER_LONG) / BITS_PER_BYTE)
230	230
231		−static inline void bitmap_zero(unsigned long *dst, unsigned int nbits)
	231	+static __always_inline void bitmap_zero(unsigned long *dst, unsigned int nbits)
232	232	{
233	233	unsigned int len = bitmap_size(nbits);
234	234
@@ -238,7 +238,7 @@ static inline void bitmap_zero(unsigned long *dst, unsigned int nbits) · 3 unmodified lines
238	238	memset(dst, 0, len);
239	239	}
240	240
241		−static inline void bitmap_fill(unsigned long *dst, unsigned int nbits)
	241	+static __always_inline void bitmap_fill(unsigned long *dst, unsigned int nbits)
242	242	{
243	243	unsigned int len = bitmap_size(nbits);
244	244
@@ -248,8 +248,8 @@ static inline void bitmap_fill(unsigned long *dst, unsigned int nbits) · 3 unmodified lines
248	248	memset(dst, 0xff, len);
249	249	}
250	250
251		−static inline void bitmap_copy(unsigned long dst, const unsigned long src,
252		− unsigned int nbits)
	251	+static __always_inline
	252	+void bitmap_copy(unsigned long dst, const unsigned long src, unsigned int nbits)
253	253	{
254	254	unsigned int len = bitmap_size(nbits);
255	255
@@ -262,8 +262,8 @@ static inline void bitmap_copy(unsigned long dst, const unsigned long src, · 6 unmodified lines
262	262	/*
263	263	* Copy bitmap and clear tail bits in last word.
264	264	*/
265		−static inline void bitmap_copy_clear_tail(unsigned long *dst,
266		− const unsigned long *src, unsigned int nbits)
	265	+static __always_inline
	266	+void bitmap_copy_clear_tail(unsigned long dst, const unsigned long src, unsigned int nbits)
267	267	{
268	268	bitmap_copy(dst, src, nbits);
269	269	if (nbits % BITS_PER_LONG)
@@ -318,16 +318,18 @@ void bitmap_to_arr64(u64 buf, const unsigned long bitmap, unsigned int nbits); · 48 unmodified lines
318	318	bitmap_copy_clear_tail((unsigned long )(buf), (const unsigned long )(bitmap), (nbits))
319	319	#endif
320	320
321		−static inline bool bitmap_and(unsigned long dst, const unsigned long src1,
322		− const unsigned long *src2, unsigned int nbits)
	321	+static __always_inline
	322	+bool bitmap_and(unsigned long dst, const unsigned long src1,
	323	+ const unsigned long *src2, unsigned int nbits)
323	324	{
324	325	if (small_const_nbits(nbits))
325	326	return (dst = src1 & *src2 & BITMAP_LAST_WORD_MASK(nbits)) != 0;
326	327	return __bitmap_and(dst, src1, src2, nbits);
327	328	}
328	329
329		−static inline void bitmap_or(unsigned long dst, const unsigned long src1,
330		− const unsigned long *src2, unsigned int nbits)
	330	+static __always_inline
	331	+void bitmap_or(unsigned long dst, const unsigned long src1,
	332	+ const unsigned long *src2, unsigned int nbits)
331	333	{
332	334	if (small_const_nbits(nbits))
333	335	dst = src1 \| *src2;
@@ -335,8 +337,9 @@ static inline void bitmap_or(unsigned long dst, const unsigned long src1, · 1 unmodified line
335	337	__bitmap_or(dst, src1, src2, nbits);
336	338	}
337	339
338		−static inline void bitmap_xor(unsigned long dst, const unsigned long src1,
339		− const unsigned long *src2, unsigned int nbits)
	340	+static __always_inline
	341	+void bitmap_xor(unsigned long dst, const unsigned long src1,
	342	+ const unsigned long *src2, unsigned int nbits)
340	343	{
341	344	if (small_const_nbits(nbits))
342	345	dst = src1 ^ *src2;
@@ -344,16 +347,17 @@ static inline void bitmap_xor(unsigned long dst, const unsigned long src1, · 1 unmodified line
344	347	__bitmap_xor(dst, src1, src2, nbits);
345	348	}
346	349
347		−static inline bool bitmap_andnot(unsigned long dst, const unsigned long src1,
348		− const unsigned long *src2, unsigned int nbits)
	350	+static __always_inline
	351	+bool bitmap_andnot(unsigned long dst, const unsigned long src1,
	352	+ const unsigned long *src2, unsigned int nbits)
349	353	{
350	354	if (small_const_nbits(nbits))
351	355	return (dst = src1 & ~(*src2) & BITMAP_LAST_WORD_MASK(nbits)) != 0;
352	356	return __bitmap_andnot(dst, src1, src2, nbits);
353	357	}
354	358
355		−static inline void bitmap_complement(unsigned long dst, const unsigned long src,
356		− unsigned int nbits)
	359	+static __always_inline
	360	+void bitmap_complement(unsigned long dst, const unsigned long src, unsigned int nbits)
357	361	{
358	362	if (small_const_nbits(nbits))
359	363	dst = ~(src);
@@ -368,8 +372,8 @@ static inline void bitmap_complement(unsigned long dst, const unsigned long sr · 8 unmodified lines
368	372	#endif
369	373	#define BITMAP_MEM_MASK (BITMAP_MEM_ALIGNMENT - 1)
370	374
371		−static inline bool bitmap_equal(const unsigned long *src1,
372		− const unsigned long *src2, unsigned int nbits)
	375	+static __always_inline
	376	+bool bitmap_equal(const unsigned long src1, const unsigned long src2, unsigned int nbits)
373	377	{
374	378	if (small_const_nbits(nbits))
375	379	return !((src1 ^ src2) & BITMAP_LAST_WORD_MASK(nbits));
@@ -388,10 +392,9 @@ static inline bool bitmap_equal(const unsigned long *src1, · 12 unmodified lines
388	392	*
389	393	* Returns: True if (@src1 \| @src2) == *@src3, false otherwise
390	394	*/
391		−static inline bool bitmap_or_equal(const unsigned long *src1,
392		− const unsigned long *src2,
393		− const unsigned long *src3,
394		− unsigned int nbits)
	395	+static __always_inline
	396	+bool bitmap_or_equal(const unsigned long src1, const unsigned long src2,
	397	+ const unsigned long *src3, unsigned int nbits)
395	398	{
396	399	if (!small_const_nbits(nbits))
397	400	return __bitmap_or_equal(src1, src2, src3, nbits);
@@ -399,9 +402,8 @@ static inline bool bitmap_or_equal(const unsigned long *src1, · 1 unmodified line
399	402	return !(((src1 \| src2) ^ *src3) & BITMAP_LAST_WORD_MASK(nbits));
400	403	}
401	404
402		−static inline bool bitmap_intersects(const unsigned long *src1,
403		− const unsigned long *src2,
404		− unsigned int nbits)
	405	+static __always_inline
	406	+bool bitmap_intersects(const unsigned long src1, const unsigned long src2, unsigned int nbits)
405	407	{
406	408	if (small_const_nbits(nbits))
407	409	return ((src1 & src2) & BITMAP_LAST_WORD_MASK(nbits)) != 0;
@@ -409,8 +411,8 @@ static inline bool bitmap_intersects(const unsigned long *src1, · 1 unmodified line
409	411	return __bitmap_intersects(src1, src2, nbits);
410	412	}
411	413
412		−static inline bool bitmap_subset(const unsigned long *src1,
413		− const unsigned long *src2, unsigned int nbits)
	414	+static __always_inline
	415	+bool bitmap_subset(const unsigned long src1, const unsigned long src2, unsigned int nbits)
414	416	{
415	417	if (small_const_nbits(nbits))
416	418	return ! ((src1 & ~(src2)) & BITMAP_LAST_WORD_MASK(nbits));
@@ -418,7 +420,8 @@ static inline bool bitmap_subset(const unsigned long *src1, · 1 unmodified line
418	420	return __bitmap_subset(src1, src2, nbits);
419	421	}
420	422
421		−static inline bool bitmap_empty(const unsigned long *src, unsigned nbits)
	423	+static __always_inline
	424	+bool bitmap_empty(const unsigned long *src, unsigned nbits)
422	425	{
423	426	if (small_const_nbits(nbits))
424	427	return ! (*src & BITMAP_LAST_WORD_MASK(nbits));
@@ -426,7 +429,8 @@ static inline bool bitmap_empty(const unsigned long *src, unsigned nbits) · 1 unmodified line
426	429	return find_first_bit(src, nbits) == nbits;
427	430	}
428	431
429		−static inline bool bitmap_full(const unsigned long *src, unsigned int nbits)
	432	+static __always_inline
	433	+bool bitmap_full(const unsigned long *src, unsigned int nbits)
430	434	{
431	435	if (small_const_nbits(nbits))
432	436	return ! (~(*src) & BITMAP_LAST_WORD_MASK(nbits));
@@ -460,8 +464,8 @@ unsigned long bitmap_weight_andnot(const unsigned long *src1, · 27 unmodified lines
460	464	return __bitmap_weight_andnot(src1, src2, nbits);
461	465	}
462	466
463		−static __always_inline void bitmap_set(unsigned long *map, unsigned int start,
464		− unsigned int nbits)
	467	+static __always_inline
	468	+void bitmap_set(unsigned long *map, unsigned int start, unsigned int nbits)
465	469	{
466	470	if (__builtin_constant_p(nbits) && nbits == 1)
467	471	__set_bit(start, map);
@@ -476,8 +480,8 @@ static __always_inline void bitmap_set(unsigned long *map, unsigned int start, · 8 unmodified lines
476	480	__bitmap_set(map, start, nbits);
477	481	}
478	482
479		−static __always_inline void bitmap_clear(unsigned long *map, unsigned int start,
480		− unsigned int nbits)
	483	+static __always_inline
	484	+void bitmap_clear(unsigned long *map, unsigned int start, unsigned int nbits)
481	485	{
482	486	if (__builtin_constant_p(nbits) && nbits == 1)
483	487	__clear_bit(start, map);
@@ -492,8 +496,9 @@ static __always_inline void bitmap_clear(unsigned long *map, unsigned int start, · 8 unmodified lines
492	496	__bitmap_clear(map, start, nbits);
493	497	}
494	498
495		−static inline void bitmap_shift_right(unsigned long dst, const unsigned long src,
496		− unsigned int shift, unsigned int nbits)
	499	+static __always_inline
	500	+void bitmap_shift_right(unsigned long dst, const unsigned long src,
	501	+ unsigned int shift, unsigned int nbits)
497	502	{
498	503	if (small_const_nbits(nbits))
499	504	dst = (src & BITMAP_LAST_WORD_MASK(nbits)) >> shift;
@@ -501,8 +506,9 @@ static inline void bitmap_shift_right(unsigned long dst, const unsigned long s · 1 unmodified line
501	506	__bitmap_shift_right(dst, src, shift, nbits);
502	507	}
503	508
504		−static inline void bitmap_shift_left(unsigned long dst, const unsigned long src,
505		− unsigned int shift, unsigned int nbits)
	509	+static __always_inline
	510	+void bitmap_shift_left(unsigned long dst, const unsigned long src,
	511	+ unsigned int shift, unsigned int nbits)
506	512	{
507	513	if (small_const_nbits(nbits))
508	514	dst = (src << shift) & BITMAP_LAST_WORD_MASK(nbits);
@@ -510,11 +516,12 @@ static inline void bitmap_shift_left(unsigned long dst, const unsigned long sr · 1 unmodified line
510	516	__bitmap_shift_left(dst, src, shift, nbits);
511	517	}
512	518
513		−static inline void bitmap_replace(unsigned long *dst,
514		− const unsigned long *old,
515		− const unsigned long *new,
516		− const unsigned long *mask,
517		− unsigned int nbits)
	519	+static __always_inline
	520	+void bitmap_replace(unsigned long *dst,
	521	+ const unsigned long *old,
	522	+ const unsigned long *new,
	523	+ const unsigned long *mask,
	524	+ unsigned int nbits)
518	525	{
519	526	if (small_const_nbits(nbits))
520	527	dst = (old & ~(mask)) \| (new & *mask);
@@ -557,8 +564,9 @@ static inline void bitmap_replace(unsigned long *dst, · 36 unmodified lines
557	564	* bitmap_gather() can be seen as the 'reverse' bitmap_scatter() operation.
558	565	* See bitmap_scatter() for details related to this relationship.
559	566	*/
560		−static inline void bitmap_scatter(unsigned long dst, const unsigned long src,
561		− const unsigned long *mask, unsigned int nbits)
	567	+static __always_inline
	568	+void bitmap_scatter(unsigned long dst, const unsigned long src,
	569	+ const unsigned long *mask, unsigned int nbits)
562	570	{
563	571	unsigned int n = 0;
564	572	unsigned int bit;
@@ -611,8 +619,9 @@ static inline void bitmap_scatter(unsigned long dst, const unsigned long src, · 46 unmodified lines
611	619	* bitmap_scatter(res, src, mask, n) and a call to
612	620	* bitmap_scatter(res, result, mask, n) will lead to the same res value.
613	621	*/
614		−static inline void bitmap_gather(unsigned long dst, const unsigned long src,
615		− const unsigned long *mask, unsigned int nbits)
	622	+static __always_inline
	623	+void bitmap_gather(unsigned long dst, const unsigned long src,
	624	+ const unsigned long *mask, unsigned int nbits)
616	625	{
617	626	unsigned int n = 0;
618	627	unsigned int bit;
@@ -623,9 +632,9 @@ static inline void bitmap_gather(unsigned long dst, const unsigned long src, · 4 unmodified lines
623	632	__assign_bit(n++, dst, test_bit(bit, src));
624	633	}
625	634
626		−static inline void bitmap_next_set_region(unsigned long *bitmap,
627		− unsigned int rs, unsigned int re,
628		− unsigned int end)
	635	+static __always_inline
	636	+void bitmap_next_set_region(unsigned long bitmap, unsigned int rs,
	637	+ unsigned int *re, unsigned int end)
629	638	{
630	639	rs = find_next_bit(bitmap, end, rs);
631	640	re = find_next_zero_bit(bitmap, end, rs + 1);
@@ -640,7 +649,8 @@ static inline void bitmap_next_set_region(unsigned long *bitmap, · 8 unmodified lines
640	649	* This is the complement to __bitmap_find_free_region() and releases
641	650	* the found region (by clearing it in the bitmap).
642	651	*/
643		−static inline void bitmap_release_region(unsigned long *bitmap, unsigned int pos, int order)
	652	+static __always_inline
	653	+void bitmap_release_region(unsigned long *bitmap, unsigned int pos, int order)
644	654	{
645	655	bitmap_clear(bitmap, pos, BIT(order));
646	656	}
@@ -656,7 +666,8 @@ static inline void bitmap_release_region(unsigned long *bitmap, unsigned int pos · 9 unmodified lines
656	666	* Returns: 0 on success, or %-EBUSY if specified region wasn't
657	667	* free (not all bits were zero).
658	668	*/
659		−static inline int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos, int order)
	669	+static __always_inline
	670	+int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos, int order)
660	671	{
661	672	unsigned int len = BIT(order);
662	673
@@ -680,7 +691,8 @@ static inline int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos · 17 unmodified lines
680	691	* Returns: the bit offset in bitmap of the allocated region,
681	692	* or -errno on failure.
682	693	*/
683		−static inline int bitmap_find_free_region(unsigned long *bitmap, unsigned int bits, int order)
	694	+static __always_inline
	695	+int bitmap_find_free_region(unsigned long *bitmap, unsigned int bits, int order)
684	696	{
685	697	unsigned int pos, end; /* scans bitmap by regions of size order */
686	698
@@ -734,7 +746,7 @@ static inline int bitmap_find_free_region(unsigned long *bitmap, unsigned int bi · 47 unmodified lines
734	746	* That is ``(u32 *)(&val)[0]`` gets the upper 32 bits,
735	747	* but we expect the lower 32-bits of u64.
736	748	*/
737		−static inline void bitmap_from_u64(unsigned long *dst, u64 mask)
	749	+static __always_inline void bitmap_from_u64(unsigned long *dst, u64 mask)
738	750	{
739	751	bitmap_from_arr64(dst, &mask, 64);
740	752	}
@@ -749,9 +761,8 @@ static inline void bitmap_from_u64(unsigned long *dst, u64 mask) · 8 unmodified lines
749	761	* @map memory region. For @nbits = 0 and @nbits > BITS_PER_LONG the return
750	762	* value is undefined.
751	763	*/
752		−static inline unsigned long bitmap_read(const unsigned long *map,
753		− unsigned long start,
754		− unsigned long nbits)
	764	+static __always_inline
	765	+unsigned long bitmap_read(const unsigned long *map, unsigned long start, unsigned long nbits)
755	766	{
756	767	size_t index = BIT_WORD(start);
757	768	unsigned long offset = start % BITS_PER_LONG;
@@ -784,8 +795,9 @@ static inline unsigned long bitmap_read(const unsigned long *map, · 26 unmodified lines
784	795	*
785	796	* For @nbits == 0 and @nbits > BITS_PER_LONG no writes are performed.
786	797	*/
787		−static inline void bitmap_write(unsigned long *map, unsigned long value,
788		− unsigned long start, unsigned long nbits)
	798	+static __always_inline
	799	+void bitmap_write(unsigned long *map, unsigned long value,
	800	+ unsigned long start, unsigned long nbits)
789	801	{
790	802	size_t index;
791	803	unsigned long offset;

include/linux/bits.h+15−0

@@ -36,4 +36,19 @@ · 35 unmodified lines
36	36	#define GENMASK_ULL(h, l) \
37	37	(GENMASK_INPUT_CHECK(h, l) + __GENMASK_ULL(h, l))
38	38
	39	+#if !defined(__ASSEMBLY__)
	40	+/*
	41	+ * Missing asm support
	42	+ *
	43	+ * __GENMASK_U128() depends on _BIT128() which would not work
	44	+ * in the asm code, as it shifts an 'unsigned __init128' data
	45	+ * type instead of direct representation of 128 bit constants
	46	+ * such as long and unsigned long. The fundamental problem is
	47	+ * that a 128 bit constant will get silently truncated by the
	48	+ * gcc compiler.
	49	+ */
	50	+#define GENMASK_U128(h, l) \
	51	+ (GENMASK_INPUT_CHECK(h, l) + __GENMASK_U128(h, l))
	52	+#endif
	53	+
39	54	#endif /* __LINUX_BITS_H */

include/linux/cpumask.h+112−100

@@ -30,7 +30,7 @@ · 29 unmodified lines
30	30	extern unsigned int nr_cpu_ids;
31	31	#endif
32	32
33		−static inline void set_nr_cpu_ids(unsigned int nr)
	33	+static __always_inline void set_nr_cpu_ids(unsigned int nr)
34	34	{
35	35	#if (NR_CPUS == 1) \|\| defined(CONFIG_FORCE_NR_CPUS)
36	36	WARN_ON(nr != nr_cpu_ids);
@@ -149,7 +149,7 @@ static __always_inline unsigned int cpumask_check(unsigned int cpu) · 112 unmodified lines
149	149	*
150	150	* Return: >= nr_cpu_ids if no cpus set.
151	151	*/
152		−static inline unsigned int cpumask_first(const struct cpumask *srcp)
	152	+static __always_inline unsigned int cpumask_first(const struct cpumask *srcp)
153	153	{
154	154	return find_first_bit(cpumask_bits(srcp), small_cpumask_bits);
155	155	}
@@ -160,7 +160,7 @@ static inline unsigned int cpumask_first(const struct cpumask *srcp) · 4 unmodified lines
160	160	*
161	161	* Return: >= nr_cpu_ids if all cpus are set.
162	162	*/
163		−static inline unsigned int cpumask_first_zero(const struct cpumask *srcp)
	163	+static __always_inline unsigned int cpumask_first_zero(const struct cpumask *srcp)
164	164	{
165	165	return find_first_zero_bit(cpumask_bits(srcp), small_cpumask_bits);
166	166	}
@@ -172,7 +172,7 @@ static inline unsigned int cpumask_first_zero(const struct cpumask *srcp) · 5 unmodified lines
172	172	*
173	173	* Return: >= nr_cpu_ids if no cpus set in both. See also cpumask_next_and().
174	174	*/
175		−static inline
	175	+static __always_inline
176	176	unsigned int cpumask_first_and(const struct cpumask srcp1, const struct cpumask srcp2)
177	177	{
178	178	return find_first_and_bit(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits);
@@ -186,7 +186,7 @@ unsigned int cpumask_first_and(const struct cpumask *srcp1, const struct cpumask · 7 unmodified lines
186	186	*
187	187	* Return: >= nr_cpu_ids if no cpus set in all.
188	188	*/
189		−static inline
	189	+static __always_inline
190	190	unsigned int cpumask_first_and_and(const struct cpumask *srcp1,
191	191	const struct cpumask *srcp2,
192	192	const struct cpumask *srcp3)
@@ -201,7 +201,7 @@ unsigned int cpumask_first_and_and(const struct cpumask *srcp1, · 8 unmodified lines
201	201	*
202	202	* Return: >= nr_cpumask_bits if no CPUs set.
203	203	*/
204		−static inline unsigned int cpumask_last(const struct cpumask *srcp)
	204	+static __always_inline unsigned int cpumask_last(const struct cpumask *srcp)
205	205	{
206	206	return find_last_bit(cpumask_bits(srcp), small_cpumask_bits);
207	207	}
@@ -213,7 +213,7 @@ static inline unsigned int cpumask_last(const struct cpumask *srcp) · 5 unmodified lines
213	213	*
214	214	* Return: >= nr_cpu_ids if no further cpus set.
215	215	*/
216		−static inline
	216	+static __always_inline
217	217	unsigned int cpumask_next(int n, const struct cpumask *srcp)
218	218	{
219	219	/* -1 is a legal arg here. */
@@ -229,7 +229,8 @@ unsigned int cpumask_next(int n, const struct cpumask *srcp) · 9 unmodified lines
229	229	*
230	230	* Return: >= nr_cpu_ids if no further cpus unset.
231	231	*/
232		−static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
	232	+static __always_inline
	233	+unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
233	234	{
234	235	/* -1 is a legal arg here. */
235	236	if (n != -1)
@@ -239,18 +240,21 @@ static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp) · 3 unmodified lines
239	240
240	241	#if NR_CPUS == 1
241	242	/* Uniprocessor: there is only one valid CPU */
242		−static inline unsigned int cpumask_local_spread(unsigned int i, int node)
	243	+static __always_inline
	244	+unsigned int cpumask_local_spread(unsigned int i, int node)
243	245	{
244	246	return 0;
245	247	}
246	248
247		−static inline unsigned int cpumask_any_and_distribute(const struct cpumask *src1p,
248		− const struct cpumask *src2p)
	249	+static __always_inline
	250	+unsigned int cpumask_any_and_distribute(const struct cpumask *src1p,
	251	+ const struct cpumask *src2p)
249	252	{
250	253	return cpumask_first_and(src1p, src2p);
251	254	}
252	255
253		−static inline unsigned int cpumask_any_distribute(const struct cpumask *srcp)
	256	+static __always_inline
	257	+unsigned int cpumask_any_distribute(const struct cpumask *srcp)
254	258	{
255	259	return cpumask_first(srcp);
256	260	}
@@ -269,9 +273,9 @@ unsigned int cpumask_any_distribute(const struct cpumask *srcp); · 12 unmodified lines
269	273	*
270	274	* Return: >= nr_cpu_ids if no further cpus set in both.
271	275	*/
272		−static inline
	276	+static __always_inline
273	277	unsigned int cpumask_next_and(int n, const struct cpumask *src1p,
274		− const struct cpumask *src2p)
	278	+ const struct cpumask *src2p)
275	279	{
276	280	/* -1 is a legal arg here. */
277	281	if (n != -1)
@@ -291,7 +295,7 @@ unsigned int cpumask_next_and(int n, const struct cpumask *src1p, · 13 unmodified lines
291	295	for_each_set_bit(cpu, cpumask_bits(mask), small_cpumask_bits)
292	296
293	297	#if NR_CPUS == 1
294		−static inline
	298	+static __always_inline
295	299	unsigned int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap)
296	300	{
297	301	cpumask_check(start);
@@ -394,7 +398,7 @@ unsigned int __pure cpumask_next_wrap(int n, const struct cpumask *mask, int sta · 96 unmodified lines
394	398	* Often used to find any cpu but smp_processor_id() in a mask.
395	399	* Return: >= nr_cpu_ids if no cpus set.
396	400	*/
397		−static inline
	401	+static __always_inline
398	402	unsigned int cpumask_any_but(const struct cpumask *mask, unsigned int cpu)
399	403	{
400	404	unsigned int i;
@@ -414,7 +418,7 @@ unsigned int cpumask_any_but(const struct cpumask *mask, unsigned int cpu) · 13 unmodified lines
414	418	*
415	419	* Returns >= nr_cpu_ids if no cpus set.
416	420	*/
417		−static inline
	421	+static __always_inline
418	422	unsigned int cpumask_any_and_but(const struct cpumask *mask1,
419	423	const struct cpumask *mask2,
420	424	unsigned int cpu)
@@ -436,7 +440,8 @@ unsigned int cpumask_any_and_but(const struct cpumask *mask1, · 15 unmodified lines
436	440	*
437	441	* Return: >= nr_cpu_ids if such cpu doesn't exist.
438	442	*/
439		−static inline unsigned int cpumask_nth(unsigned int cpu, const struct cpumask *srcp)
	443	+static __always_inline
	444	+unsigned int cpumask_nth(unsigned int cpu, const struct cpumask *srcp)
440	445	{
441	446	return find_nth_bit(cpumask_bits(srcp), small_cpumask_bits, cpumask_check(cpu));
442	447	}
@@ -449,7 +454,7 @@ static inline unsigned int cpumask_nth(unsigned int cpu, const struct cpumask *s · 6 unmodified lines
449	454	*
450	455	* Return: >= nr_cpu_ids if such cpu doesn't exist.
451	456	*/
452		−static inline
	457	+static __always_inline
453	458	unsigned int cpumask_nth_and(unsigned int cpu, const struct cpumask *srcp1,
454	459	const struct cpumask *srcp2)
455	460	{
@@ -465,7 +470,7 @@ unsigned int cpumask_nth_and(unsigned int cpu, const struct cpumask *srcp1, · 9 unmodified lines
465	470	*
466	471	* Return: >= nr_cpu_ids if such cpu doesn't exist.
467	472	*/
468		−static inline
	473	+static __always_inline
469	474	unsigned int cpumask_nth_andnot(unsigned int cpu, const struct cpumask *srcp1,
470	475	const struct cpumask *srcp2)
471	476	{
@@ -508,12 +513,14 @@ unsigned int cpumask_nth_and_andnot(unsigned int cpu, const struct cpumask *srcp · 36 unmodified lines
508	513	* @cpu: cpu number (< nr_cpu_ids)
509	514	* @dstp: the cpumask pointer
510	515	*/
511		−static __always_inline void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
	516	+static __always_inline
	517	+void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
512	518	{
513	519	set_bit(cpumask_check(cpu), cpumask_bits(dstp));
514	520	}
515	521
516		−static __always_inline void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
	522	+static __always_inline
	523	+void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
517	524	{
518	525	__set_bit(cpumask_check(cpu), cpumask_bits(dstp));
519	526	}
@@ -557,7 +564,8 @@ static __always_inline void __cpumask_assign_cpu(int cpu, struct cpumask *dstp, · 37 unmodified lines
557	564	*
558	565	* Return: true if @cpu is set in @cpumask, else returns false
559	566	*/
560		−static __always_inline bool cpumask_test_cpu(int cpu, const struct cpumask *cpumask)
	567	+static __always_inline
	568	+bool cpumask_test_cpu(int cpu, const struct cpumask *cpumask)
561	569	{
562	570	return test_bit(cpumask_check(cpu), cpumask_bits((cpumask)));
563	571	}
@@ -571,7 +579,8 @@ static __always_inline bool cpumask_test_cpu(int cpu, const struct cpumask *cpum · 7 unmodified lines
571	579	*
572	580	* Return: true if @cpu is set in old bitmap of @cpumask, else returns false
573	581	*/
574		−static __always_inline bool cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)
	582	+static __always_inline
	583	+bool cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)
575	584	{
576	585	return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask));
577	586	}
@@ -585,7 +594,8 @@ static __always_inline bool cpumask_test_and_set_cpu(int cpu, struct cpumask *cp · 7 unmodified lines
585	594	*
586	595	* Return: true if @cpu is set in old bitmap of @cpumask, else returns false
587	596	*/
588		−static __always_inline bool cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask)
	597	+static __always_inline
	598	+bool cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask)
589	599	{
590	600	return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask));
591	601	}
@@ -594,7 +604,7 @@ static __always_inline bool cpumask_test_and_clear_cpu(int cpu, struct cpumask * · 2 unmodified lines
594	604	* cpumask_setall - set all cpus (< nr_cpu_ids) in a cpumask
595	605	* @dstp: the cpumask pointer
596	606	*/
597		−static inline void cpumask_setall(struct cpumask *dstp)
	607	+static __always_inline void cpumask_setall(struct cpumask *dstp)
598	608	{
599	609	if (small_const_nbits(small_cpumask_bits)) {
600	610	cpumask_bits(dstp)[0] = BITMAP_LAST_WORD_MASK(nr_cpumask_bits);
@@ -607,7 +617,7 @@ static inline void cpumask_setall(struct cpumask *dstp) · 6 unmodified lines
607	617	* cpumask_clear - clear all cpus (< nr_cpu_ids) in a cpumask
608	618	* @dstp: the cpumask pointer
609	619	*/
610		−static inline void cpumask_clear(struct cpumask *dstp)
	620	+static __always_inline void cpumask_clear(struct cpumask *dstp)
611	621	{
612	622	bitmap_zero(cpumask_bits(dstp), large_cpumask_bits);
613	623	}
@@ -620,9 +630,9 @@ static inline void cpumask_clear(struct cpumask *dstp) · 6 unmodified lines
620	630	*
621	631	* Return: false if *@dstp is empty, else returns true
622	632	*/
623		−static inline bool cpumask_and(struct cpumask *dstp,
624		− const struct cpumask *src1p,
625		− const struct cpumask *src2p)
	633	+static __always_inline
	634	+bool cpumask_and(struct cpumask dstp, const struct cpumask src1p,
	635	+ const struct cpumask *src2p)
626	636	{
627	637	return bitmap_and(cpumask_bits(dstp), cpumask_bits(src1p),
628	638	cpumask_bits(src2p), small_cpumask_bits);
@@ -634,8 +644,9 @@ static inline bool cpumask_and(struct cpumask *dstp, · 5 unmodified lines
634	644	* @src1p: the first input
635	645	* @src2p: the second input
636	646	*/
637		−static inline void cpumask_or(struct cpumask dstp, const struct cpumask src1p,
638		− const struct cpumask *src2p)
	647	+static __always_inline
	648	+void cpumask_or(struct cpumask dstp, const struct cpumask src1p,
	649	+ const struct cpumask *src2p)
639	650	{
640	651	bitmap_or(cpumask_bits(dstp), cpumask_bits(src1p),
641	652	cpumask_bits(src2p), small_cpumask_bits);
@@ -647,9 +658,9 @@ static inline void cpumask_or(struct cpumask dstp, const struct cpumask src1p, · 5 unmodified lines
647	658	* @src1p: the first input
648	659	* @src2p: the second input
649	660	*/
650		−static inline void cpumask_xor(struct cpumask *dstp,
651		− const struct cpumask *src1p,
652		− const struct cpumask *src2p)
	661	+static __always_inline
	662	+void cpumask_xor(struct cpumask dstp, const struct cpumask src1p,
	663	+ const struct cpumask *src2p)
653	664	{
654	665	bitmap_xor(cpumask_bits(dstp), cpumask_bits(src1p),
655	666	cpumask_bits(src2p), small_cpumask_bits);
@@ -663,9 +674,9 @@ static inline void cpumask_xor(struct cpumask *dstp, · 7 unmodified lines
663	674	*
664	675	* Return: false if *@dstp is empty, else returns true
665	676	*/
666		−static inline bool cpumask_andnot(struct cpumask *dstp,
667		− const struct cpumask *src1p,
668		− const struct cpumask *src2p)
	677	+static __always_inline
	678	+bool cpumask_andnot(struct cpumask dstp, const struct cpumask src1p,
	679	+ const struct cpumask *src2p)
669	680	{
670	681	return bitmap_andnot(cpumask_bits(dstp), cpumask_bits(src1p),
671	682	cpumask_bits(src2p), small_cpumask_bits);
@@ -678,8 +689,8 @@ static inline bool cpumask_andnot(struct cpumask *dstp, · 6 unmodified lines
678	689	*
679	690	* Return: true if the cpumasks are equal, false if not
680	691	*/
681		−static inline bool cpumask_equal(const struct cpumask *src1p,
682		− const struct cpumask *src2p)
	692	+static __always_inline
	693	+bool cpumask_equal(const struct cpumask src1p, const struct cpumask src2p)
683	694	{
684	695	return bitmap_equal(cpumask_bits(src1p), cpumask_bits(src2p),
685	696	small_cpumask_bits);
@@ -694,9 +705,9 @@ static inline bool cpumask_equal(const struct cpumask *src1p, · 8 unmodified lines
694	705	* Return: true if first cpumask ORed with second cpumask == third cpumask,
695	706	* otherwise false
696	707	*/
697		−static inline bool cpumask_or_equal(const struct cpumask *src1p,
698		− const struct cpumask *src2p,
699		− const struct cpumask *src3p)
	708	+static __always_inline
	709	+bool cpumask_or_equal(const struct cpumask src1p, const struct cpumask src2p,
	710	+ const struct cpumask *src3p)
700	711	{
701	712	return bitmap_or_equal(cpumask_bits(src1p), cpumask_bits(src2p),
702	713	cpumask_bits(src3p), small_cpumask_bits);
@@ -710,8 +721,8 @@ static inline bool cpumask_or_equal(const struct cpumask *src1p, · 7 unmodified lines
710	721	* Return: true if first cpumask ANDed with second cpumask is non-empty,
711	722	* otherwise false
712	723	*/
713		−static inline bool cpumask_intersects(const struct cpumask *src1p,
714		− const struct cpumask *src2p)
	724	+static __always_inline
	725	+bool cpumask_intersects(const struct cpumask src1p, const struct cpumask src2p)
715	726	{
716	727	return bitmap_intersects(cpumask_bits(src1p), cpumask_bits(src2p),
717	728	small_cpumask_bits);
@@ -724,8 +735,8 @@ static inline bool cpumask_intersects(const struct cpumask *src1p, · 6 unmodified lines
724	735	*
725	736	* Return: true if @src1p is a subset of @src2p, else returns false
726	737	*/
727		−static inline bool cpumask_subset(const struct cpumask *src1p,
728		− const struct cpumask *src2p)
	738	+static __always_inline
	739	+bool cpumask_subset(const struct cpumask src1p, const struct cpumask src2p)
729	740	{
730	741	return bitmap_subset(cpumask_bits(src1p), cpumask_bits(src2p),
731	742	small_cpumask_bits);
@@ -737,7 +748,7 @@ static inline bool cpumask_subset(const struct cpumask *src1p, · 5 unmodified lines
737	748	*
738	749	* Return: true if srcp is empty (has no bits set), else false
739	750	*/
740		−static inline bool cpumask_empty(const struct cpumask *srcp)
	751	+static __always_inline bool cpumask_empty(const struct cpumask *srcp)
741	752	{
742	753	return bitmap_empty(cpumask_bits(srcp), small_cpumask_bits);
743	754	}
@@ -748,7 +759,7 @@ static inline bool cpumask_empty(const struct cpumask *srcp) · 4 unmodified lines
748	759	*
749	760	* Return: true if srcp is full (has all bits set), else false
750	761	*/
751		−static inline bool cpumask_full(const struct cpumask *srcp)
	762	+static __always_inline bool cpumask_full(const struct cpumask *srcp)
752	763	{
753	764	return bitmap_full(cpumask_bits(srcp), nr_cpumask_bits);
754	765	}
@@ -759,7 +770,7 @@ static inline bool cpumask_full(const struct cpumask *srcp) · 4 unmodified lines
759	770	*
760	771	* Return: count of bits set in *srcp
761	772	*/
762		−static inline unsigned int cpumask_weight(const struct cpumask *srcp)
	773	+static __always_inline unsigned int cpumask_weight(const struct cpumask *srcp)
763	774	{
764	775	return bitmap_weight(cpumask_bits(srcp), small_cpumask_bits);
765	776	}
@@ -771,8 +782,8 @@ static inline unsigned int cpumask_weight(const struct cpumask *srcp) · 5 unmodified lines
771	782	*
772	783	* Return: count of bits set in both srcp1 and srcp2
773	784	*/
774		−static inline unsigned int cpumask_weight_and(const struct cpumask *srcp1,
775		− const struct cpumask *srcp2)
	785	+static __always_inline
	786	+unsigned int cpumask_weight_and(const struct cpumask srcp1, const struct cpumask srcp2)
776	787	{
777	788	return bitmap_weight_and(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits);
778	789	}
@@ -784,8 +795,9 @@ static inline unsigned int cpumask_weight_and(const struct cpumask *srcp1, · 5 unmodified lines
784	795	*
785	796	* Return: count of bits set in both srcp1 and srcp2
786	797	*/
787		−static inline unsigned int cpumask_weight_andnot(const struct cpumask *srcp1,
788		− const struct cpumask *srcp2)
	798	+static __always_inline
	799	+unsigned int cpumask_weight_andnot(const struct cpumask *srcp1,
	800	+ const struct cpumask *srcp2)
789	801	{
790	802	return bitmap_weight_andnot(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits);
791	803	}
@@ -796,8 +808,8 @@ static inline unsigned int cpumask_weight_andnot(const struct cpumask *srcp1, · 4 unmodified lines
796	808	* @srcp: the input to shift
797	809	* @n: the number of bits to shift by
798	810	*/
799		−static inline void cpumask_shift_right(struct cpumask *dstp,
800		− const struct cpumask *srcp, int n)
	811	+static __always_inline
	812	+void cpumask_shift_right(struct cpumask dstp, const struct cpumask srcp, int n)
801	813	{
802	814	bitmap_shift_right(cpumask_bits(dstp), cpumask_bits(srcp), n,
803	815	small_cpumask_bits);
@@ -809,8 +821,8 @@ static inline void cpumask_shift_right(struct cpumask *dstp, · 5 unmodified lines
809	821	* @srcp: the input to shift
810	822	* @n: the number of bits to shift by
811	823	*/
812		−static inline void cpumask_shift_left(struct cpumask *dstp,
813		− const struct cpumask *srcp, int n)
	824	+static __always_inline
	825	+void cpumask_shift_left(struct cpumask dstp, const struct cpumask srcp, int n)
814	826	{
815	827	bitmap_shift_left(cpumask_bits(dstp), cpumask_bits(srcp), n,
816	828	nr_cpumask_bits);
@@ -821,8 +833,8 @@ static inline void cpumask_shift_left(struct cpumask *dstp, · 4 unmodified lines
821	833	* @dstp: the result
822	834	* @srcp: the input cpumask
823	835	*/
824		−static inline void cpumask_copy(struct cpumask *dstp,
825		− const struct cpumask *srcp)
	836	+static __always_inline
	837	+void cpumask_copy(struct cpumask dstp, const struct cpumask srcp)
826	838	{
827	839	bitmap_copy(cpumask_bits(dstp), cpumask_bits(srcp), large_cpumask_bits);
828	840	}
@@ -858,8 +870,8 @@ static inline void cpumask_copy(struct cpumask *dstp, · 29 unmodified lines
858	870	*
859	871	* Return: -errno, or 0 for success.
860	872	*/
861		−static inline int cpumask_parse_user(const char __user *buf, int len,
862		− struct cpumask *dstp)
	873	+static __always_inline
	874	+int cpumask_parse_user(const char __user buf, int len, struct cpumask dstp)
863	875	{
864	876	return bitmap_parse_user(buf, len, cpumask_bits(dstp), nr_cpumask_bits);
865	877	}
@@ -872,8 +884,8 @@ static inline int cpumask_parse_user(const char __user *buf, int len, · 6 unmodified lines
872	884	*
873	885	* Return: -errno, or 0 for success.
874	886	*/
875		−static inline int cpumask_parselist_user(const char __user *buf, int len,
876		− struct cpumask *dstp)
	887	+static __always_inline
	888	+int cpumask_parselist_user(const char __user buf, int len, struct cpumask dstp)
877	889	{
878	890	return bitmap_parselist_user(buf, len, cpumask_bits(dstp),
879	891	nr_cpumask_bits);
@@ -886,7 +898,7 @@ static inline int cpumask_parselist_user(const char __user *buf, int len, · 6 unmodified lines
886	898	*
887	899	* Return: -errno, or 0 for success.
888	900	*/
889		−static inline int cpumask_parse(const char buf, struct cpumask dstp)
	901	+static __always_inline int cpumask_parse(const char buf, struct cpumask dstp)
890	902	{
891	903	return bitmap_parse(buf, UINT_MAX, cpumask_bits(dstp), nr_cpumask_bits);
892	904	}
@@ -898,7 +910,7 @@ static inline int cpumask_parse(const char buf, struct cpumask dstp) · 5 unmodified lines
898	910	*
899	911	* Return: -errno, or 0 for success.
900	912	*/
901		−static inline int cpulist_parse(const char buf, struct cpumask dstp)
	913	+static __always_inline int cpulist_parse(const char buf, struct cpumask dstp)
902	914	{
903	915	return bitmap_parselist(buf, cpumask_bits(dstp), nr_cpumask_bits);
904	916	}
@@ -908,7 +920,7 @@ static inline int cpulist_parse(const char buf, struct cpumask dstp) · 3 unmodified lines
908	920	*
909	921	* Return: size to allocate for a &struct cpumask in bytes
910	922	*/
911		−static inline unsigned int cpumask_size(void)
	923	+static __always_inline unsigned int cpumask_size(void)
912	924	{
913	925	return bitmap_size(large_cpumask_bits);
914	926	}
@@ -920,7 +932,7 @@ static inline unsigned int cpumask_size(void) · 5 unmodified lines
920	932
921	933	bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
922	934
923		−static inline
	935	+static __always_inline
924	936	bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node)
925	937	{
926	938	return alloc_cpumask_var_node(mask, flags \| __GFP_ZERO, node);
@@ -938,13 +950,13 @@ bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node) · 11 unmodified lines
938	950	*
939	951	* Return: %true if allocation succeeded, %false if not
940	952	*/
941		−static inline
	953	+static __always_inline
942	954	bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
943	955	{
944	956	return alloc_cpumask_var_node(mask, flags, NUMA_NO_NODE);
945	957	}
946	958
947		−static inline
	959	+static __always_inline
948	960	bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
949	961	{
950	962	return alloc_cpumask_var(mask, flags \| __GFP_ZERO);
@@ -954,7 +966,7 @@ void alloc_bootmem_cpumask_var(cpumask_var_t *mask); · 3 unmodified lines
954	966	void free_cpumask_var(cpumask_var_t mask);
955	967	void free_bootmem_cpumask_var(cpumask_var_t mask);
956	968
957		−static inline bool cpumask_available(cpumask_var_t mask)
	969	+static __always_inline bool cpumask_available(cpumask_var_t mask)
958	970	{
959	971	return mask != NULL;
960	972	}
@@ -964,43 +976,43 @@ static inline bool cpumask_available(cpumask_var_t mask) · 3 unmodified lines
964	976	#define this_cpu_cpumask_var_ptr(x) this_cpu_ptr(x)
965	977	#define __cpumask_var_read_mostly
966	978
967		−static inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
	979	+static __always_inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
968	980	{
969	981	return true;
970	982	}
971	983
972		−static inline bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
	984	+static __always_inline bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
973	985	int node)
974	986	{
975	987	return true;
976	988	}
977	989
978		−static inline bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
	990	+static __always_inline bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
979	991	{
980	992	cpumask_clear(*mask);
981	993	return true;
982	994	}
983	995
984		−static inline bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
	996	+static __always_inline bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
985	997	int node)
986	998	{
987	999	cpumask_clear(*mask);
988	1000	return true;
989	1001	}
990	1002
991		−static inline void alloc_bootmem_cpumask_var(cpumask_var_t *mask)
	1003	+static __always_inline void alloc_bootmem_cpumask_var(cpumask_var_t *mask)
992	1004	{
993	1005	}
994	1006
995		−static inline void free_cpumask_var(cpumask_var_t mask)
	1007	+static __always_inline void free_cpumask_var(cpumask_var_t mask)
996	1008	{
997	1009	}
998	1010
999		−static inline void free_bootmem_cpumask_var(cpumask_var_t mask)
	1011	+static __always_inline void free_bootmem_cpumask_var(cpumask_var_t mask)
1000	1012	{
1001	1013	}
1002	1014
1003		−static inline bool cpumask_available(cpumask_var_t mask)
	1015	+static __always_inline bool cpumask_available(cpumask_var_t mask)
1004	1016	{
1005	1017	return true;
1006	1018	}
@@ -1058,7 +1070,7 @@ void set_cpu_online(unsigned int cpu, bool online); · 51 unmodified lines
1058	1070	((struct cpumask *)(1 ? (bitmap) \
1059	1071	: (void *)sizeof(__check_is_bitmap(bitmap))))
1060	1072
1061		−static inline int __check_is_bitmap(const unsigned long *bitmap)
	1073	+static __always_inline int __check_is_bitmap(const unsigned long *bitmap)
1062	1074	{
1063	1075	return 1;
1064	1076	}
@@ -1073,7 +1085,7 @@ static inline int __check_is_bitmap(const unsigned long *bitmap) · 8 unmodified lines
1073	1085	extern const unsigned long
1074	1086	cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)];
1075	1087
1076		−static inline const struct cpumask *get_cpu_mask(unsigned int cpu)
	1088	+static __always_inline const struct cpumask *get_cpu_mask(unsigned int cpu)
1077	1089	{
1078	1090	const unsigned long *p = cpu_bit_bitmap[1 + cpu % BITS_PER_LONG];
1079	1091	p -= cpu / BITS_PER_LONG;
@@ -1100,32 +1112,32 @@ static __always_inline unsigned int num_online_cpus(void) · 20 unmodified lines
1100	1112	#define num_present_cpus() cpumask_weight(cpu_present_mask)
1101	1113	#define num_active_cpus() cpumask_weight(cpu_active_mask)
1102	1114
1103		−static inline bool cpu_online(unsigned int cpu)
	1115	+static __always_inline bool cpu_online(unsigned int cpu)
1104	1116	{
1105	1117	return cpumask_test_cpu(cpu, cpu_online_mask);
1106	1118	}
1107	1119
1108		−static inline bool cpu_enabled(unsigned int cpu)
	1120	+static __always_inline bool cpu_enabled(unsigned int cpu)
1109	1121	{
1110	1122	return cpumask_test_cpu(cpu, cpu_enabled_mask);
1111	1123	}
1112	1124
1113		−static inline bool cpu_possible(unsigned int cpu)
	1125	+static __always_inline bool cpu_possible(unsigned int cpu)
1114	1126	{
1115	1127	return cpumask_test_cpu(cpu, cpu_possible_mask);
1116	1128	}
1117	1129
1118		−static inline bool cpu_present(unsigned int cpu)
	1130	+static __always_inline bool cpu_present(unsigned int cpu)
1119	1131	{
1120	1132	return cpumask_test_cpu(cpu, cpu_present_mask);
1121	1133	}
1122	1134
1123		−static inline bool cpu_active(unsigned int cpu)
	1135	+static __always_inline bool cpu_active(unsigned int cpu)
1124	1136	{
1125	1137	return cpumask_test_cpu(cpu, cpu_active_mask);
1126	1138	}
1127	1139
1128		−static inline bool cpu_dying(unsigned int cpu)
	1140	+static __always_inline bool cpu_dying(unsigned int cpu)
1129	1141	{
1130	1142	return cpumask_test_cpu(cpu, cpu_dying_mask);
1131	1143	}
@@ -1138,32 +1150,32 @@ static inline bool cpu_dying(unsigned int cpu) · 6 unmodified lines
1138	1150	#define num_present_cpus() 1U
1139	1151	#define num_active_cpus() 1U
1140	1152
1141		−static inline bool cpu_online(unsigned int cpu)
	1153	+static __always_inline bool cpu_online(unsigned int cpu)
1142	1154	{
1143	1155	return cpu == 0;
1144	1156	}
1145	1157
1146		−static inline bool cpu_possible(unsigned int cpu)
	1158	+static __always_inline bool cpu_possible(unsigned int cpu)
1147	1159	{
1148	1160	return cpu == 0;
1149	1161	}
1150	1162
1151		−static inline bool cpu_enabled(unsigned int cpu)
	1163	+static __always_inline bool cpu_enabled(unsigned int cpu)
1152	1164	{
1153	1165	return cpu == 0;
1154	1166	}
1155	1167
1156		−static inline bool cpu_present(unsigned int cpu)
	1168	+static __always_inline bool cpu_present(unsigned int cpu)
1157	1169	{
1158	1170	return cpu == 0;
1159	1171	}
1160	1172
1161		−static inline bool cpu_active(unsigned int cpu)
	1173	+static __always_inline bool cpu_active(unsigned int cpu)
1162	1174	{
1163	1175	return cpu == 0;
1164	1176	}
1165	1177
1166		−static inline bool cpu_dying(unsigned int cpu)
	1178	+static __always_inline bool cpu_dying(unsigned int cpu)
1167	1179	{
1168	1180	return false;
1169	1181	}
@@ -1197,7 +1209,7 @@ static inline bool cpu_dying(unsigned int cpu) · 27 unmodified lines
1197	1209	* Return: the length of the (null-terminated) @buf string, zero if
1198	1210	* nothing is copied.
1199	1211	*/
1200		−static inline ssize_t
	1212	+static __always_inline ssize_t
1201	1213	cpumap_print_to_pagebuf(bool list, char buf, const struct cpumask mask)
1202	1214	{
1203	1215	return bitmap_print_to_pagebuf(list, buf, cpumask_bits(mask),
@@ -1220,9 +1232,9 @@ cpumap_print_to_pagebuf(bool list, char buf, const struct cpumask mask) · 16 unmodified lines
1220	1232	* Return: the length of how many bytes have been copied, excluding
1221	1233	* terminating '\0'.
1222	1234	*/
1223		−static inline ssize_t
1224		−cpumap_print_bitmask_to_buf(char buf, const struct cpumask mask,
1225		− loff_t off, size_t count)
	1235	+static __always_inline
	1236	+ssize_t cpumap_print_bitmask_to_buf(char buf, const struct cpumask mask,
	1237	+ loff_t off, size_t count)
1226	1238	{
1227	1239	return bitmap_print_bitmask_to_buf(buf, cpumask_bits(mask),
1228	1240	nr_cpu_ids, off, count) - 1;
@@ -1242,9 +1254,9 @@ cpumap_print_bitmask_to_buf(char buf, const struct cpumask mask, · 13 unmodified lines
1242	1254	* Return: the length of how many bytes have been copied, excluding
1243	1255	* terminating '\0'.
1244	1256	*/
1245		−static inline ssize_t
1246		−cpumap_print_list_to_buf(char buf, const struct cpumask mask,
1247		− loff_t off, size_t count)
	1257	+static __always_inline
	1258	+ssize_t cpumap_print_list_to_buf(char buf, const struct cpumask mask,
	1259	+ loff_t off, size_t count)
1248	1260	{
1249	1261	return bitmap_print_list_to_buf(buf, cpumask_bits(mask),
1250	1262	nr_cpu_ids, off, count) - 1;

include/linux/find.h+25−25

@@ -52,7 +52,7 @@ unsigned long _find_next_bit_le(const unsigned long *addr, unsigned · 51 unmodified lines
52	52	* Returns the bit number for the next set bit
53	53	* If no bits are set, returns @size.
54	54	*/
55		−static inline
	55	+static __always_inline
56	56	unsigned long find_next_bit(const unsigned long *addr, unsigned long size,
57	57	unsigned long offset)
58	58	{
@@ -81,7 +81,7 @@ unsigned long find_next_bit(const unsigned long *addr, unsigned long size, · 22 unmodified lines
81	81	* Returns the bit number for the next set bit
82	82	* If no bits are set, returns @size.
83	83	*/
84		−static inline
	84	+static __always_inline
85	85	unsigned long find_next_and_bit(const unsigned long *addr1,
86	86	const unsigned long *addr2, unsigned long size,
87	87	unsigned long offset)
@@ -112,7 +112,7 @@ unsigned long find_next_and_bit(const unsigned long *addr1, · 24 unmodified lines
112	112	* Returns the bit number for the next set bit
113	113	* If no bits are set, returns @size.
114	114	*/
115		−static inline
	115	+static __always_inline
116	116	unsigned long find_next_andnot_bit(const unsigned long *addr1,
117	117	const unsigned long *addr2, unsigned long size,
118	118	unsigned long offset)
@@ -142,7 +142,7 @@ unsigned long find_next_andnot_bit(const unsigned long *addr1, · 23 unmodified lines
142	142	* Returns the bit number for the next set bit
143	143	* If no bits are set, returns @size.
144	144	*/
145		−static inline
	145	+static __always_inline
146	146	unsigned long find_next_or_bit(const unsigned long *addr1,
147	147	const unsigned long *addr2, unsigned long size,
148	148	unsigned long offset)
@@ -171,7 +171,7 @@ unsigned long find_next_or_bit(const unsigned long *addr1, · 22 unmodified lines
171	171	* Returns the bit number of the next zero bit
172	172	* If no bits are zero, returns @size.
173	173	*/
174		−static inline
	174	+static __always_inline
175	175	unsigned long find_next_zero_bit(const unsigned long *addr, unsigned long size,
176	176	unsigned long offset)
177	177	{
@@ -198,7 +198,7 @@ unsigned long find_next_zero_bit(const unsigned long *addr, unsigned long size, · 20 unmodified lines
198	198	* Returns the bit number of the first set bit.
199	199	* If no bits are set, returns @size.
200	200	*/
201		−static inline
	201	+static __always_inline
202	202	unsigned long find_first_bit(const unsigned long *addr, unsigned long size)
203	203	{
204	204	if (small_const_nbits(size)) {
@@ -224,7 +224,7 @@ unsigned long find_first_bit(const unsigned long *addr, unsigned long size) · 19 unmodified lines
224	224	* Returns the bit number of the N'th set bit.
225	225	* If no such, returns >= @size.
226	226	*/
227		−static inline
	227	+static __always_inline
228	228	unsigned long find_nth_bit(const unsigned long *addr, unsigned long size, unsigned long n)
229	229	{
230	230	if (n >= size)
@@ -249,7 +249,7 @@ unsigned long find_nth_bit(const unsigned long *addr, unsigned long size, unsign · 18 unmodified lines
249	249	* Returns the bit number of the N'th set bit.
250	250	* If no such, returns @size.
251	251	*/
252		−static inline
	252	+static __always_inline
253	253	unsigned long find_nth_and_bit(const unsigned long addr1, const unsigned long addr2,
254	254	unsigned long size, unsigned long n)
255	255	{
@@ -276,7 +276,7 @@ unsigned long find_nth_and_bit(const unsigned long addr1, const unsigned long · 20 unmodified lines
276	276	* Returns the bit number of the N'th set bit.
277	277	* If no such, returns @size.
278	278	*/
279		−static inline
	279	+static __always_inline
280	280	unsigned long find_nth_andnot_bit(const unsigned long addr1, const unsigned long addr2,
281	281	unsigned long size, unsigned long n)
282	282	{
@@ -332,7 +332,7 @@ unsigned long find_nth_and_andnot_bit(const unsigned long *addr1, · 49 unmodified lines
332	332	* Returns the bit number for the next set bit
333	333	* If no bits are set, returns @size.
334	334	*/
335		−static inline
	335	+static __always_inline
336	336	unsigned long find_first_and_bit(const unsigned long *addr1,
337	337	const unsigned long *addr2,
338	338	unsigned long size)
@@ -357,7 +357,7 @@ unsigned long find_first_and_bit(const unsigned long *addr1, · 18 unmodified lines
357	357	* Returns the bit number for the first set bit
358	358	* If no bits are set, returns @size.
359	359	*/
360		−static inline
	360	+static __always_inline
361	361	unsigned long find_first_and_and_bit(const unsigned long *addr1,
362	362	const unsigned long *addr2,
363	363	const unsigned long *addr3,
@@ -381,7 +381,7 @@ unsigned long find_first_and_and_bit(const unsigned long *addr1, · 17 unmodified lines
381	381	* Returns the bit number of the first cleared bit.
382	382	* If no bits are zero, returns @size.
383	383	*/
384		−static inline
	384	+static __always_inline
385	385	unsigned long find_first_zero_bit(const unsigned long *addr, unsigned long size)
386	386	{
387	387	if (small_const_nbits(size)) {
@@ -402,7 +402,7 @@ unsigned long find_first_zero_bit(const unsigned long *addr, unsigned long size) · 14 unmodified lines
402	402	*
403	403	* Returns the bit number of the last set bit, or size.
404	404	*/
405		−static inline
	405	+static __always_inline
406	406	unsigned long find_last_bit(const unsigned long *addr, unsigned long size)
407	407	{
408	408	if (small_const_nbits(size)) {
@@ -425,7 +425,7 @@ unsigned long find_last_bit(const unsigned long *addr, unsigned long size) · 16 unmodified lines
425	425	* Returns the bit number for the next set bit, or first set bit up to @offset
426	426	* If no bits are set, returns @size.
427	427	*/
428		−static inline
	428	+static __always_inline
429	429	unsigned long find_next_and_bit_wrap(const unsigned long *addr1,
430	430	const unsigned long *addr2,
431	431	unsigned long size, unsigned long offset)
@@ -448,7 +448,7 @@ unsigned long find_next_and_bit_wrap(const unsigned long *addr1, · 16 unmodified lines
448	448	* Returns the bit number for the next set bit, or first set bit up to @offset
449	449	* If no bits are set, returns @size.
450	450	*/
451		−static inline
	451	+static __always_inline
452	452	unsigned long find_next_bit_wrap(const unsigned long *addr,
453	453	unsigned long size, unsigned long offset)
454	454	{
@@ -465,7 +465,7 @@ unsigned long find_next_bit_wrap(const unsigned long *addr, · 10 unmodified lines
465	465	* Helper for for_each_set_bit_wrap(). Make sure you're doing right thing
466	466	* before using it alone.
467	467	*/
468		−static inline
	468	+static __always_inline
469	469	unsigned long __for_each_wrap(const unsigned long *bitmap, unsigned long size,
470	470	unsigned long start, unsigned long n)
471	471	{
@@ -506,20 +506,20 @@ extern unsigned long find_next_clump8(unsigned long *clump, · 34 unmodified lines
506	506
507	507	#if defined(__LITTLE_ENDIAN)
508	508
509		−static inline unsigned long find_next_zero_bit_le(const void *addr,
510		− unsigned long size, unsigned long offset)
	509	+static __always_inline
	510	+unsigned long find_next_zero_bit_le(const void *addr, unsigned long size, unsigned long offset)
511	511	{
512	512	return find_next_zero_bit(addr, size, offset);
513	513	}
514	514
515		−static inline unsigned long find_next_bit_le(const void *addr,
516		− unsigned long size, unsigned long offset)
	515	+static __always_inline
	516	+unsigned long find_next_bit_le(const void *addr, unsigned long size, unsigned long offset)
517	517	{
518	518	return find_next_bit(addr, size, offset);
519	519	}
520	520
521		−static inline unsigned long find_first_zero_bit_le(const void *addr,
522		− unsigned long size)
	521	+static __always_inline
	522	+unsigned long find_first_zero_bit_le(const void *addr, unsigned long size)
523	523	{
524	524	return find_first_zero_bit(addr, size);
525	525	}
@@ -527,7 +527,7 @@ static inline unsigned long find_first_zero_bit_le(const void *addr, · 1 unmodified line
527	527	#elif defined(__BIG_ENDIAN)
528	528
529	529	#ifndef find_next_zero_bit_le
530		−static inline
	530	+static __always_inline
531	531	unsigned long find_next_zero_bit_le(const void *addr, unsigned
532	532	long size, unsigned long offset)
533	533	{
@@ -546,7 +546,7 @@ unsigned long find_next_zero_bit_le(const void *addr, unsigned · 12 unmodified lines
546	546	#endif
547	547
548	548	#ifndef find_first_zero_bit_le
549		−static inline
	549	+static __always_inline
550	550	unsigned long find_first_zero_bit_le(const void *addr, unsigned long size)
551	551	{
552	552	if (small_const_nbits(size)) {
@@ -560,7 +560,7 @@ unsigned long find_first_zero_bit_le(const void *addr, unsigned long size) · 7 unmodified lines
560	560	#endif
561	561
562	562	#ifndef find_next_bit_le
563		−static inline
	563	+static __always_inline
564	564	unsigned long find_next_bit_le(const void *addr, unsigned
565	565	long size, unsigned long offset)
566	566	{

include/linux/nodemask.h+43−43

@@ -107,11 +107,11 @@ extern nodemask_t _unused_nodemask_arg_; · 106 unmodified lines
107	107	*/
108	108	#define nodemask_pr_args(maskp) __nodemask_pr_numnodes(maskp), \
109	109	__nodemask_pr_bits(maskp)
110		−static inline unsigned int __nodemask_pr_numnodes(const nodemask_t *m)
	110	+static __always_inline unsigned int __nodemask_pr_numnodes(const nodemask_t *m)
111	111	{
112	112	return m ? MAX_NUMNODES : 0;
113	113	}
114		−static inline const unsigned long __nodemask_pr_bits(const nodemask_t m)
	114	+static __always_inline const unsigned long __nodemask_pr_bits(const nodemask_t m)
115	115	{
116	116	return m ? m->bits : NULL;
117	117	}
@@ -132,19 +132,19 @@ static __always_inline void __node_set(int node, volatile nodemask_t *dstp) · 14 unmodified lines
132	132	}
133	133
134	134	#define node_clear(node, dst) __node_clear((node), &(dst))
135		−static inline void __node_clear(int node, volatile nodemask_t *dstp)
	135	+static __always_inline void __node_clear(int node, volatile nodemask_t *dstp)
136	136	{
137	137	clear_bit(node, dstp->bits);
138	138	}
139	139
140	140	#define nodes_setall(dst) __nodes_setall(&(dst), MAX_NUMNODES)
141		−static inline void __nodes_setall(nodemask_t *dstp, unsigned int nbits)
	141	+static __always_inline void __nodes_setall(nodemask_t *dstp, unsigned int nbits)
142	142	{
143	143	bitmap_fill(dstp->bits, nbits);
144	144	}
145	145
146	146	#define nodes_clear(dst) __nodes_clear(&(dst), MAX_NUMNODES)
147		−static inline void __nodes_clear(nodemask_t *dstp, unsigned int nbits)
	147	+static __always_inline void __nodes_clear(nodemask_t *dstp, unsigned int nbits)
148	148	{
149	149	bitmap_zero(dstp->bits, nbits);
150	150	}
@@ -154,14 +154,14 @@ static inline void __nodes_clear(nodemask_t *dstp, unsigned int nbits) · 3 unmodified lines
154	154
155	155	#define node_test_and_set(node, nodemask) \
156	156	__node_test_and_set((node), &(nodemask))
157		−static inline bool __node_test_and_set(int node, nodemask_t *addr)
	157	+static __always_inline bool __node_test_and_set(int node, nodemask_t *addr)
158	158	{
159	159	return test_and_set_bit(node, addr->bits);
160	160	}
161	161
162	162	#define nodes_and(dst, src1, src2) \
163	163	__nodes_and(&(dst), &(src1), &(src2), MAX_NUMNODES)
164		−static inline void __nodes_and(nodemask_t dstp, const nodemask_t src1p,
	164	+static __always_inline void __nodes_and(nodemask_t dstp, const nodemask_t src1p,
165	165	const nodemask_t *src2p, unsigned int nbits)
166	166	{
167	167	bitmap_and(dstp->bits, src1p->bits, src2p->bits, nbits);
@@ -169,7 +169,7 @@ static inline void __nodes_and(nodemask_t dstp, const nodemask_t src1p, · 1 unmodified line
169	169
170	170	#define nodes_or(dst, src1, src2) \
171	171	__nodes_or(&(dst), &(src1), &(src2), MAX_NUMNODES)
172		−static inline void __nodes_or(nodemask_t dstp, const nodemask_t src1p,
	172	+static __always_inline void __nodes_or(nodemask_t dstp, const nodemask_t src1p,
173	173	const nodemask_t *src2p, unsigned int nbits)
174	174	{
175	175	bitmap_or(dstp->bits, src1p->bits, src2p->bits, nbits);
@@ -177,7 +177,7 @@ static inline void __nodes_or(nodemask_t dstp, const nodemask_t src1p, · 1 unmodified line
177	177
178	178	#define nodes_xor(dst, src1, src2) \
179	179	__nodes_xor(&(dst), &(src1), &(src2), MAX_NUMNODES)
180		−static inline void __nodes_xor(nodemask_t dstp, const nodemask_t src1p,
	180	+static __always_inline void __nodes_xor(nodemask_t dstp, const nodemask_t src1p,
181	181	const nodemask_t *src2p, unsigned int nbits)
182	182	{
183	183	bitmap_xor(dstp->bits, src1p->bits, src2p->bits, nbits);
@@ -185,7 +185,7 @@ static inline void __nodes_xor(nodemask_t dstp, const nodemask_t src1p, · 1 unmodified line
185	185
186	186	#define nodes_andnot(dst, src1, src2) \
187	187	__nodes_andnot(&(dst), &(src1), &(src2), MAX_NUMNODES)
188		−static inline void __nodes_andnot(nodemask_t dstp, const nodemask_t src1p,
	188	+static __always_inline void __nodes_andnot(nodemask_t dstp, const nodemask_t src1p,
189	189	const nodemask_t *src2p, unsigned int nbits)
190	190	{
191	191	bitmap_andnot(dstp->bits, src1p->bits, src2p->bits, nbits);
@@ -193,7 +193,7 @@ static inline void __nodes_andnot(nodemask_t dstp, const nodemask_t src1p, · 1 unmodified line
193	193
194	194	#define nodes_complement(dst, src) \
195	195	__nodes_complement(&(dst), &(src), MAX_NUMNODES)
196		−static inline void __nodes_complement(nodemask_t *dstp,
	196	+static __always_inline void __nodes_complement(nodemask_t *dstp,
197	197	const nodemask_t *srcp, unsigned int nbits)
198	198	{
199	199	bitmap_complement(dstp->bits, srcp->bits, nbits);
@@ -201,7 +201,7 @@ static inline void __nodes_complement(nodemask_t *dstp, · 1 unmodified line
201	201
202	202	#define nodes_equal(src1, src2) \
203	203	__nodes_equal(&(src1), &(src2), MAX_NUMNODES)
204		−static inline bool __nodes_equal(const nodemask_t *src1p,
	204	+static __always_inline bool __nodes_equal(const nodemask_t *src1p,
205	205	const nodemask_t *src2p, unsigned int nbits)
206	206	{
207	207	return bitmap_equal(src1p->bits, src2p->bits, nbits);
@@ -209,7 +209,7 @@ static inline bool __nodes_equal(const nodemask_t *src1p, · 1 unmodified line
209	209
210	210	#define nodes_intersects(src1, src2) \
211	211	__nodes_intersects(&(src1), &(src2), MAX_NUMNODES)
212		−static inline bool __nodes_intersects(const nodemask_t *src1p,
	212	+static __always_inline bool __nodes_intersects(const nodemask_t *src1p,
213	213	const nodemask_t *src2p, unsigned int nbits)
214	214	{
215	215	return bitmap_intersects(src1p->bits, src2p->bits, nbits);
@@ -217,33 +217,33 @@ static inline bool __nodes_intersects(const nodemask_t *src1p, · 1 unmodified line
217	217
218	218	#define nodes_subset(src1, src2) \
219	219	__nodes_subset(&(src1), &(src2), MAX_NUMNODES)
220		−static inline bool __nodes_subset(const nodemask_t *src1p,
	220	+static __always_inline bool __nodes_subset(const nodemask_t *src1p,
221	221	const nodemask_t *src2p, unsigned int nbits)
222	222	{
223	223	return bitmap_subset(src1p->bits, src2p->bits, nbits);
224	224	}
225	225
226	226	#define nodes_empty(src) __nodes_empty(&(src), MAX_NUMNODES)
227		−static inline bool __nodes_empty(const nodemask_t *srcp, unsigned int nbits)
	227	+static __always_inline bool __nodes_empty(const nodemask_t *srcp, unsigned int nbits)
228	228	{
229	229	return bitmap_empty(srcp->bits, nbits);
230	230	}
231	231
232	232	#define nodes_full(nodemask) __nodes_full(&(nodemask), MAX_NUMNODES)
233		−static inline bool __nodes_full(const nodemask_t *srcp, unsigned int nbits)
	233	+static __always_inline bool __nodes_full(const nodemask_t *srcp, unsigned int nbits)
234	234	{
235	235	return bitmap_full(srcp->bits, nbits);
236	236	}
237	237
238	238	#define nodes_weight(nodemask) __nodes_weight(&(nodemask), MAX_NUMNODES)
239		−static inline int __nodes_weight(const nodemask_t *srcp, unsigned int nbits)
	239	+static __always_inline int __nodes_weight(const nodemask_t *srcp, unsigned int nbits)
240	240	{
241	241	return bitmap_weight(srcp->bits, nbits);
242	242	}
243	243
244	244	#define nodes_shift_right(dst, src, n) \
245	245	__nodes_shift_right(&(dst), &(src), (n), MAX_NUMNODES)
246		−static inline void __nodes_shift_right(nodemask_t *dstp,
	246	+static __always_inline void __nodes_shift_right(nodemask_t *dstp,
247	247	const nodemask_t *srcp, int n, int nbits)
248	248	{
249	249	bitmap_shift_right(dstp->bits, srcp->bits, n, nbits);
@@ -251,7 +251,7 @@ static inline void __nodes_shift_right(nodemask_t *dstp, · 1 unmodified line
251	251
252	252	#define nodes_shift_left(dst, src, n) \
253	253	__nodes_shift_left(&(dst), &(src), (n), MAX_NUMNODES)
254		−static inline void __nodes_shift_left(nodemask_t *dstp,
	254	+static __always_inline void __nodes_shift_left(nodemask_t *dstp,
255	255	const nodemask_t *srcp, int n, int nbits)
256	256	{
257	257	bitmap_shift_left(dstp->bits, srcp->bits, n, nbits);
@@ -261,13 +261,13 @@ static inline void __nodes_shift_left(nodemask_t *dstp, · 3 unmodified lines
261	261	> MAX_NUMNODES, then the silly min_ts could be dropped. */
262	262
263	263	#define first_node(src) __first_node(&(src))
264		−static inline unsigned int __first_node(const nodemask_t *srcp)
	264	+static __always_inline unsigned int __first_node(const nodemask_t *srcp)
265	265	{
266	266	return min_t(unsigned int, MAX_NUMNODES, find_first_bit(srcp->bits, MAX_NUMNODES));
267	267	}
268	268
269	269	#define next_node(n, src) __next_node((n), &(src))
270		−static inline unsigned int __next_node(int n, const nodemask_t *srcp)
	270	+static __always_inline unsigned int __next_node(int n, const nodemask_t *srcp)
271	271	{
272	272	return min_t(unsigned int, MAX_NUMNODES, find_next_bit(srcp->bits, MAX_NUMNODES, n+1));
273	273	}
@@ -277,7 +277,7 @@ static inline unsigned int __next_node(int n, const nodemask_t *srcp) · 3 unmodified lines
277	277	* the first node in src if needed. Returns MAX_NUMNODES if src is empty.
278	278	*/
279	279	#define next_node_in(n, src) __next_node_in((n), &(src))
280		−static inline unsigned int __next_node_in(int node, const nodemask_t *srcp)
	280	+static __always_inline unsigned int __next_node_in(int node, const nodemask_t *srcp)
281	281	{
282	282	unsigned int ret = __next_node(node, srcp);
283	283
@@ -286,7 +286,7 @@ static inline unsigned int __next_node_in(int node, const nodemask_t *srcp) · 2 unmodified lines
286	286	return ret;
287	287	}
288	288
289		−static inline void init_nodemask_of_node(nodemask_t *mask, int node)
	289	+static __always_inline void init_nodemask_of_node(nodemask_t *mask, int node)
290	290	{
291	291	nodes_clear(*mask);
292	292	node_set(node, *mask);
@@ -304,7 +304,7 @@ static inline void init_nodemask_of_node(nodemask_t *mask, int node) · 11 unmodified lines
304	304	})
305	305
306	306	#define first_unset_node(mask) __first_unset_node(&(mask))
307		−static inline unsigned int __first_unset_node(const nodemask_t *maskp)
	307	+static __always_inline unsigned int __first_unset_node(const nodemask_t *maskp)
308	308	{
309	309	return min_t(unsigned int, MAX_NUMNODES,
310	310	find_first_zero_bit(maskp->bits, MAX_NUMNODES));
@@ -338,21 +338,21 @@ static inline unsigned int __first_unset_node(const nodemask_t *maskp) · 27 unmodified lines
338	338
339	339	#define nodemask_parse_user(ubuf, ulen, dst) \
340	340	__nodemask_parse_user((ubuf), (ulen), &(dst), MAX_NUMNODES)
341		−static inline int __nodemask_parse_user(const char __user *buf, int len,
	341	+static __always_inline int __nodemask_parse_user(const char __user *buf, int len,
342	342	nodemask_t *dstp, int nbits)
343	343	{
344	344	return bitmap_parse_user(buf, len, dstp->bits, nbits);
345	345	}
346	346
347	347	#define nodelist_parse(buf, dst) __nodelist_parse((buf), &(dst), MAX_NUMNODES)
348		−static inline int __nodelist_parse(const char buf, nodemask_t dstp, int nbits)
	348	+static __always_inline int __nodelist_parse(const char buf, nodemask_t dstp, int nbits)
349	349	{
350	350	return bitmap_parselist(buf, dstp->bits, nbits);
351	351	}
352	352
353	353	#define node_remap(oldbit, old, new) \
354	354	__node_remap((oldbit), &(old), &(new), MAX_NUMNODES)
355		−static inline int __node_remap(int oldbit,
	355	+static __always_inline int __node_remap(int oldbit,
356	356	const nodemask_t oldp, const nodemask_t newp, int nbits)
357	357	{
358	358	return bitmap_bitremap(oldbit, oldp->bits, newp->bits, nbits);
@@ -360,7 +360,7 @@ static inline int __node_remap(int oldbit, · 1 unmodified line
360	360
361	361	#define nodes_remap(dst, src, old, new) \
362	362	__nodes_remap(&(dst), &(src), &(old), &(new), MAX_NUMNODES)
363		−static inline void __nodes_remap(nodemask_t dstp, const nodemask_t srcp,
	363	+static __always_inline void __nodes_remap(nodemask_t dstp, const nodemask_t srcp,
364	364	const nodemask_t oldp, const nodemask_t newp, int nbits)
365	365	{
366	366	bitmap_remap(dstp->bits, srcp->bits, oldp->bits, newp->bits, nbits);
@@ -368,7 +368,7 @@ static inline void __nodes_remap(nodemask_t dstp, const nodemask_t srcp, · 1 unmodified line
368	368
369	369	#define nodes_onto(dst, orig, relmap) \
370	370	__nodes_onto(&(dst), &(orig), &(relmap), MAX_NUMNODES)
371		−static inline void __nodes_onto(nodemask_t dstp, const nodemask_t origp,
	371	+static __always_inline void __nodes_onto(nodemask_t dstp, const nodemask_t origp,
372	372	const nodemask_t *relmapp, int nbits)
373	373	{
374	374	bitmap_onto(dstp->bits, origp->bits, relmapp->bits, nbits);
@@ -376,7 +376,7 @@ static inline void __nodes_onto(nodemask_t dstp, const nodemask_t origp, · 1 unmodified line
376	376
377	377	#define nodes_fold(dst, orig, sz) \
378	378	__nodes_fold(&(dst), &(orig), sz, MAX_NUMNODES)
379		−static inline void __nodes_fold(nodemask_t dstp, const nodemask_t origp,
	379	+static __always_inline void __nodes_fold(nodemask_t dstp, const nodemask_t origp,
380	380	int sz, int nbits)
381	381	{
382	382	bitmap_fold(dstp->bits, origp->bits, sz, nbits);
@@ -418,22 +418,22 @@ enum node_states { · 35 unmodified lines
418	418	extern nodemask_t node_states[NR_NODE_STATES];
419	419
420	420	#if MAX_NUMNODES > 1
421		−static inline int node_state(int node, enum node_states state)
	421	+static __always_inline int node_state(int node, enum node_states state)
422	422	{
423	423	return node_isset(node, node_states[state]);
424	424	}
425	425
426		−static inline void node_set_state(int node, enum node_states state)
	426	+static __always_inline void node_set_state(int node, enum node_states state)
427	427	{
428	428	__node_set(node, &node_states[state]);
429	429	}
430	430
431		−static inline void node_clear_state(int node, enum node_states state)
	431	+static __always_inline void node_clear_state(int node, enum node_states state)
432	432	{
433	433	__node_clear(node, &node_states[state]);
434	434	}
435	435
436		−static inline int num_node_state(enum node_states state)
	436	+static __always_inline int num_node_state(enum node_states state)
437	437	{
438	438	return nodes_weight(node_states[state]);
439	439	}
@@ -443,11 +443,11 @@ static inline int num_node_state(enum node_states state) · 3 unmodified lines
443	443
444	444	#define first_online_node first_node(node_states[N_ONLINE])
445	445	#define first_memory_node first_node(node_states[N_MEMORY])
446		−static inline unsigned int next_online_node(int nid)
	446	+static __always_inline unsigned int next_online_node(int nid)
447	447	{
448	448	return next_node(nid, node_states[N_ONLINE]);
449	449	}
450		−static inline unsigned int next_memory_node(int nid)
	450	+static __always_inline unsigned int next_memory_node(int nid)
451	451	{
452	452	return next_node(nid, node_states[N_MEMORY]);
453	453	}
@@ -455,13 +455,13 @@ static inline unsigned int next_memory_node(int nid) · 1 unmodified line
455	455	extern unsigned int nr_node_ids;
456	456	extern unsigned int nr_online_nodes;
457	457
458		−static inline void node_set_online(int nid)
	458	+static __always_inline void node_set_online(int nid)
459	459	{
460	460	node_set_state(nid, N_ONLINE);
461	461	nr_online_nodes = num_node_state(N_ONLINE);
462	462	}
463	463
464		−static inline void node_set_offline(int nid)
	464	+static __always_inline void node_set_offline(int nid)
465	465	{
466	466	node_clear_state(nid, N_ONLINE);
467	467	nr_online_nodes = num_node_state(N_ONLINE);
@@ -469,20 +469,20 @@ static inline void node_set_offline(int nid) · 1 unmodified line
469	469
470	470	#else
471	471
472		−static inline int node_state(int node, enum node_states state)
	472	+static __always_inline int node_state(int node, enum node_states state)
473	473	{
474	474	return node == 0;
475	475	}
476	476
477		−static inline void node_set_state(int node, enum node_states state)
	477	+static __always_inline void node_set_state(int node, enum node_states state)
478	478	{
479	479	}
480	480
481		−static inline void node_clear_state(int node, enum node_states state)
	481	+static __always_inline void node_clear_state(int node, enum node_states state)
482	482	{
483	483	}
484	484
485		−static inline int num_node_state(enum node_states state)
	485	+static __always_inline int num_node_state(enum node_states state)
486	486	{
487	487	return 1;
488	488	}
@@ -502,7 +502,7 @@ static inline int num_node_state(enum node_states state) · 13 unmodified lines
502	502
503	503	#endif
504	504
505		−static inline int node_random(const nodemask_t *maskp)
	505	+static __always_inline int node_random(const nodemask_t *maskp)
506	506	{
507	507	#if defined(CONFIG_NUMA) && (MAX_NUMNODES > 1)
508	508	int w, bit;

include/uapi/linux/bits.h+3−0

@@ -12,4 +12,7 @@ · 11 unmodified lines
12	12	(((~_ULL(0)) - (_ULL(1) << (l)) + 1) & \
13	13	(~_ULL(0) >> (__BITS_PER_LONG_LONG - 1 - (h))))
14	14
	15	+#define __GENMASK_U128(h, l) \
	16	+ ((_BIT128((h)) << 1) - (_BIT128(l)))
	17	+
15	18	#endif /* _UAPI_LINUX_BITS_H */

include/uapi/linux/const.h+17−0

@@ -28,6 +28,23 @@ · 27 unmodified lines
28	28	#define _BITUL(x) (_UL(1) << (x))
29	29	#define _BITULL(x) (_ULL(1) << (x))
30	30
	31	+#if !defined(__ASSEMBLY__)
	32	+/*
	33	+ * Missing asm support
	34	+ *
	35	+ * __BIT128() would not work in the asm code, as it shifts an
	36	+ * 'unsigned __init128' data type as direct representation of
	37	+ * 128 bit constants is not supported in the gcc compiler, as
	38	+ * they get silently truncated.
	39	+ *
	40	+ * TODO: Please revisit this implementation when gcc compiler
	41	+ * starts representing 128 bit constants directly like long
	42	+ * and unsigned long etc. Subsequently drop the comment for
	43	+ * GENMASK_U128() which would then start supporting asm code.
	44	+ */
	45	+#define _BIT128(x) ((unsigned __int128)(1) << (x))
	46	+#endif
	47	+
31	48	#define __ALIGN_KERNEL(x, a) __ALIGN_KERNEL_MASK(x, (__typeof__(x))(a) - 1)
32	49	#define __ALIGN_KERNEL_MASK(x, mask) (((x) + (mask)) & ~(mask))
33	50

lib/test_bits.c+34−0

@@ -39,6 +39,36 @@ static void genmask_ull_test(struct kunit *test) · 38 unmodified lines
39	39	#endif
40	40	}
41	41
	42	+static void genmask_u128_test(struct kunit *test)
	43	+{
	44	+#ifdef CONFIG_ARCH_SUPPORTS_INT128
	45	+ /* Below 64 bit masks */
	46	+ KUNIT_EXPECT_EQ(test, 0x0000000000000001ull, GENMASK_U128(0, 0));
	47	+ KUNIT_EXPECT_EQ(test, 0x0000000000000003ull, GENMASK_U128(1, 0));
	48	+ KUNIT_EXPECT_EQ(test, 0x0000000000000006ull, GENMASK_U128(2, 1));
	49	+ KUNIT_EXPECT_EQ(test, 0x00000000ffffffffull, GENMASK_U128(31, 0));
	50	+ KUNIT_EXPECT_EQ(test, 0x000000ffffe00000ull, GENMASK_U128(39, 21));
	51	+ KUNIT_EXPECT_EQ(test, 0xffffffffffffffffull, GENMASK_U128(63, 0));
	52	+
	53	+ /* Above 64 bit masks - only 64 bit portion can be validated once */
	54	+ KUNIT_EXPECT_EQ(test, 0xffffffffffffffffull, GENMASK_U128(64, 0) >> 1);
	55	+ KUNIT_EXPECT_EQ(test, 0x00000000ffffffffull, GENMASK_U128(81, 50) >> 50);
	56	+ KUNIT_EXPECT_EQ(test, 0x0000000000ffffffull, GENMASK_U128(87, 64) >> 64);
	57	+ KUNIT_EXPECT_EQ(test, 0x0000000000ff0000ull, GENMASK_U128(87, 80) >> 64);
	58	+
	59	+ KUNIT_EXPECT_EQ(test, 0xffffffffffffffffull, GENMASK_U128(127, 0) >> 64);
	60	+ KUNIT_EXPECT_EQ(test, 0xffffffffffffffffull, (u64)GENMASK_U128(127, 0));
	61	+ KUNIT_EXPECT_EQ(test, 0x0000000000000003ull, GENMASK_U128(127, 126) >> 126);
	62	+ KUNIT_EXPECT_EQ(test, 0x0000000000000001ull, GENMASK_U128(127, 127) >> 127);
	63	+#ifdef TEST_GENMASK_FAILURES
	64	+ /* these should fail compilation */
	65	+ GENMASK_U128(0, 1);
	66	+ GENMASK_U128(0, 10);
	67	+ GENMASK_U128(9, 10);
	68	+#endif /* TEST_GENMASK_FAILURES */
	69	+#endif /* CONFIG_ARCH_SUPPORTS_INT128 */
	70	+}
	71	+
42	72	static void genmask_input_check_test(struct kunit *test)
43	73	{
44	74	unsigned int x, y;
@@ -56,12 +86,16 @@ static void genmask_input_check_test(struct kunit *test) · 11 unmodified lines
56	86	/* Valid input */
57	87	KUNIT_EXPECT_EQ(test, 0, GENMASK_INPUT_CHECK(1, 1));
58	88	KUNIT_EXPECT_EQ(test, 0, GENMASK_INPUT_CHECK(39, 21));
	89	+ KUNIT_EXPECT_EQ(test, 0, GENMASK_INPUT_CHECK(100, 80));
	90	+ KUNIT_EXPECT_EQ(test, 0, GENMASK_INPUT_CHECK(110, 65));
	91	+ KUNIT_EXPECT_EQ(test, 0, GENMASK_INPUT_CHECK(127, 0));
59	92	}
60	93
61	94
62	95	static struct kunit_case bits_test_cases[] = {
63	96	KUNIT_CASE(genmask_test),
64	97	KUNIT_CASE(genmask_ull_test),
	98	+ KUNIT_CASE(genmask_u128_test),
65	99	KUNIT_CASE(genmask_input_check_test),
66	100	{}
67	101	};