lael-labs/linux

Documentation/core-api/mm-api.rst+4−4

@@ -22,16 +22,16 @@ Memory Allocation Controls · 21 unmodified lines
22	.. kernel-doc:: include/linux/gfp.h	22	.. kernel-doc:: include/linux/gfp.h
23	:internal:	23	:internal:
24		24
25	.. kernel-doc:: include/linux/gfp.h	25	.. kernel-doc:: include/linux/gfp_types.h
26	:doc: Page mobility and placement hints	26	:doc: Page mobility and placement hints
27		27
28	.. kernel-doc:: include/linux/gfp.h	28	.. kernel-doc:: include/linux/gfp_types.h
29	:doc: Watermark modifiers	29	:doc: Watermark modifiers
30		30
31	.. kernel-doc:: include/linux/gfp.h	31	.. kernel-doc:: include/linux/gfp_types.h
32	:doc: Reclaim modifiers	32	:doc: Reclaim modifiers
33		33
34	.. kernel-doc:: include/linux/gfp.h	34	.. kernel-doc:: include/linux/gfp_types.h
35	:doc: Useful GFP flag combinations	35	:doc: Useful GFP flag combinations
36		36
37	The Slab Cache	37	The Slab Cache

MAINTAINERS+1−1

 F:	lib/cpumask.c
 F:	lib/find_bit.c
 F:	lib/find_bit_benchmark.c
-F:	lib/nodemask.c
 F:	lib/test_bitmap.c
 F:	tools/include/linux/bitmap.h
 F:	tools/include/linux/find.h
 T:	git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
 T:	quilt git://git.kernel.org/pub/scm/linux/kernel/git/akpm/25-new
 F:	include/linux/gfp.h
+F:	include/linux/gfp_types.h
 F:	include/linux/memory_hotplug.h
 F:	include/linux/mm.h
 F:	include/linux/mmzone.h

arch/alpha/include/asm/bitops.h+17−15

 /*
  * WARNING: non atomic version.
  */
-static inline void
+static __always_inline void
-__set_bit(unsigned long nr, volatile void * addr)
+arch___set_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	int *m = ((int *) addr) + (nr >> 5);
 /*
  * WARNING: non atomic version.
  */
-static __inline__ void
+static __always_inline void
-__clear_bit(unsigned long nr, volatile void * addr)
+arch___clear_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	int *m = ((int *) addr) + (nr >> 5);
 __clear_bit_unlock(unsigned long nr, volatile void * addr)
 {
 	smp_mb();
-	__clear_bit(nr, addr);
+	arch___clear_bit(nr, addr);
 }
 static inline void
 /*
  * WARNING: non atomic version.
  */
-static __inline__ void
+static __always_inline void
-__change_bit(unsigned long nr, volatile void * addr)
+arch___change_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	int *m = ((int *) addr) + (nr >> 5);
 /*
  * WARNING: non atomic version.
  */
-static inline int
+static __always_inline bool
-__test_and_set_bit(unsigned long nr, volatile void * addr)
+arch___test_and_set_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	unsigned long mask = 1 << (nr & 0x1f);
 	int *m = ((int *) addr) + (nr >> 5);
 /*
  * WARNING: non atomic version.
  */
-static inline int
+static __always_inline bool
-__test_and_clear_bit(unsigned long nr, volatile void * addr)
+arch___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	unsigned long mask = 1 << (nr & 0x1f);
 	int *m = ((int *) addr) + (nr >> 5);
 /*
  * WARNING: non atomic version.
  */
-static __inline__ int
+static __always_inline bool
-__test_and_change_bit(unsigned long nr, volatile void * addr)
+arch___test_and_change_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	unsigned long mask = 1 << (nr & 0x1f);
 	int *m = ((int *) addr) + (nr >> 5);
 	return (old & mask) != 0;
 }
-static inline int
+static __always_inline bool
-test_bit(int nr, const volatile void * addr)
+arch_test_bit(unsigned long nr, const volatile unsigned long *addr)
 {
 	return (1UL & (((const int *) addr)[nr >> 5] >> (nr & 31))) != 0UL;
 }
 	return __ffs(tmp) + ofs;
 }
+#include <asm-generic/bitops/non-instrumented-non-atomic.h>
 #include <asm-generic/bitops/le.h>
 #include <asm-generic/bitops/ext2-atomic-setbit.h>

arch/arm/include/asm/bitops.h+10−8

@@ -160,18 +160,20 @@ extern int _test_and_change_bit(int nr, volatile unsigned long * p); · 159 unmodified lines
160	/*	160	/*
161	* Little endian assembly bitops. nr = 0 -> byte 0 bit 0.	161	* Little endian assembly bitops. nr = 0 -> byte 0 bit 0.
162	*/	162	*/
163	extern int _find_first_zero_bit_le(const unsigned long *p, unsigned size);	163	unsigned long _find_first_zero_bit_le(const unsigned long *p, unsigned long size);
164	extern int _find_next_zero_bit_le(const unsigned long *p, int size, int offset);	164	unsigned long _find_next_zero_bit_le(const unsigned long *p,
165	extern int _find_first_bit_le(const unsigned long *p, unsigned size);	165	unsigned long size, unsigned long offset);
166	extern int _find_next_bit_le(const unsigned long *p, int size, int offset);	166	unsigned long _find_first_bit_le(const unsigned long *p, unsigned long size);
		167	unsigned long _find_next_bit_le(const unsigned long *p, unsigned long size, unsigned long offset);
167		168
168	/*	169	/*
169	* Big endian assembly bitops. nr = 0 -> byte 3 bit 0.	170	* Big endian assembly bitops. nr = 0 -> byte 3 bit 0.
170	*/	171	*/
171	extern int _find_first_zero_bit_be(const unsigned long *p, unsigned size);	172	unsigned long _find_first_zero_bit_be(const unsigned long *p, unsigned long size);
172	extern int _find_next_zero_bit_be(const unsigned long *p, int size, int offset);	173	unsigned long _find_next_zero_bit_be(const unsigned long *p,
173	extern int _find_first_bit_be(const unsigned long *p, unsigned size);	174	unsigned long size, unsigned long offset);
174	extern int _find_next_bit_be(const unsigned long *p, int size, int offset);	175	unsigned long _find_first_bit_be(const unsigned long *p, unsigned long size);
		176	unsigned long _find_next_bit_be(const unsigned long *p, unsigned long size, unsigned long offset);
175		177
176	#ifndef CONFIG_SMP	178	#ifndef CONFIG_SMP
177	/*	179	/*

arch/hexagon/include/asm/bitops.h+15−9

  * be atomic, particularly for things like slab_lock and slab_unlock.
  *
  */
-static inline void __clear_bit(int nr, volatile unsigned long *addr)
+static __always_inline void
+arch___clear_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	test_and_clear_bit(nr, addr);
 }
-static inline void __set_bit(int nr, volatile unsigned long *addr)
+static __always_inline void
+arch___set_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	test_and_set_bit(nr, addr);
 }
-static inline void __change_bit(int nr, volatile unsigned long *addr)
+static __always_inline void
+arch___change_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	test_and_change_bit(nr, addr);
 }
 /*  Apparently, at least some of these are allowed to be non-atomic  */
-static inline int __test_and_clear_bit(int nr, volatile unsigned long *addr)
+static __always_inline bool
+arch___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	return test_and_clear_bit(nr, addr);
 }
-static inline int __test_and_set_bit(int nr, volatile unsigned long *addr)
+static __always_inline bool
+arch___test_and_set_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	return test_and_set_bit(nr, addr);
 }
-static inline int __test_and_change_bit(int nr, volatile unsigned long *addr)
+static __always_inline bool
+arch___test_and_change_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	return test_and_change_bit(nr, addr);
 }
-static inline int __test_bit(int nr, const volatile unsigned long *addr)
+static __always_inline bool
+arch_test_bit(unsigned long nr, const volatile unsigned long *addr)
 {
 	int retval;
 	return retval;
 }
-#define test_bit(nr, addr) __test_bit(nr, addr)
 /*
  * ffz - find first zero in word.
  * @word: The word to search
 }
 #include <asm-generic/bitops/lock.h>
+#include <asm-generic/bitops/non-instrumented-non-atomic.h>
 #include <asm-generic/bitops/fls64.h>
 #include <asm-generic/bitops/sched.h>

arch/ia64/include/asm/bitops.h+22−20

 }
 /**
- * __set_bit - Set a bit in memory
+ * arch___set_bit - Set a bit in memory
  * @nr: the bit to set
  * @addr: the address to start counting from
  *
  * If it's called on the same region of memory simultaneously, the effect
  * may be that only one operation succeeds.
  */
-static __inline__ void
+static __always_inline void
-__set_bit (int nr, volatile void *addr)
+arch___set_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	*((__u32 *) addr + (nr >> 5)) |= (1 << (nr & 31));
 }
 }
 /**
- * __clear_bit - Clears a bit in memory (non-atomic version)
+ * arch___clear_bit - Clears a bit in memory (non-atomic version)
  * @nr: the bit to clear
  * @addr: the address to start counting from
  *
  * If it's called on the same region of memory simultaneously, the effect
  * may be that only one operation succeeds.
  */
-static __inline__ void
+static __always_inline void
-__clear_bit (int nr, volatile void *addr)
+arch___clear_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	*((__u32 *) addr + (nr >> 5)) &= ~(1 << (nr & 31));
 }
 }
 /**
- * __change_bit - Toggle a bit in memory
+ * arch___change_bit - Toggle a bit in memory
  * @nr: the bit to toggle
  * @addr: the address to start counting from
  *
  * If it's called on the same region of memory simultaneously, the effect
  * may be that only one operation succeeds.
  */
-static __inline__ void
+static __always_inline void
-__change_bit (int nr, volatile void *addr)
+arch___change_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	*((__u32 *) addr + (nr >> 5)) ^= (1 << (nr & 31));
 }
 #define test_and_set_bit_lock test_and_set_bit
 /**
- * __test_and_set_bit - Set a bit and return its old value
+ * arch___test_and_set_bit - Set a bit and return its old value
  * @nr: Bit to set
  * @addr: Address to count from
  *
  * If two examples of this operation race, one can appear to succeed
  * but actually fail.  You must protect multiple accesses with a lock.
  */
-static __inline__ int
+static __always_inline bool
-__test_and_set_bit (int nr, volatile void *addr)
+arch___test_and_set_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	__u32 *p = (__u32 *) addr + (nr >> 5);
 	__u32 m = 1 << (nr & 31);
 }
 /**
- * __test_and_clear_bit - Clear a bit and return its old value
+ * arch___test_and_clear_bit - Clear a bit and return its old value
  * @nr: Bit to clear
  * @addr: Address to count from
  *
  * If two examples of this operation race, one can appear to succeed
  * but actually fail.  You must protect multiple accesses with a lock.
  */
-static __inline__ int
+static __always_inline bool
-__test_and_clear_bit(int nr, volatile void * addr)
+arch___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	__u32 *p = (__u32 *) addr + (nr >> 5);
 	__u32 m = 1 << (nr & 31);
 }
 /**
- * __test_and_change_bit - Change a bit and return its old value
+ * arch___test_and_change_bit - Change a bit and return its old value
  * @nr: Bit to change
  * @addr: Address to count from
  *
  * This operation is non-atomic and can be reordered.
  */
-static __inline__ int
+static __always_inline bool
-__test_and_change_bit (int nr, void *addr)
+arch___test_and_change_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	__u32 old, bit = (1 << (nr & 31));
 	__u32 *m = (__u32 *) addr + (nr >> 5);
 	return (old & bit) != 0;
 }
-static __inline__ int
+static __always_inline bool
-test_bit (int nr, const volatile void *addr)
+arch_test_bit(unsigned long nr, const volatile unsigned long *addr)
 {
 	return 1 & (((const volatile __u32 *) addr)[nr >> 5] >> (nr & 31));
 }
 #ifdef __KERNEL__
+#include <asm-generic/bitops/non-instrumented-non-atomic.h>
 #include <asm-generic/bitops/le.h>
 #include <asm-generic/bitops/ext2-atomic-setbit.h>

arch/ia64/include/asm/processor.h+1−1

@@ -538,7 +538,7 @@ ia64_get_irr(unsigned int vector) · 537 unmodified lines
538	{	538	{
539	unsigned int reg = vector / 64;	539	unsigned int reg = vector / 64;
540	unsigned int bit = vector % 64;	540	unsigned int bit = vector % 64;
541	u64 irr;	541	unsigned long irr;
542		542
543	switch (reg) {	543	switch (reg) {
544	case 0: irr = ia64_getreg(_IA64_REG_CR_IRR0); break;	544	case 0: irr = ia64_getreg(_IA64_REG_CR_IRR0); break;

arch/m68k/include/asm/bitops.h+34−15

@@ -65,8 +65,11 @@ static inline void bfset_mem_set_bit(int nr, volatile unsigned long *vaddr) · 64 unmodified lines
65	bfset_mem_set_bit(nr, vaddr))	65	bfset_mem_set_bit(nr, vaddr))
66	#endif	66	#endif
67		67
68	#define __set_bit(nr, vaddr) set_bit(nr, vaddr)	68	static __always_inline void
69		69	arch___set_bit(unsigned long nr, volatile unsigned long *addr)
		70	{
		71	set_bit(nr, addr);
		72	}
70		73
71	static inline void bclr_reg_clear_bit(int nr, volatile unsigned long *vaddr)	74	static inline void bclr_reg_clear_bit(int nr, volatile unsigned long *vaddr)
72	{	75	{
@@ -105,8 +108,11 @@ static inline void bfclr_mem_clear_bit(int nr, volatile unsigned long *vaddr) · 32 unmodified lines
105	bfclr_mem_clear_bit(nr, vaddr))	108	bfclr_mem_clear_bit(nr, vaddr))
106	#endif	109	#endif
107		110
108	#define __clear_bit(nr, vaddr) clear_bit(nr, vaddr)	111	static __always_inline void
109		112	arch___clear_bit(unsigned long nr, volatile unsigned long *addr)
		113	{
		114	clear_bit(nr, addr);
		115	}
110		116
111	static inline void bchg_reg_change_bit(int nr, volatile unsigned long *vaddr)	117	static inline void bchg_reg_change_bit(int nr, volatile unsigned long *vaddr)
112	{	118	{
@@ -145,14 +151,17 @@ static inline void bfchg_mem_change_bit(int nr, volatile unsigned long *vaddr) · 32 unmodified lines
145	bfchg_mem_change_bit(nr, vaddr))	151	bfchg_mem_change_bit(nr, vaddr))
146	#endif	152	#endif
147		153
148	#define __change_bit(nr, vaddr) change_bit(nr, vaddr)	154	static __always_inline void
149		155	arch___change_bit(unsigned long nr, volatile unsigned long *addr)
150
151	static inline int test_bit(int nr, const volatile unsigned long *vaddr)
152	{	156	{
153	return (vaddr[nr >> 5] & (1UL << (nr & 31))) != 0;	157	change_bit(nr, addr);
154	}	158	}
155		159
		160	static __always_inline bool
		161	arch_test_bit(unsigned long nr, const volatile unsigned long *addr)
		162	{
		163	return (addr[nr >> 5] & (1UL << (nr & 31))) != 0;
		164	}
156		165
157	static inline int bset_reg_test_and_set_bit(int nr,	166	static inline int bset_reg_test_and_set_bit(int nr,
158	volatile unsigned long *vaddr)	167	volatile unsigned long *vaddr)
@@ -201,8 +210,11 @@ static inline int bfset_mem_test_and_set_bit(int nr, · 42 unmodified lines
201	bfset_mem_test_and_set_bit(nr, vaddr))	210	bfset_mem_test_and_set_bit(nr, vaddr))
202	#endif	211	#endif
203		212
204	#define __test_and_set_bit(nr, vaddr) test_and_set_bit(nr, vaddr)	213	static __always_inline bool
205		214	arch___test_and_set_bit(unsigned long nr, volatile unsigned long *addr)
		215	{
		216	return test_and_set_bit(nr, addr);
		217	}
206		218
207	static inline int bclr_reg_test_and_clear_bit(int nr,	219	static inline int bclr_reg_test_and_clear_bit(int nr,
208	volatile unsigned long *vaddr)	220	volatile unsigned long *vaddr)
@@ -251,8 +263,11 @@ static inline int bfclr_mem_test_and_clear_bit(int nr, · 42 unmodified lines
251	bfclr_mem_test_and_clear_bit(nr, vaddr))	263	bfclr_mem_test_and_clear_bit(nr, vaddr))
252	#endif	264	#endif
253		265
254	#define __test_and_clear_bit(nr, vaddr) test_and_clear_bit(nr, vaddr)	266	static __always_inline bool
255		267	arch___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr)
		268	{
		269	return test_and_clear_bit(nr, addr);
		270	}
256		271
257	static inline int bchg_reg_test_and_change_bit(int nr,	272	static inline int bchg_reg_test_and_change_bit(int nr,
258	volatile unsigned long *vaddr)	273	volatile unsigned long *vaddr)
@@ -301,8 +316,11 @@ static inline int bfchg_mem_test_and_change_bit(int nr, · 42 unmodified lines
301	bfchg_mem_test_and_change_bit(nr, vaddr))	316	bfchg_mem_test_and_change_bit(nr, vaddr))
302	#endif	317	#endif
303		318
304	#define __test_and_change_bit(nr, vaddr) test_and_change_bit(nr, vaddr)	319	static __always_inline bool
305		320	arch___test_and_change_bit(unsigned long nr, volatile unsigned long *addr)
		321	{
		322	return test_and_change_bit(nr, addr);
		323	}
306		324
307	/*	325	/*
308	* The true 68020 and more advanced processors support the "bfffo"	326	* The true 68020 and more advanced processors support the "bfffo"
@@ -522,6 +540,7 @@ static inline unsigned long __fls(unsigned long x) · 213 unmodified lines
522	#define clear_bit_unlock clear_bit	540	#define clear_bit_unlock clear_bit
523	#define __clear_bit_unlock clear_bit_unlock	541	#define __clear_bit_unlock clear_bit_unlock
524		542
		543	#include <asm-generic/bitops/non-instrumented-non-atomic.h>
525	#include <asm-generic/bitops/ext2-atomic.h>	544	#include <asm-generic/bitops/ext2-atomic.h>
526	#include <asm-generic/bitops/fls64.h>	545	#include <asm-generic/bitops/fls64.h>
527	#include <asm-generic/bitops/sched.h>	546	#include <asm-generic/bitops/sched.h>

arch/powerpc/include/asm/archrandom.h+1−8

 #ifndef _ASM_POWERPC_ARCHRANDOM_H
 #define _ASM_POWERPC_ARCHRANDOM_H
-#include <asm/machdep.h>
 static inline size_t __must_check arch_get_random_longs(unsigned long *v, size_t max_longs)
 {
 	return 0;
 }
-static inline size_t __must_check arch_get_random_seed_longs(unsigned long *v, size_t max_longs)
+size_t __must_check arch_get_random_seed_longs(unsigned long *v, size_t max_longs);
-{
-	if (max_longs && ppc_md.get_random_seed && ppc_md.get_random_seed(v))
-		return 1;
-	return 0;
-}
 #ifdef CONFIG_PPC_POWERNV
 int pnv_get_random_long(unsigned long *v);

arch/powerpc/kernel/setup-common.c+8−0

@@ -171,6 +171,14 @@ EXPORT_SYMBOL_GPL(machine_power_off); · 170 unmodified lines
171	void (*pm_power_off)(void);	171	void (*pm_power_off)(void);
172	EXPORT_SYMBOL_GPL(pm_power_off);	172	EXPORT_SYMBOL_GPL(pm_power_off);
173		173
		174	size_t __must_check arch_get_random_seed_longs(unsigned long *v, size_t max_longs)
		175	{
		176	if (max_longs && ppc_md.get_random_seed && ppc_md.get_random_seed(v))
		177	return 1;
		178	return 0;
		179	}
		180	EXPORT_SYMBOL(arch_get_random_seed_longs);
		181
174	void machine_halt(void)	182	void machine_halt(void)
175	{	183	{
176	machine_shutdown();	184	machine_shutdown();

arch/s390/include/asm/bitops.h+31−30

@@ -113,75 +113,76 @@ static inline bool arch_test_and_change_bit(unsigned long nr, · 112 unmodified lines
113	return old & mask;	113	return old & mask;
114	}	114	}
115		115
116	static inline void arch___set_bit(unsigned long nr, volatile unsigned long *ptr)	116	static __always_inline void
		117	arch___set_bit(unsigned long nr, volatile unsigned long *addr)
117	{	118	{
118	unsigned long *addr = __bitops_word(nr, ptr);	119	unsigned long *p = __bitops_word(nr, addr);
119	unsigned long mask = __bitops_mask(nr);	120	unsigned long mask = __bitops_mask(nr);
120		121
121	*addr \|= mask;	122	*p \|= mask;
122	}	123	}
123		124
124	static inline void arch___clear_bit(unsigned long nr,	125	static __always_inline void
125	volatile unsigned long *ptr)	126	arch___clear_bit(unsigned long nr, volatile unsigned long *addr)
126	{	127	{
127	unsigned long *addr = __bitops_word(nr, ptr);	128	unsigned long *p = __bitops_word(nr, addr);
128	unsigned long mask = __bitops_mask(nr);	129	unsigned long mask = __bitops_mask(nr);
129		130
130	*addr &= ~mask;	131	*p &= ~mask;
131	}	132	}
132		133
133	static inline void arch___change_bit(unsigned long nr,	134	static __always_inline void
134	volatile unsigned long *ptr)	135	arch___change_bit(unsigned long nr, volatile unsigned long *addr)
135	{	136	{
136	unsigned long *addr = __bitops_word(nr, ptr);	137	unsigned long *p = __bitops_word(nr, addr);
137	unsigned long mask = __bitops_mask(nr);	138	unsigned long mask = __bitops_mask(nr);
138		139
139	*addr ^= mask;	140	*p ^= mask;
140	}	141	}
141		142
142	static inline bool arch___test_and_set_bit(unsigned long nr,	143	static __always_inline bool
143	volatile unsigned long *ptr)	144	arch___test_and_set_bit(unsigned long nr, volatile unsigned long *addr)
144	{	145	{
145	unsigned long *addr = __bitops_word(nr, ptr);	146	unsigned long *p = __bitops_word(nr, addr);
146	unsigned long mask = __bitops_mask(nr);	147	unsigned long mask = __bitops_mask(nr);
147	unsigned long old;	148	unsigned long old;
148		149
149	old = *addr;	150	old = *p;
150	*addr \|= mask;	151	*p \|= mask;
151	return old & mask;	152	return old & mask;
152	}	153	}
153		154
154	static inline bool arch___test_and_clear_bit(unsigned long nr,	155	static __always_inline bool
155	volatile unsigned long *ptr)	156	arch___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr)
156	{	157	{
157	unsigned long *addr = __bitops_word(nr, ptr);	158	unsigned long *p = __bitops_word(nr, addr);
158	unsigned long mask = __bitops_mask(nr);	159	unsigned long mask = __bitops_mask(nr);
159	unsigned long old;	160	unsigned long old;
160		161
161	old = *addr;	162	old = *p;
162	*addr &= ~mask;	163	*p &= ~mask;
163	return old & mask;	164	return old & mask;
164	}	165	}
165		166
166	static inline bool arch___test_and_change_bit(unsigned long nr,	167	static __always_inline bool
167	volatile unsigned long *ptr)	168	arch___test_and_change_bit(unsigned long nr, volatile unsigned long *addr)
168	{	169	{
169	unsigned long *addr = __bitops_word(nr, ptr);	170	unsigned long *p = __bitops_word(nr, addr);
170	unsigned long mask = __bitops_mask(nr);	171	unsigned long mask = __bitops_mask(nr);
171	unsigned long old;	172	unsigned long old;
172		173
173	old = *addr;	174	old = *p;
174	*addr ^= mask;	175	*p ^= mask;
175	return old & mask;	176	return old & mask;
176	}	177	}
177		178
178	static inline bool arch_test_bit(unsigned long nr,	179	static __always_inline bool
179	const volatile unsigned long *ptr)	180	arch_test_bit(unsigned long nr, const volatile unsigned long *addr)
180	{	181	{
181	const volatile unsigned long *addr = __bitops_word(nr, ptr);	182	const volatile unsigned long *p = __bitops_word(nr, addr);
182	unsigned long mask = __bitops_mask(nr);	183	unsigned long mask = __bitops_mask(nr);
183		184
184	return *addr & mask;	185	return *p & mask;
185	}	186	}
186		187
187	static inline bool arch_test_and_set_bit_lock(unsigned long nr,	188	static inline bool arch_test_and_set_bit_lock(unsigned long nr,

arch/sh/include/asm/bitops-op32.h+22−12

 #ifndef __ASM_SH_BITOPS_OP32_H
 #define __ASM_SH_BITOPS_OP32_H
+#include <linux/bits.h>
 /*
  * The bit modifying instructions on SH-2A are only capable of working
  * with a 3-bit immediate, which signifies the shift position for the bit
 #define BYTE_OFFSET(nr)		((nr) % BITS_PER_BYTE)
 #endif
-static inline void __set_bit(int nr, volatile unsigned long *addr)
+static __always_inline void
+arch___set_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	if (__builtin_constant_p(nr)) {
 		__asm__ __volatile__ (
 	}
 }
-static inline void __clear_bit(int nr, volatile unsigned long *addr)
+static __always_inline void
+arch___clear_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	if (__builtin_constant_p(nr)) {
 		__asm__ __volatile__ (
 }
 /**
- * __change_bit - Toggle a bit in memory
+ * arch___change_bit - Toggle a bit in memory
  * @nr: the bit to change
  * @addr: the address to start counting from
  *
  * If it's called on the same region of memory simultaneously, the effect
  * may be that only one operation succeeds.
  */
-static inline void __change_bit(int nr, volatile unsigned long *addr)
+static __always_inline void
+arch___change_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	if (__builtin_constant_p(nr)) {
 		__asm__ __volatile__ (
 }
 /**
- * __test_and_set_bit - Set a bit and return its old value
+ * arch___test_and_set_bit - Set a bit and return its old value
  * @nr: Bit to set
  * @addr: Address to count from
  *
  * If two examples of this operation race, one can appear to succeed
  * but actually fail.  You must protect multiple accesses with a lock.
  */
-static inline int __test_and_set_bit(int nr, volatile unsigned long *addr)
+static __always_inline bool
+arch___test_and_set_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	unsigned long mask = BIT_MASK(nr);
 	unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
 }
 /**
- * __test_and_clear_bit - Clear a bit and return its old value
+ * arch___test_and_clear_bit - Clear a bit and return its old value
  * @nr: Bit to clear
  * @addr: Address to count from
  *
  * If two examples of this operation race, one can appear to succeed
  * but actually fail.  You must protect multiple accesses with a lock.
  */
-static inline int __test_and_clear_bit(int nr, volatile unsigned long *addr)
+static __always_inline bool
+arch___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	unsigned long mask = BIT_MASK(nr);
 	unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
 }
 /* WARNING: non atomic and it can be reordered! */
-static inline int __test_and_change_bit(int nr,
+static __always_inline bool
-					    volatile unsigned long *addr)
+arch___test_and_change_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	unsigned long mask = BIT_MASK(nr);
 	unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
 }
 /**
- * test_bit - Determine whether a bit is set
+ * arch_test_bit - Determine whether a bit is set
  * @nr: bit number to test
  * @addr: Address to start counting from
  */
-static inline int test_bit(int nr, const volatile unsigned long *addr)
+static __always_inline bool
+arch_test_bit(unsigned long nr, const volatile unsigned long *addr)
 {
 	return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1)));
 }
+#include <asm-generic/bitops/non-instrumented-non-atomic.h>
 #endif /* __ASM_SH_BITOPS_OP32_H */

arch/sparc/include/asm/bitops_32.h+9−9

@@ -19,9 +19,9 @@ · 18 unmodified lines
19	#error only <linux/bitops.h> can be included directly	19	#error only <linux/bitops.h> can be included directly
20	#endif	20	#endif
21		21
22	unsigned long ___set_bit(unsigned long *addr, unsigned long mask);	22	unsigned long sp32___set_bit(unsigned long *addr, unsigned long mask);
23	unsigned long ___clear_bit(unsigned long *addr, unsigned long mask);	23	unsigned long sp32___clear_bit(unsigned long *addr, unsigned long mask);
24	unsigned long ___change_bit(unsigned long *addr, unsigned long mask);	24	unsigned long sp32___change_bit(unsigned long *addr, unsigned long mask);
25		25
26	/*	26	/*
27	* Set bit 'nr' in 32-bit quantity at address 'addr' where bit '0'	27	* Set bit 'nr' in 32-bit quantity at address 'addr' where bit '0'
@@ -36,7 +36,7 @@ static inline int test_and_set_bit(unsigned long nr, volatile unsigned long *add · 8 unmodified lines
36	ADDR = ((unsigned long *) addr) + (nr >> 5);	36	ADDR = ((unsigned long *) addr) + (nr >> 5);
37	mask = 1 << (nr & 31);	37	mask = 1 << (nr & 31);
38		38
39	return ___set_bit(ADDR, mask) != 0;	39	return sp32___set_bit(ADDR, mask) != 0;
40	}	40	}
41		41
42	static inline void set_bit(unsigned long nr, volatile unsigned long *addr)	42	static inline void set_bit(unsigned long nr, volatile unsigned long *addr)
@@ -46,7 +46,7 @@ static inline void set_bit(unsigned long nr, volatile unsigned long *addr) · 3 unmodified lines
46	ADDR = ((unsigned long *) addr) + (nr >> 5);	46	ADDR = ((unsigned long *) addr) + (nr >> 5);
47	mask = 1 << (nr & 31);	47	mask = 1 << (nr & 31);
48		48
49	(void) ___set_bit(ADDR, mask);	49	(void) sp32___set_bit(ADDR, mask);
50	}	50	}
51		51
52	static inline int test_and_clear_bit(unsigned long nr, volatile unsigned long *addr)	52	static inline int test_and_clear_bit(unsigned long nr, volatile unsigned long *addr)
@@ -56,7 +56,7 @@ static inline int test_and_clear_bit(unsigned long nr, volatile unsigned long *a · 3 unmodified lines
56	ADDR = ((unsigned long *) addr) + (nr >> 5);	56	ADDR = ((unsigned long *) addr) + (nr >> 5);
57	mask = 1 << (nr & 31);	57	mask = 1 << (nr & 31);
58		58
59	return ___clear_bit(ADDR, mask) != 0;	59	return sp32___clear_bit(ADDR, mask) != 0;
60	}	60	}
61		61
62	static inline void clear_bit(unsigned long nr, volatile unsigned long *addr)	62	static inline void clear_bit(unsigned long nr, volatile unsigned long *addr)
@@ -66,7 +66,7 @@ static inline void clear_bit(unsigned long nr, volatile unsigned long *addr) · 3 unmodified lines
66	ADDR = ((unsigned long *) addr) + (nr >> 5);	66	ADDR = ((unsigned long *) addr) + (nr >> 5);
67	mask = 1 << (nr & 31);	67	mask = 1 << (nr & 31);
68		68
69	(void) ___clear_bit(ADDR, mask);	69	(void) sp32___clear_bit(ADDR, mask);
70	}	70	}
71		71
72	static inline int test_and_change_bit(unsigned long nr, volatile unsigned long *addr)	72	static inline int test_and_change_bit(unsigned long nr, volatile unsigned long *addr)
@@ -76,7 +76,7 @@ static inline int test_and_change_bit(unsigned long nr, volatile unsigned long * · 3 unmodified lines
76	ADDR = ((unsigned long *) addr) + (nr >> 5);	76	ADDR = ((unsigned long *) addr) + (nr >> 5);
77	mask = 1 << (nr & 31);	77	mask = 1 << (nr & 31);
78		78
79	return ___change_bit(ADDR, mask) != 0;	79	return sp32___change_bit(ADDR, mask) != 0;
80	}	80	}
81		81
82	static inline void change_bit(unsigned long nr, volatile unsigned long *addr)	82	static inline void change_bit(unsigned long nr, volatile unsigned long *addr)
@@ -86,7 +86,7 @@ static inline void change_bit(unsigned long nr, volatile unsigned long *addr) · 3 unmodified lines
86	ADDR = ((unsigned long *) addr) + (nr >> 5);	86	ADDR = ((unsigned long *) addr) + (nr >> 5);
87	mask = 1 << (nr & 31);	87	mask = 1 << (nr & 31);
88		88
89	(void) ___change_bit(ADDR, mask);	89	(void) sp32___change_bit(ADDR, mask);
90	}	90	}
91		91
92	#include <asm-generic/bitops/non-atomic.h>	92	#include <asm-generic/bitops/non-atomic.h>

arch/sparc/lib/atomic32.c+6−6

@@ -120,7 +120,7 @@ void arch_atomic_set(atomic_t *v, int i) · 119 unmodified lines
120	}	120	}
121	EXPORT_SYMBOL(arch_atomic_set);	121	EXPORT_SYMBOL(arch_atomic_set);
122		122
123	unsigned long ___set_bit(unsigned long *addr, unsigned long mask)	123	unsigned long sp32___set_bit(unsigned long *addr, unsigned long mask)
124	{	124	{
125	unsigned long old, flags;	125	unsigned long old, flags;
126		126
@@ -131,9 +131,9 @@ unsigned long ___set_bit(unsigned long *addr, unsigned long mask) · 4 unmodified lines
131		131
132	return old & mask;	132	return old & mask;
133	}	133	}
134	EXPORT_SYMBOL(___set_bit);	134	EXPORT_SYMBOL(sp32___set_bit);
135		135
136	unsigned long ___clear_bit(unsigned long *addr, unsigned long mask)	136	unsigned long sp32___clear_bit(unsigned long *addr, unsigned long mask)
137	{	137	{
138	unsigned long old, flags;	138	unsigned long old, flags;
139		139
@@ -144,9 +144,9 @@ unsigned long ___clear_bit(unsigned long *addr, unsigned long mask) · 4 unmodified lines
144		144
145	return old & mask;	145	return old & mask;
146	}	146	}
147	EXPORT_SYMBOL(___clear_bit);	147	EXPORT_SYMBOL(sp32___clear_bit);
148		148
149	unsigned long ___change_bit(unsigned long *addr, unsigned long mask)	149	unsigned long sp32___change_bit(unsigned long *addr, unsigned long mask)
150	{	150	{
151	unsigned long old, flags;	151	unsigned long old, flags;
152		152
@@ -157,7 +157,7 @@ unsigned long ___change_bit(unsigned long *addr, unsigned long mask) · 4 unmodified lines
157		157
158	return old & mask;	158	return old & mask;
159	}	159	}
160	EXPORT_SYMBOL(___change_bit);	160	EXPORT_SYMBOL(sp32___change_bit);
161		161
162	unsigned long __cmpxchg_u32(volatile u32 *ptr, u32 old, u32 new)	162	unsigned long __cmpxchg_u32(volatile u32 *ptr, u32 old, u32 new)
163	{	163	{

arch/x86/include/asm/bitops.h+12−10

 }
 static __always_inline void
-arch___set_bit(long nr, volatile unsigned long *addr)
+arch___set_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	asm volatile(__ASM_SIZE(bts) " %1,%0" : : ADDR, "Ir" (nr) : "memory");
 }
 }
 static __always_inline void
-arch___clear_bit(long nr, volatile unsigned long *addr)
+arch___clear_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	asm volatile(__ASM_SIZE(btr) " %1,%0" : : ADDR, "Ir" (nr) : "memory");
 }
 }
 static __always_inline void
-arch___change_bit(long nr, volatile unsigned long *addr)
+arch___change_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	asm volatile(__ASM_SIZE(btc) " %1,%0" : : ADDR, "Ir" (nr) : "memory");
 }
 }
 static __always_inline bool
-arch___test_and_set_bit(long nr, volatile unsigned long *addr)
+arch___test_and_set_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	bool oldbit;
  * this without also updating arch/x86/kernel/kvm.c
  */
 static __always_inline bool
-arch___test_and_clear_bit(long nr, volatile unsigned long *addr)
+arch___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	bool oldbit;
 }
 static __always_inline bool
-arch___test_and_change_bit(long nr, volatile unsigned long *addr)
+arch___test_and_change_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	bool oldbit;
 	return oldbit;
 }
-#define arch_test_bit(nr, addr)			\
+static __always_inline bool
-	(__builtin_constant_p((nr))		\
+arch_test_bit(unsigned long nr, const volatile unsigned long *addr)
-	 ? constant_test_bit((nr), (addr))	\
+{
-	 : variable_test_bit((nr), (addr)))
+	return __builtin_constant_p(nr) ? constant_test_bit(nr, addr) :
+					  variable_test_bit(nr, addr);
+}
 /**
  * __ffs - find first set bit in word

arch/x86/platform/olpc/olpc-xo1-sci.c+1−1

@@ -80,7 +80,7 @@ static void send_ebook_state(void) · 79 unmodified lines
80	return;	80	return;
81	}	81	}
82		82
83	if (!!test_bit(SW_TABLET_MODE, ebook_switch_idev->sw) == state)	83	if (test_bit(SW_TABLET_MODE, ebook_switch_idev->sw) == !!state)
84	return; /* Nothing new to report. */	84	return; /* Nothing new to report. */
85		85
86	input_report_switch(ebook_switch_idev, SW_TABLET_MODE, state);	86	input_report_switch(ebook_switch_idev, SW_TABLET_MODE, state);

drivers/iommu/intel/dmar.c+1−1

@@ -494,7 +494,7 @@ static int dmar_parse_one_rhsa(struct acpi_dmar_header header, void arg) · 493 unmodified lines
494	if (drhd->reg_base_addr == rhsa->base_address) {	494	if (drhd->reg_base_addr == rhsa->base_address) {
495	int node = pxm_to_node(rhsa->proximity_domain);	495	int node = pxm_to_node(rhsa->proximity_domain);
496		496
497	if (!node_online(node))	497	if (node != NUMA_NO_NODE && !node_online(node))
498	node = NUMA_NO_NODE;	498	node = NUMA_NO_NODE;
499	drhd->iommu->node = node;	499	drhd->iommu->node = node;
500	return 0;	500	return 0;

drivers/net/ethernet/intel/ice/ice_switch.c+1−1

@@ -4971,7 +4971,7 @@ ice_find_free_recp_res_idx(struct ice_hw hw, const unsigned long profiles, · 4970 unmodified lines
4971	bitmap_zero(recipes, ICE_MAX_NUM_RECIPES);	4971	bitmap_zero(recipes, ICE_MAX_NUM_RECIPES);
4972	bitmap_zero(used_idx, ICE_MAX_FV_WORDS);	4972	bitmap_zero(used_idx, ICE_MAX_FV_WORDS);
4973		4973
4974	bitmap_set(possible_idx, 0, ICE_MAX_FV_WORDS);	4974	bitmap_fill(possible_idx, ICE_MAX_FV_WORDS);
4975		4975
4976	/* For each profile we are going to associate the recipe with, add the	4976	/* For each profile we are going to associate the recipe with, add the
4977	* recipes that are associated with that profile. This will give us	4977	* recipes that are associated with that profile. This will give us

drivers/net/ethernet/mellanox/mlx4/fw.c+1−1

@@ -463,7 +463,7 @@ int mlx4_QUERY_FUNC_CAP_wrapper(struct mlx4_dev *dev, int slave, · 462 unmodified lines
463		463
464	field = min(	464	field = min(
465	bitmap_weight(actv_ports.ports, dev->caps.num_ports),	465	bitmap_weight(actv_ports.ports, dev->caps.num_ports),
466	dev->caps.num_ports);	466	(unsigned int) dev->caps.num_ports);
467	MLX4_PUT(outbox->buf, field, QUERY_FUNC_CAP_NUM_PORTS_OFFSET);	467	MLX4_PUT(outbox->buf, field, QUERY_FUNC_CAP_NUM_PORTS_OFFSET);
468		468
469	size = dev->caps.function_caps; /* set PF behaviours */	469	size = dev->caps.function_caps; /* set PF behaviours */

include/asm-generic/bitops/generic-non-atomic.h+161−0

@@ -0,0 +1,161 @@
	1	/* SPDX-License-Identifier: GPL-2.0-only */
	2
	3	#ifndef __ASM_GENERIC_BITOPS_GENERIC_NON_ATOMIC_H
	4	#define __ASM_GENERIC_BITOPS_GENERIC_NON_ATOMIC_H
	5
	6	#include <linux/bits.h>
	7
	8	#ifndef _LINUX_BITOPS_H
	9	#error only <linux/bitops.h> can be included directly
	10	#endif
	11
	12	/*
	13	* Generic definitions for bit operations, should not be used in regular code
	14	* directly.
	15	*/
	16
	17	/**
	18	* generic___set_bit - Set a bit in memory
	19	* @nr: the bit to set
	20	* @addr: the address to start counting from
	21	*
	22	* Unlike set_bit(), this function is non-atomic and may be reordered.
	23	* If it's called on the same region of memory simultaneously, the effect
	24	* may be that only one operation succeeds.
	25	*/
	26	static __always_inline void
	27	generic___set_bit(unsigned long nr, volatile unsigned long *addr)
	28	{
	29	unsigned long mask = BIT_MASK(nr);
	30	unsigned long p = ((unsigned long )addr) + BIT_WORD(nr);
	31
	32	*p \|= mask;
	33	}
	34
	35	static __always_inline void
	36	generic___clear_bit(unsigned long nr, volatile unsigned long *addr)
	37	{
	38	unsigned long mask = BIT_MASK(nr);
	39	unsigned long p = ((unsigned long )addr) + BIT_WORD(nr);
	40
	41	*p &= ~mask;
	42	}
	43
	44	/**
	45	* generic___change_bit - Toggle a bit in memory
	46	* @nr: the bit to change
	47	* @addr: the address to start counting from
	48	*
	49	* Unlike change_bit(), this function is non-atomic and may be reordered.
	50	* If it's called on the same region of memory simultaneously, the effect
	51	* may be that only one operation succeeds.
	52	*/
	53	static __always_inline void
	54	generic___change_bit(unsigned long nr, volatile unsigned long *addr)
	55	{
	56	unsigned long mask = BIT_MASK(nr);
	57	unsigned long p = ((unsigned long )addr) + BIT_WORD(nr);
	58
	59	*p ^= mask;
	60	}
	61
	62	/**
	63	* generic___test_and_set_bit - Set a bit and return its old value
	64	* @nr: Bit to set
	65	* @addr: Address to count from
	66	*
	67	* This operation is non-atomic and can be reordered.
	68	* If two examples of this operation race, one can appear to succeed
	69	* but actually fail. You must protect multiple accesses with a lock.
	70	*/
	71	static __always_inline bool
	72	generic___test_and_set_bit(unsigned long nr, volatile unsigned long *addr)
	73	{
	74	unsigned long mask = BIT_MASK(nr);
	75	unsigned long p = ((unsigned long )addr) + BIT_WORD(nr);
	76	unsigned long old = *p;
	77
	78	*p = old \| mask;
	79	return (old & mask) != 0;
	80	}
	81
	82	/**
	83	* generic___test_and_clear_bit - Clear a bit and return its old value
	84	* @nr: Bit to clear
	85	* @addr: Address to count from
	86	*
	87	* This operation is non-atomic and can be reordered.
	88	* If two examples of this operation race, one can appear to succeed
	89	* but actually fail. You must protect multiple accesses with a lock.
	90	*/
	91	static __always_inline bool
	92	generic___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr)
	93	{
	94	unsigned long mask = BIT_MASK(nr);
	95	unsigned long p = ((unsigned long )addr) + BIT_WORD(nr);
	96	unsigned long old = *p;
	97
	98	*p = old & ~mask;
	99	return (old & mask) != 0;
	100	}
	101
	102	/* WARNING: non atomic and it can be reordered! */
	103	static __always_inline bool
	104	generic___test_and_change_bit(unsigned long nr, volatile unsigned long *addr)
	105	{
	106	unsigned long mask = BIT_MASK(nr);
	107	unsigned long p = ((unsigned long )addr) + BIT_WORD(nr);
	108	unsigned long old = *p;
	109
	110	*p = old ^ mask;
	111	return (old & mask) != 0;
	112	}
	113
	114	/**
	115	* generic_test_bit - Determine whether a bit is set
	116	* @nr: bit number to test
	117	* @addr: Address to start counting from
	118	*/
	119	static __always_inline bool
	120	generic_test_bit(unsigned long nr, const volatile unsigned long *addr)
	121	{
	122	/*
	123	* Unlike the bitops with the '__' prefix above, this one is atomic,
	124	* so `volatile` must always stay here with no cast-aways. See
	125	* `Documentation/atomic_bitops.txt` for the details.
	126	*/
	127	return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1)));
	128	}
	129
	130	/*
	131	* const_*() definitions provide good compile-time optimizations when
	132	* the passed arguments can be resolved at compile time.
	133	*/
	134	#define const___set_bit generic___set_bit
	135	#define const___clear_bit generic___clear_bit
	136	#define const___change_bit generic___change_bit
	137	#define const___test_and_set_bit generic___test_and_set_bit
	138	#define const___test_and_clear_bit generic___test_and_clear_bit
	139	#define const___test_and_change_bit generic___test_and_change_bit
	140
	141	/**
	142	* const_test_bit - Determine whether a bit is set
	143	* @nr: bit number to test
	144	* @addr: Address to start counting from
	145	*
	146	* A version of generic_test_bit() which discards the `volatile` qualifier to
	147	* allow a compiler to optimize code harder. Non-atomic and to be called only
	148	* for testing compile-time constants, e.g. by the corresponding macros, not
	149	* directly from "regular" code.
	150	*/
	151	static __always_inline bool
	152	const_test_bit(unsigned long nr, const volatile unsigned long *addr)
	153	{
	154	const unsigned long p = (const unsigned long )addr + BIT_WORD(nr);
	155	unsigned long mask = BIT_MASK(nr);
	156	unsigned long val = *p;
	157
	158	return !!(val & mask);
	159	}
	160
	161	#endif /* __ASM_GENERIC_BITOPS_GENERIC_NON_ATOMIC_H */

include/asm-generic/bitops/instrumented-non-atomic.h+21−14

 #include <linux/instrumented.h>
 /**
- * __set_bit - Set a bit in memory
+ * ___set_bit - Set a bit in memory
  * @nr: the bit to set
  * @addr: the address to start counting from
  *
  * region of memory concurrently, the effect may be that only one operation
  * succeeds.
  */
-static __always_inline void __set_bit(long nr, volatile unsigned long *addr)
+static __always_inline void
+___set_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	instrument_write(addr + BIT_WORD(nr), sizeof(long));
 	arch___set_bit(nr, addr);
 }
 /**
- * __clear_bit - Clears a bit in memory
+ * ___clear_bit - Clears a bit in memory
  * @nr: the bit to clear
  * @addr: the address to start counting from
  *
  * region of memory concurrently, the effect may be that only one operation
  * succeeds.
  */
-static __always_inline void __clear_bit(long nr, volatile unsigned long *addr)
+static __always_inline void
+___clear_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	instrument_write(addr + BIT_WORD(nr), sizeof(long));
 	arch___clear_bit(nr, addr);
 }
 /**
- * __change_bit - Toggle a bit in memory
+ * ___change_bit - Toggle a bit in memory
  * @nr: the bit to change
  * @addr: the address to start counting from
  *
  * region of memory concurrently, the effect may be that only one operation
  * succeeds.
  */
-static __always_inline void __change_bit(long nr, volatile unsigned long *addr)
+static __always_inline void
+___change_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	instrument_write(addr + BIT_WORD(nr), sizeof(long));
 	arch___change_bit(nr, addr);
 }
 /**
- * __test_and_set_bit - Set a bit and return its old value
+ * ___test_and_set_bit - Set a bit and return its old value
  * @nr: Bit to set
  * @addr: Address to count from
  *
  * This operation is non-atomic. If two instances of this operation race, one
  * can appear to succeed but actually fail.
  */
-static __always_inline bool __test_and_set_bit(long nr, volatile unsigned long *addr)
+static __always_inline bool
+___test_and_set_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	__instrument_read_write_bitop(nr, addr);
 	return arch___test_and_set_bit(nr, addr);
 }
 /**
- * __test_and_clear_bit - Clear a bit and return its old value
+ * ___test_and_clear_bit - Clear a bit and return its old value
  * @nr: Bit to clear
  * @addr: Address to count from
  *
  * This operation is non-atomic. If two instances of this operation race, one
  * can appear to succeed but actually fail.
  */
-static __always_inline bool __test_and_clear_bit(long nr, volatile unsigned long *addr)
+static __always_inline bool
+___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	__instrument_read_write_bitop(nr, addr);
 	return arch___test_and_clear_bit(nr, addr);
 }
 /**
- * __test_and_change_bit - Change a bit and return its old value
+ * ___test_and_change_bit - Change a bit and return its old value
  * @nr: Bit to change
  * @addr: Address to count from
  *
  * This operation is non-atomic. If two instances of this operation race, one
  * can appear to succeed but actually fail.
  */
-static __always_inline bool __test_and_change_bit(long nr, volatile unsigned long *addr)
+static __always_inline bool
+___test_and_change_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	__instrument_read_write_bitop(nr, addr);
 	return arch___test_and_change_bit(nr, addr);
 }
 /**
- * test_bit - Determine whether a bit is set
+ * _test_bit - Determine whether a bit is set
  * @nr: bit number to test
  * @addr: Address to start counting from
  */
-static __always_inline bool test_bit(long nr, const volatile unsigned long *addr)
+static __always_inline bool
+_test_bit(unsigned long nr, const volatile unsigned long *addr)
 {
 	instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long));
 	return arch_test_bit(nr, addr);

include/asm-generic/bitops/non-atomic.h+9−112

@@ -2,121 +2,18 @@ · 1 unmodified line
2	#ifndef _ASM_GENERIC_BITOPS_NON_ATOMIC_H_	2	#ifndef _ASM_GENERIC_BITOPS_NON_ATOMIC_H_
3	#define _ASM_GENERIC_BITOPS_NON_ATOMIC_H_	3	#define _ASM_GENERIC_BITOPS_NON_ATOMIC_H_
4		4
5	#include <asm/types.h>	5	#include <asm-generic/bitops/generic-non-atomic.h>
6		6
7	/**	7	#define arch___set_bit generic___set_bit
8	* arch___set_bit - Set a bit in memory	8	#define arch___clear_bit generic___clear_bit
9	* @nr: the bit to set	9	#define arch___change_bit generic___change_bit
10	* @addr: the address to start counting from
11	*
12	* Unlike set_bit(), this function is non-atomic and may be reordered.
13	* If it's called on the same region of memory simultaneously, the effect
14	* may be that only one operation succeeds.
15	*/
16	static __always_inline void
17	arch___set_bit(unsigned int nr, volatile unsigned long *addr)
18	{
19	unsigned long mask = BIT_MASK(nr);
20	unsigned long p = ((unsigned long )addr) + BIT_WORD(nr);
21		10
22	*p \|= mask;	11	#define arch___test_and_set_bit generic___test_and_set_bit
23	}	12	#define arch___test_and_clear_bit generic___test_and_clear_bit
24	#define __set_bit arch___set_bit	13	#define arch___test_and_change_bit generic___test_and_change_bit
25		14
26	static __always_inline void	15	#define arch_test_bit generic_test_bit
27	arch___clear_bit(unsigned int nr, volatile unsigned long *addr)
28	{
29	unsigned long mask = BIT_MASK(nr);
30	unsigned long p = ((unsigned long )addr) + BIT_WORD(nr);
31		16
32	*p &= ~mask;	17	#include <asm-generic/bitops/non-instrumented-non-atomic.h>
33	}
34	#define __clear_bit arch___clear_bit
35
36	/**
37	* arch___change_bit - Toggle a bit in memory
38	* @nr: the bit to change
39	* @addr: the address to start counting from
40	*
41	* Unlike change_bit(), this function is non-atomic and may be reordered.
42	* If it's called on the same region of memory simultaneously, the effect
43	* may be that only one operation succeeds.
44	*/
45	static __always_inline
46	void arch___change_bit(unsigned int nr, volatile unsigned long *addr)
47	{
48	unsigned long mask = BIT_MASK(nr);
49	unsigned long p = ((unsigned long )addr) + BIT_WORD(nr);
50
51	*p ^= mask;
52	}
53	#define __change_bit arch___change_bit
54
55	/**
56	* arch___test_and_set_bit - Set a bit and return its old value
57	* @nr: Bit to set
58	* @addr: Address to count from
59	*
60	* This operation is non-atomic and can be reordered.
61	* If two examples of this operation race, one can appear to succeed
62	* but actually fail. You must protect multiple accesses with a lock.
63	*/
64	static __always_inline int
65	arch___test_and_set_bit(unsigned int nr, volatile unsigned long *addr)
66	{
67	unsigned long mask = BIT_MASK(nr);
68	unsigned long p = ((unsigned long )addr) + BIT_WORD(nr);
69	unsigned long old = *p;
70
71	*p = old \| mask;
72	return (old & mask) != 0;
73	}
74	#define __test_and_set_bit arch___test_and_set_bit
75
76	/**
77	* arch___test_and_clear_bit - Clear a bit and return its old value
78	* @nr: Bit to clear
79	* @addr: Address to count from
80	*
81	* This operation is non-atomic and can be reordered.
82	* If two examples of this operation race, one can appear to succeed
83	* but actually fail. You must protect multiple accesses with a lock.
84	*/
85	static __always_inline int
86	arch___test_and_clear_bit(unsigned int nr, volatile unsigned long *addr)
87	{
88	unsigned long mask = BIT_MASK(nr);
89	unsigned long p = ((unsigned long )addr) + BIT_WORD(nr);
90	unsigned long old = *p;
91
92	*p = old & ~mask;
93	return (old & mask) != 0;
94	}
95	#define __test_and_clear_bit arch___test_and_clear_bit
96
97	/* WARNING: non atomic and it can be reordered! */
98	static __always_inline int
99	arch___test_and_change_bit(unsigned int nr, volatile unsigned long *addr)
100	{
101	unsigned long mask = BIT_MASK(nr);
102	unsigned long p = ((unsigned long )addr) + BIT_WORD(nr);
103	unsigned long old = *p;
104
105	*p = old ^ mask;
106	return (old & mask) != 0;
107	}
108	#define __test_and_change_bit arch___test_and_change_bit
109
110	/**
111	* arch_test_bit - Determine whether a bit is set
112	* @nr: bit number to test
113	* @addr: Address to start counting from
114	*/
115	static __always_inline int
116	arch_test_bit(unsigned int nr, const volatile unsigned long *addr)
117	{
118	return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1)));
119	}
120	#define test_bit arch_test_bit
121		18
122	#endif /* _ASM_GENERIC_BITOPS_NON_ATOMIC_H_ */	19	#endif /* _ASM_GENERIC_BITOPS_NON_ATOMIC_H_ */

include/asm-generic/bitops/non-instrumented-non-atomic.h+16−0

@@ -0,0 +1,16 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2
	3	#ifndef __ASM_GENERIC_BITOPS_NON_INSTRUMENTED_NON_ATOMIC_H
	4	#define __ASM_GENERIC_BITOPS_NON_INSTRUMENTED_NON_ATOMIC_H
	5
	6	#define ___set_bit arch___set_bit
	7	#define ___clear_bit arch___clear_bit
	8	#define ___change_bit arch___change_bit
	9
	10	#define ___test_and_set_bit arch___test_and_set_bit
	11	#define ___test_and_clear_bit arch___test_and_clear_bit
	12	#define ___test_and_change_bit arch___test_and_change_bit
	13
	14	#define _test_bit arch_test_bit
	15
	16	#endif /* __ASM_GENERIC_BITOPS_NON_INSTRUMENTED_NON_ATOMIC_H */

include/linux/bitmap.h+27−10

@@ -71,9 +71,9 @@ struct device; · 70 unmodified lines
71	* bitmap_release_region(bitmap, pos, order) Free specified bit region	71	* bitmap_release_region(bitmap, pos, order) Free specified bit region
72	* bitmap_allocate_region(bitmap, pos, order) Allocate specified bit region	72	* bitmap_allocate_region(bitmap, pos, order) Allocate specified bit region
73	* bitmap_from_arr32(dst, buf, nbits) Copy nbits from u32[] buf to dst	73	* bitmap_from_arr32(dst, buf, nbits) Copy nbits from u32[] buf to dst
		74	* bitmap_from_arr64(dst, buf, nbits) Copy nbits from u64[] buf to dst
74	* bitmap_to_arr32(buf, src, nbits) Copy nbits from buf to u32[] dst	75	* bitmap_to_arr32(buf, src, nbits) Copy nbits from buf to u32[] dst
75	* bitmap_to_arr64(buf, src, nbits) Copy nbits from buf to u64[] dst	76	* bitmap_to_arr64(buf, src, nbits) Copy nbits from buf to u64[] dst
76	* bitmap_to_arr64(buf, src, nbits) Copy nbits from buf to u64[] dst
77	* bitmap_get_value8(map, start) Get 8bit value from map at start	77	* bitmap_get_value8(map, start) Get 8bit value from map at start
78	* bitmap_set_value8(map, value, start) Set 8bit value to map at start	78	* bitmap_set_value8(map, value, start) Set 8bit value to map at start
79	*	79	*
@@ -148,13 +148,13 @@ void __bitmap_shift_left(unsigned long dst, const unsigned long src, · 68 unmodified lines
148	unsigned int shift, unsigned int nbits);	148	unsigned int shift, unsigned int nbits);
149	void bitmap_cut(unsigned long dst, const unsigned long src,	149	void bitmap_cut(unsigned long dst, const unsigned long src,
150	unsigned int first, unsigned int cut, unsigned int nbits);	150	unsigned int first, unsigned int cut, unsigned int nbits);
151	int __bitmap_and(unsigned long dst, const unsigned long bitmap1,	151	bool __bitmap_and(unsigned long dst, const unsigned long bitmap1,
152	const unsigned long *bitmap2, unsigned int nbits);	152	const unsigned long *bitmap2, unsigned int nbits);
153	void __bitmap_or(unsigned long dst, const unsigned long bitmap1,	153	void __bitmap_or(unsigned long dst, const unsigned long bitmap1,
154	const unsigned long *bitmap2, unsigned int nbits);	154	const unsigned long *bitmap2, unsigned int nbits);
155	void __bitmap_xor(unsigned long dst, const unsigned long bitmap1,	155	void __bitmap_xor(unsigned long dst, const unsigned long bitmap1,
156	const unsigned long *bitmap2, unsigned int nbits);	156	const unsigned long *bitmap2, unsigned int nbits);
157	int __bitmap_andnot(unsigned long dst, const unsigned long bitmap1,	157	bool __bitmap_andnot(unsigned long dst, const unsigned long bitmap1,
158	const unsigned long *bitmap2, unsigned int nbits);	158	const unsigned long *bitmap2, unsigned int nbits);
159	void __bitmap_replace(unsigned long *dst,	159	void __bitmap_replace(unsigned long *dst,
160	const unsigned long old, const unsigned long new,	160	const unsigned long old, const unsigned long new,
@@ -163,7 +163,7 @@ bool __bitmap_intersects(const unsigned long *bitmap1, · 2 unmodified lines
163	const unsigned long *bitmap2, unsigned int nbits);	163	const unsigned long *bitmap2, unsigned int nbits);
164	bool __bitmap_subset(const unsigned long *bitmap1,	164	bool __bitmap_subset(const unsigned long *bitmap1,
165	const unsigned long *bitmap2, unsigned int nbits);	165	const unsigned long *bitmap2, unsigned int nbits);
166	int __bitmap_weight(const unsigned long *bitmap, unsigned int nbits);	166	unsigned int __bitmap_weight(const unsigned long *bitmap, unsigned int nbits);
167	void __bitmap_set(unsigned long *map, unsigned int start, int len);	167	void __bitmap_set(unsigned long *map, unsigned int start, int len);
168	void __bitmap_clear(unsigned long *map, unsigned int start, int len);	168	void __bitmap_clear(unsigned long *map, unsigned int start, int len);
169		169
@@ -238,20 +238,32 @@ extern int bitmap_print_list_to_buf(char buf, const unsigned long maskp, · 68 unmodified lines
238	static inline void bitmap_zero(unsigned long *dst, unsigned int nbits)	238	static inline void bitmap_zero(unsigned long *dst, unsigned int nbits)
239	{	239	{
240	unsigned int len = BITS_TO_LONGS(nbits) * sizeof(unsigned long);	240	unsigned int len = BITS_TO_LONGS(nbits) * sizeof(unsigned long);
241	memset(dst, 0, len);	241
		242	if (small_const_nbits(nbits))
		243	*dst = 0;
		244	else
		245	memset(dst, 0, len);
242	}	246	}
243		247
244	static inline void bitmap_fill(unsigned long *dst, unsigned int nbits)	248	static inline void bitmap_fill(unsigned long *dst, unsigned int nbits)
245	{	249	{
246	unsigned int len = BITS_TO_LONGS(nbits) * sizeof(unsigned long);	250	unsigned int len = BITS_TO_LONGS(nbits) * sizeof(unsigned long);
247	memset(dst, 0xff, len);	251
		252	if (small_const_nbits(nbits))
		253	*dst = ~0UL;
		254	else
		255	memset(dst, 0xff, len);
248	}	256	}
249		257
250	static inline void bitmap_copy(unsigned long dst, const unsigned long src,	258	static inline void bitmap_copy(unsigned long dst, const unsigned long src,
251	unsigned int nbits)	259	unsigned int nbits)
252	{	260	{
253	unsigned int len = BITS_TO_LONGS(nbits) * sizeof(unsigned long);	261	unsigned int len = BITS_TO_LONGS(nbits) * sizeof(unsigned long);
254	memcpy(dst, src, len);	262
		263	if (small_const_nbits(nbits))
		264	dst = src;
		265	else
		266	memcpy(dst, src, len);
255	}	267	}
256		268
257	/*	269	/*
@@ -303,7 +315,7 @@ void bitmap_to_arr64(u64 buf, const unsigned long bitmap, unsigned int nbits); · 45 unmodified lines
303	bitmap_copy_clear_tail((unsigned long )(buf), (const unsigned long )(bitmap), (nbits))	315	bitmap_copy_clear_tail((unsigned long )(buf), (const unsigned long )(bitmap), (nbits))
304	#endif	316	#endif
305		317
306	static inline int bitmap_and(unsigned long dst, const unsigned long src1,	318	static inline bool bitmap_and(unsigned long dst, const unsigned long src1,
307	const unsigned long *src2, unsigned int nbits)	319	const unsigned long *src2, unsigned int nbits)
308	{	320	{
309	if (small_const_nbits(nbits))	321	if (small_const_nbits(nbits))
@@ -329,7 +341,7 @@ static inline void bitmap_xor(unsigned long dst, const unsigned long src1, · 19 unmodified lines
329	__bitmap_xor(dst, src1, src2, nbits);	341	__bitmap_xor(dst, src1, src2, nbits);
330	}	342	}
331		343
332	static inline int bitmap_andnot(unsigned long dst, const unsigned long src1,	344	static inline bool bitmap_andnot(unsigned long dst, const unsigned long src1,
333	const unsigned long *src2, unsigned int nbits)	345	const unsigned long *src2, unsigned int nbits)
334	{	346	{
335	if (small_const_nbits(nbits))	347	if (small_const_nbits(nbits))
@@ -419,7 +431,8 @@ static inline bool bitmap_full(const unsigned long *src, unsigned int nbits) · 83 unmodified lines
419	return find_first_zero_bit(src, nbits) == nbits;	431	return find_first_zero_bit(src, nbits) == nbits;
420	}	432	}
421		433
422	static __always_inline int bitmap_weight(const unsigned long *src, unsigned int nbits)	434	static __always_inline
		435	unsigned int bitmap_weight(const unsigned long *src, unsigned int nbits)
423	{	436	{
424	if (small_const_nbits(nbits))	437	if (small_const_nbits(nbits))
425	return hweight_long(*src & BITMAP_LAST_WORD_MASK(nbits));	438	return hweight_long(*src & BITMAP_LAST_WORD_MASK(nbits));
@@ -431,6 +444,8 @@ static __always_inline void bitmap_set(unsigned long *map, unsigned int start, · 5 unmodified lines
431	{	444	{
432	if (__builtin_constant_p(nbits) && nbits == 1)	445	if (__builtin_constant_p(nbits) && nbits == 1)
433	__set_bit(start, map);	446	__set_bit(start, map);
		447	else if (small_const_nbits(start + nbits))
		448	*map \|= GENMASK(start + nbits - 1, start);
434	else if (__builtin_constant_p(start & BITMAP_MEM_MASK) &&	449	else if (__builtin_constant_p(start & BITMAP_MEM_MASK) &&
435	IS_ALIGNED(start, BITMAP_MEM_ALIGNMENT) &&	450	IS_ALIGNED(start, BITMAP_MEM_ALIGNMENT) &&
436	__builtin_constant_p(nbits & BITMAP_MEM_MASK) &&	451	__builtin_constant_p(nbits & BITMAP_MEM_MASK) &&
@@ -445,6 +460,8 @@ static __always_inline void bitmap_clear(unsigned long *map, unsigned int start, · 8 unmodified lines
445	{	460	{
446	if (__builtin_constant_p(nbits) && nbits == 1)	461	if (__builtin_constant_p(nbits) && nbits == 1)
447	__clear_bit(start, map);	462	__clear_bit(start, map);
		463	else if (small_const_nbits(start + nbits))
		464	*map &= ~GENMASK(start + nbits - 1, start);
448	else if (__builtin_constant_p(start & BITMAP_MEM_MASK) &&	465	else if (__builtin_constant_p(start & BITMAP_MEM_MASK) &&
449	IS_ALIGNED(start, BITMAP_MEM_ALIGNMENT) &&	466	IS_ALIGNED(start, BITMAP_MEM_ALIGNMENT) &&
450	__builtin_constant_p(nbits & BITMAP_MEM_MASK) &&	467	__builtin_constant_p(nbits & BITMAP_MEM_MASK) &&

include/linux/bitops.h+50−0

@@ -26,12 +26,62 @@ extern unsigned int __sw_hweight16(unsigned int w); · 25 unmodified lines
26	extern unsigned int __sw_hweight32(unsigned int w);	26	extern unsigned int __sw_hweight32(unsigned int w);
27	extern unsigned long __sw_hweight64(__u64 w);	27	extern unsigned long __sw_hweight64(__u64 w);
28		28
		29	/*
		30	* Defined here because those may be needed by architecture-specific static
		31	* inlines.
		32	*/
		33
		34	#include <asm-generic/bitops/generic-non-atomic.h>
		35
		36	/*
		37	* Many architecture-specific non-atomic bitops contain inline asm code and due
		38	* to that the compiler can't optimize them to compile-time expressions or
		39	* constants. In contrary, generic_*() helpers are defined in pure C and
		40	* compilers optimize them just well.
		41	* Therefore, to make `unsigned long foo = 0; __set_bit(BAR, &foo)` effectively
		42	* equal to `unsigned long foo = BIT(BAR)`, pick the generic C alternative when
		43	* the arguments can be resolved at compile time. That expression itself is a
		44	* constant and doesn't bring any functional changes to the rest of cases.
		45	* The casts to `uintptr_t` are needed to mitigate `-Waddress` warnings when
		46	* passing a bitmap from .bss or .data (-> `!!addr` is always true).
		47	*/
		48	#define bitop(op, nr, addr) \
		49	((__builtin_constant_p(nr) && \
		50	__builtin_constant_p((uintptr_t)(addr) != (uintptr_t)NULL) && \
		51	(uintptr_t)(addr) != (uintptr_t)NULL && \
		52	__builtin_constant_p((const unsigned long )(addr))) ? \
		53	const##op(nr, addr) : op(nr, addr))
		54
		55	#define __set_bit(nr, addr) bitop(___set_bit, nr, addr)
		56	#define __clear_bit(nr, addr) bitop(___clear_bit, nr, addr)
		57	#define __change_bit(nr, addr) bitop(___change_bit, nr, addr)
		58	#define __test_and_set_bit(nr, addr) bitop(___test_and_set_bit, nr, addr)
		59	#define __test_and_clear_bit(nr, addr) bitop(___test_and_clear_bit, nr, addr)
		60	#define __test_and_change_bit(nr, addr) bitop(___test_and_change_bit, nr, addr)
		61	#define test_bit(nr, addr) bitop(_test_bit, nr, addr)
		62
29	/*	63	/*
30	* Include this here because some architectures need generic_ffs/fls in	64	* Include this here because some architectures need generic_ffs/fls in
31	* scope	65	* scope
32	*/	66	*/
33	#include <asm/bitops.h>	67	#include <asm/bitops.h>
34		68
		69	/* Check that the bitops prototypes are sane */
		70	#define __check_bitop_pr(name) \
		71	static_assert(__same_type(arch_##name, generic_##name) && \
		72	__same_type(const_##name, generic_##name) && \
		73	__same_type(_##name, generic_##name))
		74
		75	__check_bitop_pr(__set_bit);
		76	__check_bitop_pr(__clear_bit);
		77	__check_bitop_pr(__change_bit);
		78	__check_bitop_pr(__test_and_set_bit);
		79	__check_bitop_pr(__test_and_clear_bit);
		80	__check_bitop_pr(__test_and_change_bit);
		81	__check_bitop_pr(test_bit);
		82
		83	#undef __check_bitop_pr
		84
35	static inline int get_bitmask_order(unsigned int count)	85	static inline int get_bitmask_order(unsigned int count)
36	{	86	{
37	int order;	87	int order;

include/linux/cpumask.h+100−22

@@ -12,6 +12,8 @@ · 11 unmodified lines
12	#include <linux/bitmap.h>	12	#include <linux/bitmap.h>
13	#include <linux/atomic.h>	13	#include <linux/atomic.h>
14	#include <linux/bug.h>	14	#include <linux/bug.h>
		15	#include <linux/gfp_types.h>
		16	#include <linux/numa.h>
15		17
16	/* Don't assign or return these: may not be this big! */	18	/* Don't assign or return these: may not be this big! */
17	typedef struct cpumask { DECLARE_BITMAP(bits, NR_CPUS); } cpumask_t;	19	typedef struct cpumask { DECLARE_BITMAP(bits, NR_CPUS); } cpumask_t;
@@ -162,7 +164,21 @@ static inline unsigned int cpumask_last(const struct cpumask *srcp) · 144 unmodified lines
162	return find_last_bit(cpumask_bits(srcp), nr_cpumask_bits);	164	return find_last_bit(cpumask_bits(srcp), nr_cpumask_bits);
163	}	165	}
164		166
165	unsigned int __pure cpumask_next(int n, const struct cpumask *srcp);	167	/**
		168	* cpumask_next - get the next cpu in a cpumask
		169	* @n: the cpu prior to the place to search (ie. return will be > @n)
		170	* @srcp: the cpumask pointer
		171	*
		172	* Returns >= nr_cpu_ids if no further cpus set.
		173	*/
		174	static inline
		175	unsigned int cpumask_next(int n, const struct cpumask *srcp)
		176	{
		177	/* -1 is a legal arg here. */
		178	if (n != -1)
		179	cpumask_check(n);
		180	return find_next_bit(cpumask_bits(srcp), nr_cpumask_bits, n + 1);
		181	}
166		182
167	/**	183	/**
168	* cpumask_next_zero - get the next unset cpu in a cpumask	184	* cpumask_next_zero - get the next unset cpu in a cpumask
@@ -179,9 +195,6 @@ static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp) · 10 unmodified lines
179	return find_next_zero_bit(cpumask_bits(srcp), nr_cpumask_bits, n+1);	195	return find_next_zero_bit(cpumask_bits(srcp), nr_cpumask_bits, n+1);
180	}	196	}
181		197
182	int __pure cpumask_next_and(int n, const struct cpumask , const struct cpumask );
183	int __pure cpumask_any_but(const struct cpumask *mask, unsigned int cpu);
184
185	#if NR_CPUS == 1	198	#if NR_CPUS == 1
186	/* Uniprocessor: there is only one valid CPU */	199	/* Uniprocessor: there is only one valid CPU */
187	static inline unsigned int cpumask_local_spread(unsigned int i, int node)	200	static inline unsigned int cpumask_local_spread(unsigned int i, int node)
@@ -200,11 +213,30 @@ static inline int cpumask_any_distribute(const struct cpumask *srcp) · 12 unmodified lines
200	}	213	}
201	#else	214	#else
202	unsigned int cpumask_local_spread(unsigned int i, int node);	215	unsigned int cpumask_local_spread(unsigned int i, int node);
203	int cpumask_any_and_distribute(const struct cpumask *src1p,	216	unsigned int cpumask_any_and_distribute(const struct cpumask *src1p,
204	const struct cpumask *src2p);	217	const struct cpumask *src2p);
205	int cpumask_any_distribute(const struct cpumask *srcp);	218	unsigned int cpumask_any_distribute(const struct cpumask *srcp);
206	#endif /* NR_CPUS */	219	#endif /* NR_CPUS */
207		220
		221	/**
		222	* cpumask_next_and - get the next cpu in src1p & src2p
		223	* @n: the cpu prior to the place to search (ie. return will be > @n)
		224	* @src1p: the first cpumask pointer
		225	* @src2p: the second cpumask pointer
		226	*
		227	* Returns >= nr_cpu_ids if no further cpus set in both.
		228	*/
		229	static inline
		230	unsigned int cpumask_next_and(int n, const struct cpumask *src1p,
		231	const struct cpumask *src2p)
		232	{
		233	/* -1 is a legal arg here. */
		234	if (n != -1)
		235	cpumask_check(n);
		236	return find_next_and_bit(cpumask_bits(src1p), cpumask_bits(src2p),
		237	nr_cpumask_bits, n + 1);
		238	}
		239
208	/**	240	/**
209	* for_each_cpu - iterate over every cpu in a mask	241	* for_each_cpu - iterate over every cpu in a mask
210	* @cpu: the (optionally unsigned) integer iterator	242	* @cpu: the (optionally unsigned) integer iterator
@@ -229,7 +261,7 @@ int cpumask_any_distribute(const struct cpumask *srcp); · 18 unmodified lines
229	(cpu) = cpumask_next_zero((cpu), (mask)), \	261	(cpu) = cpumask_next_zero((cpu), (mask)), \
230	(cpu) < nr_cpu_ids;)	262	(cpu) < nr_cpu_ids;)
231		263
232	int __pure cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap);	264	unsigned int __pure cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap);
233		265
234	/**	266	/**
235	* for_each_cpu_wrap - iterate over every cpu in a mask, starting at a specified location	267	* for_each_cpu_wrap - iterate over every cpu in a mask, starting at a specified location
@@ -265,6 +297,26 @@ int __pure cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool · 29 unmodified lines
265	(cpu) = cpumask_next_and((cpu), (mask1), (mask2)), \	297	(cpu) = cpumask_next_and((cpu), (mask1), (mask2)), \
266	(cpu) < nr_cpu_ids;)	298	(cpu) < nr_cpu_ids;)
267		299
		300	/**
		301	* cpumask_any_but - return a "random" in a cpumask, but not this one.
		302	* @mask: the cpumask to search
		303	* @cpu: the cpu to ignore.
		304	*
		305	* Often used to find any cpu but smp_processor_id() in a mask.
		306	* Returns >= nr_cpu_ids if no cpus set.
		307	*/
		308	static inline
		309	unsigned int cpumask_any_but(const struct cpumask *mask, unsigned int cpu)
		310	{
		311	unsigned int i;
		312
		313	cpumask_check(cpu);
		314	for_each_cpu(i, mask)
		315	if (i != cpu)
		316	break;
		317	return i;
		318	}
		319
268	#define CPU_BITS_NONE \	320	#define CPU_BITS_NONE \
269	{ \	321	{ \
270	[0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \	322	[0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \
@@ -311,9 +363,9 @@ static __always_inline void __cpumask_clear_cpu(int cpu, struct cpumask *dstp) · 40 unmodified lines
311	* @cpu: cpu number (< nr_cpu_ids)	363	* @cpu: cpu number (< nr_cpu_ids)
312	* @cpumask: the cpumask pointer	364	* @cpumask: the cpumask pointer
313	*	365	*
314	* Returns 1 if @cpu is set in @cpumask, else returns 0	366	* Returns true if @cpu is set in @cpumask, else returns false
315	*/	367	*/
316	static __always_inline int cpumask_test_cpu(int cpu, const struct cpumask *cpumask)	368	static __always_inline bool cpumask_test_cpu(int cpu, const struct cpumask *cpumask)
317	{	369	{
318	return test_bit(cpumask_check(cpu), cpumask_bits((cpumask)));	370	return test_bit(cpumask_check(cpu), cpumask_bits((cpumask)));
319	}	371	}
@@ -323,11 +375,11 @@ static __always_inline int cpumask_test_cpu(int cpu, const struct cpumask *cpuma · 3 unmodified lines
323	* @cpu: cpu number (< nr_cpu_ids)	375	* @cpu: cpu number (< nr_cpu_ids)
324	* @cpumask: the cpumask pointer	376	* @cpumask: the cpumask pointer
325	*	377	*
326	* Returns 1 if @cpu is set in old bitmap of @cpumask, else returns 0	378	* Returns true if @cpu is set in old bitmap of @cpumask, else returns false
327	*	379	*
328	* test_and_set_bit wrapper for cpumasks.	380	* test_and_set_bit wrapper for cpumasks.
329	*/	381	*/
330	static __always_inline int cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)	382	static __always_inline bool cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)
331	{	383	{
332	return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask));	384	return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask));
333	}	385	}
@@ -337,11 +389,11 @@ static __always_inline int cpumask_test_and_set_cpu(int cpu, struct cpumask *cpu · 3 unmodified lines
337	* @cpu: cpu number (< nr_cpu_ids)	389	* @cpu: cpu number (< nr_cpu_ids)
338	* @cpumask: the cpumask pointer	390	* @cpumask: the cpumask pointer
339	*	391	*
340	* Returns 1 if @cpu is set in old bitmap of @cpumask, else returns 0	392	* Returns true if @cpu is set in old bitmap of @cpumask, else returns false
341	*	393	*
342	* test_and_clear_bit wrapper for cpumasks.	394	* test_and_clear_bit wrapper for cpumasks.
343	*/	395	*/
344	static __always_inline int cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask)	396	static __always_inline bool cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask)
345	{	397	{
346	return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask));	398	return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask));
347	}	399	}
@@ -370,9 +422,9 @@ static inline void cpumask_clear(struct cpumask *dstp) · 22 unmodified lines
370	* @src1p: the first input	422	* @src1p: the first input
371	* @src2p: the second input	423	* @src2p: the second input
372	*	424	*
373	* If *@dstp is empty, returns 0, else returns 1	425	* If *@dstp is empty, returns false, else returns true
374	*/	426	*/
375	static inline int cpumask_and(struct cpumask *dstp,	427	static inline bool cpumask_and(struct cpumask *dstp,
376	const struct cpumask *src1p,	428	const struct cpumask *src1p,
377	const struct cpumask *src2p)	429	const struct cpumask *src2p)
378	{	430	{
@@ -413,9 +465,9 @@ static inline void cpumask_xor(struct cpumask *dstp, · 34 unmodified lines
413	* @src1p: the first input	465	* @src1p: the first input
414	* @src2p: the second input	466	* @src2p: the second input
415	*	467	*
416	* If *@dstp is empty, returns 0, else returns 1	468	* If *@dstp is empty, returns false, else returns true
417	*/	469	*/
418	static inline int cpumask_andnot(struct cpumask *dstp,	470	static inline bool cpumask_andnot(struct cpumask *dstp,
419	const struct cpumask *src1p,	471	const struct cpumask *src1p,
420	const struct cpumask *src2p)	472	const struct cpumask *src2p)
421	{	473	{
@@ -478,9 +530,9 @@ static inline bool cpumask_intersects(const struct cpumask *src1p, · 56 unmodified lines
478	* @src1p: the first input	530	* @src1p: the first input
479	* @src2p: the second input	531	* @src2p: the second input
480	*	532	*
481	* Returns 1 if @src1p is a subset of @src2p, else returns 0	533	* Returns true if @src1p is a subset of @src2p, else returns false
482	*/	534	*/
483	static inline int cpumask_subset(const struct cpumask *src1p,	535	static inline bool cpumask_subset(const struct cpumask *src1p,
484	const struct cpumask *src2p)	536	const struct cpumask *src2p)
485	{	537	{
486	return bitmap_subset(cpumask_bits(src1p), cpumask_bits(src2p),	538	return bitmap_subset(cpumask_bits(src1p), cpumask_bits(src2p),
@@ -682,9 +734,35 @@ typedef struct cpumask *cpumask_var_t; · 195 unmodified lines
682	#define __cpumask_var_read_mostly __read_mostly	734	#define __cpumask_var_read_mostly __read_mostly
683		735
684	bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);	736	bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
685	bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags);	737
686	bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);	738	static inline
687	bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags);	739	bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node)
		740	{
		741	return alloc_cpumask_var_node(mask, flags \| __GFP_ZERO, node);
		742	}
		743
		744	/**
		745	* alloc_cpumask_var - allocate a struct cpumask
		746	* @mask: pointer to cpumask_var_t where the cpumask is returned
		747	* @flags: GFP_ flags
		748	*
		749	* Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is
		750	* a nop returning a constant 1 (in <linux/cpumask.h>).
		751	*
		752	* See alloc_cpumask_var_node.
		753	*/
		754	static inline
		755	bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
		756	{
		757	return alloc_cpumask_var_node(mask, flags, NUMA_NO_NODE);
		758	}
		759
		760	static inline
		761	bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
		762	{
		763	return alloc_cpumask_var(mask, flags \| __GFP_ZERO);
		764	}
		765
688	void alloc_bootmem_cpumask_var(cpumask_var_t *mask);	766	void alloc_bootmem_cpumask_var(cpumask_var_t *mask);
689	void free_cpumask_var(cpumask_var_t mask);	767	void free_cpumask_var(cpumask_var_t mask);
690	void free_bootmem_cpumask_var(cpumask_var_t mask);	768	void free_bootmem_cpumask_var(cpumask_var_t mask);

include/linux/gfp.h+2−346

@@ -2,357 +2,13 @@ · 1 unmodified line
2	#ifndef __LINUX_GFP_H	2	#ifndef __LINUX_GFP_H
3	#define __LINUX_GFP_H	3	#define __LINUX_GFP_H
4		4
5	#include <linux/mmdebug.h>	5	#include <linux/gfp_types.h>
		6
6	#include <linux/mmzone.h>	7	#include <linux/mmzone.h>
7	#include <linux/stddef.h>
8	#include <linux/linkage.h>
9	#include <linux/topology.h>	8	#include <linux/topology.h>
10		9
11	/* The typedef is in types.h but we want the documentation here */
12	#if 0
13	/**
14	* typedef gfp_t - Memory allocation flags.
15	*
16	* GFP flags are commonly used throughout Linux to indicate how memory
17	* should be allocated. The GFP acronym stands for get_free_pages(),
18	* the underlying memory allocation function. Not every GFP flag is
19	* supported by every function which may allocate memory. Most users
20	* will want to use a plain ``GFP_KERNEL``.
21	*/
22	typedef unsigned int __bitwise gfp_t;
23	#endif
24
25	struct vm_area_struct;	10	struct vm_area_struct;
26		11
27	/*
28	* In case of changes, please don't forget to update
29	* include/trace/events/mmflags.h and tools/perf/builtin-kmem.c
30	*/
31
32	/* Plain integer GFP bitmasks. Do not use this directly. */
33	#define ___GFP_DMA 0x01u
34	#define ___GFP_HIGHMEM 0x02u
35	#define ___GFP_DMA32 0x04u
36	#define ___GFP_MOVABLE 0x08u
37	#define ___GFP_RECLAIMABLE 0x10u
38	#define ___GFP_HIGH 0x20u
39	#define ___GFP_IO 0x40u
40	#define ___GFP_FS 0x80u
41	#define ___GFP_ZERO 0x100u
42	#define ___GFP_ATOMIC 0x200u
43	#define ___GFP_DIRECT_RECLAIM 0x400u
44	#define ___GFP_KSWAPD_RECLAIM 0x800u
45	#define ___GFP_WRITE 0x1000u
46	#define ___GFP_NOWARN 0x2000u
47	#define ___GFP_RETRY_MAYFAIL 0x4000u
48	#define ___GFP_NOFAIL 0x8000u
49	#define ___GFP_NORETRY 0x10000u
50	#define ___GFP_MEMALLOC 0x20000u
51	#define ___GFP_COMP 0x40000u
52	#define ___GFP_NOMEMALLOC 0x80000u
53	#define ___GFP_HARDWALL 0x100000u
54	#define ___GFP_THISNODE 0x200000u
55	#define ___GFP_ACCOUNT 0x400000u
56	#define ___GFP_ZEROTAGS 0x800000u
57	#ifdef CONFIG_KASAN_HW_TAGS
58	#define ___GFP_SKIP_ZERO 0x1000000u
59	#define ___GFP_SKIP_KASAN_UNPOISON 0x2000000u
60	#define ___GFP_SKIP_KASAN_POISON 0x4000000u
61	#else
62	#define ___GFP_SKIP_ZERO 0
63	#define ___GFP_SKIP_KASAN_UNPOISON 0
64	#define ___GFP_SKIP_KASAN_POISON 0
65	#endif
66	#ifdef CONFIG_LOCKDEP
67	#define ___GFP_NOLOCKDEP 0x8000000u
68	#else
69	#define ___GFP_NOLOCKDEP 0
70	#endif
71	/* If the above are modified, __GFP_BITS_SHIFT may need updating */
72
73	/*
74	* Physical address zone modifiers (see linux/mmzone.h - low four bits)
75	*
76	* Do not put any conditional on these. If necessary modify the definitions
77	* without the underscores and use them consistently. The definitions here may
78	* be used in bit comparisons.
79	*/
80	#define __GFP_DMA ((__force gfp_t)___GFP_DMA)
81	#define __GFP_HIGHMEM ((__force gfp_t)___GFP_HIGHMEM)
82	#define __GFP_DMA32 ((__force gfp_t)___GFP_DMA32)
83	#define __GFP_MOVABLE ((__force gfp_t)___GFP_MOVABLE) /* ZONE_MOVABLE allowed */
84	#define GFP_ZONEMASK (__GFP_DMA\|__GFP_HIGHMEM\|__GFP_DMA32\|__GFP_MOVABLE)
85
86	/**
87	* DOC: Page mobility and placement hints
88	*
89	* Page mobility and placement hints
90	* ---------------------------------
91	*
92	* These flags provide hints about how mobile the page is. Pages with similar
93	* mobility are placed within the same pageblocks to minimise problems due
94	* to external fragmentation.
95	*
96	* %__GFP_MOVABLE (also a zone modifier) indicates that the page can be
97	* moved by page migration during memory compaction or can be reclaimed.
98	*
99	* %__GFP_RECLAIMABLE is used for slab allocations that specify
100	* SLAB_RECLAIM_ACCOUNT and whose pages can be freed via shrinkers.
101	*
102	* %__GFP_WRITE indicates the caller intends to dirty the page. Where possible,
103	* these pages will be spread between local zones to avoid all the dirty
104	* pages being in one zone (fair zone allocation policy).
105	*
106	* %__GFP_HARDWALL enforces the cpuset memory allocation policy.
107	*
108	* %__GFP_THISNODE forces the allocation to be satisfied from the requested
109	* node with no fallbacks or placement policy enforcements.
110	*
111	* %__GFP_ACCOUNT causes the allocation to be accounted to kmemcg.
112	*/
113	#define __GFP_RECLAIMABLE ((__force gfp_t)___GFP_RECLAIMABLE)
114	#define __GFP_WRITE ((__force gfp_t)___GFP_WRITE)
115	#define __GFP_HARDWALL ((__force gfp_t)___GFP_HARDWALL)
116	#define __GFP_THISNODE ((__force gfp_t)___GFP_THISNODE)
117	#define __GFP_ACCOUNT ((__force gfp_t)___GFP_ACCOUNT)
118
119	/**
120	* DOC: Watermark modifiers
121	*
122	* Watermark modifiers -- controls access to emergency reserves
123	* ------------------------------------------------------------
124	*
125	* %__GFP_HIGH indicates that the caller is high-priority and that granting
126	* the request is necessary before the system can make forward progress.
127	* For example, creating an IO context to clean pages.
128	*
129	* %__GFP_ATOMIC indicates that the caller cannot reclaim or sleep and is
130	* high priority. Users are typically interrupt handlers. This may be
131	* used in conjunction with %__GFP_HIGH
132	*
133	* %__GFP_MEMALLOC allows access to all memory. This should only be used when
134	* the caller guarantees the allocation will allow more memory to be freed
135	* very shortly e.g. process exiting or swapping. Users either should
136	* be the MM or co-ordinating closely with the VM (e.g. swap over NFS).
137	* Users of this flag have to be extremely careful to not deplete the reserve
138	* completely and implement a throttling mechanism which controls the
139	* consumption of the reserve based on the amount of freed memory.
140	* Usage of a pre-allocated pool (e.g. mempool) should be always considered
141	* before using this flag.
142	*
143	* %__GFP_NOMEMALLOC is used to explicitly forbid access to emergency reserves.
144	* This takes precedence over the %__GFP_MEMALLOC flag if both are set.
145	*/
146	#define __GFP_ATOMIC ((__force gfp_t)___GFP_ATOMIC)
147	#define __GFP_HIGH ((__force gfp_t)___GFP_HIGH)
148	#define __GFP_MEMALLOC ((__force gfp_t)___GFP_MEMALLOC)
149	#define __GFP_NOMEMALLOC ((__force gfp_t)___GFP_NOMEMALLOC)
150
151	/**
152	* DOC: Reclaim modifiers
153	*
154	* Reclaim modifiers
155	* -----------------
156	* Please note that all the following flags are only applicable to sleepable
157	* allocations (e.g. %GFP_NOWAIT and %GFP_ATOMIC will ignore them).
158	*
159	* %__GFP_IO can start physical IO.
160	*
161	* %__GFP_FS can call down to the low-level FS. Clearing the flag avoids the
162	* allocator recursing into the filesystem which might already be holding
163	* locks.
164	*
165	* %__GFP_DIRECT_RECLAIM indicates that the caller may enter direct reclaim.
166	* This flag can be cleared to avoid unnecessary delays when a fallback
167	* option is available.
168	*
169	* %__GFP_KSWAPD_RECLAIM indicates that the caller wants to wake kswapd when
170	* the low watermark is reached and have it reclaim pages until the high
171	* watermark is reached. A caller may wish to clear this flag when fallback
172	* options are available and the reclaim is likely to disrupt the system. The
173	* canonical example is THP allocation where a fallback is cheap but
174	* reclaim/compaction may cause indirect stalls.
175	*
176	* %__GFP_RECLAIM is shorthand to allow/forbid both direct and kswapd reclaim.
177	*
178	* The default allocator behavior depends on the request size. We have a concept
179	* of so called costly allocations (with order > %PAGE_ALLOC_COSTLY_ORDER).
180	* !costly allocations are too essential to fail so they are implicitly
181	* non-failing by default (with some exceptions like OOM victims might fail so
182	* the caller still has to check for failures) while costly requests try to be
183	* not disruptive and back off even without invoking the OOM killer.
184	* The following three modifiers might be used to override some of these
185	* implicit rules
186	*
187	* %__GFP_NORETRY: The VM implementation will try only very lightweight
188	* memory direct reclaim to get some memory under memory pressure (thus
189	* it can sleep). It will avoid disruptive actions like OOM killer. The
190	* caller must handle the failure which is quite likely to happen under
191	* heavy memory pressure. The flag is suitable when failure can easily be
192	* handled at small cost, such as reduced throughput
193	*
194	* %__GFP_RETRY_MAYFAIL: The VM implementation will retry memory reclaim
195	* procedures that have previously failed if there is some indication
196	* that progress has been made else where. It can wait for other
197	* tasks to attempt high level approaches to freeing memory such as
198	* compaction (which removes fragmentation) and page-out.
199	* There is still a definite limit to the number of retries, but it is
200	* a larger limit than with %__GFP_NORETRY.
201	* Allocations with this flag may fail, but only when there is
202	* genuinely little unused memory. While these allocations do not
203	* directly trigger the OOM killer, their failure indicates that
204	* the system is likely to need to use the OOM killer soon. The
205	* caller must handle failure, but can reasonably do so by failing
206	* a higher-level request, or completing it only in a much less
207	* efficient manner.
208	* If the allocation does fail, and the caller is in a position to
209	* free some non-essential memory, doing so could benefit the system
210	* as a whole.
211	*
212	* %__GFP_NOFAIL: The VM implementation _must_ retry infinitely: the caller
213	* cannot handle allocation failures. The allocation could block
214	* indefinitely but will never return with failure. Testing for
215	* failure is pointless.
216	* New users should be evaluated carefully (and the flag should be
217	* used only when there is no reasonable failure policy) but it is
218	* definitely preferable to use the flag rather than opencode endless
219	* loop around allocator.
220	* Using this flag for costly allocations is _highly_ discouraged.
221	*/
222	#define __GFP_IO ((__force gfp_t)___GFP_IO)
223	#define __GFP_FS ((__force gfp_t)___GFP_FS)
224	#define __GFP_DIRECT_RECLAIM ((__force gfp_t)___GFP_DIRECT_RECLAIM) /* Caller can reclaim */
225	#define __GFP_KSWAPD_RECLAIM ((__force gfp_t)___GFP_KSWAPD_RECLAIM) /* kswapd can wake */
226	#define __GFP_RECLAIM ((__force gfp_t)(___GFP_DIRECT_RECLAIM\|___GFP_KSWAPD_RECLAIM))
227	#define __GFP_RETRY_MAYFAIL ((__force gfp_t)___GFP_RETRY_MAYFAIL)
228	#define __GFP_NOFAIL ((__force gfp_t)___GFP_NOFAIL)
229	#define __GFP_NORETRY ((__force gfp_t)___GFP_NORETRY)
230
231	/**
232	* DOC: Action modifiers
233	*
234	* Action modifiers
235	* ----------------
236	*
237	* %__GFP_NOWARN suppresses allocation failure reports.
238	*
239	* %__GFP_COMP address compound page metadata.
240	*
241	* %__GFP_ZERO returns a zeroed page on success.
242	*
243	* %__GFP_ZEROTAGS zeroes memory tags at allocation time if the memory itself
244	* is being zeroed (either via __GFP_ZERO or via init_on_alloc, provided that
245	* __GFP_SKIP_ZERO is not set). This flag is intended for optimization: setting
246	* memory tags at the same time as zeroing memory has minimal additional
247	* performace impact.
248	*
249	* %__GFP_SKIP_KASAN_UNPOISON makes KASAN skip unpoisoning on page allocation.
250	* Only effective in HW_TAGS mode.
251	*
252	* %__GFP_SKIP_KASAN_POISON makes KASAN skip poisoning on page deallocation.
253	* Typically, used for userspace pages. Only effective in HW_TAGS mode.
254	*/
255	#define __GFP_NOWARN ((__force gfp_t)___GFP_NOWARN)
256	#define __GFP_COMP ((__force gfp_t)___GFP_COMP)
257	#define __GFP_ZERO ((__force gfp_t)___GFP_ZERO)
258	#define __GFP_ZEROTAGS ((__force gfp_t)___GFP_ZEROTAGS)
259	#define __GFP_SKIP_ZERO ((__force gfp_t)___GFP_SKIP_ZERO)
260	#define __GFP_SKIP_KASAN_UNPOISON ((__force gfp_t)___GFP_SKIP_KASAN_UNPOISON)
261	#define __GFP_SKIP_KASAN_POISON ((__force gfp_t)___GFP_SKIP_KASAN_POISON)
262
263	/* Disable lockdep for GFP context tracking */
264	#define __GFP_NOLOCKDEP ((__force gfp_t)___GFP_NOLOCKDEP)
265
266	/* Room for N __GFP_FOO bits */
267	#define __GFP_BITS_SHIFT (27 + IS_ENABLED(CONFIG_LOCKDEP))
268	#define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1))
269
270	/**
271	* DOC: Useful GFP flag combinations
272	*
273	* Useful GFP flag combinations
274	* ----------------------------
275	*
276	* Useful GFP flag combinations that are commonly used. It is recommended
277	* that subsystems start with one of these combinations and then set/clear
278	* %__GFP_FOO flags as necessary.
279	*
280	* %GFP_ATOMIC users can not sleep and need the allocation to succeed. A lower
281	* watermark is applied to allow access to "atomic reserves".
282	* The current implementation doesn't support NMI and few other strict
283	* non-preemptive contexts (e.g. raw_spin_lock). The same applies to %GFP_NOWAIT.
284	*
285	* %GFP_KERNEL is typical for kernel-internal allocations. The caller requires
286	* %ZONE_NORMAL or a lower zone for direct access but can direct reclaim.
287	*
288	* %GFP_KERNEL_ACCOUNT is the same as GFP_KERNEL, except the allocation is
289	* accounted to kmemcg.
290	*
291	* %GFP_NOWAIT is for kernel allocations that should not stall for direct
292	* reclaim, start physical IO or use any filesystem callback.
293	*
294	* %GFP_NOIO will use direct reclaim to discard clean pages or slab pages
295	* that do not require the starting of any physical IO.
296	* Please try to avoid using this flag directly and instead use
297	* memalloc_noio_{save,restore} to mark the whole scope which cannot
298	* perform any IO with a short explanation why. All allocation requests
299	* will inherit GFP_NOIO implicitly.
300	*
301	* %GFP_NOFS will use direct reclaim but will not use any filesystem interfaces.
302	* Please try to avoid using this flag directly and instead use
303	* memalloc_nofs_{save,restore} to mark the whole scope which cannot/shouldn't
304	* recurse into the FS layer with a short explanation why. All allocation
305	* requests will inherit GFP_NOFS implicitly.
306	*
307	* %GFP_USER is for userspace allocations that also need to be directly
308	* accessibly by the kernel or hardware. It is typically used by hardware
309	* for buffers that are mapped to userspace (e.g. graphics) that hardware
310	* still must DMA to. cpuset limits are enforced for these allocations.
311	*
312	* %GFP_DMA exists for historical reasons and should be avoided where possible.
313	* The flags indicates that the caller requires that the lowest zone be
314	* used (%ZONE_DMA or 16M on x86-64). Ideally, this would be removed but
315	* it would require careful auditing as some users really require it and
316	* others use the flag to avoid lowmem reserves in %ZONE_DMA and treat the
317	* lowest zone as a type of emergency reserve.
318	*
319	* %GFP_DMA32 is similar to %GFP_DMA except that the caller requires a 32-bit
320	* address. Note that kmalloc(..., GFP_DMA32) does not return DMA32 memory
321	* because the DMA32 kmalloc cache array is not implemented.
322	* (Reason: there is no such user in kernel).
323	*
324	* %GFP_HIGHUSER is for userspace allocations that may be mapped to userspace,
325	* do not need to be directly accessible by the kernel but that cannot
326	* move once in use. An example may be a hardware allocation that maps
327	* data directly into userspace but has no addressing limitations.
328	*
329	* %GFP_HIGHUSER_MOVABLE is for userspace allocations that the kernel does not
330	* need direct access to but can use kmap() when access is required. They
331	* are expected to be movable via page reclaim or page migration. Typically,
332	* pages on the LRU would also be allocated with %GFP_HIGHUSER_MOVABLE.
333	*
334	* %GFP_TRANSHUGE and %GFP_TRANSHUGE_LIGHT are used for THP allocations. They
335	* are compound allocations that will generally fail quickly if memory is not
336	* available and will not wake kswapd/kcompactd on failure. The _LIGHT
337	* version does not attempt reclaim/compaction at all and is by default used
338	* in page fault path, while the non-light is used by khugepaged.
339	*/
340	#define GFP_ATOMIC (__GFP_HIGH\|__GFP_ATOMIC\|__GFP_KSWAPD_RECLAIM)
341	#define GFP_KERNEL (__GFP_RECLAIM \| __GFP_IO \| __GFP_FS)
342	#define GFP_KERNEL_ACCOUNT (GFP_KERNEL \| __GFP_ACCOUNT)
343	#define GFP_NOWAIT (__GFP_KSWAPD_RECLAIM)
344	#define GFP_NOIO (__GFP_RECLAIM)
345	#define GFP_NOFS (__GFP_RECLAIM \| __GFP_IO)
346	#define GFP_USER (__GFP_RECLAIM \| __GFP_IO \| __GFP_FS \| __GFP_HARDWALL)
347	#define GFP_DMA __GFP_DMA
348	#define GFP_DMA32 __GFP_DMA32
349	#define GFP_HIGHUSER (GFP_USER \| __GFP_HIGHMEM)
350	#define GFP_HIGHUSER_MOVABLE (GFP_HIGHUSER \| __GFP_MOVABLE \| \
351	__GFP_SKIP_KASAN_POISON \| __GFP_SKIP_KASAN_UNPOISON)
352	#define GFP_TRANSHUGE_LIGHT ((GFP_HIGHUSER_MOVABLE \| __GFP_COMP \| \
353	__GFP_NOMEMALLOC \| __GFP_NOWARN) & ~__GFP_RECLAIM)
354	#define GFP_TRANSHUGE (GFP_TRANSHUGE_LIGHT \| __GFP_DIRECT_RECLAIM)
355
356	/* Convert GFP flags to their corresponding migrate type */	12	/* Convert GFP flags to their corresponding migrate type */
357	#define GFP_MOVABLE_MASK (__GFP_RECLAIMABLE\|__GFP_MOVABLE)	13	#define GFP_MOVABLE_MASK (__GFP_RECLAIMABLE\|__GFP_MOVABLE)
358	#define GFP_MOVABLE_SHIFT 3	14	#define GFP_MOVABLE_SHIFT 3

include/linux/gfp_types.h+348−0

@@ -0,0 +1,348 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	#ifndef __LINUX_GFP_TYPES_H
	3	#define __LINUX_GFP_TYPES_H
	4
	5	/* The typedef is in types.h but we want the documentation here */
	6	#if 0
	7	/**
	8	* typedef gfp_t - Memory allocation flags.
	9	*
	10	* GFP flags are commonly used throughout Linux to indicate how memory
	11	* should be allocated. The GFP acronym stands for get_free_pages(),
	12	* the underlying memory allocation function. Not every GFP flag is
	13	* supported by every function which may allocate memory. Most users
	14	* will want to use a plain ``GFP_KERNEL``.
	15	*/
	16	typedef unsigned int __bitwise gfp_t;
	17	#endif
	18
	19	/*
	20	* In case of changes, please don't forget to update
	21	* include/trace/events/mmflags.h and tools/perf/builtin-kmem.c
	22	*/
	23
	24	/* Plain integer GFP bitmasks. Do not use this directly. */
	25	#define ___GFP_DMA 0x01u
	26	#define ___GFP_HIGHMEM 0x02u
	27	#define ___GFP_DMA32 0x04u
	28	#define ___GFP_MOVABLE 0x08u
	29	#define ___GFP_RECLAIMABLE 0x10u
	30	#define ___GFP_HIGH 0x20u
	31	#define ___GFP_IO 0x40u
	32	#define ___GFP_FS 0x80u
	33	#define ___GFP_ZERO 0x100u
	34	#define ___GFP_ATOMIC 0x200u
	35	#define ___GFP_DIRECT_RECLAIM 0x400u
	36	#define ___GFP_KSWAPD_RECLAIM 0x800u
	37	#define ___GFP_WRITE 0x1000u
	38	#define ___GFP_NOWARN 0x2000u
	39	#define ___GFP_RETRY_MAYFAIL 0x4000u
	40	#define ___GFP_NOFAIL 0x8000u
	41	#define ___GFP_NORETRY 0x10000u
	42	#define ___GFP_MEMALLOC 0x20000u
	43	#define ___GFP_COMP 0x40000u
	44	#define ___GFP_NOMEMALLOC 0x80000u
	45	#define ___GFP_HARDWALL 0x100000u
	46	#define ___GFP_THISNODE 0x200000u
	47	#define ___GFP_ACCOUNT 0x400000u
	48	#define ___GFP_ZEROTAGS 0x800000u
	49	#ifdef CONFIG_KASAN_HW_TAGS
	50	#define ___GFP_SKIP_ZERO 0x1000000u
	51	#define ___GFP_SKIP_KASAN_UNPOISON 0x2000000u
	52	#define ___GFP_SKIP_KASAN_POISON 0x4000000u
	53	#else
	54	#define ___GFP_SKIP_ZERO 0
	55	#define ___GFP_SKIP_KASAN_UNPOISON 0
	56	#define ___GFP_SKIP_KASAN_POISON 0
	57	#endif
	58	#ifdef CONFIG_LOCKDEP
	59	#define ___GFP_NOLOCKDEP 0x8000000u
	60	#else
	61	#define ___GFP_NOLOCKDEP 0
	62	#endif
	63	/* If the above are modified, __GFP_BITS_SHIFT may need updating */
	64
	65	/*
	66	* Physical address zone modifiers (see linux/mmzone.h - low four bits)
	67	*
	68	* Do not put any conditional on these. If necessary modify the definitions
	69	* without the underscores and use them consistently. The definitions here may
	70	* be used in bit comparisons.
	71	*/
	72	#define __GFP_DMA ((__force gfp_t)___GFP_DMA)
	73	#define __GFP_HIGHMEM ((__force gfp_t)___GFP_HIGHMEM)
	74	#define __GFP_DMA32 ((__force gfp_t)___GFP_DMA32)
	75	#define __GFP_MOVABLE ((__force gfp_t)___GFP_MOVABLE) /* ZONE_MOVABLE allowed */
	76	#define GFP_ZONEMASK (__GFP_DMA\|__GFP_HIGHMEM\|__GFP_DMA32\|__GFP_MOVABLE)
	77
	78	/**
	79	* DOC: Page mobility and placement hints
	80	*
	81	* Page mobility and placement hints
	82	* ---------------------------------
	83	*
	84	* These flags provide hints about how mobile the page is. Pages with similar
	85	* mobility are placed within the same pageblocks to minimise problems due
	86	* to external fragmentation.
	87	*
	88	* %__GFP_MOVABLE (also a zone modifier) indicates that the page can be
	89	* moved by page migration during memory compaction or can be reclaimed.
	90	*
	91	* %__GFP_RECLAIMABLE is used for slab allocations that specify
	92	* SLAB_RECLAIM_ACCOUNT and whose pages can be freed via shrinkers.
	93	*
	94	* %__GFP_WRITE indicates the caller intends to dirty the page. Where possible,
	95	* these pages will be spread between local zones to avoid all the dirty
	96	* pages being in one zone (fair zone allocation policy).
	97	*
	98	* %__GFP_HARDWALL enforces the cpuset memory allocation policy.
	99	*
	100	* %__GFP_THISNODE forces the allocation to be satisfied from the requested
	101	* node with no fallbacks or placement policy enforcements.
	102	*
	103	* %__GFP_ACCOUNT causes the allocation to be accounted to kmemcg.
	104	*/
	105	#define __GFP_RECLAIMABLE ((__force gfp_t)___GFP_RECLAIMABLE)
	106	#define __GFP_WRITE ((__force gfp_t)___GFP_WRITE)
	107	#define __GFP_HARDWALL ((__force gfp_t)___GFP_HARDWALL)
	108	#define __GFP_THISNODE ((__force gfp_t)___GFP_THISNODE)
	109	#define __GFP_ACCOUNT ((__force gfp_t)___GFP_ACCOUNT)
	110
	111	/**
	112	* DOC: Watermark modifiers
	113	*
	114	* Watermark modifiers -- controls access to emergency reserves
	115	* ------------------------------------------------------------
	116	*
	117	* %__GFP_HIGH indicates that the caller is high-priority and that granting
	118	* the request is necessary before the system can make forward progress.
	119	* For example, creating an IO context to clean pages.
	120	*
	121	* %__GFP_ATOMIC indicates that the caller cannot reclaim or sleep and is
	122	* high priority. Users are typically interrupt handlers. This may be
	123	* used in conjunction with %__GFP_HIGH
	124	*
	125	* %__GFP_MEMALLOC allows access to all memory. This should only be used when
	126	* the caller guarantees the allocation will allow more memory to be freed
	127	* very shortly e.g. process exiting or swapping. Users either should
	128	* be the MM or co-ordinating closely with the VM (e.g. swap over NFS).
	129	* Users of this flag have to be extremely careful to not deplete the reserve
	130	* completely and implement a throttling mechanism which controls the
	131	* consumption of the reserve based on the amount of freed memory.
	132	* Usage of a pre-allocated pool (e.g. mempool) should be always considered
	133	* before using this flag.
	134	*
	135	* %__GFP_NOMEMALLOC is used to explicitly forbid access to emergency reserves.
	136	* This takes precedence over the %__GFP_MEMALLOC flag if both are set.
	137	*/
	138	#define __GFP_ATOMIC ((__force gfp_t)___GFP_ATOMIC)
	139	#define __GFP_HIGH ((__force gfp_t)___GFP_HIGH)
	140	#define __GFP_MEMALLOC ((__force gfp_t)___GFP_MEMALLOC)
	141	#define __GFP_NOMEMALLOC ((__force gfp_t)___GFP_NOMEMALLOC)
	142
	143	/**
	144	* DOC: Reclaim modifiers
	145	*
	146	* Reclaim modifiers
	147	* -----------------
	148	* Please note that all the following flags are only applicable to sleepable
	149	* allocations (e.g. %GFP_NOWAIT and %GFP_ATOMIC will ignore them).
	150	*
	151	* %__GFP_IO can start physical IO.
	152	*
	153	* %__GFP_FS can call down to the low-level FS. Clearing the flag avoids the
	154	* allocator recursing into the filesystem which might already be holding
	155	* locks.
	156	*
	157	* %__GFP_DIRECT_RECLAIM indicates that the caller may enter direct reclaim.
	158	* This flag can be cleared to avoid unnecessary delays when a fallback
	159	* option is available.
	160	*
	161	* %__GFP_KSWAPD_RECLAIM indicates that the caller wants to wake kswapd when
	162	* the low watermark is reached and have it reclaim pages until the high
	163	* watermark is reached. A caller may wish to clear this flag when fallback
	164	* options are available and the reclaim is likely to disrupt the system. The
	165	* canonical example is THP allocation where a fallback is cheap but
	166	* reclaim/compaction may cause indirect stalls.
	167	*
	168	* %__GFP_RECLAIM is shorthand to allow/forbid both direct and kswapd reclaim.
	169	*
	170	* The default allocator behavior depends on the request size. We have a concept
	171	* of so called costly allocations (with order > %PAGE_ALLOC_COSTLY_ORDER).
	172	* !costly allocations are too essential to fail so they are implicitly
	173	* non-failing by default (with some exceptions like OOM victims might fail so
	174	* the caller still has to check for failures) while costly requests try to be
	175	* not disruptive and back off even without invoking the OOM killer.
	176	* The following three modifiers might be used to override some of these
	177	* implicit rules
	178	*
	179	* %__GFP_NORETRY: The VM implementation will try only very lightweight
	180	* memory direct reclaim to get some memory under memory pressure (thus
	181	* it can sleep). It will avoid disruptive actions like OOM killer. The
	182	* caller must handle the failure which is quite likely to happen under
	183	* heavy memory pressure. The flag is suitable when failure can easily be
	184	* handled at small cost, such as reduced throughput
	185	*
	186	* %__GFP_RETRY_MAYFAIL: The VM implementation will retry memory reclaim
	187	* procedures that have previously failed if there is some indication
	188	* that progress has been made else where. It can wait for other
	189	* tasks to attempt high level approaches to freeing memory such as
	190	* compaction (which removes fragmentation) and page-out.
	191	* There is still a definite limit to the number of retries, but it is
	192	* a larger limit than with %__GFP_NORETRY.
	193	* Allocations with this flag may fail, but only when there is
	194	* genuinely little unused memory. While these allocations do not
	195	* directly trigger the OOM killer, their failure indicates that
	196	* the system is likely to need to use the OOM killer soon. The
	197	* caller must handle failure, but can reasonably do so by failing
	198	* a higher-level request, or completing it only in a much less
	199	* efficient manner.
	200	* If the allocation does fail, and the caller is in a position to
	201	* free some non-essential memory, doing so could benefit the system
	202	* as a whole.
	203	*
	204	* %__GFP_NOFAIL: The VM implementation _must_ retry infinitely: the caller
	205	* cannot handle allocation failures. The allocation could block
	206	* indefinitely but will never return with failure. Testing for
	207	* failure is pointless.
	208	* New users should be evaluated carefully (and the flag should be
	209	* used only when there is no reasonable failure policy) but it is
	210	* definitely preferable to use the flag rather than opencode endless
	211	* loop around allocator.
	212	* Using this flag for costly allocations is _highly_ discouraged.
	213	*/
	214	#define __GFP_IO ((__force gfp_t)___GFP_IO)
	215	#define __GFP_FS ((__force gfp_t)___GFP_FS)
	216	#define __GFP_DIRECT_RECLAIM ((__force gfp_t)___GFP_DIRECT_RECLAIM) /* Caller can reclaim */
	217	#define __GFP_KSWAPD_RECLAIM ((__force gfp_t)___GFP_KSWAPD_RECLAIM) /* kswapd can wake */
	218	#define __GFP_RECLAIM ((__force gfp_t)(___GFP_DIRECT_RECLAIM\|___GFP_KSWAPD_RECLAIM))
	219	#define __GFP_RETRY_MAYFAIL ((__force gfp_t)___GFP_RETRY_MAYFAIL)
	220	#define __GFP_NOFAIL ((__force gfp_t)___GFP_NOFAIL)
	221	#define __GFP_NORETRY ((__force gfp_t)___GFP_NORETRY)
	222
	223	/**
	224	* DOC: Action modifiers
	225	*
	226	* Action modifiers
	227	* ----------------
	228	*
	229	* %__GFP_NOWARN suppresses allocation failure reports.
	230	*
	231	* %__GFP_COMP address compound page metadata.
	232	*
	233	* %__GFP_ZERO returns a zeroed page on success.
	234	*
	235	* %__GFP_ZEROTAGS zeroes memory tags at allocation time if the memory itself
	236	* is being zeroed (either via __GFP_ZERO or via init_on_alloc, provided that
	237	* __GFP_SKIP_ZERO is not set). This flag is intended for optimization: setting
	238	* memory tags at the same time as zeroing memory has minimal additional
	239	* performace impact.
	240	*
	241	* %__GFP_SKIP_KASAN_UNPOISON makes KASAN skip unpoisoning on page allocation.
	242	* Only effective in HW_TAGS mode.
	243	*
	244	* %__GFP_SKIP_KASAN_POISON makes KASAN skip poisoning on page deallocation.
	245	* Typically, used for userspace pages. Only effective in HW_TAGS mode.
	246	*/
	247	#define __GFP_NOWARN ((__force gfp_t)___GFP_NOWARN)
	248	#define __GFP_COMP ((__force gfp_t)___GFP_COMP)
	249	#define __GFP_ZERO ((__force gfp_t)___GFP_ZERO)
	250	#define __GFP_ZEROTAGS ((__force gfp_t)___GFP_ZEROTAGS)
	251	#define __GFP_SKIP_ZERO ((__force gfp_t)___GFP_SKIP_ZERO)
	252	#define __GFP_SKIP_KASAN_UNPOISON ((__force gfp_t)___GFP_SKIP_KASAN_UNPOISON)
	253	#define __GFP_SKIP_KASAN_POISON ((__force gfp_t)___GFP_SKIP_KASAN_POISON)
	254
	255	/* Disable lockdep for GFP context tracking */
	256	#define __GFP_NOLOCKDEP ((__force gfp_t)___GFP_NOLOCKDEP)
	257
	258	/* Room for N __GFP_FOO bits */
	259	#define __GFP_BITS_SHIFT (27 + IS_ENABLED(CONFIG_LOCKDEP))
	260	#define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1))
	261
	262	/**
	263	* DOC: Useful GFP flag combinations
	264	*
	265	* Useful GFP flag combinations
	266	* ----------------------------
	267	*
	268	* Useful GFP flag combinations that are commonly used. It is recommended
	269	* that subsystems start with one of these combinations and then set/clear
	270	* %__GFP_FOO flags as necessary.
	271	*
	272	* %GFP_ATOMIC users can not sleep and need the allocation to succeed. A lower
	273	* watermark is applied to allow access to "atomic reserves".
	274	* The current implementation doesn't support NMI and few other strict
	275	* non-preemptive contexts (e.g. raw_spin_lock). The same applies to %GFP_NOWAIT.
	276	*
	277	* %GFP_KERNEL is typical for kernel-internal allocations. The caller requires
	278	* %ZONE_NORMAL or a lower zone for direct access but can direct reclaim.
	279	*
	280	* %GFP_KERNEL_ACCOUNT is the same as GFP_KERNEL, except the allocation is
	281	* accounted to kmemcg.
	282	*
	283	* %GFP_NOWAIT is for kernel allocations that should not stall for direct
	284	* reclaim, start physical IO or use any filesystem callback.
	285	*
	286	* %GFP_NOIO will use direct reclaim to discard clean pages or slab pages
	287	* that do not require the starting of any physical IO.
	288	* Please try to avoid using this flag directly and instead use
	289	* memalloc_noio_{save,restore} to mark the whole scope which cannot
	290	* perform any IO with a short explanation why. All allocation requests
	291	* will inherit GFP_NOIO implicitly.
	292	*
	293	* %GFP_NOFS will use direct reclaim but will not use any filesystem interfaces.
	294	* Please try to avoid using this flag directly and instead use
	295	* memalloc_nofs_{save,restore} to mark the whole scope which cannot/shouldn't
	296	* recurse into the FS layer with a short explanation why. All allocation
	297	* requests will inherit GFP_NOFS implicitly.
	298	*
	299	* %GFP_USER is for userspace allocations that also need to be directly
	300	* accessibly by the kernel or hardware. It is typically used by hardware
	301	* for buffers that are mapped to userspace (e.g. graphics) that hardware
	302	* still must DMA to. cpuset limits are enforced for these allocations.
	303	*
	304	* %GFP_DMA exists for historical reasons and should be avoided where possible.
	305	* The flags indicates that the caller requires that the lowest zone be
	306	* used (%ZONE_DMA or 16M on x86-64). Ideally, this would be removed but
	307	* it would require careful auditing as some users really require it and
	308	* others use the flag to avoid lowmem reserves in %ZONE_DMA and treat the
	309	* lowest zone as a type of emergency reserve.
	310	*
	311	* %GFP_DMA32 is similar to %GFP_DMA except that the caller requires a 32-bit
	312	* address. Note that kmalloc(..., GFP_DMA32) does not return DMA32 memory
	313	* because the DMA32 kmalloc cache array is not implemented.
	314	* (Reason: there is no such user in kernel).
	315	*
	316	* %GFP_HIGHUSER is for userspace allocations that may be mapped to userspace,
	317	* do not need to be directly accessible by the kernel but that cannot
	318	* move once in use. An example may be a hardware allocation that maps
	319	* data directly into userspace but has no addressing limitations.
	320	*
	321	* %GFP_HIGHUSER_MOVABLE is for userspace allocations that the kernel does not
	322	* need direct access to but can use kmap() when access is required. They
	323	* are expected to be movable via page reclaim or page migration. Typically,
	324	* pages on the LRU would also be allocated with %GFP_HIGHUSER_MOVABLE.
	325	*
	326	* %GFP_TRANSHUGE and %GFP_TRANSHUGE_LIGHT are used for THP allocations. They
	327	* are compound allocations that will generally fail quickly if memory is not
	328	* available and will not wake kswapd/kcompactd on failure. The _LIGHT
	329	* version does not attempt reclaim/compaction at all and is by default used
	330	* in page fault path, while the non-light is used by khugepaged.
	331	*/
	332	#define GFP_ATOMIC (__GFP_HIGH\|__GFP_ATOMIC\|__GFP_KSWAPD_RECLAIM)
	333	#define GFP_KERNEL (__GFP_RECLAIM \| __GFP_IO \| __GFP_FS)
	334	#define GFP_KERNEL_ACCOUNT (GFP_KERNEL \| __GFP_ACCOUNT)
	335	#define GFP_NOWAIT (__GFP_KSWAPD_RECLAIM)
	336	#define GFP_NOIO (__GFP_RECLAIM)
	337	#define GFP_NOFS (__GFP_RECLAIM \| __GFP_IO)
	338	#define GFP_USER (__GFP_RECLAIM \| __GFP_IO \| __GFP_FS \| __GFP_HARDWALL)
	339	#define GFP_DMA __GFP_DMA
	340	#define GFP_DMA32 __GFP_DMA32
	341	#define GFP_HIGHUSER (GFP_USER \| __GFP_HIGHMEM)
	342	#define GFP_HIGHUSER_MOVABLE (GFP_HIGHUSER \| __GFP_MOVABLE \| \
	343	__GFP_SKIP_KASAN_POISON \| __GFP_SKIP_KASAN_UNPOISON)
	344	#define GFP_TRANSHUGE_LIGHT ((GFP_HIGHUSER_MOVABLE \| __GFP_COMP \| \
	345	__GFP_NOMEMALLOC \| __GFP_NOWARN) & ~__GFP_RECLAIM)
	346	#define GFP_TRANSHUGE (GFP_TRANSHUGE_LIGHT \| __GFP_DIRECT_RECLAIM)
	347
	348	#endif /* __LINUX_GFP_TYPES_H */

include/linux/nodemask.h+19−5

 #include <linux/bitmap.h>
 #include <linux/minmax.h>
 #include <linux/numa.h>
+#include <linux/random.h>
 typedef struct { DECLARE_BITMAP(bits, MAX_NUMNODES); } nodemask_t;
 extern nodemask_t _unused_nodemask_arg_;
  * the first node in src if needed.  Returns MAX_NUMNODES if src is empty.
  */
 #define next_node_in(n, src) __next_node_in((n), &(src))
-unsigned int __next_node_in(int node, const nodemask_t *srcp);
+static inline unsigned int __next_node_in(int node, const nodemask_t *srcp)
+{
+	unsigned int ret = __next_node(node, srcp);
+	if (ret == MAX_NUMNODES)
+		ret = __first_node(srcp);
+	return ret;
+}
 static inline void init_nodemask_of_node(nodemask_t *mask, int node)
 {
 #endif
+static inline int node_random(const nodemask_t *maskp)
+{
 #if defined(CONFIG_NUMA) && (MAX_NUMNODES > 1)
-extern int node_random(const nodemask_t *maskp);
+	int w, bit = NUMA_NO_NODE;
+	w = nodes_weight(*maskp);
+	if (w)
+		bit = bitmap_ord_to_pos(maskp->bits,
+			get_random_int() % w, MAX_NUMNODES);
+	return bit;
 #else
-static inline int node_random(const nodemask_t *mask)
-{
 	return 0;
-}
 #endif
+}
 #define node_online_map 	node_states[N_ONLINE]
 #define node_possible_map 	node_states[N_POSSIBLE]

lib/Makefile+1−1

@@ -33,7 +33,7 @@ lib-y := ctype.o string.o vsprintf.o cmdline.o \ · 32 unmodified lines
33	flex_proportions.o ratelimit.o show_mem.o \	33	flex_proportions.o ratelimit.o show_mem.o \
34	is_single_threaded.o plist.o decompress.o kobject_uevent.o \	34	is_single_threaded.o plist.o decompress.o kobject_uevent.o \
35	earlycpio.o seq_buf.o siphash.o dec_and_lock.o \	35	earlycpio.o seq_buf.o siphash.o dec_and_lock.o \
36	nmi_backtrace.o nodemask.o win_minmax.o memcat_p.o \	36	nmi_backtrace.o win_minmax.o memcat_p.o \
37	buildid.o cpumask.o	37	buildid.o cpumask.o
38		38
39	lib-$(CONFIG_PRINTK) += dump_stack.o	39	lib-$(CONFIG_PRINTK) += dump_stack.o

lib/bitmap.c+5−6

 }
 EXPORT_SYMBOL(bitmap_cut);
-int __bitmap_and(unsigned long *dst, const unsigned long *bitmap1,
+bool __bitmap_and(unsigned long *dst, const unsigned long *bitmap1,
 				const unsigned long *bitmap2, unsigned int bits)
 {
 	unsigned int k;
 }
 EXPORT_SYMBOL(__bitmap_xor);
-int __bitmap_andnot(unsigned long *dst, const unsigned long *bitmap1,
+bool __bitmap_andnot(unsigned long *dst, const unsigned long *bitmap1,
 				const unsigned long *bitmap2, unsigned int bits)
 {
 	unsigned int k;
 }
 EXPORT_SYMBOL(__bitmap_subset);
-int __bitmap_weight(const unsigned long *bitmap, unsigned int bits)
+unsigned int __bitmap_weight(const unsigned long *bitmap, unsigned int bits)
 {
-	unsigned int k, lim = bits/BITS_PER_LONG;
+	unsigned int k, lim = bits/BITS_PER_LONG, w = 0;
-	int w = 0;
 	for (k = 0; k < lim; k++)
 		w += hweight_long(bitmap[k]);
 	/* Clear tail bits in the last element of array beyond nbits. */
 	if (nbits % 64)
-		buf[-1] &= GENMASK_ULL(nbits % 64, 0);
+		buf[-1] &= GENMASK_ULL((nbits - 1) % 64, 0);
 }
 EXPORT_SYMBOL(bitmap_to_arr64);
 #endif

lib/cpumask.c+7−90

@@ -7,61 +7,6 @@ · 6 unmodified lines
7	#include <linux/memblock.h>	7	#include <linux/memblock.h>
8	#include <linux/numa.h>	8	#include <linux/numa.h>
9		9
10	/**
11	* cpumask_next - get the next cpu in a cpumask
12	* @n: the cpu prior to the place to search (ie. return will be > @n)
13	* @srcp: the cpumask pointer
14	*
15	* Returns >= nr_cpu_ids if no further cpus set.
16	*/
17	unsigned int cpumask_next(int n, const struct cpumask *srcp)
18	{
19	/* -1 is a legal arg here. */
20	if (n != -1)
21	cpumask_check(n);
22	return find_next_bit(cpumask_bits(srcp), nr_cpumask_bits, n + 1);
23	}
24	EXPORT_SYMBOL(cpumask_next);
25
26	/**
27	* cpumask_next_and - get the next cpu in src1p & src2p
28	* @n: the cpu prior to the place to search (ie. return will be > @n)
29	* @src1p: the first cpumask pointer
30	* @src2p: the second cpumask pointer
31	*
32	* Returns >= nr_cpu_ids if no further cpus set in both.
33	*/
34	int cpumask_next_and(int n, const struct cpumask *src1p,
35	const struct cpumask *src2p)
36	{
37	/* -1 is a legal arg here. */
38	if (n != -1)
39	cpumask_check(n);
40	return find_next_and_bit(cpumask_bits(src1p), cpumask_bits(src2p),
41	nr_cpumask_bits, n + 1);
42	}
43	EXPORT_SYMBOL(cpumask_next_and);
44
45	/**
46	* cpumask_any_but - return a "random" in a cpumask, but not this one.
47	* @mask: the cpumask to search
48	* @cpu: the cpu to ignore.
49	*
50	* Often used to find any cpu but smp_processor_id() in a mask.
51	* Returns >= nr_cpu_ids if no cpus set.
52	*/
53	int cpumask_any_but(const struct cpumask *mask, unsigned int cpu)
54	{
55	unsigned int i;
56
57	cpumask_check(cpu);
58	for_each_cpu(i, mask)
59	if (i != cpu)
60	break;
61	return i;
62	}
63	EXPORT_SYMBOL(cpumask_any_but);
64
65	/**	10	/**
66	* cpumask_next_wrap - helper to implement for_each_cpu_wrap	11	* cpumask_next_wrap - helper to implement for_each_cpu_wrap
67	* @n: the cpu prior to the place to search	12	* @n: the cpu prior to the place to search
@@ -74,9 +19,9 @@ EXPORT_SYMBOL(cpumask_any_but); · 6 unmodified lines
74	* Note: the @wrap argument is required for the start condition when	19	* Note: the @wrap argument is required for the start condition when
75	* we cannot assume @start is set in @mask.	20	* we cannot assume @start is set in @mask.
76	*/	21	*/
77	int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap)	22	unsigned int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap)
78	{	23	{
79	int next;	24	unsigned int next;
80		25
81	again:	26	again:
82	next = cpumask_next(n, mask);	27	next = cpumask_next(n, mask);
@@ -125,34 +70,6 @@ bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node) · 42 unmodified lines
125	}	70	}
126	EXPORT_SYMBOL(alloc_cpumask_var_node);	71	EXPORT_SYMBOL(alloc_cpumask_var_node);
127		72
128	bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node)
129	{
130	return alloc_cpumask_var_node(mask, flags \| __GFP_ZERO, node);
131	}
132	EXPORT_SYMBOL(zalloc_cpumask_var_node);
133
134	/**
135	* alloc_cpumask_var - allocate a struct cpumask
136	* @mask: pointer to cpumask_var_t where the cpumask is returned
137	* @flags: GFP_ flags
138	*
139	* Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is
140	* a nop returning a constant 1 (in <linux/cpumask.h>).
141	*
142	* See alloc_cpumask_var_node.
143	*/
144	bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
145	{
146	return alloc_cpumask_var_node(mask, flags, NUMA_NO_NODE);
147	}
148	EXPORT_SYMBOL(alloc_cpumask_var);
149
150	bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
151	{
152	return alloc_cpumask_var(mask, flags \| __GFP_ZERO);
153	}
154	EXPORT_SYMBOL(zalloc_cpumask_var);
155
156	/**	73	/**
157	* alloc_bootmem_cpumask_var - allocate a struct cpumask from the bootmem arena.	74	* alloc_bootmem_cpumask_var - allocate a struct cpumask from the bootmem arena.
158	* @mask: pointer to cpumask_var_t where the cpumask is returned	75	* @mask: pointer to cpumask_var_t where the cpumask is returned
@@ -206,7 +123,7 @@ void __init free_bootmem_cpumask_var(cpumask_var_t mask) · 47 unmodified lines
206	*/	123	*/
207	unsigned int cpumask_local_spread(unsigned int i, int node)	124	unsigned int cpumask_local_spread(unsigned int i, int node)
208	{	125	{
209	int cpu;	126	unsigned int cpu;
210		127
211	/* Wrap: we always want a cpu. */	128	/* Wrap: we always want a cpu. */
212	i %= num_online_cpus();	129	i %= num_online_cpus();
@@ -244,10 +161,10 @@ static DEFINE_PER_CPU(int, distribute_cpu_mask_prev); · 31 unmodified lines
244	*	161	*
245	* Returns >= nr_cpu_ids if the intersection is empty.	162	* Returns >= nr_cpu_ids if the intersection is empty.
246	*/	163	*/
247	int cpumask_any_and_distribute(const struct cpumask *src1p,	164	unsigned int cpumask_any_and_distribute(const struct cpumask *src1p,
248	const struct cpumask *src2p)	165	const struct cpumask *src2p)
249	{	166	{
250	int next, prev;	167	unsigned int next, prev;
251		168
252	/* NOTE: our first selection will skip 0. */	169	/* NOTE: our first selection will skip 0. */
253	prev = __this_cpu_read(distribute_cpu_mask_prev);	170	prev = __this_cpu_read(distribute_cpu_mask_prev);
@@ -263,9 +180,9 @@ int cpumask_any_and_distribute(const struct cpumask *src1p, · 9 unmodified lines
263	}	180	}
264	EXPORT_SYMBOL(cpumask_any_and_distribute);	181	EXPORT_SYMBOL(cpumask_any_and_distribute);
265		182
266	int cpumask_any_distribute(const struct cpumask *srcp)	183	unsigned int cpumask_any_distribute(const struct cpumask *srcp)
267	{	184	{
268	int next, prev;	185	unsigned int next, prev;
269		186
270	/* NOTE: our first selection will skip 0. */	187	/* NOTE: our first selection will skip 0. */
271	prev = __this_cpu_read(distribute_cpu_mask_prev);	188	prev = __this_cpu_read(distribute_cpu_mask_prev);

lib/nodemask.c+0−8

@@ -3,14 +3,6 @@ · 2 unmodified lines
3	#include <linux/module.h>	3	#include <linux/module.h>
4	#include <linux/random.h>	4	#include <linux/random.h>
5		5
6	unsigned int __next_node_in(int node, const nodemask_t *srcp)
7	{
8	unsigned int ret = __next_node(node, srcp);
9
10	if (ret == MAX_NUMNODES)
11	ret = __first_node(srcp);
12	return ret;
13	}
14	EXPORT_SYMBOL(__next_node_in);	6	EXPORT_SYMBOL(__next_node_in);
15		7
16	#ifdef CONFIG_NUMA	8	#ifdef CONFIG_NUMA

lib/test_bitmap.c+68−0

@@ -604,6 +604,12 @@ static void __init test_bitmap_arr64(void) · 603 unmodified lines
604	pr_err("bitmap_copy_arr64(nbits == %d:"	604	pr_err("bitmap_copy_arr64(nbits == %d:"
605	" tail is not safely cleared: %d\n", nbits, next_bit);	605	" tail is not safely cleared: %d\n", nbits, next_bit);
606		606
		607	if ((nbits % 64) &&
		608	(arr[(nbits - 1) / 64] & ~GENMASK_ULL((nbits - 1) % 64, 0)))
		609	pr_err("bitmap_to_arr64(nbits == %d): tail is not safely cleared: 0x%016llx (must be 0x%016llx)\n",
		610	nbits, arr[(nbits - 1) / 64],
		611	GENMASK_ULL((nbits - 1) % 64, 0));
		612
607	if (nbits < EXP1_IN_BITS - 64)	613	if (nbits < EXP1_IN_BITS - 64)
608	expect_eq_uint(arr[DIV_ROUND_UP(nbits, 64)], 0xa5a5a5a5);	614	expect_eq_uint(arr[DIV_ROUND_UP(nbits, 64)], 0xa5a5a5a5);
609	}	615	}
@@ -869,6 +875,67 @@ static void __init test_bitmap_print_buf(void) · 259 unmodified lines
869	}	875	}
870	}	876	}
871		877
		878	static void __init test_bitmap_const_eval(void)
		879	{
		880	DECLARE_BITMAP(bitmap, BITS_PER_LONG);
		881	unsigned long initvar = BIT(2);
		882	unsigned long bitopvar = 0;
		883	unsigned long var = 0;
		884	int res;
		885
		886	/*
		887	* Compilers must be able to optimize all of those to compile-time
		888	* constants on any supported optimization level (-O2, -Os) and any
		889	* architecture. Otherwise, trigger a build bug.
		890	* The whole function gets optimized out then, there's nothing to do
		891	* in runtime.
		892	*/
		893
		894	/*
		895	* Equals to `unsigned long bitmap[1] = { GENMASK(6, 5), }`.
		896	* Clang on s390 optimizes bitops at compile-time as intended, but at
		897	* the same time stops treating @bitmap and @bitopvar as compile-time
		898	* constants after regular test_bit() is executed, thus triggering the
		899	* build bugs below. So, call const_test_bit() there directly until
		900	* the compiler is fixed.
		901	*/
		902	bitmap_clear(bitmap, 0, BITS_PER_LONG);
		903	#if defined(__s390__) && defined(__clang__)
		904	if (!const_test_bit(7, bitmap))
		905	#else
		906	if (!test_bit(7, bitmap))
		907	#endif
		908	bitmap_set(bitmap, 5, 2);
		909
		910	/* Equals to `unsigned long bitopvar = BIT(20)` */
		911	__change_bit(31, &bitopvar);
		912	bitmap_shift_right(&bitopvar, &bitopvar, 11, BITS_PER_LONG);
		913
		914	/* Equals to `unsigned long var = BIT(25)` */
		915	var \|= BIT(25);
		916	if (var & BIT(0))
		917	var ^= GENMASK(9, 6);
		918
		919	/* __const_hweight<32\|64>(GENMASK(6, 5)) == 2 */
		920	res = bitmap_weight(bitmap, 20);
		921	BUILD_BUG_ON(!__builtin_constant_p(res));
		922	BUILD_BUG_ON(res != 2);
		923
		924	/* !(BIT(31) & BIT(18)) == 1 */
		925	res = !test_bit(18, &bitopvar);
		926	BUILD_BUG_ON(!__builtin_constant_p(res));
		927	BUILD_BUG_ON(!res);
		928
		929	/* BIT(2) & GENMASK(14, 8) == 0 */
		930	res = initvar & GENMASK(14, 8);
		931	BUILD_BUG_ON(!__builtin_constant_p(res));
		932	BUILD_BUG_ON(res);
		933
		934	/* ~BIT(25) */
		935	BUILD_BUG_ON(!__builtin_constant_p(~var));
		936	BUILD_BUG_ON(~var != ~BIT(25));
		937	}
		938
872	static void __init selftest(void)	939	static void __init selftest(void)
873	{	940	{
874	test_zero_clear();	941	test_zero_clear();
@@ -884,6 +951,7 @@ static void __init selftest(void) · 9 unmodified lines
884	test_for_each_set_clump8();	951	test_for_each_set_clump8();
885	test_bitmap_cut();	952	test_bitmap_cut();
886	test_bitmap_print_buf();	953	test_bitmap_print_buf();
		954	test_bitmap_const_eval();
887	}	955	}
888		956
889	KSTM_MODULE_LOADERS(test_bitmap);	957	KSTM_MODULE_LOADERS(test_bitmap);

tools/include/asm-generic/bitops/non-atomic.h+20−14

 #ifndef _ASM_GENERIC_BITOPS_NON_ATOMIC_H_
 #define _ASM_GENERIC_BITOPS_NON_ATOMIC_H_
-#include <asm/types.h>
+#include <linux/bits.h>
 /**
- * __set_bit - Set a bit in memory
+ * ___set_bit - Set a bit in memory
  * @nr: the bit to set
  * @addr: the address to start counting from
  *
  * If it's called on the same region of memory simultaneously, the effect
  * may be that only one operation succeeds.
  */
-static inline void __set_bit(int nr, volatile unsigned long *addr)
+static __always_inline void
+___set_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	unsigned long mask = BIT_MASK(nr);
 	unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
 	*p  |= mask;
 }
-static inline void __clear_bit(int nr, volatile unsigned long *addr)
+static __always_inline void
+___clear_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	unsigned long mask = BIT_MASK(nr);
 	unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
 }
 /**
- * __change_bit - Toggle a bit in memory
+ * ___change_bit - Toggle a bit in memory
  * @nr: the bit to change
  * @addr: the address to start counting from
  *
  * If it's called on the same region of memory simultaneously, the effect
  * may be that only one operation succeeds.
  */
-static inline void __change_bit(int nr, volatile unsigned long *addr)
+static __always_inline void
+___change_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	unsigned long mask = BIT_MASK(nr);
 	unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
 }
 /**
- * __test_and_set_bit - Set a bit and return its old value
+ * ___test_and_set_bit - Set a bit and return its old value
  * @nr: Bit to set
  * @addr: Address to count from
  *
  * If two examples of this operation race, one can appear to succeed
  * but actually fail.  You must protect multiple accesses with a lock.
  */
-static inline int __test_and_set_bit(int nr, volatile unsigned long *addr)
+static __always_inline bool
+___test_and_set_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	unsigned long mask = BIT_MASK(nr);
 	unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
 }
 /**
- * __test_and_clear_bit - Clear a bit and return its old value
+ * ___test_and_clear_bit - Clear a bit and return its old value
  * @nr: Bit to clear
  * @addr: Address to count from
  *
  * If two examples of this operation race, one can appear to succeed
  * but actually fail.  You must protect multiple accesses with a lock.
  */
-static inline int __test_and_clear_bit(int nr, volatile unsigned long *addr)
+static __always_inline bool
+___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	unsigned long mask = BIT_MASK(nr);
 	unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
 }
 /* WARNING: non atomic and it can be reordered! */
-static inline int __test_and_change_bit(int nr,
+static __always_inline bool
-					    volatile unsigned long *addr)
+___test_and_change_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	unsigned long mask = BIT_MASK(nr);
 	unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
 }
 /**
- * test_bit - Determine whether a bit is set
+ * _test_bit - Determine whether a bit is set
  * @nr: bit number to test
  * @addr: Address to start counting from
  */
-static inline int test_bit(int nr, const volatile unsigned long *addr)
+static __always_inline bool
+_test_bit(unsigned long nr, const volatile unsigned long *addr)
 {
 	return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1)));
 }

tools/include/linux/bitmap.h+6−6

tools/include/linux/bitops.h+16−0

@@ -25,6 +25,22 @@ extern unsigned int __sw_hweight16(unsigned int w); · 24 unmodified lines
25	extern unsigned int __sw_hweight32(unsigned int w);	25	extern unsigned int __sw_hweight32(unsigned int w);
26	extern unsigned long __sw_hweight64(__u64 w);	26	extern unsigned long __sw_hweight64(__u64 w);
27		27
		28	/*
		29	* Defined here because those may be needed by architecture-specific static
		30	* inlines.
		31	*/
		32
		33	#define bitop(op, nr, addr) \
		34	op(nr, addr)
		35
		36	#define __set_bit(nr, addr) bitop(___set_bit, nr, addr)
		37	#define __clear_bit(nr, addr) bitop(___clear_bit, nr, addr)
		38	#define __change_bit(nr, addr) bitop(___change_bit, nr, addr)
		39	#define __test_and_set_bit(nr, addr) bitop(___test_and_set_bit, nr, addr)
		40	#define __test_and_clear_bit(nr, addr) bitop(___test_and_clear_bit, nr, addr)
		41	#define __test_and_change_bit(nr, addr) bitop(___test_and_change_bit, nr, addr)
		42	#define test_bit(nr, addr) bitop(_test_bit, nr, addr)
		43
28	/*	44	/*
29	* Include this here because some architectures need generic_ffs/fls in	45	* Include this here because some architectures need generic_ffs/fls in
30	* scope	46	* scope

tools/lib/bitmap.c+3−3

@@ -5,9 +5,9 @@ · 4 unmodified lines
5	*/	5	*/
6	#include <linux/bitmap.h>	6	#include <linux/bitmap.h>
7		7
8	int __bitmap_weight(const unsigned long *bitmap, int bits)	8	unsigned int __bitmap_weight(const unsigned long *bitmap, int bits)
9	{	9	{
10	int k, w = 0, lim = bits/BITS_PER_LONG;	10	unsigned int k, w = 0, lim = bits/BITS_PER_LONG;
11		11
12	for (k = 0; k < lim; k++)	12	for (k = 0; k < lim; k++)
13	w += hweight_long(bitmap[k]);	13	w += hweight_long(bitmap[k]);
@@ -57,7 +57,7 @@ size_t bitmap_scnprintf(unsigned long *bitmap, unsigned int nbits, · 43 unmodified lines
57	return ret;	57	return ret;
58	}	58	}
59		59
60	int __bitmap_and(unsigned long dst, const unsigned long bitmap1,	60	bool __bitmap_and(unsigned long dst, const unsigned long bitmap1,
61	const unsigned long *bitmap2, unsigned int bits)	61	const unsigned long *bitmap2, unsigned int bits)
62	{	62	{
63	unsigned int k;	63	unsigned int k;

@@ -11,10 +11,10 @@ · 10 unmodified lines
11	#define DECLARE_BITMAP(name,bits) \	11	#define DECLARE_BITMAP(name,bits) \
12	unsigned long name[BITS_TO_LONGS(bits)]	12	unsigned long name[BITS_TO_LONGS(bits)]
13		13
14	int __bitmap_weight(const unsigned long *bitmap, int bits);	14	unsigned int __bitmap_weight(const unsigned long *bitmap, int bits);
15	void __bitmap_or(unsigned long dst, const unsigned long bitmap1,	15	void __bitmap_or(unsigned long dst, const unsigned long bitmap1,
16	const unsigned long *bitmap2, int bits);	16	const unsigned long *bitmap2, int bits);
17	int __bitmap_and(unsigned long dst, const unsigned long bitmap1,	17	bool __bitmap_and(unsigned long dst, const unsigned long bitmap1,
18	const unsigned long *bitmap2, unsigned int bits);	18	const unsigned long *bitmap2, unsigned int bits);
19	bool __bitmap_equal(const unsigned long *bitmap1,	19	bool __bitmap_equal(const unsigned long *bitmap1,
20	const unsigned long *bitmap2, unsigned int bits);	20	const unsigned long *bitmap2, unsigned int bits);
@@ -45,7 +45,7 @@ static inline void bitmap_fill(unsigned long *dst, unsigned int nbits) · 24 unmodified lines
45	dst[nlongs - 1] = BITMAP_LAST_WORD_MASK(nbits);	45	dst[nlongs - 1] = BITMAP_LAST_WORD_MASK(nbits);
46	}	46	}
47		47
48	static inline int bitmap_empty(const unsigned long *src, unsigned nbits)	48	static inline bool bitmap_empty(const unsigned long *src, unsigned int nbits)
49	{	49	{
50	if (small_const_nbits(nbits))	50	if (small_const_nbits(nbits))
51	return ! (*src & BITMAP_LAST_WORD_MASK(nbits));	51	return ! (*src & BITMAP_LAST_WORD_MASK(nbits));
@@ -53,7 +53,7 @@ static inline int bitmap_empty(const unsigned long *src, unsigned nbits) · 1 unmodified line
53	return find_first_bit(src, nbits) == nbits;	53	return find_first_bit(src, nbits) == nbits;
54	}	54	}
55		55
56	static inline int bitmap_full(const unsigned long *src, unsigned int nbits)	56	static inline bool bitmap_full(const unsigned long *src, unsigned int nbits)
57	{	57	{
58	if (small_const_nbits(nbits))	58	if (small_const_nbits(nbits))
59	return ! (~(*src) & BITMAP_LAST_WORD_MASK(nbits));	59	return ! (~(*src) & BITMAP_LAST_WORD_MASK(nbits));
@@ -61,7 +61,7 @@ static inline int bitmap_full(const unsigned long *src, unsigned int nbits) · 1 unmodified line
61	return find_first_zero_bit(src, nbits) == nbits;	61	return find_first_zero_bit(src, nbits) == nbits;
62	}	62	}
63		63
64	static inline int bitmap_weight(const unsigned long *src, unsigned int nbits)	64	static inline unsigned int bitmap_weight(const unsigned long *src, unsigned int nbits)
65	{	65	{
66	if (small_const_nbits(nbits))	66	if (small_const_nbits(nbits))
67	return hweight_long(*src & BITMAP_LAST_WORD_MASK(nbits));	67	return hweight_long(*src & BITMAP_LAST_WORD_MASK(nbits));
@@ -146,7 +146,7 @@ size_t bitmap_scnprintf(unsigned long *bitmap, unsigned int nbits, · 78 unmodified lines
146	* @src2: operand 2	146	* @src2: operand 2
147	* @nbits: size of bitmap	147	* @nbits: size of bitmap
148	*/	148	*/
149	static inline int bitmap_and(unsigned long dst, const unsigned long src1,	149	static inline bool bitmap_and(unsigned long dst, const unsigned long src1,
150	const unsigned long *src2, unsigned int nbits)	150	const unsigned long *src2, unsigned int nbits)
151	{	151	{
152	if (small_const_nbits(nbits))	152	if (small_const_nbits(nbits))