percpu: add raw_cpu_ops

The kernel has never been audited to ensure that this_cpu operations are consistently used throughout the kernel. The code generated in many places can be improved through the use of this_cpu operations (which uses a segment register for relocation of per cpu offsets instead of performing address calculations). The patch set also addresses various consistency issues in general with the per cpu macros. A. The semantics of __this_cpu_ptr() differs from this_cpu_ptr only because checks are skipped. This is typically shown through a raw_ prefix. So this patch set changes the places where __this_cpu_ptr() is used to raw_cpu_ptr(). B. There has been the long term wish by some that __this_cpu operations would check for preemption. However, there are cases where preemption checks need to be skipped. This patch set adds raw_cpu operations that do not check for preemption and then adds preemption checks to the __this_cpu operations. C. The use of __get_cpu_var is always a reference to a percpu variable that can also be handled via a this_cpu operation. This patch set replaces all uses of __get_cpu_var with this_cpu operations. D. We can then use this_cpu RMW operations in various places replacing sequences of instructions by a single one. E. The use of this_cpu operations throughout will allow other arches than x86 to implement optimized references and RMV operations to work with per cpu local data. F. The use of this_cpu operations opens up the possibility to further optimize code that relies on synchronization through per cpu data. The patch set works in a couple of stages: I. Patch 1 adds the additional raw_cpu operations and raw_cpu_ptr(). Also converts the existing __this_cpu_xx_# primitive in the x86 code to raw_cpu_xx_#. II. Patch 2-4 use the raw_cpu operations in places that would give us false positives once they are enabled. III. Patch 5 adds preemption checks to __this_cpu operations to allow checking if preemption is properly disabled when these functions are used. IV. Patches 6-20 are patches that simply replace uses of __get_cpu_var with this_cpu_ptr. They do not depend on any changes to the percpu code. No preemption tests are skipped if they are applied. V. Patches 21-46 are conversion patches that use this_cpu operations in various kernel subsystems/drivers or arch code. VI. Patches 47/48 (not included in this series) remove no longer used functions (__this_cpu_ptr and __get_cpu_var). These should only be applied after all the conversion patches have made it and after we have done additional passes through the kernel to ensure that none of the uses of these functions remain. This patch (of 46): The patches following this one will add preemption checks to __this_cpu ops so we need to have an alternative way to use this_cpu operations without preemption checks. raw_cpu_ops will be the basis for all other ops since these will be the operations that do not implement any checks. Primitive operations are renamed by this patch from __this_cpu_xxx to raw_cpu_xxxx. Also change the uses of the x86 percpu primitives in preempt.h. These depend directly on asm/percpu.h (header #include nesting issue).

Christoph Lameter committed 12 years ago

Commit b3ca1c11 parent 54b6a734 files · +260 −198Browse files

Unified · Split

4 files

arch/x86/include/asm/percpu.h +49 −49
arch/x86/include/asm/preempt.h +8 −8
include/asm-generic/percpu.h +8 −5
include/linux/percpu.h +195 −136

arch/x86/include/asm/percpu.h+49−49

@@ -52,7 +52,7 @@ · 51 unmodified lines
52	52	* Compared to the generic __my_cpu_offset version, the following
53	53	* saves one instruction and avoids clobbering a temp register.
54	54	*/
55		−#define __this_cpu_ptr(ptr) \
	55	+#define raw_cpu_ptr(ptr) \
56	56	({ \
57	57	unsigned long tcp_ptr__; \
58	58	__verify_pcpu_ptr(ptr); \
@@ -362,25 +362,25 @@ do { \ · 303 unmodified lines
362	362	*/
363	363	#define this_cpu_read_stable(var) percpu_from_op("mov", var, "p" (&(var)))
364	364
365		−#define __this_cpu_read_1(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
366		−#define __this_cpu_read_2(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
367		−#define __this_cpu_read_4(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
368		−
369		−#define __this_cpu_write_1(pcp, val) percpu_to_op("mov", (pcp), val)
370		−#define __this_cpu_write_2(pcp, val) percpu_to_op("mov", (pcp), val)
371		−#define __this_cpu_write_4(pcp, val) percpu_to_op("mov", (pcp), val)
372		−#define __this_cpu_add_1(pcp, val) percpu_add_op((pcp), val)
373		−#define __this_cpu_add_2(pcp, val) percpu_add_op((pcp), val)
374		−#define __this_cpu_add_4(pcp, val) percpu_add_op((pcp), val)
375		−#define __this_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val)
376		−#define __this_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val)
377		−#define __this_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val)
378		−#define __this_cpu_or_1(pcp, val) percpu_to_op("or", (pcp), val)
379		−#define __this_cpu_or_2(pcp, val) percpu_to_op("or", (pcp), val)
380		−#define __this_cpu_or_4(pcp, val) percpu_to_op("or", (pcp), val)
381		−#define __this_cpu_xchg_1(pcp, val) percpu_xchg_op(pcp, val)
382		−#define __this_cpu_xchg_2(pcp, val) percpu_xchg_op(pcp, val)
383		−#define __this_cpu_xchg_4(pcp, val) percpu_xchg_op(pcp, val)
	365	+#define raw_cpu_read_1(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
	366	+#define raw_cpu_read_2(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
	367	+#define raw_cpu_read_4(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
	368	+
	369	+#define raw_cpu_write_1(pcp, val) percpu_to_op("mov", (pcp), val)
	370	+#define raw_cpu_write_2(pcp, val) percpu_to_op("mov", (pcp), val)
	371	+#define raw_cpu_write_4(pcp, val) percpu_to_op("mov", (pcp), val)
	372	+#define raw_cpu_add_1(pcp, val) percpu_add_op((pcp), val)
	373	+#define raw_cpu_add_2(pcp, val) percpu_add_op((pcp), val)
	374	+#define raw_cpu_add_4(pcp, val) percpu_add_op((pcp), val)
	375	+#define raw_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val)
	376	+#define raw_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val)
	377	+#define raw_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val)
	378	+#define raw_cpu_or_1(pcp, val) percpu_to_op("or", (pcp), val)
	379	+#define raw_cpu_or_2(pcp, val) percpu_to_op("or", (pcp), val)
	380	+#define raw_cpu_or_4(pcp, val) percpu_to_op("or", (pcp), val)
	381	+#define raw_cpu_xchg_1(pcp, val) percpu_xchg_op(pcp, val)
	382	+#define raw_cpu_xchg_2(pcp, val) percpu_xchg_op(pcp, val)
	383	+#define raw_cpu_xchg_4(pcp, val) percpu_xchg_op(pcp, val)
384	384
385	385	#define this_cpu_read_1(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
386	386	#define this_cpu_read_2(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
@@ -401,16 +401,16 @@ do { \ · 14 unmodified lines
401	401	#define this_cpu_xchg_2(pcp, nval) percpu_xchg_op(pcp, nval)
402	402	#define this_cpu_xchg_4(pcp, nval) percpu_xchg_op(pcp, nval)
403	403
404		−#define __this_cpu_add_return_1(pcp, val) percpu_add_return_op(pcp, val)
405		−#define __this_cpu_add_return_2(pcp, val) percpu_add_return_op(pcp, val)
406		−#define __this_cpu_add_return_4(pcp, val) percpu_add_return_op(pcp, val)
407		−#define __this_cpu_cmpxchg_1(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval)
408		−#define __this_cpu_cmpxchg_2(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval)
409		−#define __this_cpu_cmpxchg_4(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval)
	404	+#define raw_cpu_add_return_1(pcp, val) percpu_add_return_op(pcp, val)
	405	+#define raw_cpu_add_return_2(pcp, val) percpu_add_return_op(pcp, val)
	406	+#define raw_cpu_add_return_4(pcp, val) percpu_add_return_op(pcp, val)
	407	+#define raw_cpu_cmpxchg_1(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval)
	408	+#define raw_cpu_cmpxchg_2(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval)
	409	+#define raw_cpu_cmpxchg_4(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval)
410	410
411		−#define this_cpu_add_return_1(pcp, val) percpu_add_return_op(pcp, val)
412		−#define this_cpu_add_return_2(pcp, val) percpu_add_return_op(pcp, val)
413		−#define this_cpu_add_return_4(pcp, val) percpu_add_return_op(pcp, val)
	411	+#define this_cpu_add_return_1(pcp, val) percpu_add_return_op(pcp, val)
	412	+#define this_cpu_add_return_2(pcp, val) percpu_add_return_op(pcp, val)
	413	+#define this_cpu_add_return_4(pcp, val) percpu_add_return_op(pcp, val)
414	414	#define this_cpu_cmpxchg_1(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval)
415	415	#define this_cpu_cmpxchg_2(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval)
416	416	#define this_cpu_cmpxchg_4(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval)
@@ -427,7 +427,7 @@ do { \ · 10 unmodified lines
427	427	__ret; \
428	428	})
429	429
430		−#define __this_cpu_cmpxchg_double_4 percpu_cmpxchg8b_double
	430	+#define raw_cpu_cmpxchg_double_4 percpu_cmpxchg8b_double
431	431	#define this_cpu_cmpxchg_double_4 percpu_cmpxchg8b_double
432	432	#endif /* CONFIG_X86_CMPXCHG64 */
433	433
@@ -436,22 +436,22 @@ do { \ · 2 unmodified lines
436	436	* 32 bit must fall back to generic operations.
437	437	*/
438	438	#ifdef CONFIG_X86_64
439		−#define __this_cpu_read_8(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
440		−#define __this_cpu_write_8(pcp, val) percpu_to_op("mov", (pcp), val)
441		−#define __this_cpu_add_8(pcp, val) percpu_add_op((pcp), val)
442		−#define __this_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val)
443		−#define __this_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val)
444		−#define __this_cpu_add_return_8(pcp, val) percpu_add_return_op(pcp, val)
445		−#define __this_cpu_xchg_8(pcp, nval) percpu_xchg_op(pcp, nval)
446		−#define __this_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval)
447		−
448		−#define this_cpu_read_8(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
449		−#define this_cpu_write_8(pcp, val) percpu_to_op("mov", (pcp), val)
450		−#define this_cpu_add_8(pcp, val) percpu_add_op((pcp), val)
451		−#define this_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val)
452		−#define this_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val)
453		−#define this_cpu_add_return_8(pcp, val) percpu_add_return_op(pcp, val)
454		−#define this_cpu_xchg_8(pcp, nval) percpu_xchg_op(pcp, nval)
	439	+#define raw_cpu_read_8(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
	440	+#define raw_cpu_write_8(pcp, val) percpu_to_op("mov", (pcp), val)
	441	+#define raw_cpu_add_8(pcp, val) percpu_add_op((pcp), val)
	442	+#define raw_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val)
	443	+#define raw_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val)
	444	+#define raw_cpu_add_return_8(pcp, val) percpu_add_return_op(pcp, val)
	445	+#define raw_cpu_xchg_8(pcp, nval) percpu_xchg_op(pcp, nval)
	446	+#define raw_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval)
	447	+
	448	+#define this_cpu_read_8(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
	449	+#define this_cpu_write_8(pcp, val) percpu_to_op("mov", (pcp), val)
	450	+#define this_cpu_add_8(pcp, val) percpu_add_op((pcp), val)
	451	+#define this_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val)
	452	+#define this_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val)
	453	+#define this_cpu_add_return_8(pcp, val) percpu_add_return_op(pcp, val)
	454	+#define this_cpu_xchg_8(pcp, nval) percpu_xchg_op(pcp, nval)
455	455	#define this_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval)
456	456
457	457	/*
@@ -474,7 +474,7 @@ do { \ · 16 unmodified lines
474	474	__ret; \
475	475	})
476	476
477		−#define __this_cpu_cmpxchg_double_8 percpu_cmpxchg16b_double
	477	+#define raw_cpu_cmpxchg_double_8 percpu_cmpxchg16b_double
478	478	#define this_cpu_cmpxchg_double_8 percpu_cmpxchg16b_double
479	479
480	480	#endif
@@ -495,9 +495,9 @@ static __always_inline int x86_this_cpu_constant_test_bit(unsigned int nr, · 14 unmodified lines
495	495	unsigned long __percpu a = (unsigned long )addr + nr / BITS_PER_LONG;
496	496
497	497	#ifdef CONFIG_X86_64
498		− return ((1UL << (nr % BITS_PER_LONG)) & __this_cpu_read_8(*a)) != 0;
	498	+ return ((1UL << (nr % BITS_PER_LONG)) & raw_cpu_read_8(*a)) != 0;
499	499	#else
500		− return ((1UL << (nr % BITS_PER_LONG)) & __this_cpu_read_4(*a)) != 0;
	500	+ return ((1UL << (nr % BITS_PER_LONG)) & raw_cpu_read_4(*a)) != 0;
501	501	#endif
502	502	}
503	503

arch/x86/include/asm/preempt.h+8−8

@@ -19,12 +19,12 @@ DECLARE_PER_CPU(int, __preempt_count); · 18 unmodified lines
19	19	*/
20	20	static __always_inline int preempt_count(void)
21	21	{
22		− return __this_cpu_read_4(__preempt_count) & ~PREEMPT_NEED_RESCHED;
	22	+ return raw_cpu_read_4(__preempt_count) & ~PREEMPT_NEED_RESCHED;
23	23	}
24	24
25	25	static __always_inline void preempt_count_set(int pc)
26	26	{
27		− __this_cpu_write_4(__preempt_count, pc);
	27	+ raw_cpu_write_4(__preempt_count, pc);
28	28	}
29	29
30	30	/*
@@ -53,17 +53,17 @@ static __always_inline void preempt_count_set(int pc) · 22 unmodified lines
53	53
54	54	static __always_inline void set_preempt_need_resched(void)
55	55	{
56		− __this_cpu_and_4(__preempt_count, ~PREEMPT_NEED_RESCHED);
	56	+ raw_cpu_and_4(__preempt_count, ~PREEMPT_NEED_RESCHED);
57	57	}
58	58
59	59	static __always_inline void clear_preempt_need_resched(void)
60	60	{
61		− __this_cpu_or_4(__preempt_count, PREEMPT_NEED_RESCHED);
	61	+ raw_cpu_or_4(__preempt_count, PREEMPT_NEED_RESCHED);
62	62	}
63	63
64	64	static __always_inline bool test_preempt_need_resched(void)
65	65	{
66		− return !(__this_cpu_read_4(__preempt_count) & PREEMPT_NEED_RESCHED);
	66	+ return !(raw_cpu_read_4(__preempt_count) & PREEMPT_NEED_RESCHED);
67	67	}
68	68
69	69	/*
@@ -72,12 +72,12 @@ static __always_inline bool test_preempt_need_resched(void) · 2 unmodified lines
72	72
73	73	static __always_inline void __preempt_count_add(int val)
74	74	{
75		− __this_cpu_add_4(__preempt_count, val);
	75	+ raw_cpu_add_4(__preempt_count, val);
76	76	}
77	77
78	78	static __always_inline void __preempt_count_sub(int val)
79	79	{
80		− __this_cpu_add_4(__preempt_count, -val);
	80	+ raw_cpu_add_4(__preempt_count, -val);
81	81	}
82	82
83	83	/*
@@ -95,7 +95,7 @@ static __always_inline bool __preempt_count_dec_and_test(void) · 11 unmodified lines
95	95	*/
96	96	static __always_inline bool should_resched(void)
97	97	{
98		− return unlikely(!__this_cpu_read_4(__preempt_count));
	98	+ return unlikely(!raw_cpu_read_4(__preempt_count));
99	99	}
100	100
101	101	#ifdef CONFIG_PREEMPT

include/asm-generic/percpu.h+8−5

@@ -56,17 +56,17 @@ extern unsigned long __per_cpu_offset[NR_CPUS]; · 55 unmodified lines
56	56	#define per_cpu(var, cpu) \
57	57	(*SHIFT_PERCPU_PTR(&(var), per_cpu_offset(cpu)))
58	58
59		−#ifndef __this_cpu_ptr
60		−#define __this_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
	59	+#ifndef raw_cpu_ptr
	60	+#define raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
61	61	#endif
62	62	#ifdef CONFIG_DEBUG_PREEMPT
63	63	#define this_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, my_cpu_offset)
64	64	#else
65		−#define this_cpu_ptr(ptr) __this_cpu_ptr(ptr)
	65	+#define this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
66	66	#endif
67	67
68	68	#define __get_cpu_var(var) (*this_cpu_ptr(&(var)))
69		−#define __raw_get_cpu_var(var) (*__this_cpu_ptr(&(var)))
	69	+#define __raw_get_cpu_var(var) (*raw_cpu_ptr(&(var)))
70	70
71	71	#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
72	72	extern void setup_per_cpu_areas(void);
@@ -83,7 +83,7 @@ extern void setup_per_cpu_areas(void); · 10 unmodified lines
83	83	#define __get_cpu_var(var) (*VERIFY_PERCPU_PTR(&(var)))
84	84	#define __raw_get_cpu_var(var) (*VERIFY_PERCPU_PTR(&(var)))
85	85	#define this_cpu_ptr(ptr) per_cpu_ptr(ptr, 0)
86		−#define __this_cpu_ptr(ptr) this_cpu_ptr(ptr)
	86	+#define raw_cpu_ptr(ptr) this_cpu_ptr(ptr)
87	87
88	88	#endif /* SMP */
89	89
@@ -122,4 +122,7 @@ extern void setup_per_cpu_areas(void); · 32 unmodified lines
122	122	#define PER_CPU_DEF_ATTRIBUTES
123	123	#endif
124	124
	125	+/* Keep until we have removed all uses of __this_cpu_ptr */
	126	+#define __this_cpu_ptr raw_cpu_ptr
	127	+
125	128	#endif /* _ASM_GENERIC_PERCPU_H_ */

include/linux/percpu.h+195−136

@@ -243,6 +243,8 @@ do { \ · 242 unmodified lines
243	243	} while (0)
244	244
245	245	/*
	246	+ * this_cpu operations (C) 2008-2013 Christoph Lameter <cl@linux.com>
	247	+ *
246	248	* Optimized manipulation for memory allocated through the per cpu
247	249	* allocator or for addresses of per cpu variables.
248	250	*
@@ -296,7 +298,7 @@ do { \ · 47 unmodified lines
296	298	do { \
297	299	unsigned long flags; \
298	300	raw_local_irq_save(flags); \
299		− *__this_cpu_ptr(&(pcp)) op val; \
	301	+ *raw_cpu_ptr(&(pcp)) op val; \
300	302	raw_local_irq_restore(flags); \
301	303	} while (0)
302	304
@@ -381,8 +383,8 @@ do { \ · 78 unmodified lines
381	383	typeof(pcp) ret__; \
382	384	unsigned long flags; \
383	385	raw_local_irq_save(flags); \
384		− __this_cpu_add(pcp, val); \
385		− ret__ = __this_cpu_read(pcp); \
	386	+ raw_cpu_add(pcp, val); \
	387	+ ret__ = raw_cpu_read(pcp); \
386	388	raw_local_irq_restore(flags); \
387	389	ret__; \
388	390	})
@@ -411,8 +413,8 @@ do { \ · 22 unmodified lines
411	413	({ typeof(pcp) ret__; \
412	414	unsigned long flags; \
413	415	raw_local_irq_save(flags); \
414		− ret__ = __this_cpu_read(pcp); \
415		− __this_cpu_write(pcp, nval); \
	416	+ ret__ = raw_cpu_read(pcp); \
	417	+ raw_cpu_write(pcp, nval); \
416	418	raw_local_irq_restore(flags); \
417	419	ret__; \
418	420	})
@@ -439,9 +441,9 @@ do { \ · 20 unmodified lines
439	441	typeof(pcp) ret__; \
440	442	unsigned long flags; \
441	443	raw_local_irq_save(flags); \
442		− ret__ = __this_cpu_read(pcp); \
	444	+ ret__ = raw_cpu_read(pcp); \
443	445	if (ret__ == (oval)) \
444		− __this_cpu_write(pcp, nval); \
	446	+ raw_cpu_write(pcp, nval); \
445	447	raw_local_irq_restore(flags); \
446	448	ret__; \
447	449	})
@@ -476,7 +478,7 @@ do { \ · 28 unmodified lines
476	478	int ret__; \
477	479	unsigned long flags; \
478	480	raw_local_irq_save(flags); \
479		− ret__ = __this_cpu_generic_cmpxchg_double(pcp1, pcp2, \
	481	+ ret__ = raw_cpu_generic_cmpxchg_double(pcp1, pcp2, \
480	482	oval1, oval2, nval1, nval2); \
481	483	raw_local_irq_restore(flags); \
482	484	ret__; \
@@ -504,12 +506,8 @@ do { \ · 21 unmodified lines
504	506	#endif
505	507
506	508	/*
507		− * Generic percpu operations for context that are safe from preemption/interrupts.
508		− * Either we do not care about races or the caller has the
509		− * responsibility of handling preemption/interrupt issues. Arch code can still
510		− * override these instructions since the arch per cpu code may be more
511		− * efficient and may actually get race freeness for free (that is the
512		− * case for x86 for example).
	509	+ * Generic percpu operations for contexts where we do not want to do
	510	+ * any checks for preemptiosn.
513	511	*
514	512	* If there is no other protection through preempt disable and/or
515	513	* disabling interupts then one of these RMW operations can show unexpected
@@ -517,211 +515,272 @@ do { \ · 1 unmodified line
517	515	* or an interrupt occurred and the same percpu variable was modified from
518	516	* the interrupt context.
519	517	*/
520		−#ifndef __this_cpu_read
521		−# ifndef __this_cpu_read_1
522		−# define __this_cpu_read_1(pcp) (*__this_cpu_ptr(&(pcp)))
	518	+#ifndef raw_cpu_read
	519	+# ifndef raw_cpu_read_1
	520	+# define raw_cpu_read_1(pcp) (*raw_cpu_ptr(&(pcp)))
523	521	# endif
524		−# ifndef __this_cpu_read_2
525		−# define __this_cpu_read_2(pcp) (*__this_cpu_ptr(&(pcp)))
	522	+# ifndef raw_cpu_read_2
	523	+# define raw_cpu_read_2(pcp) (*raw_cpu_ptr(&(pcp)))
526	524	# endif
527		−# ifndef __this_cpu_read_4
528		−# define __this_cpu_read_4(pcp) (*__this_cpu_ptr(&(pcp)))
	525	+# ifndef raw_cpu_read_4
	526	+# define raw_cpu_read_4(pcp) (*raw_cpu_ptr(&(pcp)))
529	527	# endif
530		−# ifndef __this_cpu_read_8
531		−# define __this_cpu_read_8(pcp) (*__this_cpu_ptr(&(pcp)))
	528	+# ifndef raw_cpu_read_8
	529	+# define raw_cpu_read_8(pcp) (*raw_cpu_ptr(&(pcp)))
532	530	# endif
533		−# define __this_cpu_read(pcp) __pcpu_size_call_return(__this_cpu_read_, (pcp))
	531	+# define raw_cpu_read(pcp) __pcpu_size_call_return(raw_cpu_read_, (pcp))
534	532	#endif
535	533
536		−#define __this_cpu_generic_to_op(pcp, val, op) \
	534	+#define raw_cpu_generic_to_op(pcp, val, op) \
537	535	do { \
538		− *__this_cpu_ptr(&(pcp)) op val; \
	536	+ *raw_cpu_ptr(&(pcp)) op val; \
539	537	} while (0)
540	538
541		−#ifndef __this_cpu_write
542		−# ifndef __this_cpu_write_1
543		−# define __this_cpu_write_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), =)
	539	+
	540	+#ifndef raw_cpu_write
	541	+# ifndef raw_cpu_write_1
	542	+# define raw_cpu_write_1(pcp, val) raw_cpu_generic_to_op((pcp), (val), =)
544	543	# endif
545		−# ifndef __this_cpu_write_2
546		−# define __this_cpu_write_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), =)
	544	+# ifndef raw_cpu_write_2
	545	+# define raw_cpu_write_2(pcp, val) raw_cpu_generic_to_op((pcp), (val), =)
547	546	# endif
548		−# ifndef __this_cpu_write_4
549		−# define __this_cpu_write_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), =)
	547	+# ifndef raw_cpu_write_4
	548	+# define raw_cpu_write_4(pcp, val) raw_cpu_generic_to_op((pcp), (val), =)
550	549	# endif
551		−# ifndef __this_cpu_write_8
552		−# define __this_cpu_write_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), =)
	550	+# ifndef raw_cpu_write_8
	551	+# define raw_cpu_write_8(pcp, val) raw_cpu_generic_to_op((pcp), (val), =)
553	552	# endif
554		−# define __this_cpu_write(pcp, val) __pcpu_size_call(__this_cpu_write_, (pcp), (val))
	553	+# define raw_cpu_write(pcp, val) __pcpu_size_call(raw_cpu_write_, (pcp), (val))
555	554	#endif
556	555
557		−#ifndef __this_cpu_add
558		−# ifndef __this_cpu_add_1
559		−# define __this_cpu_add_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), +=)
	556	+#ifndef raw_cpu_add
	557	+# ifndef raw_cpu_add_1
	558	+# define raw_cpu_add_1(pcp, val) raw_cpu_generic_to_op((pcp), (val), +=)
560	559	# endif
561		−# ifndef __this_cpu_add_2
562		−# define __this_cpu_add_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), +=)
	560	+# ifndef raw_cpu_add_2
	561	+# define raw_cpu_add_2(pcp, val) raw_cpu_generic_to_op((pcp), (val), +=)
563	562	# endif
564		−# ifndef __this_cpu_add_4
565		−# define __this_cpu_add_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), +=)
	563	+# ifndef raw_cpu_add_4
	564	+# define raw_cpu_add_4(pcp, val) raw_cpu_generic_to_op((pcp), (val), +=)
566	565	# endif
567		−# ifndef __this_cpu_add_8
568		−# define __this_cpu_add_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), +=)
	566	+# ifndef raw_cpu_add_8
	567	+# define raw_cpu_add_8(pcp, val) raw_cpu_generic_to_op((pcp), (val), +=)
569	568	# endif
570		−# define __this_cpu_add(pcp, val) __pcpu_size_call(__this_cpu_add_, (pcp), (val))
	569	+# define raw_cpu_add(pcp, val) __pcpu_size_call(raw_cpu_add_, (pcp), (val))
571	570	#endif
572	571
573		−#ifndef __this_cpu_sub
574		−# define __this_cpu_sub(pcp, val) __this_cpu_add((pcp), -(typeof(pcp))(val))
	572	+#ifndef raw_cpu_sub
	573	+# define raw_cpu_sub(pcp, val) raw_cpu_add((pcp), -(val))
575	574	#endif
576	575
577		−#ifndef __this_cpu_inc
578		−# define __this_cpu_inc(pcp) __this_cpu_add((pcp), 1)
	576	+#ifndef raw_cpu_inc
	577	+# define raw_cpu_inc(pcp) raw_cpu_add((pcp), 1)
579	578	#endif
580	579
581		−#ifndef __this_cpu_dec
582		−# define __this_cpu_dec(pcp) __this_cpu_sub((pcp), 1)
	580	+#ifndef raw_cpu_dec
	581	+# define raw_cpu_dec(pcp) raw_cpu_sub((pcp), 1)
583	582	#endif
584	583
585		−#ifndef __this_cpu_and
586		−# ifndef __this_cpu_and_1
587		−# define __this_cpu_and_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), &=)
	584	+#ifndef raw_cpu_and
	585	+# ifndef raw_cpu_and_1
	586	+# define raw_cpu_and_1(pcp, val) raw_cpu_generic_to_op((pcp), (val), &=)
588	587	# endif
589		−# ifndef __this_cpu_and_2
590		−# define __this_cpu_and_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), &=)
	588	+# ifndef raw_cpu_and_2
	589	+# define raw_cpu_and_2(pcp, val) raw_cpu_generic_to_op((pcp), (val), &=)
591	590	# endif
592		−# ifndef __this_cpu_and_4
593		−# define __this_cpu_and_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), &=)
	591	+# ifndef raw_cpu_and_4
	592	+# define raw_cpu_and_4(pcp, val) raw_cpu_generic_to_op((pcp), (val), &=)
594	593	# endif
595		−# ifndef __this_cpu_and_8
596		−# define __this_cpu_and_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), &=)
	594	+# ifndef raw_cpu_and_8
	595	+# define raw_cpu_and_8(pcp, val) raw_cpu_generic_to_op((pcp), (val), &=)
597	596	# endif
598		−# define __this_cpu_and(pcp, val) __pcpu_size_call(__this_cpu_and_, (pcp), (val))
	597	+# define raw_cpu_and(pcp, val) __pcpu_size_call(raw_cpu_and_, (pcp), (val))
599	598	#endif
600	599
601		−#ifndef __this_cpu_or
602		−# ifndef __this_cpu_or_1
603		−# define __this_cpu_or_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), \|=)
	600	+#ifndef raw_cpu_or
	601	+# ifndef raw_cpu_or_1
	602	+# define raw_cpu_or_1(pcp, val) raw_cpu_generic_to_op((pcp), (val), \|=)
604	603	# endif
605		−# ifndef __this_cpu_or_2
606		−# define __this_cpu_or_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), \|=)
	604	+# ifndef raw_cpu_or_2
	605	+# define raw_cpu_or_2(pcp, val) raw_cpu_generic_to_op((pcp), (val), \|=)
607	606	# endif
608		−# ifndef __this_cpu_or_4
609		−# define __this_cpu_or_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), \|=)
	607	+# ifndef raw_cpu_or_4
	608	+# define raw_cpu_or_4(pcp, val) raw_cpu_generic_to_op((pcp), (val), \|=)
610	609	# endif
611		−# ifndef __this_cpu_or_8
612		−# define __this_cpu_or_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), \|=)
	610	+# ifndef raw_cpu_or_8
	611	+# define raw_cpu_or_8(pcp, val) raw_cpu_generic_to_op((pcp), (val), \|=)
613	612	# endif
614		−# define __this_cpu_or(pcp, val) __pcpu_size_call(__this_cpu_or_, (pcp), (val))
	613	+# define raw_cpu_or(pcp, val) __pcpu_size_call(raw_cpu_or_, (pcp), (val))
615	614	#endif
616	615
617		−#define __this_cpu_generic_add_return(pcp, val) \
	616	+#define raw_cpu_generic_add_return(pcp, val) \
618	617	({ \
619		− __this_cpu_add(pcp, val); \
620		− __this_cpu_read(pcp); \
	618	+ raw_cpu_add(pcp, val); \
	619	+ raw_cpu_read(pcp); \
621	620	})
622	621
623		−#ifndef __this_cpu_add_return
624		−# ifndef __this_cpu_add_return_1
625		−# define __this_cpu_add_return_1(pcp, val) __this_cpu_generic_add_return(pcp, val)
	622	+#ifndef raw_cpu_add_return
	623	+# ifndef raw_cpu_add_return_1
	624	+# define raw_cpu_add_return_1(pcp, val) raw_cpu_generic_add_return(pcp, val)
626	625	# endif
627		−# ifndef __this_cpu_add_return_2
628		−# define __this_cpu_add_return_2(pcp, val) __this_cpu_generic_add_return(pcp, val)
	626	+# ifndef raw_cpu_add_return_2
	627	+# define raw_cpu_add_return_2(pcp, val) raw_cpu_generic_add_return(pcp, val)
629	628	# endif
630		−# ifndef __this_cpu_add_return_4
631		−# define __this_cpu_add_return_4(pcp, val) __this_cpu_generic_add_return(pcp, val)
	629	+# ifndef raw_cpu_add_return_4
	630	+# define raw_cpu_add_return_4(pcp, val) raw_cpu_generic_add_return(pcp, val)
632	631	# endif
633		−# ifndef __this_cpu_add_return_8
634		−# define __this_cpu_add_return_8(pcp, val) __this_cpu_generic_add_return(pcp, val)
	632	+# ifndef raw_cpu_add_return_8
	633	+# define raw_cpu_add_return_8(pcp, val) raw_cpu_generic_add_return(pcp, val)
635	634	# endif
636		−# define __this_cpu_add_return(pcp, val) \
637		− __pcpu_size_call_return2(__this_cpu_add_return_, pcp, val)
	635	+# define raw_cpu_add_return(pcp, val) \
	636	+ __pcpu_size_call_return2(raw_add_return_, pcp, val)
638	637	#endif
639	638
640		−#define __this_cpu_sub_return(pcp, val) __this_cpu_add_return(pcp, -(typeof(pcp))(val))
641		−#define __this_cpu_inc_return(pcp) __this_cpu_add_return(pcp, 1)
642		−#define __this_cpu_dec_return(pcp) __this_cpu_add_return(pcp, -1)
	639	+#define raw_cpu_sub_return(pcp, val) raw_cpu_add_return(pcp, -(typeof(pcp))(val))
	640	+#define raw_cpu_inc_return(pcp) raw_cpu_add_return(pcp, 1)
	641	+#define raw_cpu_dec_return(pcp) raw_cpu_add_return(pcp, -1)
643	642
644		−#define __this_cpu_generic_xchg(pcp, nval) \
	643	+#define raw_cpu_generic_xchg(pcp, nval) \
645	644	({ typeof(pcp) ret__; \
646		− ret__ = __this_cpu_read(pcp); \
647		− __this_cpu_write(pcp, nval); \
	645	+ ret__ = raw_cpu_read(pcp); \
	646	+ raw_cpu_write(pcp, nval); \
648	647	ret__; \
649	648	})
650	649
651		−#ifndef __this_cpu_xchg
652		−# ifndef __this_cpu_xchg_1
653		−# define __this_cpu_xchg_1(pcp, nval) __this_cpu_generic_xchg(pcp, nval)
	650	+#ifndef raw_cpu_xchg
	651	+# ifndef raw_cpu_xchg_1
	652	+# define raw_cpu_xchg_1(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
654	653	# endif
655		−# ifndef __this_cpu_xchg_2
656		−# define __this_cpu_xchg_2(pcp, nval) __this_cpu_generic_xchg(pcp, nval)
	654	+# ifndef raw_cpu_xchg_2
	655	+# define raw_cpu_xchg_2(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
657	656	# endif
658		−# ifndef __this_cpu_xchg_4
659		−# define __this_cpu_xchg_4(pcp, nval) __this_cpu_generic_xchg(pcp, nval)
	657	+# ifndef raw_cpu_xchg_4
	658	+# define raw_cpu_xchg_4(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
660	659	# endif
661		−# ifndef __this_cpu_xchg_8
662		−# define __this_cpu_xchg_8(pcp, nval) __this_cpu_generic_xchg(pcp, nval)
	660	+# ifndef raw_cpu_xchg_8
	661	+# define raw_cpu_xchg_8(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
663	662	# endif
664		−# define __this_cpu_xchg(pcp, nval) \
665		− __pcpu_size_call_return2(__this_cpu_xchg_, (pcp), nval)
	663	+# define raw_cpu_xchg(pcp, nval) \
	664	+ __pcpu_size_call_return2(raw_cpu_xchg_, (pcp), nval)
666	665	#endif
667	666
668		−#define __this_cpu_generic_cmpxchg(pcp, oval, nval) \
	667	+#define raw_cpu_generic_cmpxchg(pcp, oval, nval) \
669	668	({ \
670	669	typeof(pcp) ret__; \
671		− ret__ = __this_cpu_read(pcp); \
	670	+ ret__ = raw_cpu_read(pcp); \
672	671	if (ret__ == (oval)) \
673		− __this_cpu_write(pcp, nval); \
	672	+ raw_cpu_write(pcp, nval); \
674	673	ret__; \
675	674	})
676	675
677		−#ifndef __this_cpu_cmpxchg
678		−# ifndef __this_cpu_cmpxchg_1
679		−# define __this_cpu_cmpxchg_1(pcp, oval, nval) __this_cpu_generic_cmpxchg(pcp, oval, nval)
	676	+#ifndef raw_cpu_cmpxchg
	677	+# ifndef raw_cpu_cmpxchg_1
	678	+# define raw_cpu_cmpxchg_1(pcp, oval, nval) raw_cpu_generic_cmpxchg(pcp, oval, nval)
680	679	# endif
681		−# ifndef __this_cpu_cmpxchg_2
682		−# define __this_cpu_cmpxchg_2(pcp, oval, nval) __this_cpu_generic_cmpxchg(pcp, oval, nval)
	680	+# ifndef raw_cpu_cmpxchg_2
	681	+# define raw_cpu_cmpxchg_2(pcp, oval, nval) raw_cpu_generic_cmpxchg(pcp, oval, nval)
683	682	# endif
684		−# ifndef __this_cpu_cmpxchg_4
685		−# define __this_cpu_cmpxchg_4(pcp, oval, nval) __this_cpu_generic_cmpxchg(pcp, oval, nval)
	683	+# ifndef raw_cpu_cmpxchg_4
	684	+# define raw_cpu_cmpxchg_4(pcp, oval, nval) raw_cpu_generic_cmpxchg(pcp, oval, nval)
686	685	# endif
687		−# ifndef __this_cpu_cmpxchg_8
688		−# define __this_cpu_cmpxchg_8(pcp, oval, nval) __this_cpu_generic_cmpxchg(pcp, oval, nval)
	686	+# ifndef raw_cpu_cmpxchg_8
	687	+# define raw_cpu_cmpxchg_8(pcp, oval, nval) raw_cpu_generic_cmpxchg(pcp, oval, nval)
689	688	# endif
690		−# define __this_cpu_cmpxchg(pcp, oval, nval) \
691		− __pcpu_size_call_return2(__this_cpu_cmpxchg_, pcp, oval, nval)
	689	+# define raw_cpu_cmpxchg(pcp, oval, nval) \
	690	+ __pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval)
692	691	#endif
693	692
694		−#define __this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
	693	+#define raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
695	694	({ \
696	695	int __ret = 0; \
697		− if (__this_cpu_read(pcp1) == (oval1) && \
698		− __this_cpu_read(pcp2) == (oval2)) { \
699		− __this_cpu_write(pcp1, (nval1)); \
700		− __this_cpu_write(pcp2, (nval2)); \
	696	+ if (raw_cpu_read(pcp1) == (oval1) && \
	697	+ raw_cpu_read(pcp2) == (oval2)) { \
	698	+ raw_cpu_write(pcp1, (nval1)); \
	699	+ raw_cpu_write(pcp2, (nval2)); \
701	700	__ret = 1; \
702	701	} \
703	702	(__ret); \
704	703	})
705	704
706		−#ifndef __this_cpu_cmpxchg_double
707		−# ifndef __this_cpu_cmpxchg_double_1
708		−# define __this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
709		− __this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
	705	+#ifndef raw_cpu_cmpxchg_double
	706	+# ifndef raw_cpu_cmpxchg_double_1
	707	+# define raw_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
	708	+ raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
710	709	# endif
711		−# ifndef __this_cpu_cmpxchg_double_2
712		−# define __this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
713		− __this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
	710	+# ifndef raw_cpu_cmpxchg_double_2
	711	+# define raw_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
	712	+ raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
714	713	# endif
715		−# ifndef __this_cpu_cmpxchg_double_4
716		−# define __this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
717		− __this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
	714	+# ifndef raw_cpu_cmpxchg_double_4
	715	+# define raw_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
	716	+ raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
718	717	# endif
719		−# ifndef __this_cpu_cmpxchg_double_8
720		−# define __this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
721		− __this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
	718	+# ifndef raw_cpu_cmpxchg_double_8
	719	+# define raw_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
	720	+ raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
722	721	# endif
	722	+# define raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
	723	+ __pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2))
	724	+#endif
	725	+
	726	+/*
	727	+ * Generic percpu operations for context that are safe from preemption/interrupts.
	728	+ * Checks will be added here soon.
	729	+ */
	730	+#ifndef __this_cpu_read
	731	+# define __this_cpu_read(pcp) __pcpu_size_call_return(raw_cpu_read_, (pcp))
	732	+#endif
	733	+
	734	+#ifndef __this_cpu_write
	735	+# define __this_cpu_write(pcp, val) __pcpu_size_call(raw_cpu_write_, (pcp), (val))
	736	+#endif
	737	+
	738	+#ifndef __this_cpu_add
	739	+# define __this_cpu_add(pcp, val) __pcpu_size_call(raw_cpu_add_, (pcp), (val))
	740	+#endif
	741	+
	742	+#ifndef __this_cpu_sub
	743	+# define __this_cpu_sub(pcp, val) __this_cpu_add((pcp), -(typeof(pcp))(val))
	744	+#endif
	745	+
	746	+#ifndef __this_cpu_inc
	747	+# define __this_cpu_inc(pcp) __this_cpu_add((pcp), 1)
	748	+#endif
	749	+
	750	+#ifndef __this_cpu_dec
	751	+# define __this_cpu_dec(pcp) __this_cpu_sub((pcp), 1)
	752	+#endif
	753	+
	754	+#ifndef __this_cpu_and
	755	+# define __this_cpu_and(pcp, val) __pcpu_size_call(raw_cpu_and_, (pcp), (val))
	756	+#endif
	757	+
	758	+#ifndef __this_cpu_or
	759	+# define __this_cpu_or(pcp, val) __pcpu_size_call(raw_cpu_or_, (pcp), (val))
	760	+#endif
	761	+
	762	+#ifndef __this_cpu_add_return
	763	+# define __this_cpu_add_return(pcp, val) \
	764	+ __pcpu_size_call_return2(raw_cpu_add_return_, pcp, val)
	765	+#endif
	766	+
	767	+#define __this_cpu_sub_return(pcp, val) __this_cpu_add_return(pcp, -(typeof(pcp))(val))
	768	+#define __this_cpu_inc_return(pcp) __this_cpu_add_return(pcp, 1)
	769	+#define __this_cpu_dec_return(pcp) __this_cpu_add_return(pcp, -1)
	770	+
	771	+#ifndef __this_cpu_xchg
	772	+# define __this_cpu_xchg(pcp, nval) \
	773	+ __pcpu_size_call_return2(raw_cpu_xchg_, (pcp), nval)
	774	+#endif
	775	+
	776	+#ifndef __this_cpu_cmpxchg
	777	+# define __this_cpu_cmpxchg(pcp, oval, nval) \
	778	+ __pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval)
	779	+#endif
	780	+
	781	+#ifndef __this_cpu_cmpxchg_double
723	782	# define __this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
724		− __pcpu_double_call_return_bool(__this_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2))
	783	+ __pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2))
725	784	#endif
726	785
727	786	#endif /* __LINUX_PERCPU_H */