History

asm-generic/atomic: Use __always_inline for fallback wrappers

Use __always_inline for atomic fallback wrappers. When building for size (CC_OPTIMIZE_FOR_SIZE), some compilers appear to be less inclined to inline even relatively small static inline functions that are assumed to be inlinable such as atomic ops. This can cause problems, for example in UACCESS regions. While the fallback wrappers aren't pure wrappers, they are trivial nonetheless, and the function they wrap should determine the final inlining policy. For x86 tinyconfig we observe: - vmlinux baseline: 1315988 - vmlinux with patch: 1315928 (-60 bytes)

Marco Elver committed 6 years ago

Commit 944bc9c1 parent c02039521 files · +192 −188Browse files

Showing the changes to include/linux (1 of 21 files). Show all

Unified · Split

include/linux/atomic-fallback.h+171−169
@@ -6,6 +6,8 @@ · 5 unmodified lines
6#ifndef _LINUX_ATOMIC_FALLBACK_H6#ifndef _LINUX_ATOMIC_FALLBACK_H
7#define _LINUX_ATOMIC_FALLBACK_H7#define _LINUX_ATOMIC_FALLBACK_H
8 8
9#include <linux/compiler.h>
10
9#ifndef xchg_relaxed11#ifndef xchg_relaxed
10#define xchg_relaxed xchg12#define xchg_relaxed xchg
11#define xchg_acquire xchg13#define xchg_acquire xchg
@@ -76,7 +78,7 @@ · 64 unmodified lines
76#endif /* cmpxchg64_relaxed */78#endif /* cmpxchg64_relaxed */
77 79
78#ifndef atomic_read_acquire80#ifndef atomic_read_acquire
79static inline int81static __always_inline int
80atomic_read_acquire(const atomic_t *v)82atomic_read_acquire(const atomic_t *v)
81{83{
82 return smp_load_acquire(&(v)->counter);84 return smp_load_acquire(&(v)->counter);
@@ -85,7 +87,7 @@ atomic_read_acquire(const atomic_t *v) · 2 unmodified lines
85#endif87#endif
86 88
87#ifndef atomic_set_release89#ifndef atomic_set_release
88static inline void90static __always_inline void
89atomic_set_release(atomic_t *v, int i)91atomic_set_release(atomic_t *v, int i)
90{92{
91 smp_store_release(&(v)->counter, i);93 smp_store_release(&(v)->counter, i);
@@ -100,7 +102,7 @@ atomic_set_release(atomic_t *v, int i) · 8 unmodified lines
100#else /* atomic_add_return_relaxed */102#else /* atomic_add_return_relaxed */
101 103
102#ifndef atomic_add_return_acquire104#ifndef atomic_add_return_acquire
103static inline int105static __always_inline int
104atomic_add_return_acquire(int i, atomic_t *v)106atomic_add_return_acquire(int i, atomic_t *v)
105{107{
106 int ret = atomic_add_return_relaxed(i, v);108 int ret = atomic_add_return_relaxed(i, v);
@@ -111,7 +113,7 @@ atomic_add_return_acquire(int i, atomic_t *v) · 4 unmodified lines
111#endif113#endif
112 114
113#ifndef atomic_add_return_release115#ifndef atomic_add_return_release
114static inline int116static __always_inline int
115atomic_add_return_release(int i, atomic_t *v)117atomic_add_return_release(int i, atomic_t *v)
116{118{
117 __atomic_release_fence();119 __atomic_release_fence();
@@ -121,7 +123,7 @@ atomic_add_return_release(int i, atomic_t *v) · 3 unmodified lines
121#endif123#endif
122 124
123#ifndef atomic_add_return125#ifndef atomic_add_return
124static inline int126static __always_inline int
125atomic_add_return(int i, atomic_t *v)127atomic_add_return(int i, atomic_t *v)
126{128{
127 int ret;129 int ret;
@@ -142,7 +144,7 @@ atomic_add_return(int i, atomic_t *v) · 14 unmodified lines
142#else /* atomic_fetch_add_relaxed */144#else /* atomic_fetch_add_relaxed */
143 145
144#ifndef atomic_fetch_add_acquire146#ifndef atomic_fetch_add_acquire
145static inline int147static __always_inline int
146atomic_fetch_add_acquire(int i, atomic_t *v)148atomic_fetch_add_acquire(int i, atomic_t *v)
147{149{
148 int ret = atomic_fetch_add_relaxed(i, v);150 int ret = atomic_fetch_add_relaxed(i, v);
@@ -153,7 +155,7 @@ atomic_fetch_add_acquire(int i, atomic_t *v) · 4 unmodified lines
153#endif155#endif
154 156
155#ifndef atomic_fetch_add_release157#ifndef atomic_fetch_add_release
156static inline int158static __always_inline int
157atomic_fetch_add_release(int i, atomic_t *v)159atomic_fetch_add_release(int i, atomic_t *v)
158{160{
159 __atomic_release_fence();161 __atomic_release_fence();
@@ -163,7 +165,7 @@ atomic_fetch_add_release(int i, atomic_t *v) · 3 unmodified lines
163#endif165#endif
164 166
165#ifndef atomic_fetch_add167#ifndef atomic_fetch_add
166static inline int168static __always_inline int
167atomic_fetch_add(int i, atomic_t *v)169atomic_fetch_add(int i, atomic_t *v)
168{170{
169 int ret;171 int ret;
@@ -184,7 +186,7 @@ atomic_fetch_add(int i, atomic_t *v) · 14 unmodified lines
184#else /* atomic_sub_return_relaxed */186#else /* atomic_sub_return_relaxed */
185 187
186#ifndef atomic_sub_return_acquire188#ifndef atomic_sub_return_acquire
187static inline int189static __always_inline int
188atomic_sub_return_acquire(int i, atomic_t *v)190atomic_sub_return_acquire(int i, atomic_t *v)
189{191{
190 int ret = atomic_sub_return_relaxed(i, v);192 int ret = atomic_sub_return_relaxed(i, v);
@@ -195,7 +197,7 @@ atomic_sub_return_acquire(int i, atomic_t *v) · 4 unmodified lines
195#endif197#endif
196 198
197#ifndef atomic_sub_return_release199#ifndef atomic_sub_return_release
198static inline int200static __always_inline int
199atomic_sub_return_release(int i, atomic_t *v)201atomic_sub_return_release(int i, atomic_t *v)
200{202{
201 __atomic_release_fence();203 __atomic_release_fence();
@@ -205,7 +207,7 @@ atomic_sub_return_release(int i, atomic_t *v) · 3 unmodified lines
205#endif207#endif
206 208
207#ifndef atomic_sub_return209#ifndef atomic_sub_return
208static inline int210static __always_inline int
209atomic_sub_return(int i, atomic_t *v)211atomic_sub_return(int i, atomic_t *v)
210{212{
211 int ret;213 int ret;
@@ -226,7 +228,7 @@ atomic_sub_return(int i, atomic_t *v) · 14 unmodified lines
226#else /* atomic_fetch_sub_relaxed */228#else /* atomic_fetch_sub_relaxed */
227 229
228#ifndef atomic_fetch_sub_acquire230#ifndef atomic_fetch_sub_acquire
229static inline int231static __always_inline int
230atomic_fetch_sub_acquire(int i, atomic_t *v)232atomic_fetch_sub_acquire(int i, atomic_t *v)
231{233{
232 int ret = atomic_fetch_sub_relaxed(i, v);234 int ret = atomic_fetch_sub_relaxed(i, v);
@@ -237,7 +239,7 @@ atomic_fetch_sub_acquire(int i, atomic_t *v) · 4 unmodified lines
237#endif239#endif
238 240
239#ifndef atomic_fetch_sub_release241#ifndef atomic_fetch_sub_release
240static inline int242static __always_inline int
241atomic_fetch_sub_release(int i, atomic_t *v)243atomic_fetch_sub_release(int i, atomic_t *v)
242{244{
243 __atomic_release_fence();245 __atomic_release_fence();
@@ -247,7 +249,7 @@ atomic_fetch_sub_release(int i, atomic_t *v) · 3 unmodified lines
247#endif249#endif
248 250
249#ifndef atomic_fetch_sub251#ifndef atomic_fetch_sub
250static inline int252static __always_inline int
251atomic_fetch_sub(int i, atomic_t *v)253atomic_fetch_sub(int i, atomic_t *v)
252{254{
253 int ret;255 int ret;
@@ -262,7 +264,7 @@ atomic_fetch_sub(int i, atomic_t *v) · 8 unmodified lines
262#endif /* atomic_fetch_sub_relaxed */264#endif /* atomic_fetch_sub_relaxed */
263 265
264#ifndef atomic_inc266#ifndef atomic_inc
265static inline void267static __always_inline void
266atomic_inc(atomic_t *v)268atomic_inc(atomic_t *v)
267{269{
268 atomic_add(1, v);270 atomic_add(1, v);
@@ -278,7 +280,7 @@ atomic_inc(atomic_t *v) · 9 unmodified lines
278#endif /* atomic_inc_return */280#endif /* atomic_inc_return */
279 281
280#ifndef atomic_inc_return282#ifndef atomic_inc_return
281static inline int283static __always_inline int
282atomic_inc_return(atomic_t *v)284atomic_inc_return(atomic_t *v)
283{285{
284 return atomic_add_return(1, v);286 return atomic_add_return(1, v);
@@ -287,7 +289,7 @@ atomic_inc_return(atomic_t *v) · 2 unmodified lines
287#endif289#endif
288 290
289#ifndef atomic_inc_return_acquire291#ifndef atomic_inc_return_acquire
290static inline int292static __always_inline int
291atomic_inc_return_acquire(atomic_t *v)293atomic_inc_return_acquire(atomic_t *v)
292{294{
293 return atomic_add_return_acquire(1, v);295 return atomic_add_return_acquire(1, v);
@@ -296,7 +298,7 @@ atomic_inc_return_acquire(atomic_t *v) · 2 unmodified lines
296#endif298#endif
297 299
298#ifndef atomic_inc_return_release300#ifndef atomic_inc_return_release
299static inline int301static __always_inline int
300atomic_inc_return_release(atomic_t *v)302atomic_inc_return_release(atomic_t *v)
301{303{
302 return atomic_add_return_release(1, v);304 return atomic_add_return_release(1, v);
@@ -305,7 +307,7 @@ atomic_inc_return_release(atomic_t *v) · 2 unmodified lines
305#endif307#endif
306 308
307#ifndef atomic_inc_return_relaxed309#ifndef atomic_inc_return_relaxed
308static inline int310static __always_inline int
309atomic_inc_return_relaxed(atomic_t *v)311atomic_inc_return_relaxed(atomic_t *v)
310{312{
311 return atomic_add_return_relaxed(1, v);313 return atomic_add_return_relaxed(1, v);
@@ -316,7 +318,7 @@ atomic_inc_return_relaxed(atomic_t *v) · 4 unmodified lines
316#else /* atomic_inc_return_relaxed */318#else /* atomic_inc_return_relaxed */
317 319
318#ifndef atomic_inc_return_acquire320#ifndef atomic_inc_return_acquire
319static inline int321static __always_inline int
320atomic_inc_return_acquire(atomic_t *v)322atomic_inc_return_acquire(atomic_t *v)
321{323{
322 int ret = atomic_inc_return_relaxed(v);324 int ret = atomic_inc_return_relaxed(v);
@@ -327,7 +329,7 @@ atomic_inc_return_acquire(atomic_t *v) · 4 unmodified lines
327#endif329#endif
328 330
329#ifndef atomic_inc_return_release331#ifndef atomic_inc_return_release
330static inline int332static __always_inline int
331atomic_inc_return_release(atomic_t *v)333atomic_inc_return_release(atomic_t *v)
332{334{
333 __atomic_release_fence();335 __atomic_release_fence();
@@ -337,7 +339,7 @@ atomic_inc_return_release(atomic_t *v) · 3 unmodified lines
337#endif339#endif
338 340
339#ifndef atomic_inc_return341#ifndef atomic_inc_return
340static inline int342static __always_inline int
341atomic_inc_return(atomic_t *v)343atomic_inc_return(atomic_t *v)
342{344{
343 int ret;345 int ret;
@@ -359,7 +361,7 @@ atomic_inc_return(atomic_t *v) · 15 unmodified lines
359#endif /* atomic_fetch_inc */361#endif /* atomic_fetch_inc */
360 362
361#ifndef atomic_fetch_inc363#ifndef atomic_fetch_inc
362static inline int364static __always_inline int
363atomic_fetch_inc(atomic_t *v)365atomic_fetch_inc(atomic_t *v)
364{366{
365 return atomic_fetch_add(1, v);367 return atomic_fetch_add(1, v);
@@ -368,7 +370,7 @@ atomic_fetch_inc(atomic_t *v) · 2 unmodified lines
368#endif370#endif
369 371
370#ifndef atomic_fetch_inc_acquire372#ifndef atomic_fetch_inc_acquire
371static inline int373static __always_inline int
372atomic_fetch_inc_acquire(atomic_t *v)374atomic_fetch_inc_acquire(atomic_t *v)
373{375{
374 return atomic_fetch_add_acquire(1, v);376 return atomic_fetch_add_acquire(1, v);
@@ -377,7 +379,7 @@ atomic_fetch_inc_acquire(atomic_t *v) · 2 unmodified lines
377#endif379#endif
378 380
379#ifndef atomic_fetch_inc_release381#ifndef atomic_fetch_inc_release
380static inline int382static __always_inline int
381atomic_fetch_inc_release(atomic_t *v)383atomic_fetch_inc_release(atomic_t *v)
382{384{
383 return atomic_fetch_add_release(1, v);385 return atomic_fetch_add_release(1, v);
@@ -386,7 +388,7 @@ atomic_fetch_inc_release(atomic_t *v) · 2 unmodified lines
386#endif388#endif
387 389
388#ifndef atomic_fetch_inc_relaxed390#ifndef atomic_fetch_inc_relaxed
389static inline int391static __always_inline int
390atomic_fetch_inc_relaxed(atomic_t *v)392atomic_fetch_inc_relaxed(atomic_t *v)
391{393{
392 return atomic_fetch_add_relaxed(1, v);394 return atomic_fetch_add_relaxed(1, v);
@@ -397,7 +399,7 @@ atomic_fetch_inc_relaxed(atomic_t *v) · 4 unmodified lines
397#else /* atomic_fetch_inc_relaxed */399#else /* atomic_fetch_inc_relaxed */
398 400
399#ifndef atomic_fetch_inc_acquire401#ifndef atomic_fetch_inc_acquire
400static inline int402static __always_inline int
401atomic_fetch_inc_acquire(atomic_t *v)403atomic_fetch_inc_acquire(atomic_t *v)
402{404{
403 int ret = atomic_fetch_inc_relaxed(v);405 int ret = atomic_fetch_inc_relaxed(v);
@@ -408,7 +410,7 @@ atomic_fetch_inc_acquire(atomic_t *v) · 4 unmodified lines
408#endif410#endif
409 411
410#ifndef atomic_fetch_inc_release412#ifndef atomic_fetch_inc_release
411static inline int413static __always_inline int
412atomic_fetch_inc_release(atomic_t *v)414atomic_fetch_inc_release(atomic_t *v)
413{415{
414 __atomic_release_fence();416 __atomic_release_fence();
@@ -418,7 +420,7 @@ atomic_fetch_inc_release(atomic_t *v) · 3 unmodified lines
418#endif420#endif
419 421
420#ifndef atomic_fetch_inc422#ifndef atomic_fetch_inc
421static inline int423static __always_inline int
422atomic_fetch_inc(atomic_t *v)424atomic_fetch_inc(atomic_t *v)
423{425{
424 int ret;426 int ret;
@@ -433,7 +435,7 @@ atomic_fetch_inc(atomic_t *v) · 8 unmodified lines
433#endif /* atomic_fetch_inc_relaxed */435#endif /* atomic_fetch_inc_relaxed */
434 436
435#ifndef atomic_dec437#ifndef atomic_dec
436static inline void438static __always_inline void
437atomic_dec(atomic_t *v)439atomic_dec(atomic_t *v)
438{440{
439 atomic_sub(1, v);441 atomic_sub(1, v);
@@ -449,7 +451,7 @@ atomic_dec(atomic_t *v) · 9 unmodified lines
449#endif /* atomic_dec_return */451#endif /* atomic_dec_return */
450 452
451#ifndef atomic_dec_return453#ifndef atomic_dec_return
452static inline int454static __always_inline int
453atomic_dec_return(atomic_t *v)455atomic_dec_return(atomic_t *v)
454{456{
455 return atomic_sub_return(1, v);457 return atomic_sub_return(1, v);
@@ -458,7 +460,7 @@ atomic_dec_return(atomic_t *v) · 2 unmodified lines
458#endif460#endif
459 461
460#ifndef atomic_dec_return_acquire462#ifndef atomic_dec_return_acquire
461static inline int463static __always_inline int
462atomic_dec_return_acquire(atomic_t *v)464atomic_dec_return_acquire(atomic_t *v)
463{465{
464 return atomic_sub_return_acquire(1, v);466 return atomic_sub_return_acquire(1, v);
@@ -467,7 +469,7 @@ atomic_dec_return_acquire(atomic_t *v) · 2 unmodified lines
467#endif469#endif
468 470
469#ifndef atomic_dec_return_release471#ifndef atomic_dec_return_release
470static inline int472static __always_inline int
471atomic_dec_return_release(atomic_t *v)473atomic_dec_return_release(atomic_t *v)
472{474{
473 return atomic_sub_return_release(1, v);475 return atomic_sub_return_release(1, v);
@@ -476,7 +478,7 @@ atomic_dec_return_release(atomic_t *v) · 2 unmodified lines
476#endif478#endif
477 479
478#ifndef atomic_dec_return_relaxed480#ifndef atomic_dec_return_relaxed
479static inline int481static __always_inline int
480atomic_dec_return_relaxed(atomic_t *v)482atomic_dec_return_relaxed(atomic_t *v)
481{483{
482 return atomic_sub_return_relaxed(1, v);484 return atomic_sub_return_relaxed(1, v);
@@ -487,7 +489,7 @@ atomic_dec_return_relaxed(atomic_t *v) · 4 unmodified lines
487#else /* atomic_dec_return_relaxed */489#else /* atomic_dec_return_relaxed */
488 490
489#ifndef atomic_dec_return_acquire491#ifndef atomic_dec_return_acquire
490static inline int492static __always_inline int
491atomic_dec_return_acquire(atomic_t *v)493atomic_dec_return_acquire(atomic_t *v)
492{494{
493 int ret = atomic_dec_return_relaxed(v);495 int ret = atomic_dec_return_relaxed(v);
@@ -498,7 +500,7 @@ atomic_dec_return_acquire(atomic_t *v) · 4 unmodified lines
498#endif500#endif
499 501
500#ifndef atomic_dec_return_release502#ifndef atomic_dec_return_release
501static inline int503static __always_inline int
502atomic_dec_return_release(atomic_t *v)504atomic_dec_return_release(atomic_t *v)
503{505{
504 __atomic_release_fence();506 __atomic_release_fence();
@@ -508,7 +510,7 @@ atomic_dec_return_release(atomic_t *v) · 3 unmodified lines
508#endif510#endif
509 511
510#ifndef atomic_dec_return512#ifndef atomic_dec_return
511static inline int513static __always_inline int
512atomic_dec_return(atomic_t *v)514atomic_dec_return(atomic_t *v)
513{515{
514 int ret;516 int ret;
@@ -530,7 +532,7 @@ atomic_dec_return(atomic_t *v) · 15 unmodified lines
530#endif /* atomic_fetch_dec */532#endif /* atomic_fetch_dec */
531 533
532#ifndef atomic_fetch_dec534#ifndef atomic_fetch_dec
533static inline int535static __always_inline int
534atomic_fetch_dec(atomic_t *v)536atomic_fetch_dec(atomic_t *v)
535{537{
536 return atomic_fetch_sub(1, v);538 return atomic_fetch_sub(1, v);
@@ -539,7 +541,7 @@ atomic_fetch_dec(atomic_t *v) · 2 unmodified lines
539#endif541#endif
540 542
541#ifndef atomic_fetch_dec_acquire543#ifndef atomic_fetch_dec_acquire
542static inline int544static __always_inline int
543atomic_fetch_dec_acquire(atomic_t *v)545atomic_fetch_dec_acquire(atomic_t *v)
544{546{
545 return atomic_fetch_sub_acquire(1, v);547 return atomic_fetch_sub_acquire(1, v);
@@ -548,7 +550,7 @@ atomic_fetch_dec_acquire(atomic_t *v) · 2 unmodified lines
548#endif550#endif
549 551
550#ifndef atomic_fetch_dec_release552#ifndef atomic_fetch_dec_release
551static inline int553static __always_inline int
552atomic_fetch_dec_release(atomic_t *v)554atomic_fetch_dec_release(atomic_t *v)
553{555{
554 return atomic_fetch_sub_release(1, v);556 return atomic_fetch_sub_release(1, v);
@@ -557,7 +559,7 @@ atomic_fetch_dec_release(atomic_t *v) · 2 unmodified lines
557#endif559#endif
558 560
559#ifndef atomic_fetch_dec_relaxed561#ifndef atomic_fetch_dec_relaxed
560static inline int562static __always_inline int
561atomic_fetch_dec_relaxed(atomic_t *v)563atomic_fetch_dec_relaxed(atomic_t *v)
562{564{
563 return atomic_fetch_sub_relaxed(1, v);565 return atomic_fetch_sub_relaxed(1, v);
@@ -568,7 +570,7 @@ atomic_fetch_dec_relaxed(atomic_t *v) · 4 unmodified lines
568#else /* atomic_fetch_dec_relaxed */570#else /* atomic_fetch_dec_relaxed */
569 571
570#ifndef atomic_fetch_dec_acquire572#ifndef atomic_fetch_dec_acquire
571static inline int573static __always_inline int
572atomic_fetch_dec_acquire(atomic_t *v)574atomic_fetch_dec_acquire(atomic_t *v)
573{575{
574 int ret = atomic_fetch_dec_relaxed(v);576 int ret = atomic_fetch_dec_relaxed(v);
@@ -579,7 +581,7 @@ atomic_fetch_dec_acquire(atomic_t *v) · 4 unmodified lines
579#endif581#endif
580 582
581#ifndef atomic_fetch_dec_release583#ifndef atomic_fetch_dec_release
582static inline int584static __always_inline int
583atomic_fetch_dec_release(atomic_t *v)585atomic_fetch_dec_release(atomic_t *v)
584{586{
585 __atomic_release_fence();587 __atomic_release_fence();
@@ -589,7 +591,7 @@ atomic_fetch_dec_release(atomic_t *v) · 3 unmodified lines
589#endif591#endif
590 592
591#ifndef atomic_fetch_dec593#ifndef atomic_fetch_dec
592static inline int594static __always_inline int
593atomic_fetch_dec(atomic_t *v)595atomic_fetch_dec(atomic_t *v)
594{596{
595 int ret;597 int ret;
@@ -610,7 +612,7 @@ atomic_fetch_dec(atomic_t *v) · 14 unmodified lines
610#else /* atomic_fetch_and_relaxed */612#else /* atomic_fetch_and_relaxed */
611 613
612#ifndef atomic_fetch_and_acquire614#ifndef atomic_fetch_and_acquire
613static inline int615static __always_inline int
614atomic_fetch_and_acquire(int i, atomic_t *v)616atomic_fetch_and_acquire(int i, atomic_t *v)
615{617{
616 int ret = atomic_fetch_and_relaxed(i, v);618 int ret = atomic_fetch_and_relaxed(i, v);
@@ -621,7 +623,7 @@ atomic_fetch_and_acquire(int i, atomic_t *v) · 4 unmodified lines
621#endif623#endif
622 624
623#ifndef atomic_fetch_and_release625#ifndef atomic_fetch_and_release
624static inline int626static __always_inline int
625atomic_fetch_and_release(int i, atomic_t *v)627atomic_fetch_and_release(int i, atomic_t *v)
626{628{
627 __atomic_release_fence();629 __atomic_release_fence();
@@ -631,7 +633,7 @@ atomic_fetch_and_release(int i, atomic_t *v) · 3 unmodified lines
631#endif633#endif
632 634
633#ifndef atomic_fetch_and635#ifndef atomic_fetch_and
634static inline int636static __always_inline int
635atomic_fetch_and(int i, atomic_t *v)637atomic_fetch_and(int i, atomic_t *v)
636{638{
637 int ret;639 int ret;
@@ -646,7 +648,7 @@ atomic_fetch_and(int i, atomic_t *v) · 8 unmodified lines
646#endif /* atomic_fetch_and_relaxed */648#endif /* atomic_fetch_and_relaxed */
647 649
648#ifndef atomic_andnot650#ifndef atomic_andnot
649static inline void651static __always_inline void
650atomic_andnot(int i, atomic_t *v)652atomic_andnot(int i, atomic_t *v)
651{653{
652 atomic_and(~i, v);654 atomic_and(~i, v);
@@ -662,7 +664,7 @@ atomic_andnot(int i, atomic_t *v) · 9 unmodified lines
662#endif /* atomic_fetch_andnot */664#endif /* atomic_fetch_andnot */
663 665
664#ifndef atomic_fetch_andnot666#ifndef atomic_fetch_andnot
665static inline int667static __always_inline int
666atomic_fetch_andnot(int i, atomic_t *v)668atomic_fetch_andnot(int i, atomic_t *v)
667{669{
668 return atomic_fetch_and(~i, v);670 return atomic_fetch_and(~i, v);
@@ -671,7 +673,7 @@ atomic_fetch_andnot(int i, atomic_t *v) · 2 unmodified lines
671#endif673#endif
672 674
673#ifndef atomic_fetch_andnot_acquire675#ifndef atomic_fetch_andnot_acquire
674static inline int676static __always_inline int
675atomic_fetch_andnot_acquire(int i, atomic_t *v)677atomic_fetch_andnot_acquire(int i, atomic_t *v)
676{678{
677 return atomic_fetch_and_acquire(~i, v);679 return atomic_fetch_and_acquire(~i, v);
@@ -680,7 +682,7 @@ atomic_fetch_andnot_acquire(int i, atomic_t *v) · 2 unmodified lines
680#endif682#endif
681 683
682#ifndef atomic_fetch_andnot_release684#ifndef atomic_fetch_andnot_release
683static inline int685static __always_inline int
684atomic_fetch_andnot_release(int i, atomic_t *v)686atomic_fetch_andnot_release(int i, atomic_t *v)
685{687{
686 return atomic_fetch_and_release(~i, v);688 return atomic_fetch_and_release(~i, v);
@@ -689,7 +691,7 @@ atomic_fetch_andnot_release(int i, atomic_t *v) · 2 unmodified lines
689#endif691#endif
690 692
691#ifndef atomic_fetch_andnot_relaxed693#ifndef atomic_fetch_andnot_relaxed
692static inline int694static __always_inline int
693atomic_fetch_andnot_relaxed(int i, atomic_t *v)695atomic_fetch_andnot_relaxed(int i, atomic_t *v)
694{696{
695 return atomic_fetch_and_relaxed(~i, v);697 return atomic_fetch_and_relaxed(~i, v);
@@ -700,7 +702,7 @@ atomic_fetch_andnot_relaxed(int i, atomic_t *v) · 4 unmodified lines
700#else /* atomic_fetch_andnot_relaxed */702#else /* atomic_fetch_andnot_relaxed */
701 703
702#ifndef atomic_fetch_andnot_acquire704#ifndef atomic_fetch_andnot_acquire
703static inline int705static __always_inline int
704atomic_fetch_andnot_acquire(int i, atomic_t *v)706atomic_fetch_andnot_acquire(int i, atomic_t *v)
705{707{
706 int ret = atomic_fetch_andnot_relaxed(i, v);708 int ret = atomic_fetch_andnot_relaxed(i, v);
@@ -711,7 +713,7 @@ atomic_fetch_andnot_acquire(int i, atomic_t *v) · 4 unmodified lines
711#endif713#endif
712 714
713#ifndef atomic_fetch_andnot_release715#ifndef atomic_fetch_andnot_release
714static inline int716static __always_inline int
715atomic_fetch_andnot_release(int i, atomic_t *v)717atomic_fetch_andnot_release(int i, atomic_t *v)
716{718{
717 __atomic_release_fence();719 __atomic_release_fence();
@@ -721,7 +723,7 @@ atomic_fetch_andnot_release(int i, atomic_t *v) · 3 unmodified lines
721#endif723#endif
722 724
723#ifndef atomic_fetch_andnot725#ifndef atomic_fetch_andnot
724static inline int726static __always_inline int
725atomic_fetch_andnot(int i, atomic_t *v)727atomic_fetch_andnot(int i, atomic_t *v)
726{728{
727 int ret;729 int ret;
@@ -742,7 +744,7 @@ atomic_fetch_andnot(int i, atomic_t *v) · 14 unmodified lines
742#else /* atomic_fetch_or_relaxed */744#else /* atomic_fetch_or_relaxed */
743 745
744#ifndef atomic_fetch_or_acquire746#ifndef atomic_fetch_or_acquire
745static inline int747static __always_inline int
746atomic_fetch_or_acquire(int i, atomic_t *v)748atomic_fetch_or_acquire(int i, atomic_t *v)
747{749{
748 int ret = atomic_fetch_or_relaxed(i, v);750 int ret = atomic_fetch_or_relaxed(i, v);
@@ -753,7 +755,7 @@ atomic_fetch_or_acquire(int i, atomic_t *v) · 4 unmodified lines
753#endif755#endif
754 756
755#ifndef atomic_fetch_or_release757#ifndef atomic_fetch_or_release
756static inline int758static __always_inline int
757atomic_fetch_or_release(int i, atomic_t *v)759atomic_fetch_or_release(int i, atomic_t *v)
758{760{
759 __atomic_release_fence();761 __atomic_release_fence();
@@ -763,7 +765,7 @@ atomic_fetch_or_release(int i, atomic_t *v) · 3 unmodified lines
763#endif765#endif
764 766
765#ifndef atomic_fetch_or767#ifndef atomic_fetch_or
766static inline int768static __always_inline int
767atomic_fetch_or(int i, atomic_t *v)769atomic_fetch_or(int i, atomic_t *v)
768{770{
769 int ret;771 int ret;
@@ -784,7 +786,7 @@ atomic_fetch_or(int i, atomic_t *v) · 14 unmodified lines
784#else /* atomic_fetch_xor_relaxed */786#else /* atomic_fetch_xor_relaxed */
785 787
786#ifndef atomic_fetch_xor_acquire788#ifndef atomic_fetch_xor_acquire
787static inline int789static __always_inline int
788atomic_fetch_xor_acquire(int i, atomic_t *v)790atomic_fetch_xor_acquire(int i, atomic_t *v)
789{791{
790 int ret = atomic_fetch_xor_relaxed(i, v);792 int ret = atomic_fetch_xor_relaxed(i, v);
@@ -795,7 +797,7 @@ atomic_fetch_xor_acquire(int i, atomic_t *v) · 4 unmodified lines
795#endif797#endif
796 798
797#ifndef atomic_fetch_xor_release799#ifndef atomic_fetch_xor_release
798static inline int800static __always_inline int
799atomic_fetch_xor_release(int i, atomic_t *v)801atomic_fetch_xor_release(int i, atomic_t *v)
800{802{
801 __atomic_release_fence();803 __atomic_release_fence();
@@ -805,7 +807,7 @@ atomic_fetch_xor_release(int i, atomic_t *v) · 3 unmodified lines
805#endif807#endif
806 808
807#ifndef atomic_fetch_xor809#ifndef atomic_fetch_xor
808static inline int810static __always_inline int
809atomic_fetch_xor(int i, atomic_t *v)811atomic_fetch_xor(int i, atomic_t *v)
810{812{
811 int ret;813 int ret;
@@ -826,7 +828,7 @@ atomic_fetch_xor(int i, atomic_t *v) · 14 unmodified lines
826#else /* atomic_xchg_relaxed */828#else /* atomic_xchg_relaxed */
827 829
828#ifndef atomic_xchg_acquire830#ifndef atomic_xchg_acquire
829static inline int831static __always_inline int
830atomic_xchg_acquire(atomic_t *v, int i)832atomic_xchg_acquire(atomic_t *v, int i)
831{833{
832 int ret = atomic_xchg_relaxed(v, i);834 int ret = atomic_xchg_relaxed(v, i);
@@ -837,7 +839,7 @@ atomic_xchg_acquire(atomic_t *v, int i) · 4 unmodified lines
837#endif839#endif
838 840
839#ifndef atomic_xchg_release841#ifndef atomic_xchg_release
840static inline int842static __always_inline int
841atomic_xchg_release(atomic_t *v, int i)843atomic_xchg_release(atomic_t *v, int i)
842{844{
843 __atomic_release_fence();845 __atomic_release_fence();
@@ -847,7 +849,7 @@ atomic_xchg_release(atomic_t *v, int i) · 3 unmodified lines
847#endif849#endif
848 850
849#ifndef atomic_xchg851#ifndef atomic_xchg
850static inline int852static __always_inline int
851atomic_xchg(atomic_t *v, int i)853atomic_xchg(atomic_t *v, int i)
852{854{
853 int ret;855 int ret;
@@ -868,7 +870,7 @@ atomic_xchg(atomic_t *v, int i) · 14 unmodified lines
868#else /* atomic_cmpxchg_relaxed */870#else /* atomic_cmpxchg_relaxed */
869 871
870#ifndef atomic_cmpxchg_acquire872#ifndef atomic_cmpxchg_acquire
871static inline int873static __always_inline int
872atomic_cmpxchg_acquire(atomic_t *v, int old, int new)874atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
873{875{
874 int ret = atomic_cmpxchg_relaxed(v, old, new);876 int ret = atomic_cmpxchg_relaxed(v, old, new);
@@ -879,7 +881,7 @@ atomic_cmpxchg_acquire(atomic_t *v, int old, int new) · 4 unmodified lines
879#endif881#endif
880 882
881#ifndef atomic_cmpxchg_release883#ifndef atomic_cmpxchg_release
882static inline int884static __always_inline int
883atomic_cmpxchg_release(atomic_t *v, int old, int new)885atomic_cmpxchg_release(atomic_t *v, int old, int new)
884{886{
885 __atomic_release_fence();887 __atomic_release_fence();
@@ -889,7 +891,7 @@ atomic_cmpxchg_release(atomic_t *v, int old, int new) · 3 unmodified lines
889#endif891#endif
890 892
891#ifndef atomic_cmpxchg893#ifndef atomic_cmpxchg
892static inline int894static __always_inline int
893atomic_cmpxchg(atomic_t *v, int old, int new)895atomic_cmpxchg(atomic_t *v, int old, int new)
894{896{
895 int ret;897 int ret;
@@ -911,7 +913,7 @@ atomic_cmpxchg(atomic_t *v, int old, int new) · 15 unmodified lines
911#endif /* atomic_try_cmpxchg */913#endif /* atomic_try_cmpxchg */
912 914
913#ifndef atomic_try_cmpxchg915#ifndef atomic_try_cmpxchg
914static inline bool916static __always_inline bool
915atomic_try_cmpxchg(atomic_t *v, int *old, int new)917atomic_try_cmpxchg(atomic_t *v, int *old, int new)
916{918{
917 int r, o = *old;919 int r, o = *old;
@@ -924,7 +926,7 @@ atomic_try_cmpxchg(atomic_t *v, int *old, int new) · 6 unmodified lines
924#endif926#endif
925 927
926#ifndef atomic_try_cmpxchg_acquire928#ifndef atomic_try_cmpxchg_acquire
927static inline bool929static __always_inline bool
928atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)930atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
929{931{
930 int r, o = *old;932 int r, o = *old;
@@ -937,7 +939,7 @@ atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) · 6 unmodified lines
937#endif939#endif
938 940
939#ifndef atomic_try_cmpxchg_release941#ifndef atomic_try_cmpxchg_release
940static inline bool942static __always_inline bool
941atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)943atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
942{944{
943 int r, o = *old;945 int r, o = *old;
@@ -950,7 +952,7 @@ atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) · 6 unmodified lines
950#endif952#endif
951 953
952#ifndef atomic_try_cmpxchg_relaxed954#ifndef atomic_try_cmpxchg_relaxed
953static inline bool955static __always_inline bool
954atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)956atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
955{957{
956 int r, o = *old;958 int r, o = *old;
@@ -965,7 +967,7 @@ atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) · 8 unmodified lines
965#else /* atomic_try_cmpxchg_relaxed */967#else /* atomic_try_cmpxchg_relaxed */
966 968
967#ifndef atomic_try_cmpxchg_acquire969#ifndef atomic_try_cmpxchg_acquire
968static inline bool970static __always_inline bool
969atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)971atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
970{972{
971 bool ret = atomic_try_cmpxchg_relaxed(v, old, new);973 bool ret = atomic_try_cmpxchg_relaxed(v, old, new);
@@ -976,7 +978,7 @@ atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) · 4 unmodified lines
976#endif978#endif
977 979
978#ifndef atomic_try_cmpxchg_release980#ifndef atomic_try_cmpxchg_release
979static inline bool981static __always_inline bool
980atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)982atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
981{983{
982 __atomic_release_fence();984 __atomic_release_fence();
@@ -986,7 +988,7 @@ atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) · 3 unmodified lines
986#endif988#endif
987 989
988#ifndef atomic_try_cmpxchg990#ifndef atomic_try_cmpxchg
989static inline bool991static __always_inline bool
990atomic_try_cmpxchg(atomic_t *v, int *old, int new)992atomic_try_cmpxchg(atomic_t *v, int *old, int new)
991{993{
992 bool ret;994 bool ret;
@@ -1010,7 +1012,7 @@ atomic_try_cmpxchg(atomic_t *v, int *old, int new) · 17 unmodified lines
1010 * true if the result is zero, or false for all1012 * true if the result is zero, or false for all
1011 * other cases.1013 * other cases.
1012 */1014 */
1013static inline bool1015static __always_inline bool
1014atomic_sub_and_test(int i, atomic_t *v)1016atomic_sub_and_test(int i, atomic_t *v)
1015{1017{
1016 return atomic_sub_return(i, v) == 0;1018 return atomic_sub_return(i, v) == 0;
@@ -1027,7 +1029,7 @@ atomic_sub_and_test(int i, atomic_t *v) · 10 unmodified lines
1027 * returns true if the result is 0, or false for all other1029 * returns true if the result is 0, or false for all other
1028 * cases.1030 * cases.
1029 */1031 */
1030static inline bool1032static __always_inline bool
1031atomic_dec_and_test(atomic_t *v)1033atomic_dec_and_test(atomic_t *v)
1032{1034{
1033 return atomic_dec_return(v) == 0;1035 return atomic_dec_return(v) == 0;
@@ -1044,7 +1046,7 @@ atomic_dec_and_test(atomic_t *v) · 10 unmodified lines
1044 * and returns true if the result is zero, or false for all1046 * and returns true if the result is zero, or false for all
1045 * other cases.1047 * other cases.
1046 */1048 */
1047static inline bool1049static __always_inline bool
1048atomic_inc_and_test(atomic_t *v)1050atomic_inc_and_test(atomic_t *v)
1049{1051{
1050 return atomic_inc_return(v) == 0;1052 return atomic_inc_return(v) == 0;
@@ -1062,7 +1064,7 @@ atomic_inc_and_test(atomic_t *v) · 11 unmodified lines
1062 * if the result is negative, or false when1064 * if the result is negative, or false when
1063 * result is greater than or equal to zero.1065 * result is greater than or equal to zero.
1064 */1066 */
1065static inline bool1067static __always_inline bool
1066atomic_add_negative(int i, atomic_t *v)1068atomic_add_negative(int i, atomic_t *v)
1067{1069{
1068 return atomic_add_return(i, v) < 0;1070 return atomic_add_return(i, v) < 0;
@@ -1080,7 +1082,7 @@ atomic_add_negative(int i, atomic_t *v) · 11 unmodified lines
1080 * Atomically adds @a to @v, so long as @v was not already @u.1082 * Atomically adds @a to @v, so long as @v was not already @u.
1081 * Returns original value of @v1083 * Returns original value of @v
1082 */1084 */
1083static inline int1085static __always_inline int
1084atomic_fetch_add_unless(atomic_t *v, int a, int u)1086atomic_fetch_add_unless(atomic_t *v, int a, int u)
1085{1087{
1086 int c = atomic_read(v);1088 int c = atomic_read(v);
@@ -1105,7 +1107,7 @@ atomic_fetch_add_unless(atomic_t *v, int a, int u) · 18 unmodified lines
1105 * Atomically adds @a to @v, if @v was not already @u.1107 * Atomically adds @a to @v, if @v was not already @u.
1106 * Returns true if the addition was done.1108 * Returns true if the addition was done.
1107 */1109 */
1108static inline bool1110static __always_inline bool
1109atomic_add_unless(atomic_t *v, int a, int u)1111atomic_add_unless(atomic_t *v, int a, int u)
1110{1112{
1111 return atomic_fetch_add_unless(v, a, u) != u;1113 return atomic_fetch_add_unless(v, a, u) != u;
@@ -1121,7 +1123,7 @@ atomic_add_unless(atomic_t *v, int a, int u) · 9 unmodified lines
1121 * Atomically increments @v by 1, if @v is non-zero.1123 * Atomically increments @v by 1, if @v is non-zero.
1122 * Returns true if the increment was done.1124 * Returns true if the increment was done.
1123 */1125 */
1124static inline bool1126static __always_inline bool
1125atomic_inc_not_zero(atomic_t *v)1127atomic_inc_not_zero(atomic_t *v)
1126{1128{
1127 return atomic_add_unless(v, 1, 0);1129 return atomic_add_unless(v, 1, 0);
@@ -1130,7 +1132,7 @@ atomic_inc_not_zero(atomic_t *v) · 2 unmodified lines
1130#endif1132#endif
1131 1133
1132#ifndef atomic_inc_unless_negative1134#ifndef atomic_inc_unless_negative
1133static inline bool1135static __always_inline bool
1134atomic_inc_unless_negative(atomic_t *v)1136atomic_inc_unless_negative(atomic_t *v)
1135{1137{
1136 int c = atomic_read(v);1138 int c = atomic_read(v);
@@ -1146,7 +1148,7 @@ atomic_inc_unless_negative(atomic_t *v) · 9 unmodified lines
1146#endif1148#endif
1147 1149
1148#ifndef atomic_dec_unless_positive1150#ifndef atomic_dec_unless_positive
1149static inline bool1151static __always_inline bool
1150atomic_dec_unless_positive(atomic_t *v)1152atomic_dec_unless_positive(atomic_t *v)
1151{1153{
1152 int c = atomic_read(v);1154 int c = atomic_read(v);
@@ -1162,7 +1164,7 @@ atomic_dec_unless_positive(atomic_t *v) · 9 unmodified lines
1162#endif1164#endif
1163 1165
1164#ifndef atomic_dec_if_positive1166#ifndef atomic_dec_if_positive
1165static inline int1167static __always_inline int
1166atomic_dec_if_positive(atomic_t *v)1168atomic_dec_if_positive(atomic_t *v)
1167{1169{
1168 int dec, c = atomic_read(v);1170 int dec, c = atomic_read(v);
@@ -1186,7 +1188,7 @@ atomic_dec_if_positive(atomic_t *v) · 17 unmodified lines
1186#endif1188#endif
1187 1189
1188#ifndef atomic64_read_acquire1190#ifndef atomic64_read_acquire
1189static inline s641191static __always_inline s64
1190atomic64_read_acquire(const atomic64_t *v)1192atomic64_read_acquire(const atomic64_t *v)
1191{1193{
1192 return smp_load_acquire(&(v)->counter);1194 return smp_load_acquire(&(v)->counter);
@@ -1195,7 +1197,7 @@ atomic64_read_acquire(const atomic64_t *v) · 2 unmodified lines
1195#endif1197#endif
1196 1198
1197#ifndef atomic64_set_release1199#ifndef atomic64_set_release
1198static inline void1200static __always_inline void
1199atomic64_set_release(atomic64_t *v, s64 i)1201atomic64_set_release(atomic64_t *v, s64 i)
1200{1202{
1201 smp_store_release(&(v)->counter, i);1203 smp_store_release(&(v)->counter, i);
@@ -1210,7 +1212,7 @@ atomic64_set_release(atomic64_t *v, s64 i) · 8 unmodified lines
1210#else /* atomic64_add_return_relaxed */1212#else /* atomic64_add_return_relaxed */
1211 1213
1212#ifndef atomic64_add_return_acquire1214#ifndef atomic64_add_return_acquire
1213static inline s641215static __always_inline s64
1214atomic64_add_return_acquire(s64 i, atomic64_t *v)1216atomic64_add_return_acquire(s64 i, atomic64_t *v)
1215{1217{
1216 s64 ret = atomic64_add_return_relaxed(i, v);1218 s64 ret = atomic64_add_return_relaxed(i, v);
@@ -1221,7 +1223,7 @@ atomic64_add_return_acquire(s64 i, atomic64_t *v) · 4 unmodified lines
1221#endif1223#endif
1222 1224
1223#ifndef atomic64_add_return_release1225#ifndef atomic64_add_return_release
1224static inline s641226static __always_inline s64
1225atomic64_add_return_release(s64 i, atomic64_t *v)1227atomic64_add_return_release(s64 i, atomic64_t *v)
1226{1228{
1227 __atomic_release_fence();1229 __atomic_release_fence();
@@ -1231,7 +1233,7 @@ atomic64_add_return_release(s64 i, atomic64_t *v) · 3 unmodified lines
1231#endif1233#endif
1232 1234
1233#ifndef atomic64_add_return1235#ifndef atomic64_add_return
1234static inline s641236static __always_inline s64
1235atomic64_add_return(s64 i, atomic64_t *v)1237atomic64_add_return(s64 i, atomic64_t *v)
1236{1238{
1237 s64 ret;1239 s64 ret;
@@ -1252,7 +1254,7 @@ atomic64_add_return(s64 i, atomic64_t *v) · 14 unmodified lines
1252#else /* atomic64_fetch_add_relaxed */1254#else /* atomic64_fetch_add_relaxed */
1253 1255
1254#ifndef atomic64_fetch_add_acquire1256#ifndef atomic64_fetch_add_acquire
1255static inline s641257static __always_inline s64
1256atomic64_fetch_add_acquire(s64 i, atomic64_t *v)1258atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
1257{1259{
1258 s64 ret = atomic64_fetch_add_relaxed(i, v);1260 s64 ret = atomic64_fetch_add_relaxed(i, v);
@@ -1263,7 +1265,7 @@ atomic64_fetch_add_acquire(s64 i, atomic64_t *v) · 4 unmodified lines
1263#endif1265#endif
1264 1266
1265#ifndef atomic64_fetch_add_release1267#ifndef atomic64_fetch_add_release
1266static inline s641268static __always_inline s64
1267atomic64_fetch_add_release(s64 i, atomic64_t *v)1269atomic64_fetch_add_release(s64 i, atomic64_t *v)
1268{1270{
1269 __atomic_release_fence();1271 __atomic_release_fence();
@@ -1273,7 +1275,7 @@ atomic64_fetch_add_release(s64 i, atomic64_t *v) · 3 unmodified lines
1273#endif1275#endif
1274 1276
1275#ifndef atomic64_fetch_add1277#ifndef atomic64_fetch_add
1276static inline s641278static __always_inline s64
1277atomic64_fetch_add(s64 i, atomic64_t *v)1279atomic64_fetch_add(s64 i, atomic64_t *v)
1278{1280{
1279 s64 ret;1281 s64 ret;
@@ -1294,7 +1296,7 @@ atomic64_fetch_add(s64 i, atomic64_t *v) · 14 unmodified lines
1294#else /* atomic64_sub_return_relaxed */1296#else /* atomic64_sub_return_relaxed */
1295 1297
1296#ifndef atomic64_sub_return_acquire1298#ifndef atomic64_sub_return_acquire
1297static inline s641299static __always_inline s64
1298atomic64_sub_return_acquire(s64 i, atomic64_t *v)1300atomic64_sub_return_acquire(s64 i, atomic64_t *v)
1299{1301{
1300 s64 ret = atomic64_sub_return_relaxed(i, v);1302 s64 ret = atomic64_sub_return_relaxed(i, v);
@@ -1305,7 +1307,7 @@ atomic64_sub_return_acquire(s64 i, atomic64_t *v) · 4 unmodified lines
1305#endif1307#endif
1306 1308
1307#ifndef atomic64_sub_return_release1309#ifndef atomic64_sub_return_release
1308static inline s641310static __always_inline s64
1309atomic64_sub_return_release(s64 i, atomic64_t *v)1311atomic64_sub_return_release(s64 i, atomic64_t *v)
1310{1312{
1311 __atomic_release_fence();1313 __atomic_release_fence();
@@ -1315,7 +1317,7 @@ atomic64_sub_return_release(s64 i, atomic64_t *v) · 3 unmodified lines
1315#endif1317#endif
1316 1318
1317#ifndef atomic64_sub_return1319#ifndef atomic64_sub_return
1318static inline s641320static __always_inline s64
1319atomic64_sub_return(s64 i, atomic64_t *v)1321atomic64_sub_return(s64 i, atomic64_t *v)
1320{1322{
1321 s64 ret;1323 s64 ret;
@@ -1336,7 +1338,7 @@ atomic64_sub_return(s64 i, atomic64_t *v) · 14 unmodified lines
1336#else /* atomic64_fetch_sub_relaxed */1338#else /* atomic64_fetch_sub_relaxed */
1337 1339
1338#ifndef atomic64_fetch_sub_acquire1340#ifndef atomic64_fetch_sub_acquire
1339static inline s641341static __always_inline s64
1340atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)1342atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
1341{1343{
1342 s64 ret = atomic64_fetch_sub_relaxed(i, v);1344 s64 ret = atomic64_fetch_sub_relaxed(i, v);
@@ -1347,7 +1349,7 @@ atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) · 4 unmodified lines
1347#endif1349#endif
1348 1350
1349#ifndef atomic64_fetch_sub_release1351#ifndef atomic64_fetch_sub_release
1350static inline s641352static __always_inline s64
1351atomic64_fetch_sub_release(s64 i, atomic64_t *v)1353atomic64_fetch_sub_release(s64 i, atomic64_t *v)
1352{1354{
1353 __atomic_release_fence();1355 __atomic_release_fence();
@@ -1357,7 +1359,7 @@ atomic64_fetch_sub_release(s64 i, atomic64_t *v) · 3 unmodified lines
1357#endif1359#endif
1358 1360
1359#ifndef atomic64_fetch_sub1361#ifndef atomic64_fetch_sub
1360static inline s641362static __always_inline s64
1361atomic64_fetch_sub(s64 i, atomic64_t *v)1363atomic64_fetch_sub(s64 i, atomic64_t *v)
1362{1364{
1363 s64 ret;1365 s64 ret;
@@ -1372,7 +1374,7 @@ atomic64_fetch_sub(s64 i, atomic64_t *v) · 8 unmodified lines
1372#endif /* atomic64_fetch_sub_relaxed */1374#endif /* atomic64_fetch_sub_relaxed */
1373 1375
1374#ifndef atomic64_inc1376#ifndef atomic64_inc
1375static inline void1377static __always_inline void
1376atomic64_inc(atomic64_t *v)1378atomic64_inc(atomic64_t *v)
1377{1379{
1378 atomic64_add(1, v);1380 atomic64_add(1, v);
@@ -1388,7 +1390,7 @@ atomic64_inc(atomic64_t *v) · 9 unmodified lines
1388#endif /* atomic64_inc_return */1390#endif /* atomic64_inc_return */
1389 1391
1390#ifndef atomic64_inc_return1392#ifndef atomic64_inc_return
1391static inline s641393static __always_inline s64
1392atomic64_inc_return(atomic64_t *v)1394atomic64_inc_return(atomic64_t *v)
1393{1395{
1394 return atomic64_add_return(1, v);1396 return atomic64_add_return(1, v);
@@ -1397,7 +1399,7 @@ atomic64_inc_return(atomic64_t *v) · 2 unmodified lines
1397#endif1399#endif
1398 1400
1399#ifndef atomic64_inc_return_acquire1401#ifndef atomic64_inc_return_acquire
1400static inline s641402static __always_inline s64
1401atomic64_inc_return_acquire(atomic64_t *v)1403atomic64_inc_return_acquire(atomic64_t *v)
1402{1404{
1403 return atomic64_add_return_acquire(1, v);1405 return atomic64_add_return_acquire(1, v);
@@ -1406,7 +1408,7 @@ atomic64_inc_return_acquire(atomic64_t *v) · 2 unmodified lines
1406#endif1408#endif
1407 1409
1408#ifndef atomic64_inc_return_release1410#ifndef atomic64_inc_return_release
1409static inline s641411static __always_inline s64
1410atomic64_inc_return_release(atomic64_t *v)1412atomic64_inc_return_release(atomic64_t *v)
1411{1413{
1412 return atomic64_add_return_release(1, v);1414 return atomic64_add_return_release(1, v);
@@ -1415,7 +1417,7 @@ atomic64_inc_return_release(atomic64_t *v) · 2 unmodified lines
1415#endif1417#endif
1416 1418
1417#ifndef atomic64_inc_return_relaxed1419#ifndef atomic64_inc_return_relaxed
1418static inline s641420static __always_inline s64
1419atomic64_inc_return_relaxed(atomic64_t *v)1421atomic64_inc_return_relaxed(atomic64_t *v)
1420{1422{
1421 return atomic64_add_return_relaxed(1, v);1423 return atomic64_add_return_relaxed(1, v);
@@ -1426,7 +1428,7 @@ atomic64_inc_return_relaxed(atomic64_t *v) · 4 unmodified lines
1426#else /* atomic64_inc_return_relaxed */1428#else /* atomic64_inc_return_relaxed */
1427 1429
1428#ifndef atomic64_inc_return_acquire1430#ifndef atomic64_inc_return_acquire
1429static inline s641431static __always_inline s64
1430atomic64_inc_return_acquire(atomic64_t *v)1432atomic64_inc_return_acquire(atomic64_t *v)
1431{1433{
1432 s64 ret = atomic64_inc_return_relaxed(v);1434 s64 ret = atomic64_inc_return_relaxed(v);
@@ -1437,7 +1439,7 @@ atomic64_inc_return_acquire(atomic64_t *v) · 4 unmodified lines
1437#endif1439#endif
1438 1440
1439#ifndef atomic64_inc_return_release1441#ifndef atomic64_inc_return_release
1440static inline s641442static __always_inline s64
1441atomic64_inc_return_release(atomic64_t *v)1443atomic64_inc_return_release(atomic64_t *v)
1442{1444{
1443 __atomic_release_fence();1445 __atomic_release_fence();
@@ -1447,7 +1449,7 @@ atomic64_inc_return_release(atomic64_t *v) · 3 unmodified lines
1447#endif1449#endif
1448 1450
1449#ifndef atomic64_inc_return1451#ifndef atomic64_inc_return
1450static inline s641452static __always_inline s64
1451atomic64_inc_return(atomic64_t *v)1453atomic64_inc_return(atomic64_t *v)
1452{1454{
1453 s64 ret;1455 s64 ret;
@@ -1469,7 +1471,7 @@ atomic64_inc_return(atomic64_t *v) · 15 unmodified lines
1469#endif /* atomic64_fetch_inc */1471#endif /* atomic64_fetch_inc */
1470 1472
1471#ifndef atomic64_fetch_inc1473#ifndef atomic64_fetch_inc
1472static inline s641474static __always_inline s64
1473atomic64_fetch_inc(atomic64_t *v)1475atomic64_fetch_inc(atomic64_t *v)
1474{1476{
1475 return atomic64_fetch_add(1, v);1477 return atomic64_fetch_add(1, v);
@@ -1478,7 +1480,7 @@ atomic64_fetch_inc(atomic64_t *v) · 2 unmodified lines
1478#endif1480#endif
1479 1481
1480#ifndef atomic64_fetch_inc_acquire1482#ifndef atomic64_fetch_inc_acquire
1481static inline s641483static __always_inline s64
1482atomic64_fetch_inc_acquire(atomic64_t *v)1484atomic64_fetch_inc_acquire(atomic64_t *v)
1483{1485{
1484 return atomic64_fetch_add_acquire(1, v);1486 return atomic64_fetch_add_acquire(1, v);
@@ -1487,7 +1489,7 @@ atomic64_fetch_inc_acquire(atomic64_t *v) · 2 unmodified lines
1487#endif1489#endif
1488 1490
1489#ifndef atomic64_fetch_inc_release1491#ifndef atomic64_fetch_inc_release
1490static inline s641492static __always_inline s64
1491atomic64_fetch_inc_release(atomic64_t *v)1493atomic64_fetch_inc_release(atomic64_t *v)
1492{1494{
1493 return atomic64_fetch_add_release(1, v);1495 return atomic64_fetch_add_release(1, v);
@@ -1496,7 +1498,7 @@ atomic64_fetch_inc_release(atomic64_t *v) · 2 unmodified lines
1496#endif1498#endif
1497 1499
1498#ifndef atomic64_fetch_inc_relaxed1500#ifndef atomic64_fetch_inc_relaxed
1499static inline s641501static __always_inline s64
1500atomic64_fetch_inc_relaxed(atomic64_t *v)1502atomic64_fetch_inc_relaxed(atomic64_t *v)
1501{1503{
1502 return atomic64_fetch_add_relaxed(1, v);1504 return atomic64_fetch_add_relaxed(1, v);
@@ -1507,7 +1509,7 @@ atomic64_fetch_inc_relaxed(atomic64_t *v) · 4 unmodified lines
1507#else /* atomic64_fetch_inc_relaxed */1509#else /* atomic64_fetch_inc_relaxed */
1508 1510
1509#ifndef atomic64_fetch_inc_acquire1511#ifndef atomic64_fetch_inc_acquire
1510static inline s641512static __always_inline s64
1511atomic64_fetch_inc_acquire(atomic64_t *v)1513atomic64_fetch_inc_acquire(atomic64_t *v)
1512{1514{
1513 s64 ret = atomic64_fetch_inc_relaxed(v);1515 s64 ret = atomic64_fetch_inc_relaxed(v);
@@ -1518,7 +1520,7 @@ atomic64_fetch_inc_acquire(atomic64_t *v) · 4 unmodified lines
1518#endif1520#endif
1519 1521
1520#ifndef atomic64_fetch_inc_release1522#ifndef atomic64_fetch_inc_release
1521static inline s641523static __always_inline s64
1522atomic64_fetch_inc_release(atomic64_t *v)1524atomic64_fetch_inc_release(atomic64_t *v)
1523{1525{
1524 __atomic_release_fence();1526 __atomic_release_fence();
@@ -1528,7 +1530,7 @@ atomic64_fetch_inc_release(atomic64_t *v) · 3 unmodified lines
1528#endif1530#endif
1529 1531
1530#ifndef atomic64_fetch_inc1532#ifndef atomic64_fetch_inc
1531static inline s641533static __always_inline s64
1532atomic64_fetch_inc(atomic64_t *v)1534atomic64_fetch_inc(atomic64_t *v)
1533{1535{
1534 s64 ret;1536 s64 ret;
@@ -1543,7 +1545,7 @@ atomic64_fetch_inc(atomic64_t *v) · 8 unmodified lines
1543#endif /* atomic64_fetch_inc_relaxed */1545#endif /* atomic64_fetch_inc_relaxed */
1544 1546
1545#ifndef atomic64_dec1547#ifndef atomic64_dec
1546static inline void1548static __always_inline void
1547atomic64_dec(atomic64_t *v)1549atomic64_dec(atomic64_t *v)
1548{1550{
1549 atomic64_sub(1, v);1551 atomic64_sub(1, v);
@@ -1559,7 +1561,7 @@ atomic64_dec(atomic64_t *v) · 9 unmodified lines
1559#endif /* atomic64_dec_return */1561#endif /* atomic64_dec_return */
1560 1562
1561#ifndef atomic64_dec_return1563#ifndef atomic64_dec_return
1562static inline s641564static __always_inline s64
1563atomic64_dec_return(atomic64_t *v)1565atomic64_dec_return(atomic64_t *v)
1564{1566{
1565 return atomic64_sub_return(1, v);1567 return atomic64_sub_return(1, v);
@@ -1568,7 +1570,7 @@ atomic64_dec_return(atomic64_t *v) · 2 unmodified lines
1568#endif1570#endif
1569 1571
1570#ifndef atomic64_dec_return_acquire1572#ifndef atomic64_dec_return_acquire
1571static inline s641573static __always_inline s64
1572atomic64_dec_return_acquire(atomic64_t *v)1574atomic64_dec_return_acquire(atomic64_t *v)
1573{1575{
1574 return atomic64_sub_return_acquire(1, v);1576 return atomic64_sub_return_acquire(1, v);
@@ -1577,7 +1579,7 @@ atomic64_dec_return_acquire(atomic64_t *v) · 2 unmodified lines
1577#endif1579#endif
1578 1580
1579#ifndef atomic64_dec_return_release1581#ifndef atomic64_dec_return_release
1580static inline s641582static __always_inline s64
1581atomic64_dec_return_release(atomic64_t *v)1583atomic64_dec_return_release(atomic64_t *v)
1582{1584{
1583 return atomic64_sub_return_release(1, v);1585 return atomic64_sub_return_release(1, v);
@@ -1586,7 +1588,7 @@ atomic64_dec_return_release(atomic64_t *v) · 2 unmodified lines
1586#endif1588#endif
1587 1589
1588#ifndef atomic64_dec_return_relaxed1590#ifndef atomic64_dec_return_relaxed
1589static inline s641591static __always_inline s64
1590atomic64_dec_return_relaxed(atomic64_t *v)1592atomic64_dec_return_relaxed(atomic64_t *v)
1591{1593{
1592 return atomic64_sub_return_relaxed(1, v);1594 return atomic64_sub_return_relaxed(1, v);
@@ -1597,7 +1599,7 @@ atomic64_dec_return_relaxed(atomic64_t *v) · 4 unmodified lines
1597#else /* atomic64_dec_return_relaxed */1599#else /* atomic64_dec_return_relaxed */
1598 1600
1599#ifndef atomic64_dec_return_acquire1601#ifndef atomic64_dec_return_acquire
1600static inline s641602static __always_inline s64
1601atomic64_dec_return_acquire(atomic64_t *v)1603atomic64_dec_return_acquire(atomic64_t *v)
1602{1604{
1603 s64 ret = atomic64_dec_return_relaxed(v);1605 s64 ret = atomic64_dec_return_relaxed(v);
@@ -1608,7 +1610,7 @@ atomic64_dec_return_acquire(atomic64_t *v) · 4 unmodified lines
1608#endif1610#endif
1609 1611
1610#ifndef atomic64_dec_return_release1612#ifndef atomic64_dec_return_release
1611static inline s641613static __always_inline s64
1612atomic64_dec_return_release(atomic64_t *v)1614atomic64_dec_return_release(atomic64_t *v)
1613{1615{
1614 __atomic_release_fence();1616 __atomic_release_fence();
@@ -1618,7 +1620,7 @@ atomic64_dec_return_release(atomic64_t *v) · 3 unmodified lines
1618#endif1620#endif
1619 1621
1620#ifndef atomic64_dec_return1622#ifndef atomic64_dec_return
1621static inline s641623static __always_inline s64
1622atomic64_dec_return(atomic64_t *v)1624atomic64_dec_return(atomic64_t *v)
1623{1625{
1624 s64 ret;1626 s64 ret;
@@ -1640,7 +1642,7 @@ atomic64_dec_return(atomic64_t *v) · 15 unmodified lines
1640#endif /* atomic64_fetch_dec */1642#endif /* atomic64_fetch_dec */
1641 1643
1642#ifndef atomic64_fetch_dec1644#ifndef atomic64_fetch_dec
1643static inline s641645static __always_inline s64
1644atomic64_fetch_dec(atomic64_t *v)1646atomic64_fetch_dec(atomic64_t *v)
1645{1647{
1646 return atomic64_fetch_sub(1, v);1648 return atomic64_fetch_sub(1, v);
@@ -1649,7 +1651,7 @@ atomic64_fetch_dec(atomic64_t *v) · 2 unmodified lines
1649#endif1651#endif
1650 1652
1651#ifndef atomic64_fetch_dec_acquire1653#ifndef atomic64_fetch_dec_acquire
1652static inline s641654static __always_inline s64
1653atomic64_fetch_dec_acquire(atomic64_t *v)1655atomic64_fetch_dec_acquire(atomic64_t *v)
1654{1656{
1655 return atomic64_fetch_sub_acquire(1, v);1657 return atomic64_fetch_sub_acquire(1, v);
@@ -1658,7 +1660,7 @@ atomic64_fetch_dec_acquire(atomic64_t *v) · 2 unmodified lines
1658#endif1660#endif
1659 1661
1660#ifndef atomic64_fetch_dec_release1662#ifndef atomic64_fetch_dec_release
1661static inline s641663static __always_inline s64
1662atomic64_fetch_dec_release(atomic64_t *v)1664atomic64_fetch_dec_release(atomic64_t *v)
1663{1665{
1664 return atomic64_fetch_sub_release(1, v);1666 return atomic64_fetch_sub_release(1, v);
@@ -1667,7 +1669,7 @@ atomic64_fetch_dec_release(atomic64_t *v) · 2 unmodified lines
1667#endif1669#endif
1668 1670
1669#ifndef atomic64_fetch_dec_relaxed1671#ifndef atomic64_fetch_dec_relaxed
1670static inline s641672static __always_inline s64
1671atomic64_fetch_dec_relaxed(atomic64_t *v)1673atomic64_fetch_dec_relaxed(atomic64_t *v)
1672{1674{
1673 return atomic64_fetch_sub_relaxed(1, v);1675 return atomic64_fetch_sub_relaxed(1, v);
@@ -1678,7 +1680,7 @@ atomic64_fetch_dec_relaxed(atomic64_t *v) · 4 unmodified lines
1678#else /* atomic64_fetch_dec_relaxed */1680#else /* atomic64_fetch_dec_relaxed */
1679 1681
1680#ifndef atomic64_fetch_dec_acquire1682#ifndef atomic64_fetch_dec_acquire
1681static inline s641683static __always_inline s64
1682atomic64_fetch_dec_acquire(atomic64_t *v)1684atomic64_fetch_dec_acquire(atomic64_t *v)
1683{1685{
1684 s64 ret = atomic64_fetch_dec_relaxed(v);1686 s64 ret = atomic64_fetch_dec_relaxed(v);
@@ -1689,7 +1691,7 @@ atomic64_fetch_dec_acquire(atomic64_t *v) · 4 unmodified lines
1689#endif1691#endif
1690 1692
1691#ifndef atomic64_fetch_dec_release1693#ifndef atomic64_fetch_dec_release
1692static inline s641694static __always_inline s64
1693atomic64_fetch_dec_release(atomic64_t *v)1695atomic64_fetch_dec_release(atomic64_t *v)
1694{1696{
1695 __atomic_release_fence();1697 __atomic_release_fence();
@@ -1699,7 +1701,7 @@ atomic64_fetch_dec_release(atomic64_t *v) · 3 unmodified lines
1699#endif1701#endif
1700 1702
1701#ifndef atomic64_fetch_dec1703#ifndef atomic64_fetch_dec
1702static inline s641704static __always_inline s64
1703atomic64_fetch_dec(atomic64_t *v)1705atomic64_fetch_dec(atomic64_t *v)
1704{1706{
1705 s64 ret;1707 s64 ret;
@@ -1720,7 +1722,7 @@ atomic64_fetch_dec(atomic64_t *v) · 14 unmodified lines
1720#else /* atomic64_fetch_and_relaxed */1722#else /* atomic64_fetch_and_relaxed */
1721 1723
1722#ifndef atomic64_fetch_and_acquire1724#ifndef atomic64_fetch_and_acquire
1723static inline s641725static __always_inline s64
1724atomic64_fetch_and_acquire(s64 i, atomic64_t *v)1726atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
1725{1727{
1726 s64 ret = atomic64_fetch_and_relaxed(i, v);1728 s64 ret = atomic64_fetch_and_relaxed(i, v);
@@ -1731,7 +1733,7 @@ atomic64_fetch_and_acquire(s64 i, atomic64_t *v) · 4 unmodified lines
1731#endif1733#endif
1732 1734
1733#ifndef atomic64_fetch_and_release1735#ifndef atomic64_fetch_and_release
1734static inline s641736static __always_inline s64
1735atomic64_fetch_and_release(s64 i, atomic64_t *v)1737atomic64_fetch_and_release(s64 i, atomic64_t *v)
1736{1738{
1737 __atomic_release_fence();1739 __atomic_release_fence();
@@ -1741,7 +1743,7 @@ atomic64_fetch_and_release(s64 i, atomic64_t *v) · 3 unmodified lines
1741#endif1743#endif
1742 1744
1743#ifndef atomic64_fetch_and1745#ifndef atomic64_fetch_and
1744static inline s641746static __always_inline s64
1745atomic64_fetch_and(s64 i, atomic64_t *v)1747atomic64_fetch_and(s64 i, atomic64_t *v)
1746{1748{
1747 s64 ret;1749 s64 ret;
@@ -1756,7 +1758,7 @@ atomic64_fetch_and(s64 i, atomic64_t *v) · 8 unmodified lines
1756#endif /* atomic64_fetch_and_relaxed */1758#endif /* atomic64_fetch_and_relaxed */
1757 1759
1758#ifndef atomic64_andnot1760#ifndef atomic64_andnot
1759static inline void1761static __always_inline void
1760atomic64_andnot(s64 i, atomic64_t *v)1762atomic64_andnot(s64 i, atomic64_t *v)
1761{1763{
1762 atomic64_and(~i, v);1764 atomic64_and(~i, v);
@@ -1772,7 +1774,7 @@ atomic64_andnot(s64 i, atomic64_t *v) · 9 unmodified lines
1772#endif /* atomic64_fetch_andnot */1774#endif /* atomic64_fetch_andnot */
1773 1775
1774#ifndef atomic64_fetch_andnot1776#ifndef atomic64_fetch_andnot
1775static inline s641777static __always_inline s64
1776atomic64_fetch_andnot(s64 i, atomic64_t *v)1778atomic64_fetch_andnot(s64 i, atomic64_t *v)
1777{1779{
1778 return atomic64_fetch_and(~i, v);1780 return atomic64_fetch_and(~i, v);
@@ -1781,7 +1783,7 @@ atomic64_fetch_andnot(s64 i, atomic64_t *v) · 2 unmodified lines
1781#endif1783#endif
1782 1784
1783#ifndef atomic64_fetch_andnot_acquire1785#ifndef atomic64_fetch_andnot_acquire
1784static inline s641786static __always_inline s64
1785atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)1787atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
1786{1788{
1787 return atomic64_fetch_and_acquire(~i, v);1789 return atomic64_fetch_and_acquire(~i, v);
@@ -1790,7 +1792,7 @@ atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) · 2 unmodified lines
1790#endif1792#endif
1791 1793
1792#ifndef atomic64_fetch_andnot_release1794#ifndef atomic64_fetch_andnot_release
1793static inline s641795static __always_inline s64
1794atomic64_fetch_andnot_release(s64 i, atomic64_t *v)1796atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1795{1797{
1796 return atomic64_fetch_and_release(~i, v);1798 return atomic64_fetch_and_release(~i, v);
@@ -1799,7 +1801,7 @@ atomic64_fetch_andnot_release(s64 i, atomic64_t *v) · 2 unmodified lines
1799#endif1801#endif
1800 1802
1801#ifndef atomic64_fetch_andnot_relaxed1803#ifndef atomic64_fetch_andnot_relaxed
1802static inline s641804static __always_inline s64
1803atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)1805atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
1804{1806{
1805 return atomic64_fetch_and_relaxed(~i, v);1807 return atomic64_fetch_and_relaxed(~i, v);
@@ -1810,7 +1812,7 @@ atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) · 4 unmodified lines
1810#else /* atomic64_fetch_andnot_relaxed */1812#else /* atomic64_fetch_andnot_relaxed */
1811 1813
1812#ifndef atomic64_fetch_andnot_acquire1814#ifndef atomic64_fetch_andnot_acquire
1813static inline s641815static __always_inline s64
1814atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)1816atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
1815{1817{
1816 s64 ret = atomic64_fetch_andnot_relaxed(i, v);1818 s64 ret = atomic64_fetch_andnot_relaxed(i, v);
@@ -1821,7 +1823,7 @@ atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) · 4 unmodified lines
1821#endif1823#endif
1822 1824
1823#ifndef atomic64_fetch_andnot_release1825#ifndef atomic64_fetch_andnot_release
1824static inline s641826static __always_inline s64
1825atomic64_fetch_andnot_release(s64 i, atomic64_t *v)1827atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1826{1828{
1827 __atomic_release_fence();1829 __atomic_release_fence();
@@ -1831,7 +1833,7 @@ atomic64_fetch_andnot_release(s64 i, atomic64_t *v) · 3 unmodified lines
1831#endif1833#endif
1832 1834
1833#ifndef atomic64_fetch_andnot1835#ifndef atomic64_fetch_andnot
1834static inline s641836static __always_inline s64
1835atomic64_fetch_andnot(s64 i, atomic64_t *v)1837atomic64_fetch_andnot(s64 i, atomic64_t *v)
1836{1838{
1837 s64 ret;1839 s64 ret;
@@ -1852,7 +1854,7 @@ atomic64_fetch_andnot(s64 i, atomic64_t *v) · 14 unmodified lines
1852#else /* atomic64_fetch_or_relaxed */1854#else /* atomic64_fetch_or_relaxed */
1853 1855
1854#ifndef atomic64_fetch_or_acquire1856#ifndef atomic64_fetch_or_acquire
1855static inline s641857static __always_inline s64
1856atomic64_fetch_or_acquire(s64 i, atomic64_t *v)1858atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
1857{1859{
1858 s64 ret = atomic64_fetch_or_relaxed(i, v);1860 s64 ret = atomic64_fetch_or_relaxed(i, v);
@@ -1863,7 +1865,7 @@ atomic64_fetch_or_acquire(s64 i, atomic64_t *v) · 4 unmodified lines
1863#endif1865#endif
1864 1866
1865#ifndef atomic64_fetch_or_release1867#ifndef atomic64_fetch_or_release
1866static inline s641868static __always_inline s64
1867atomic64_fetch_or_release(s64 i, atomic64_t *v)1869atomic64_fetch_or_release(s64 i, atomic64_t *v)
1868{1870{
1869 __atomic_release_fence();1871 __atomic_release_fence();
@@ -1873,7 +1875,7 @@ atomic64_fetch_or_release(s64 i, atomic64_t *v) · 3 unmodified lines
1873#endif1875#endif
1874 1876
1875#ifndef atomic64_fetch_or1877#ifndef atomic64_fetch_or
1876static inline s641878static __always_inline s64
1877atomic64_fetch_or(s64 i, atomic64_t *v)1879atomic64_fetch_or(s64 i, atomic64_t *v)
1878{1880{
1879 s64 ret;1881 s64 ret;
@@ -1894,7 +1896,7 @@ atomic64_fetch_or(s64 i, atomic64_t *v) · 14 unmodified lines
1894#else /* atomic64_fetch_xor_relaxed */1896#else /* atomic64_fetch_xor_relaxed */
1895 1897
1896#ifndef atomic64_fetch_xor_acquire1898#ifndef atomic64_fetch_xor_acquire
1897static inline s641899static __always_inline s64
1898atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)1900atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
1899{1901{
1900 s64 ret = atomic64_fetch_xor_relaxed(i, v);1902 s64 ret = atomic64_fetch_xor_relaxed(i, v);
@@ -1905,7 +1907,7 @@ atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) · 4 unmodified lines
1905#endif1907#endif
1906 1908
1907#ifndef atomic64_fetch_xor_release1909#ifndef atomic64_fetch_xor_release
1908static inline s641910static __always_inline s64
1909atomic64_fetch_xor_release(s64 i, atomic64_t *v)1911atomic64_fetch_xor_release(s64 i, atomic64_t *v)
1910{1912{
1911 __atomic_release_fence();1913 __atomic_release_fence();
@@ -1915,7 +1917,7 @@ atomic64_fetch_xor_release(s64 i, atomic64_t *v) · 3 unmodified lines
1915#endif1917#endif
1916 1918
1917#ifndef atomic64_fetch_xor1919#ifndef atomic64_fetch_xor
1918static inline s641920static __always_inline s64
1919atomic64_fetch_xor(s64 i, atomic64_t *v)1921atomic64_fetch_xor(s64 i, atomic64_t *v)
1920{1922{
1921 s64 ret;1923 s64 ret;
@@ -1936,7 +1938,7 @@ atomic64_fetch_xor(s64 i, atomic64_t *v) · 14 unmodified lines
1936#else /* atomic64_xchg_relaxed */1938#else /* atomic64_xchg_relaxed */
1937 1939
1938#ifndef atomic64_xchg_acquire1940#ifndef atomic64_xchg_acquire
1939static inline s641941static __always_inline s64
1940atomic64_xchg_acquire(atomic64_t *v, s64 i)1942atomic64_xchg_acquire(atomic64_t *v, s64 i)
1941{1943{
1942 s64 ret = atomic64_xchg_relaxed(v, i);1944 s64 ret = atomic64_xchg_relaxed(v, i);
@@ -1947,7 +1949,7 @@ atomic64_xchg_acquire(atomic64_t *v, s64 i) · 4 unmodified lines
1947#endif1949#endif
1948 1950
1949#ifndef atomic64_xchg_release1951#ifndef atomic64_xchg_release
1950static inline s641952static __always_inline s64
1951atomic64_xchg_release(atomic64_t *v, s64 i)1953atomic64_xchg_release(atomic64_t *v, s64 i)
1952{1954{
1953 __atomic_release_fence();1955 __atomic_release_fence();
@@ -1957,7 +1959,7 @@ atomic64_xchg_release(atomic64_t *v, s64 i) · 3 unmodified lines
1957#endif1959#endif
1958 1960
1959#ifndef atomic64_xchg1961#ifndef atomic64_xchg
1960static inline s641962static __always_inline s64
1961atomic64_xchg(atomic64_t *v, s64 i)1963atomic64_xchg(atomic64_t *v, s64 i)
1962{1964{
1963 s64 ret;1965 s64 ret;
@@ -1978,7 +1980,7 @@ atomic64_xchg(atomic64_t *v, s64 i) · 14 unmodified lines
1978#else /* atomic64_cmpxchg_relaxed */1980#else /* atomic64_cmpxchg_relaxed */
1979 1981
1980#ifndef atomic64_cmpxchg_acquire1982#ifndef atomic64_cmpxchg_acquire
1981static inline s641983static __always_inline s64
1982atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)1984atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
1983{1985{
1984 s64 ret = atomic64_cmpxchg_relaxed(v, old, new);1986 s64 ret = atomic64_cmpxchg_relaxed(v, old, new);
@@ -1989,7 +1991,7 @@ atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) · 4 unmodified lines
1989#endif1991#endif
1990 1992
1991#ifndef atomic64_cmpxchg_release1993#ifndef atomic64_cmpxchg_release
1992static inline s641994static __always_inline s64
1993atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)1995atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
1994{1996{
1995 __atomic_release_fence();1997 __atomic_release_fence();
@@ -1999,7 +2001,7 @@ atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) · 3 unmodified lines
1999#endif2001#endif
2000 2002
2001#ifndef atomic64_cmpxchg2003#ifndef atomic64_cmpxchg
2002static inline s642004static __always_inline s64
2003atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)2005atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
2004{2006{
2005 s64 ret;2007 s64 ret;
@@ -2021,7 +2023,7 @@ atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) · 15 unmodified lines
2021#endif /* atomic64_try_cmpxchg */2023#endif /* atomic64_try_cmpxchg */
2022 2024
2023#ifndef atomic64_try_cmpxchg2025#ifndef atomic64_try_cmpxchg
2024static inline bool2026static __always_inline bool
2025atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)2027atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
2026{2028{
2027 s64 r, o = *old;2029 s64 r, o = *old;
@@ -2034,7 +2036,7 @@ atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) · 6 unmodified lines
2034#endif2036#endif
2035 2037
2036#ifndef atomic64_try_cmpxchg_acquire2038#ifndef atomic64_try_cmpxchg_acquire
2037static inline bool2039static __always_inline bool
2038atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)2040atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
2039{2041{
2040 s64 r, o = *old;2042 s64 r, o = *old;
@@ -2047,7 +2049,7 @@ atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) · 6 unmodified lines
2047#endif2049#endif
2048 2050
2049#ifndef atomic64_try_cmpxchg_release2051#ifndef atomic64_try_cmpxchg_release
2050static inline bool2052static __always_inline bool
2051atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)2053atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
2052{2054{
2053 s64 r, o = *old;2055 s64 r, o = *old;
@@ -2060,7 +2062,7 @@ atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) · 6 unmodified lines
2060#endif2062#endif
2061 2063
2062#ifndef atomic64_try_cmpxchg_relaxed2064#ifndef atomic64_try_cmpxchg_relaxed
2063static inline bool2065static __always_inline bool
2064atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)2066atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
2065{2067{
2066 s64 r, o = *old;2068 s64 r, o = *old;
@@ -2075,7 +2077,7 @@ atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) · 8 unmodified lines
2075#else /* atomic64_try_cmpxchg_relaxed */2077#else /* atomic64_try_cmpxchg_relaxed */
2076 2078
2077#ifndef atomic64_try_cmpxchg_acquire2079#ifndef atomic64_try_cmpxchg_acquire
2078static inline bool2080static __always_inline bool
2079atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)2081atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
2080{2082{
2081 bool ret = atomic64_try_cmpxchg_relaxed(v, old, new);2083 bool ret = atomic64_try_cmpxchg_relaxed(v, old, new);
@@ -2086,7 +2088,7 @@ atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) · 4 unmodified lines
2086#endif2088#endif
2087 2089
2088#ifndef atomic64_try_cmpxchg_release2090#ifndef atomic64_try_cmpxchg_release
2089static inline bool2091static __always_inline bool
2090atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)2092atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
2091{2093{
2092 __atomic_release_fence();2094 __atomic_release_fence();
@@ -2096,7 +2098,7 @@ atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) · 3 unmodified lines
2096#endif2098#endif
2097 2099
2098#ifndef atomic64_try_cmpxchg2100#ifndef atomic64_try_cmpxchg
2099static inline bool2101static __always_inline bool
2100atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)2102atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
2101{2103{
2102 bool ret;2104 bool ret;
@@ -2120,7 +2122,7 @@ atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) · 17 unmodified lines
2120 * true if the result is zero, or false for all2122 * true if the result is zero, or false for all
2121 * other cases.2123 * other cases.
2122 */2124 */
2123static inline bool2125static __always_inline bool
2124atomic64_sub_and_test(s64 i, atomic64_t *v)2126atomic64_sub_and_test(s64 i, atomic64_t *v)
2125{2127{
2126 return atomic64_sub_return(i, v) == 0;2128 return atomic64_sub_return(i, v) == 0;
@@ -2137,7 +2139,7 @@ atomic64_sub_and_test(s64 i, atomic64_t *v) · 10 unmodified lines
2137 * returns true if the result is 0, or false for all other2139 * returns true if the result is 0, or false for all other
2138 * cases.2140 * cases.
2139 */2141 */
2140static inline bool2142static __always_inline bool
2141atomic64_dec_and_test(atomic64_t *v)2143atomic64_dec_and_test(atomic64_t *v)
2142{2144{
2143 return atomic64_dec_return(v) == 0;2145 return atomic64_dec_return(v) == 0;
@@ -2154,7 +2156,7 @@ atomic64_dec_and_test(atomic64_t *v) · 10 unmodified lines
2154 * and returns true if the result is zero, or false for all2156 * and returns true if the result is zero, or false for all
2155 * other cases.2157 * other cases.
2156 */2158 */
2157static inline bool2159static __always_inline bool
2158atomic64_inc_and_test(atomic64_t *v)2160atomic64_inc_and_test(atomic64_t *v)
2159{2161{
2160 return atomic64_inc_return(v) == 0;2162 return atomic64_inc_return(v) == 0;
@@ -2172,7 +2174,7 @@ atomic64_inc_and_test(atomic64_t *v) · 11 unmodified lines
2172 * if the result is negative, or false when2174 * if the result is negative, or false when
2173 * result is greater than or equal to zero.2175 * result is greater than or equal to zero.
2174 */2176 */
2175static inline bool2177static __always_inline bool
2176atomic64_add_negative(s64 i, atomic64_t *v)2178atomic64_add_negative(s64 i, atomic64_t *v)
2177{2179{
2178 return atomic64_add_return(i, v) < 0;2180 return atomic64_add_return(i, v) < 0;
@@ -2190,7 +2192,7 @@ atomic64_add_negative(s64 i, atomic64_t *v) · 11 unmodified lines
2190 * Atomically adds @a to @v, so long as @v was not already @u.2192 * Atomically adds @a to @v, so long as @v was not already @u.
2191 * Returns original value of @v2193 * Returns original value of @v
2192 */2194 */
2193static inline s642195static __always_inline s64
2194atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)2196atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
2195{2197{
2196 s64 c = atomic64_read(v);2198 s64 c = atomic64_read(v);
@@ -2215,7 +2217,7 @@ atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) · 18 unmodified lines
2215 * Atomically adds @a to @v, if @v was not already @u.2217 * Atomically adds @a to @v, if @v was not already @u.
2216 * Returns true if the addition was done.2218 * Returns true if the addition was done.
2217 */2219 */
2218static inline bool2220static __always_inline bool
2219atomic64_add_unless(atomic64_t *v, s64 a, s64 u)2221atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
2220{2222{
2221 return atomic64_fetch_add_unless(v, a, u) != u;2223 return atomic64_fetch_add_unless(v, a, u) != u;
@@ -2231,7 +2233,7 @@ atomic64_add_unless(atomic64_t *v, s64 a, s64 u) · 9 unmodified lines
2231 * Atomically increments @v by 1, if @v is non-zero.2233 * Atomically increments @v by 1, if @v is non-zero.
2232 * Returns true if the increment was done.2234 * Returns true if the increment was done.
2233 */2235 */
2234static inline bool2236static __always_inline bool
2235atomic64_inc_not_zero(atomic64_t *v)2237atomic64_inc_not_zero(atomic64_t *v)
2236{2238{
2237 return atomic64_add_unless(v, 1, 0);2239 return atomic64_add_unless(v, 1, 0);
@@ -2240,7 +2242,7 @@ atomic64_inc_not_zero(atomic64_t *v) · 2 unmodified lines
2240#endif2242#endif
2241 2243
2242#ifndef atomic64_inc_unless_negative2244#ifndef atomic64_inc_unless_negative
2243static inline bool2245static __always_inline bool
2244atomic64_inc_unless_negative(atomic64_t *v)2246atomic64_inc_unless_negative(atomic64_t *v)
2245{2247{
2246 s64 c = atomic64_read(v);2248 s64 c = atomic64_read(v);
@@ -2256,7 +2258,7 @@ atomic64_inc_unless_negative(atomic64_t *v) · 9 unmodified lines
2256#endif2258#endif
2257 2259
2258#ifndef atomic64_dec_unless_positive2260#ifndef atomic64_dec_unless_positive
2259static inline bool2261static __always_inline bool
2260atomic64_dec_unless_positive(atomic64_t *v)2262atomic64_dec_unless_positive(atomic64_t *v)
2261{2263{
2262 s64 c = atomic64_read(v);2264 s64 c = atomic64_read(v);
@@ -2272,7 +2274,7 @@ atomic64_dec_unless_positive(atomic64_t *v) · 9 unmodified lines
2272#endif2274#endif
2273 2275
2274#ifndef atomic64_dec_if_positive2276#ifndef atomic64_dec_if_positive
2275static inline s642277static __always_inline s64
2276atomic64_dec_if_positive(atomic64_t *v)2278atomic64_dec_if_positive(atomic64_t *v)
2277{2279{
2278 s64 dec, c = atomic64_read(v);2280 s64 dec, c = atomic64_read(v);
@@ -2292,4 +2294,4 @@ atomic64_dec_if_positive(atomic64_t *v) · 13 unmodified lines
2292#define atomic64_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c))2294#define atomic64_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c))
2293 2295
2294#endif /* _LINUX_ATOMIC_FALLBACK_H */2296#endif /* _LINUX_ATOMIC_FALLBACK_H */
2295// 25de4a2804d70f57e994fe3b419148658bb5378a2297// baaf45f4c24ed88ceae58baca39d7fd80bb8101b