Having the refcount saturation and warnings inline bloats the text, despite the fact that these paths should never be executed in normal operation. Move the refcount saturation and warnings out of line to reduce the image size when refcount_t checking is enabled. Relative to an x86_64 defconfig, the sizes reported by bloat-o-meter are: # defconfig+REFCOUNT_FULL, inline saturation (i.e. before this patch) Total: Before=14762076, After=14915442, chg +1.04% # defconfig+REFCOUNT_FULL, out-of-line saturation (i.e. after this patch) Total: Before=14762076, After=14835497, chg +0.50% A side-effect of this change is that we now only get one warning per refcount saturation type, rather than one per problematic call-site.
Showing the changes to include/linux (1 of 2 files). Show all
Unified · Split
| @@ -23,6 +23,16 @@ typedef struct refcount_struct { · 22 unmodified lines | ||
| 23 | 23 | |
| 24 | 24 | #define REFCOUNT_INIT(n) { .refs = ATOMIC_INIT(n), } |
| 25 | 25 | |
| 26 | +enum refcount_saturation_type { | |
| 27 | + REFCOUNT_ADD_NOT_ZERO_OVF, | |
| 28 | + REFCOUNT_ADD_OVF, | |
| 29 | + REFCOUNT_ADD_UAF, | |
| 30 | + REFCOUNT_SUB_UAF, | |
| 31 | + REFCOUNT_DEC_LEAK, | |
| 32 | +}; | |
| 33 | + | |
| 34 | +void refcount_warn_saturate(refcount_t *r, enum refcount_saturation_type t); | |
| 35 | + | |
| 26 | 36 | /** |
| 27 | 37 | * refcount_set - set a refcount's value |
| 28 | 38 | * @r: the refcount |
| @@ -154,10 +164,8 @@ static inline __must_check bool refcount_add_not_zero(int i, refcount_t *r) · 125 unmodified lines | ||
| 154 | 164 | break; |
| 155 | 165 | } while (!atomic_try_cmpxchg_relaxed(&r->refs, &old, old + i)); |
| 156 | 166 | |
| 157 | − if (unlikely(old < 0 || old + i < 0)) { | |
| 158 | − refcount_set(r, REFCOUNT_SATURATED); | |
| 159 | − WARN_ONCE(1, "refcount_t: saturated; leaking memory.\n"); | |
| 160 | − } | |
| 167 | + if (unlikely(old < 0 || old + i < 0)) | |
| 168 | + refcount_warn_saturate(r, REFCOUNT_ADD_NOT_ZERO_OVF); | |
| 161 | 169 | |
| 162 | 170 | return old; |
| 163 | 171 | } |
| @@ -182,11 +190,10 @@ static inline void refcount_add(int i, refcount_t *r) · 18 unmodified lines | ||
| 182 | 190 | { |
| 183 | 191 | int old = atomic_fetch_add_relaxed(i, &r->refs); |
| 184 | 192 | |
| 185 | − WARN_ONCE(!old, "refcount_t: addition on 0; use-after-free.\n"); | |
| 186 | − if (unlikely(old <= 0 || old + i <= 0)) { | |
| 187 | − refcount_set(r, REFCOUNT_SATURATED); | |
| 188 | − WARN_ONCE(old, "refcount_t: saturated; leaking memory.\n"); | |
| 189 | − } | |
| 193 | + if (unlikely(!old)) | |
| 194 | + refcount_warn_saturate(r, REFCOUNT_ADD_UAF); | |
| 195 | + else if (unlikely(old < 0 || old + i < 0)) | |
| 196 | + refcount_warn_saturate(r, REFCOUNT_ADD_OVF); | |
| 190 | 197 | } |
| 191 | 198 | |
| 192 | 199 | /** |
| @@ -253,10 +260,8 @@ static inline __must_check bool refcount_sub_and_test(int i, refcount_t *r) · 60 unmodified lines | ||
| 253 | 260 | return true; |
| 254 | 261 | } |
| 255 | 262 | |
| 256 | − if (unlikely(old < 0 || old - i < 0)) { | |
| 257 | − refcount_set(r, REFCOUNT_SATURATED); | |
| 258 | − WARN_ONCE(1, "refcount_t: underflow; use-after-free.\n"); | |
| 259 | − } | |
| 263 | + if (unlikely(old < 0 || old - i < 0)) | |
| 264 | + refcount_warn_saturate(r, REFCOUNT_SUB_UAF); | |
| 260 | 265 | |
| 261 | 266 | return false; |
| 262 | 267 | } |
| @@ -291,12 +296,8 @@ static inline __must_check bool refcount_dec_and_test(refcount_t *r) · 28 unmodified lines | ||
| 291 | 296 | */ |
| 292 | 297 | static inline void refcount_dec(refcount_t *r) |
| 293 | 298 | { |
| 294 | − int old = atomic_fetch_sub_release(1, &r->refs); | |
| 295 | − | |
| 296 | − if (unlikely(old <= 1)) { | |
| 297 | − refcount_set(r, REFCOUNT_SATURATED); | |
| 298 | − WARN_ONCE(1, "refcount_t: decrement hit 0; leaking memory.\n"); | |
| 299 | − } | |
| 299 | + if (unlikely(atomic_fetch_sub_release(1, &r->refs) <= 1)) | |
| 300 | + refcount_warn_saturate(r, REFCOUNT_DEC_LEAK); | |
| 300 | 301 | } |
| 301 | 302 | #else /* CONFIG_REFCOUNT_FULL */ |
| 302 | 303 | |