Only the last sbitmap_word can have different depth, and all the others must have same depth of 1U << sb->shift, so not necessary to store it in sbitmap_word, and it can be retrieved easily and efficiently by adding one internal helper of __map_depth(sb, index). Remove 'depth' field from sbitmap_word, then the annotation of ____cacheline_aligned_in_smp for 'word' isn't needed any more. Not see performance effect when running high parallel IOPS test on null_blk. This way saves us one cacheline(usually 64 words) per each sbitmap_word.
Showing the changes to include (1 of 2 files). Show all
Unified · Split
| @@ -27,15 +27,10 @@ struct seq_file; · 26 unmodified lines | ||
| 27 | 27 | * struct sbitmap_word - Word in a &struct sbitmap. |
| 28 | 28 | */ |
| 29 | 29 | struct sbitmap_word { |
| 30 | − /** | |
| 31 | − * @depth: Number of bits being used in @word/@cleared | |
| 32 | − */ | |
| 33 | − unsigned long depth; | |
| 34 | − | |
| 35 | 30 | /** |
| 36 | 31 | * @word: word holding free bits |
| 37 | 32 | */ |
| 38 | − unsigned long word ____cacheline_aligned_in_smp; | |
| 33 | + unsigned long word; | |
| 39 | 34 | |
| 40 | 35 | /** |
| 41 | 36 | * @cleared: word holding cleared bits |
| @@ -164,6 +159,14 @@ struct sbitmap_queue { · 122 unmodified lines | ||
| 164 | 159 | int sbitmap_init_node(struct sbitmap *sb, unsigned int depth, int shift, |
| 165 | 160 | gfp_t flags, int node, bool round_robin, bool alloc_hint); |
| 166 | 161 | |
| 162 | +/* sbitmap internal helper */ | |
| 163 | +static inline unsigned int __map_depth(const struct sbitmap *sb, int index) | |
| 164 | +{ | |
| 165 | + if (index == sb->map_nr - 1) | |
| 166 | + return sb->depth - (index << sb->shift); | |
| 167 | + return 1U << sb->shift; | |
| 168 | +} | |
| 169 | + | |
| 167 | 170 | /** |
| 168 | 171 | * sbitmap_free() - Free memory used by a &struct sbitmap. |
| 169 | 172 | * @sb: Bitmap to free. |
| @@ -251,7 +254,7 @@ static inline void __sbitmap_for_each_set(struct sbitmap *sb, · 81 unmodified lines | ||
| 251 | 254 | while (scanned < sb->depth) { |
| 252 | 255 | unsigned long word; |
| 253 | 256 | unsigned int depth = min_t(unsigned int, |
| 254 | − sb->map[index].depth - nr, | |
| 257 | + __map_depth(sb, index) - nr, | |
| 255 | 258 | sb->depth - scanned); |
| 256 | 259 | |
| 257 | 260 | scanned += depth; |