Only the last sbitmap_word can have different depth, and all the others must have same depth of 1U << sb->shift, so not necessary to store it in sbitmap_word, and it can be retrieved easily and efficiently by adding one internal helper of __map_depth(sb, index). Remove 'depth' field from sbitmap_word, then the annotation of ____cacheline_aligned_in_smp for 'word' isn't needed any more. Not see performance effect when running high parallel IOPS test on null_blk. This way saves us one cacheline(usually 64 words) per each sbitmap_word.
Showing the changes to include (1 of 2 files). Show all
Unified · Split
| @@ -27,15 +27,10 @@ struct seq_file; · 26 unmodified lines | |||
| 27 | * struct sbitmap_word - Word in a &struct sbitmap. | 27 | * struct sbitmap_word - Word in a &struct sbitmap. |
| 28 | */ | 28 | */ |
| 29 | struct sbitmap_word { | 29 | struct sbitmap_word { |
| 30 | /** | ||
| 31 | * @depth: Number of bits being used in @word/@cleared | ||
| 32 | */ | ||
| 33 | unsigned long depth; | ||
| 34 | |||
| 35 | /** | 30 | /** |
| 36 | * @word: word holding free bits | 31 | * @word: word holding free bits |
| 37 | */ | 32 | */ |
| 38 | unsigned long word ____cacheline_aligned_in_smp; | 33 | unsigned long word; |
| 39 | 34 | ||
| 40 | /** | 35 | /** |
| 41 | * @cleared: word holding cleared bits | 36 | * @cleared: word holding cleared bits |
| @@ -164,6 +159,14 @@ struct sbitmap_queue { · 122 unmodified lines | |||
| 164 | int sbitmap_init_node(struct sbitmap *sb, unsigned int depth, int shift, | 159 | int sbitmap_init_node(struct sbitmap *sb, unsigned int depth, int shift, |
| 165 | gfp_t flags, int node, bool round_robin, bool alloc_hint); | 160 | gfp_t flags, int node, bool round_robin, bool alloc_hint); |
| 166 | 161 | ||
| 162 | /* sbitmap internal helper */ | ||
| 163 | static inline unsigned int __map_depth(const struct sbitmap *sb, int index) | ||
| 164 | { | ||
| 165 | if (index == sb->map_nr - 1) | ||
| 166 | return sb->depth - (index << sb->shift); | ||
| 167 | return 1U << sb->shift; | ||
| 168 | } | ||
| 169 | |||
| 167 | /** | 170 | /** |
| 168 | * sbitmap_free() - Free memory used by a &struct sbitmap. | 171 | * sbitmap_free() - Free memory used by a &struct sbitmap. |
| 169 | * @sb: Bitmap to free. | 172 | * @sb: Bitmap to free. |
| @@ -251,7 +254,7 @@ static inline void __sbitmap_for_each_set(struct sbitmap *sb, · 81 unmodified lines | |||
| 251 | while (scanned < sb->depth) { | 254 | while (scanned < sb->depth) { |
| 252 | unsigned long word; | 255 | unsigned long word; |
| 253 | unsigned int depth = min_t(unsigned int, | 256 | unsigned int depth = min_t(unsigned int, |
| 254 | sb->map[index].depth - nr, | 257 | __map_depth(sb, index) - nr, |
| 255 | sb->depth - scanned); | 258 | sb->depth - scanned); |
| 256 | 259 | ||
| 257 | scanned += depth; | 260 | scanned += depth; |