Only the last sbitmap_word can have different depth, and all the others must have same depth of 1U << sb->shift, so not necessary to store it in sbitmap_word, and it can be retrieved easily and efficiently by adding one internal helper of __map_depth(sb, index). Remove 'depth' field from sbitmap_word, then the annotation of ____cacheline_aligned_in_smp for 'word' isn't needed any more. Not see performance effect when running high parallel IOPS test on null_blk. This way saves us one cacheline(usually 64 words) per each sbitmap_word.
Unified · Split
| @@ -27,15 +27,10 @@ struct seq_file; · 26 unmodified lines | ||
| 27 | 27 | * struct sbitmap_word - Word in a &struct sbitmap. |
| 28 | 28 | */ |
| 29 | 29 | struct sbitmap_word { |
| 30 | − /** | |
| 31 | − * @depth: Number of bits being used in @word/@cleared | |
| 32 | − */ | |
| 33 | − unsigned long depth; | |
| 34 | − | |
| 35 | 30 | /** |
| 36 | 31 | * @word: word holding free bits |
| 37 | 32 | */ |
| 38 | − unsigned long word ____cacheline_aligned_in_smp; | |
| 33 | + unsigned long word; | |
| 39 | 34 | |
| 40 | 35 | /** |
| 41 | 36 | * @cleared: word holding cleared bits |
| @@ -164,6 +159,14 @@ struct sbitmap_queue { · 122 unmodified lines | ||
| 164 | 159 | int sbitmap_init_node(struct sbitmap *sb, unsigned int depth, int shift, |
| 165 | 160 | gfp_t flags, int node, bool round_robin, bool alloc_hint); |
| 166 | 161 | |
| 162 | +/* sbitmap internal helper */ | |
| 163 | +static inline unsigned int __map_depth(const struct sbitmap *sb, int index) | |
| 164 | +{ | |
| 165 | + if (index == sb->map_nr - 1) | |
| 166 | + return sb->depth - (index << sb->shift); | |
| 167 | + return 1U << sb->shift; | |
| 168 | +} | |
| 169 | + | |
| 167 | 170 | /** |
| 168 | 171 | * sbitmap_free() - Free memory used by a &struct sbitmap. |
| 169 | 172 | * @sb: Bitmap to free. |
| @@ -251,7 +254,7 @@ static inline void __sbitmap_for_each_set(struct sbitmap *sb, · 81 unmodified lines | ||
| 251 | 254 | while (scanned < sb->depth) { |
| 252 | 255 | unsigned long word; |
| 253 | 256 | unsigned int depth = min_t(unsigned int, |
| 254 | − sb->map[index].depth - nr, | |
| 257 | + __map_depth(sb, index) - nr, | |
| 255 | 258 | sb->depth - scanned); |
| 256 | 259 | |
| 257 | 260 | scanned += depth; |
| @@ -85,7 +85,6 @@ int sbitmap_init_node(struct sbitmap *sb, unsigned int depth, int shift, · 84 unmodified lines | ||
| 85 | 85 | bool alloc_hint) |
| 86 | 86 | { |
| 87 | 87 | unsigned int bits_per_word; |
| 88 | − unsigned int i; | |
| 89 | 88 | |
| 90 | 89 | if (shift < 0) |
| 91 | 90 | shift = sbitmap_calculate_shift(depth); |
| @@ -117,10 +116,6 @@ int sbitmap_init_node(struct sbitmap *sb, unsigned int depth, int shift, · 25 unmodified lines | ||
| 117 | 116 | return -ENOMEM; |
| 118 | 117 | } |
| 119 | 118 | |
| 120 | − for (i = 0; i < sb->map_nr; i++) { | |
| 121 | − sb->map[i].depth = min(depth, bits_per_word); | |
| 122 | − depth -= sb->map[i].depth; | |
| 123 | − } | |
| 124 | 119 | return 0; |
| 125 | 120 | } |
| 126 | 121 | EXPORT_SYMBOL_GPL(sbitmap_init_node); |
| @@ -135,11 +130,6 @@ void sbitmap_resize(struct sbitmap *sb, unsigned int depth) · 8 unmodified lines | ||
| 135 | 130 | |
| 136 | 131 | sb->depth = depth; |
| 137 | 132 | sb->map_nr = DIV_ROUND_UP(sb->depth, bits_per_word); |
| 138 | − | |
| 139 | − for (i = 0; i < sb->map_nr; i++) { | |
| 140 | − sb->map[i].depth = min(depth, bits_per_word); | |
| 141 | − depth -= sb->map[i].depth; | |
| 142 | − } | |
| 143 | 133 | } |
| 144 | 134 | EXPORT_SYMBOL_GPL(sbitmap_resize); |
| 145 | 135 | |
| @@ -184,8 +174,8 @@ static int sbitmap_find_bit_in_index(struct sbitmap *sb, int index, · 38 unmodified lines | ||
| 184 | 174 | int nr; |
| 185 | 175 | |
| 186 | 176 | do { |
| 187 | − nr = __sbitmap_get_word(&map->word, map->depth, alloc_hint, | |
| 188 | − !sb->round_robin); | |
| 177 | + nr = __sbitmap_get_word(&map->word, __map_depth(sb, index), | |
| 178 | + alloc_hint, !sb->round_robin); | |
| 189 | 179 | if (nr != -1) |
| 190 | 180 | break; |
| 191 | 181 | if (!sbitmap_deferred_clear(map)) |
| @@ -257,7 +247,9 @@ static int __sbitmap_get_shallow(struct sbitmap *sb, · 65 unmodified lines | ||
| 257 | 247 | for (i = 0; i < sb->map_nr; i++) { |
| 258 | 248 | again: |
| 259 | 249 | nr = __sbitmap_get_word(&sb->map[index].word, |
| 260 | − min(sb->map[index].depth, shallow_depth), | |
| 250 | + min_t(unsigned int, | |
| 251 | + __map_depth(sb, index), | |
| 252 | + shallow_depth), | |
| 261 | 253 | SB_NR_TO_BIT(sb, alloc_hint), true); |
| 262 | 254 | if (nr != -1) { |
| 263 | 255 | nr += index << sb->shift; |
| @@ -315,11 +307,12 @@ static unsigned int __sbitmap_weight(const struct sbitmap *sb, bool set) · 51 unmodified lines | ||
| 315 | 307 | |
| 316 | 308 | for (i = 0; i < sb->map_nr; i++) { |
| 317 | 309 | const struct sbitmap_word *word = &sb->map[i]; |
| 310 | + unsigned int word_depth = __map_depth(sb, i); | |
| 318 | 311 | |
| 319 | 312 | if (set) |
| 320 | − weight += bitmap_weight(&word->word, word->depth); | |
| 313 | + weight += bitmap_weight(&word->word, word_depth); | |
| 321 | 314 | else |
| 322 | − weight += bitmap_weight(&word->cleared, word->depth); | |
| 315 | + weight += bitmap_weight(&word->cleared, word_depth); | |
| 323 | 316 | } |
| 324 | 317 | return weight; |
| 325 | 318 | } |
| @@ -367,7 +360,7 @@ void sbitmap_bitmap_show(struct sbitmap *sb, struct seq_file *m) · 41 unmodified lines | ||
| 367 | 360 | for (i = 0; i < sb->map_nr; i++) { |
| 368 | 361 | unsigned long word = READ_ONCE(sb->map[i].word); |
| 369 | 362 | unsigned long cleared = READ_ONCE(sb->map[i].cleared); |
| 370 | − unsigned int word_bits = READ_ONCE(sb->map[i].depth); | |
| 363 | + unsigned int word_bits = __map_depth(sb, i); | |
| 371 | 364 | |
| 372 | 365 | word &= ~cleared; |
| 373 | 366 | |
| @@ -531,15 +524,16 @@ unsigned long __sbitmap_queue_get_batch(struct sbitmap_queue *sbq, int nr_tags, · 157 unmodified lines | ||
| 531 | 524 | for (i = 0; i < sb->map_nr; i++) { |
| 532 | 525 | struct sbitmap_word *map = &sb->map[index]; |
| 533 | 526 | unsigned long get_mask; |
| 527 | + unsigned int map_depth = __map_depth(sb, index); | |
| 534 | 528 | |
| 535 | 529 | sbitmap_deferred_clear(map); |
| 536 | − if (map->word == (1UL << (map->depth - 1)) - 1) | |
| 530 | + if (map->word == (1UL << (map_depth - 1)) - 1) | |
| 537 | 531 | continue; |
| 538 | 532 | |
| 539 | − nr = find_first_zero_bit(&map->word, map->depth); | |
| 540 | − if (nr + nr_tags <= map->depth) { | |
| 533 | + nr = find_first_zero_bit(&map->word, map_depth); | |
| 534 | + if (nr + nr_tags <= map_depth) { | |
| 541 | 535 | atomic_long_t *ptr = (atomic_long_t *) &map->word; |
| 542 | − int map_tags = min_t(int, nr_tags, map->depth); | |
| 536 | + int map_tags = min_t(int, nr_tags, map_depth); | |
| 543 | 537 | unsigned long val, ret; |
| 544 | 538 | |
| 545 | 539 | get_mask = ((1UL << map_tags) - 1) << nr; |