| @@ -32,13 +32,11 @@ · 31 unmodified lines |
| 32 | 32 | |
| 33 | 33 | #include <linux/mlx5/driver.h> |
| 34 | 34 | #include <linux/mlx5/fs.h> |
| 35 | | −#include <linux/rbtree.h> |
| 36 | 35 | #include "mlx5_core.h" |
| 37 | 36 | #include "fs_core.h" |
| 38 | 37 | #include "fs_cmd.h" |
| 39 | 38 | |
| 40 | 39 | #define MLX5_FC_STATS_PERIOD msecs_to_jiffies(1000) |
| 41 | | −#define MLX5_FC_BULK_QUERY_ALLOC_PERIOD msecs_to_jiffies(180 * 1000) |
| 42 | 40 | /* Max number of counters to query in bulk read is 32K */ |
| 43 | 41 | #define MLX5_SW_MAX_COUNTERS_BULK BIT(15) |
| 44 | 42 | #define MLX5_INIT_COUNTERS_BULK 8 |
| @@ -52,21 +50,37 @@ struct mlx5_fc_cache { · 7 unmodified lines |
| 52 | 50 | }; |
| 53 | 51 | |
| 54 | 52 | struct mlx5_fc { |
| 55 | | − struct list_head list; |
| 56 | | − struct llist_node addlist; |
| 57 | | − struct llist_node dellist; |
| 58 | | − |
| 59 | | − /* last{packets,bytes} members are used when calculating the delta since |
| 60 | | − * last reading |
| 61 | | − */ |
| 53 | + u32 id; |
| 54 | + bool aging; |
| 55 | + struct mlx5_fc_bulk *bulk; |
| 56 | + struct mlx5_fc_cache cache; |
| 57 | + /* last{packets,bytes} are used for calculating deltas since last reading. */ |
| 62 | 58 | u64 lastpackets; |
| 63 | 59 | u64 lastbytes; |
| 60 | +}; |
| 64 | 61 | |
| 65 | | − struct mlx5_fc_bulk *bulk; |
| 66 | | − u32 id; |
| 67 | | − bool aging; |
| 62 | +struct mlx5_fc_pool { |
| 63 | + struct mlx5_core_dev *dev; |
| 64 | + struct mutex pool_lock; /* protects pool lists */ |
| 65 | + struct list_head fully_used; |
| 66 | + struct list_head partially_used; |
| 67 | + struct list_head unused; |
| 68 | + int available_fcs; |
| 69 | + int used_fcs; |
| 70 | + int threshold; |
| 71 | +}; |
| 68 | 72 | |
| 69 | | − struct mlx5_fc_cache cache ____cacheline_aligned_in_smp; |
| 73 | +struct mlx5_fc_stats { |
| 74 | + struct xarray counters; |
| 75 | + |
| 76 | + struct workqueue_struct *wq; |
| 77 | + struct delayed_work work; |
| 78 | + unsigned long sampling_interval; /* jiffies */ |
| 79 | + u32 *bulk_query_out; |
| 80 | + int bulk_query_len; |
| 81 | + bool bulk_query_alloc_failed; |
| 82 | + unsigned long next_bulk_query_alloc; |
| 83 | + struct mlx5_fc_pool fc_pool; |
| 70 | 84 | }; |
| 71 | 85 | |
| 72 | 86 | static void mlx5_fc_pool_init(struct mlx5_fc_pool *fc_pool, struct mlx5_core_dev *dev); |
| @@ -74,78 +88,6 @@ static void mlx5_fc_pool_cleanup(struct mlx5_fc_pool *fc_pool); · 1 unmodified line |
| 74 | 88 | static struct mlx5_fc *mlx5_fc_pool_acquire_counter(struct mlx5_fc_pool *fc_pool); |
| 75 | 89 | static void mlx5_fc_pool_release_counter(struct mlx5_fc_pool *fc_pool, struct mlx5_fc *fc); |
| 76 | 90 | |
| 77 | | −/* locking scheme: |
| 78 | | − * |
| 79 | | − * It is the responsibility of the user to prevent concurrent calls or bad |
| 80 | | − * ordering to mlx5_fc_create(), mlx5_fc_destroy() and accessing a reference |
| 81 | | − * to struct mlx5_fc. |
| 82 | | − * e.g en_tc.c is protected by RTNL lock of its caller, and will never call a |
| 83 | | − * dump (access to struct mlx5_fc) after a counter is destroyed. |
| 84 | | − * |
| 85 | | − * access to counter list: |
| 86 | | − * - create (user context) |
| 87 | | − * - mlx5_fc_create() only adds to an addlist to be used by |
| 88 | | − * mlx5_fc_stats_work(). addlist is a lockless single linked list |
| 89 | | − * that doesn't require any additional synchronization when adding single |
| 90 | | − * node. |
| 91 | | − * - spawn thread to do the actual destroy |
| 92 | | − * |
| 93 | | − * - destroy (user context) |
| 94 | | − * - add a counter to lockless dellist |
| 95 | | − * - spawn thread to do the actual del |
| 96 | | − * |
| 97 | | − * - dump (user context) |
| 98 | | − * user should not call dump after destroy |
| 99 | | − * |
| 100 | | − * - query (single thread workqueue context) |
| 101 | | − * destroy/dump - no conflict (see destroy) |
| 102 | | − * query/dump - packets and bytes might be inconsistent (since update is not |
| 103 | | − * atomic) |
| 104 | | − * query/create - no conflict (see create) |
| 105 | | − * since every create/destroy spawn the work, only after necessary time has |
| 106 | | − * elapsed, the thread will actually query the hardware. |
| 107 | | − */ |
| 108 | | − |
| 109 | | −static struct list_head *mlx5_fc_counters_lookup_next(struct mlx5_core_dev *dev, |
| 110 | | − u32 id) |
| 111 | | −{ |
| 112 | | − struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats; |
| 113 | | − unsigned long next_id = (unsigned long)id + 1; |
| 114 | | − struct mlx5_fc *counter; |
| 115 | | − unsigned long tmp; |
| 116 | | − |
| 117 | | − rcu_read_lock(); |
| 118 | | − /* skip counters that are in idr, but not yet in counters list */ |
| 119 | | − idr_for_each_entry_continue_ul(&fc_stats->counters_idr, |
| 120 | | − counter, tmp, next_id) { |
| 121 | | − if (!list_empty(&counter->list)) |
| 122 | | − break; |
| 123 | | − } |
| 124 | | − rcu_read_unlock(); |
| 125 | | − |
| 126 | | − return counter ? &counter->list : &fc_stats->counters; |
| 127 | | −} |
| 128 | | − |
| 129 | | −static void mlx5_fc_stats_insert(struct mlx5_core_dev *dev, |
| 130 | | − struct mlx5_fc *counter) |
| 131 | | −{ |
| 132 | | − struct list_head *next = mlx5_fc_counters_lookup_next(dev, counter->id); |
| 133 | | − |
| 134 | | − list_add_tail(&counter->list, next); |
| 135 | | −} |
| 136 | | − |
| 137 | | −static void mlx5_fc_stats_remove(struct mlx5_core_dev *dev, |
| 138 | | − struct mlx5_fc *counter) |
| 139 | | −{ |
| 140 | | − struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats; |
| 141 | | − |
| 142 | | − list_del(&counter->list); |
| 143 | | − |
| 144 | | − spin_lock(&fc_stats->counters_idr_lock); |
| 145 | | − WARN_ON(!idr_remove(&fc_stats->counters_idr, counter->id)); |
| 146 | | − spin_unlock(&fc_stats->counters_idr_lock); |
| 147 | | −} |
| 148 | | − |
| 149 | 91 | static int get_init_bulk_query_len(struct mlx5_core_dev *dev) |
| 150 | 92 | { |
| 151 | 93 | return min_t(int, MLX5_INIT_COUNTERS_BULK, |
| @@ -174,47 +116,64 @@ static void update_counter_cache(int index, u32 *bulk_raw_data, · 22 unmodified lines |
| 174 | 116 | cache->lastuse = jiffies; |
| 175 | 117 | } |
| 176 | 118 | |
| 177 | | −static void mlx5_fc_stats_query_counter_range(struct mlx5_core_dev *dev, |
| 178 | | − struct mlx5_fc *first, |
| 179 | | − u32 last_id) |
| 119 | +/* Synchronization notes |
| 120 | + * |
| 121 | + * Access to counter array: |
| 122 | + * - create - mlx5_fc_create() (user context) |
| 123 | + * - inserts the counter into the xarray. |
| 124 | + * |
| 125 | + * - destroy - mlx5_fc_destroy() (user context) |
| 126 | + * - erases the counter from the xarray and releases it. |
| 127 | + * |
| 128 | + * - query mlx5_fc_query(), mlx5_fc_query_cached{,_raw}() (user context) |
| 129 | + * - user should not access a counter after destroy. |
| 130 | + * |
| 131 | + * - bulk query (single thread workqueue context) |
| 132 | + * - create: query relies on 'lastuse' to avoid updating counters added |
| 133 | + * around the same time as the current bulk cmd. |
| 134 | + * - destroy: destroyed counters will not be accessed, even if they are |
| 135 | + * destroyed during a bulk query command. |
| 136 | + */ |
| 137 | +static void mlx5_fc_stats_query_all_counters(struct mlx5_core_dev *dev) |
| 180 | 138 | { |
| 181 | | − struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats; |
| 182 | | − bool query_more_counters = (first->id <= last_id); |
| 183 | | − int cur_bulk_len = fc_stats->bulk_query_len; |
| 139 | + struct mlx5_fc_stats *fc_stats = dev->priv.fc_stats; |
| 140 | + u32 bulk_len = fc_stats->bulk_query_len; |
| 141 | + XA_STATE(xas, &fc_stats->counters, 0); |
| 184 | 142 | u32 *data = fc_stats->bulk_query_out; |
| 185 | | − struct mlx5_fc *counter = first; |
| 143 | + struct mlx5_fc *counter; |
| 144 | + u32 last_bulk_id = 0; |
| 145 | + u64 bulk_query_time; |
| 186 | 146 | u32 bulk_base_id; |
| 187 | | − int bulk_len; |
| 188 | 147 | int err; |
| 189 | 148 | |
| 190 | | − while (query_more_counters) { |
| 191 | | − /* first id must be aligned to 4 when using bulk query */ |
| 192 | | − bulk_base_id = counter->id & ~0x3; |
| 193 | | − |
| 194 | | − /* number of counters to query inc. the last counter */ |
| 195 | | − bulk_len = min_t(int, cur_bulk_len, |
| 196 | | − ALIGN(last_id - bulk_base_id + 1, 4)); |
| 197 | | − |
| 198 | | − err = mlx5_cmd_fc_bulk_query(dev, bulk_base_id, bulk_len, |
| 199 | | − data); |
| 200 | | − if (err) { |
| 201 | | − mlx5_core_err(dev, "Error doing bulk query: %d\n", err); |
| 202 | | − return; |
| 203 | | − } |
| 204 | | − query_more_counters = false; |
| 205 | | − |
| 206 | | − list_for_each_entry_from(counter, &fc_stats->counters, list) { |
| 207 | | − int counter_index = counter->id - bulk_base_id; |
| 208 | | − struct mlx5_fc_cache *cache = &counter->cache; |
| 209 | | − |
| 210 | | − if (counter->id >= bulk_base_id + bulk_len) { |
| 211 | | − query_more_counters = true; |
| 212 | | − break; |
| 149 | + xas_lock(&xas); |
| 150 | + xas_for_each(&xas, counter, U32_MAX) { |
| 151 | + if (xas_retry(&xas, counter)) |
| 152 | + continue; |
| 153 | + if (unlikely(counter->id >= last_bulk_id)) { |
| 154 | + /* Start new bulk query. */ |
| 155 | + /* First id must be aligned to 4 when using bulk query. */ |
| 156 | + bulk_base_id = counter->id & ~0x3; |
| 157 | + last_bulk_id = bulk_base_id + bulk_len; |
| 158 | + /* The lock is released while querying the hw and reacquired after. */ |
| 159 | + xas_unlock(&xas); |
| 160 | + /* The same id needs to be processed again in the next loop iteration. */ |
| 161 | + xas_reset(&xas); |
| 162 | + bulk_query_time = jiffies; |
| 163 | + err = mlx5_cmd_fc_bulk_query(dev, bulk_base_id, bulk_len, data); |
| 164 | + if (err) { |
| 165 | + mlx5_core_err(dev, "Error doing bulk query: %d\n", err); |
| 166 | + return; |
| 213 | 167 | } |
| 214 | | − |
| 215 | | − update_counter_cache(counter_index, data, cache); |
| 168 | + xas_lock(&xas); |
| 169 | + continue; |
| 216 | 170 | } |
| 171 | + /* Do not update counters added after bulk query was started. */ |
| 172 | + if (time_after64(bulk_query_time, counter->cache.lastuse)) |
| 173 | + update_counter_cache(counter->id - bulk_base_id, data, |
| 174 | + &counter->cache); |
| 217 | 175 | } |
| 176 | + xas_unlock(&xas); |
| 218 | 177 | } |
| 219 | 178 | |
| 220 | 179 | static void mlx5_fc_free(struct mlx5_core_dev *dev, struct mlx5_fc *counter) |
| @@ -225,7 +184,7 @@ static void mlx5_fc_free(struct mlx5_core_dev *dev, struct mlx5_fc *counter) · 4 unmodified lines |
| 225 | 184 | |
| 226 | 185 | static void mlx5_fc_release(struct mlx5_core_dev *dev, struct mlx5_fc *counter) |
| 227 | 186 | { |
| 228 | | − struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats; |
| 187 | + struct mlx5_fc_stats *fc_stats = dev->priv.fc_stats; |
| 229 | 188 | |
| 230 | 189 | if (counter->bulk) |
| 231 | 190 | mlx5_fc_pool_release_counter(&fc_stats->fc_pool, counter); |
| @@ -233,85 +192,55 @@ static void mlx5_fc_release(struct mlx5_core_dev *dev, struct mlx5_fc *counter) · 1 unmodified line |
| 233 | 192 | mlx5_fc_free(dev, counter); |
| 234 | 193 | } |
| 235 | 194 | |
| 236 | | −static void mlx5_fc_stats_bulk_query_size_increase(struct mlx5_core_dev *dev) |
| 195 | +static void mlx5_fc_stats_bulk_query_buf_realloc(struct mlx5_core_dev *dev, |
| 196 | + int bulk_query_len) |
| 237 | 197 | { |
| 238 | | − struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats; |
| 239 | | − int max_bulk_len = get_max_bulk_query_len(dev); |
| 240 | | − unsigned long now = jiffies; |
| 198 | + struct mlx5_fc_stats *fc_stats = dev->priv.fc_stats; |
| 241 | 199 | u32 *bulk_query_out_tmp; |
| 242 | | − int max_out_len; |
| 243 | | − |
| 244 | | − if (fc_stats->bulk_query_alloc_failed && |
| 245 | | − time_before(now, fc_stats->next_bulk_query_alloc)) |
| 246 | | − return; |
| 200 | + int out_len; |
| 247 | 201 | |
| 248 | | − max_out_len = mlx5_cmd_fc_get_bulk_query_out_len(max_bulk_len); |
| 249 | | − bulk_query_out_tmp = kzalloc(max_out_len, GFP_KERNEL); |
| 202 | + out_len = mlx5_cmd_fc_get_bulk_query_out_len(bulk_query_len); |
| 203 | + bulk_query_out_tmp = kvzalloc(out_len, GFP_KERNEL); |
| 250 | 204 | if (!bulk_query_out_tmp) { |
| 251 | 205 | mlx5_core_warn_once(dev, |
| 252 | | − "Can't increase flow counters bulk query buffer size, insufficient memory, bulk_size(%d)\n", |
| 253 | | − max_bulk_len); |
| 254 | | − fc_stats->bulk_query_alloc_failed = true; |
| 255 | | − fc_stats->next_bulk_query_alloc = |
| 256 | | − now + MLX5_FC_BULK_QUERY_ALLOC_PERIOD; |
| 206 | + "Can't increase flow counters bulk query buffer size, alloc failed, bulk_query_len(%d)\n", |
| 207 | + bulk_query_len); |
| 257 | 208 | return; |
| 258 | 209 | } |
| 259 | 210 | |
| 260 | | − kfree(fc_stats->bulk_query_out); |
| 211 | + kvfree(fc_stats->bulk_query_out); |
| 261 | 212 | fc_stats->bulk_query_out = bulk_query_out_tmp; |
| 262 | | − fc_stats->bulk_query_len = max_bulk_len; |
| 263 | | − if (fc_stats->bulk_query_alloc_failed) { |
| 264 | | − mlx5_core_info(dev, |
| 265 | | − "Flow counters bulk query buffer size increased, bulk_size(%d)\n", |
| 266 | | − max_bulk_len); |
| 267 | | − fc_stats->bulk_query_alloc_failed = false; |
| 268 | | − } |
| 213 | + fc_stats->bulk_query_len = bulk_query_len; |
| 214 | + mlx5_core_info(dev, |
| 215 | + "Flow counters bulk query buffer size increased, bulk_query_len(%d)\n", |
| 216 | + bulk_query_len); |
| 269 | 217 | } |
| 270 | 218 | |
| 271 | | −static void mlx5_fc_stats_work(struct work_struct *work) |
| 219 | +static int mlx5_fc_num_counters(struct mlx5_fc_stats *fc_stats) |
| 272 | 220 | { |
| 273 | | − struct mlx5_core_dev *dev = container_of(work, struct mlx5_core_dev, |
| 274 | | − priv.fc_stats.work.work); |
| 275 | | − struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats; |
| 276 | | − /* Take dellist first to ensure that counters cannot be deleted before |
| 277 | | − * they are inserted. |
| 278 | | − */ |
| 279 | | − struct llist_node *dellist = llist_del_all(&fc_stats->dellist); |
| 280 | | − struct llist_node *addlist = llist_del_all(&fc_stats->addlist); |
| 281 | | − struct mlx5_fc *counter = NULL, *last = NULL, *tmp; |
| 282 | | − unsigned long now = jiffies; |
| 283 | | − |
| 284 | | − if (addlist || !list_empty(&fc_stats->counters)) |
| 285 | | − queue_delayed_work(fc_stats->wq, &fc_stats->work, |
| 286 | | − fc_stats->sampling_interval); |
| 287 | | − |
| 288 | | − llist_for_each_entry(counter, addlist, addlist) { |
| 289 | | − mlx5_fc_stats_insert(dev, counter); |
| 290 | | − fc_stats->num_counters++; |
| 291 | | − } |
| 292 | | − |
| 293 | | − llist_for_each_entry_safe(counter, tmp, dellist, dellist) { |
| 294 | | − mlx5_fc_stats_remove(dev, counter); |
| 221 | + struct mlx5_fc *counter; |
| 222 | + int num_counters = 0; |
| 223 | + unsigned long id; |
| 295 | 224 | |
| 296 | | − mlx5_fc_release(dev, counter); |
| 297 | | − fc_stats->num_counters--; |
| 298 | | − } |
| 225 | + xa_for_each(&fc_stats->counters, id, counter) |
| 226 | + num_counters++; |
| 227 | + return num_counters; |
| 228 | +} |
| 299 | 229 | |
| 300 | | − if (fc_stats->bulk_query_len < get_max_bulk_query_len(dev) && |
| 301 | | − fc_stats->num_counters > get_init_bulk_query_len(dev)) |
| 302 | | − mlx5_fc_stats_bulk_query_size_increase(dev); |
| 230 | +static void mlx5_fc_stats_work(struct work_struct *work) |
| 231 | +{ |
| 232 | + struct mlx5_fc_stats *fc_stats = container_of(work, struct mlx5_fc_stats, |
| 233 | + work.work); |
| 234 | + struct mlx5_core_dev *dev = fc_stats->fc_pool.dev; |
| 303 | 235 | |
| 304 | | − if (time_before(now, fc_stats->next_query) || |
| 305 | | − list_empty(&fc_stats->counters)) |
| 306 | | − return; |
| 307 | | − last = list_last_entry(&fc_stats->counters, struct mlx5_fc, list); |
| 236 | + queue_delayed_work(fc_stats->wq, &fc_stats->work, fc_stats->sampling_interval); |
| 308 | 237 | |
| 309 | | − counter = list_first_entry(&fc_stats->counters, struct mlx5_fc, |
| 310 | | − list); |
| 311 | | − if (counter) |
| 312 | | − mlx5_fc_stats_query_counter_range(dev, counter, last->id); |
| 238 | + /* Grow the bulk query buffer to max if not maxed and enough counters are present. */ |
| 239 | + if (unlikely(fc_stats->bulk_query_len < get_max_bulk_query_len(dev) && |
| 240 | + mlx5_fc_num_counters(fc_stats) > get_init_bulk_query_len(dev))) |
| 241 | + mlx5_fc_stats_bulk_query_buf_realloc(dev, get_max_bulk_query_len(dev)); |
| 313 | 242 | |
| 314 | | − fc_stats->next_query = now + fc_stats->sampling_interval; |
| 243 | + mlx5_fc_stats_query_all_counters(dev); |
| 315 | 244 | } |
| 316 | 245 | |
| 317 | 246 | static struct mlx5_fc *mlx5_fc_single_alloc(struct mlx5_core_dev *dev) |
| @@ -334,7 +263,7 @@ static struct mlx5_fc *mlx5_fc_single_alloc(struct mlx5_core_dev *dev) · 16 unmodified lines |
| 334 | 263 | |
| 335 | 264 | static struct mlx5_fc *mlx5_fc_acquire(struct mlx5_core_dev *dev, bool aging) |
| 336 | 265 | { |
| 337 | | − struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats; |
| 266 | + struct mlx5_fc_stats *fc_stats = dev->priv.fc_stats; |
| 338 | 267 | struct mlx5_fc *counter; |
| 339 | 268 | |
| 340 | 269 | if (aging && MLX5_CAP_GEN(dev, flow_counter_bulk_alloc) != 0) { |
| @@ -346,16 +275,15 @@ static struct mlx5_fc *mlx5_fc_acquire(struct mlx5_core_dev *dev, bool aging) · 5 unmodified lines |
| 346 | 275 | return mlx5_fc_single_alloc(dev); |
| 347 | 276 | } |
| 348 | 277 | |
| 349 | | −struct mlx5_fc *mlx5_fc_create_ex(struct mlx5_core_dev *dev, bool aging) |
| 278 | +struct mlx5_fc *mlx5_fc_create(struct mlx5_core_dev *dev, bool aging) |
| 350 | 279 | { |
| 351 | 280 | struct mlx5_fc *counter = mlx5_fc_acquire(dev, aging); |
| 352 | | − struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats; |
| 281 | + struct mlx5_fc_stats *fc_stats = dev->priv.fc_stats; |
| 353 | 282 | int err; |
| 354 | 283 | |
| 355 | 284 | if (IS_ERR(counter)) |
| 356 | 285 | return counter; |
| 357 | 286 | |
| 358 | | − INIT_LIST_HEAD(&counter->list); |
| 359 | 287 | counter->aging = aging; |
| 360 | 288 | |
| 361 | 289 | if (aging) { |
| @@ -365,18 +293,9 @@ struct mlx5_fc *mlx5_fc_create_ex(struct mlx5_core_dev *dev, bool aging) · 3 unmodified lines |
| 365 | 293 | counter->lastbytes = counter->cache.bytes; |
| 366 | 294 | counter->lastpackets = counter->cache.packets; |
| 367 | 295 | |
| 368 | | − idr_preload(GFP_KERNEL); |
| 369 | | − spin_lock(&fc_stats->counters_idr_lock); |
| 370 | | − |
| 371 | | − err = idr_alloc_u32(&fc_stats->counters_idr, counter, &id, id, |
| 372 | | − GFP_NOWAIT); |
| 373 | | − |
| 374 | | − spin_unlock(&fc_stats->counters_idr_lock); |
| 375 | | − idr_preload_end(); |
| 376 | | − if (err) |
| 296 | + err = xa_err(xa_store(&fc_stats->counters, id, counter, GFP_KERNEL)); |
| 297 | + if (err != 0) |
| 377 | 298 | goto err_out_alloc; |
| 378 | | − |
| 379 | | − llist_add(&counter->addlist, &fc_stats->addlist); |
| 380 | 299 | } |
| 381 | 300 | |
| 382 | 301 | return counter; |
| @@ -385,16 +304,6 @@ err_out_alloc: · 2 unmodified lines |
| 385 | 304 | mlx5_fc_release(dev, counter); |
| 386 | 305 | return ERR_PTR(err); |
| 387 | 306 | } |
| 388 | | − |
| 389 | | −struct mlx5_fc *mlx5_fc_create(struct mlx5_core_dev *dev, bool aging) |
| 390 | | −{ |
| 391 | | − struct mlx5_fc *counter = mlx5_fc_create_ex(dev, aging); |
| 392 | | − struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats; |
| 393 | | − |
| 394 | | − if (aging) |
| 395 | | − mod_delayed_work(fc_stats->wq, &fc_stats->work, 0); |
| 396 | | − return counter; |
| 397 | | −} |
| 398 | 307 | EXPORT_SYMBOL(mlx5_fc_create); |
| 399 | 308 | |
| 400 | 309 | u32 mlx5_fc_id(struct mlx5_fc *counter) |
| @@ -405,39 +314,32 @@ EXPORT_SYMBOL(mlx5_fc_id); · 4 unmodified lines |
| 405 | 314 | |
| 406 | 315 | void mlx5_fc_destroy(struct mlx5_core_dev *dev, struct mlx5_fc *counter) |
| 407 | 316 | { |
| 408 | | − struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats; |
| 317 | + struct mlx5_fc_stats *fc_stats = dev->priv.fc_stats; |
| 409 | 318 | |
| 410 | 319 | if (!counter) |
| 411 | 320 | return; |
| 412 | 321 | |
| 413 | | − if (counter->aging) { |
| 414 | | − llist_add(&counter->dellist, &fc_stats->dellist); |
| 415 | | − mod_delayed_work(fc_stats->wq, &fc_stats->work, 0); |
| 416 | | − return; |
| 417 | | − } |
| 418 | | − |
| 322 | + if (counter->aging) |
| 323 | + xa_erase(&fc_stats->counters, counter->id); |
| 419 | 324 | mlx5_fc_release(dev, counter); |
| 420 | 325 | } |
| 421 | 326 | EXPORT_SYMBOL(mlx5_fc_destroy); |
| 422 | 327 | |
| 423 | 328 | int mlx5_init_fc_stats(struct mlx5_core_dev *dev) |
| 424 | 329 | { |
| 425 | | − struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats; |
| 426 | | − int init_bulk_len; |
| 427 | | − int init_out_len; |
| 330 | + struct mlx5_fc_stats *fc_stats; |
| 331 | + |
| 332 | + fc_stats = kzalloc(sizeof(*fc_stats), GFP_KERNEL); |
| 333 | + if (!fc_stats) |
| 334 | + return -ENOMEM; |
| 335 | + dev->priv.fc_stats = fc_stats; |
| 428 | 336 | |
| 429 | | − spin_lock_init(&fc_stats->counters_idr_lock); |
| 430 | | − idr_init(&fc_stats->counters_idr); |
| 431 | | − INIT_LIST_HEAD(&fc_stats->counters); |
| 432 | | − init_llist_head(&fc_stats->addlist); |
| 433 | | − init_llist_head(&fc_stats->dellist); |
| 337 | + xa_init(&fc_stats->counters); |
| 434 | 338 | |
| 435 | | − init_bulk_len = get_init_bulk_query_len(dev); |
| 436 | | − init_out_len = mlx5_cmd_fc_get_bulk_query_out_len(init_bulk_len); |
| 437 | | − fc_stats->bulk_query_out = kzalloc(init_out_len, GFP_KERNEL); |
| 339 | + /* Allocate initial (small) bulk query buffer. */ |
| 340 | + mlx5_fc_stats_bulk_query_buf_realloc(dev, get_init_bulk_query_len(dev)); |
| 438 | 341 | if (!fc_stats->bulk_query_out) |
| 439 | | − return -ENOMEM; |
| 440 | | − fc_stats->bulk_query_len = init_bulk_len; |
| 342 | + goto err_bulk; |
| 441 | 343 | |
| 442 | 344 | fc_stats->wq = create_singlethread_workqueue("mlx5_fc"); |
| 443 | 345 | if (!fc_stats->wq) |
| @@ -447,34 +349,35 @@ int mlx5_init_fc_stats(struct mlx5_core_dev *dev) · 3 unmodified lines |
| 447 | 349 | INIT_DELAYED_WORK(&fc_stats->work, mlx5_fc_stats_work); |
| 448 | 350 | |
| 449 | 351 | mlx5_fc_pool_init(&fc_stats->fc_pool, dev); |
| 352 | + queue_delayed_work(fc_stats->wq, &fc_stats->work, MLX5_FC_STATS_PERIOD); |
| 450 | 353 | return 0; |
| 451 | 354 | |
| 452 | 355 | err_wq_create: |
| 453 | | − kfree(fc_stats->bulk_query_out); |
| 356 | + kvfree(fc_stats->bulk_query_out); |
| 357 | +err_bulk: |
| 358 | + kfree(fc_stats); |
| 454 | 359 | return -ENOMEM; |
| 455 | 360 | } |
| 456 | 361 | |
| 457 | 362 | void mlx5_cleanup_fc_stats(struct mlx5_core_dev *dev) |
| 458 | 363 | { |
| 459 | | − struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats; |
| 460 | | − struct llist_node *tmplist; |
| 364 | + struct mlx5_fc_stats *fc_stats = dev->priv.fc_stats; |
| 461 | 365 | struct mlx5_fc *counter; |
| 462 | | − struct mlx5_fc *tmp; |
| 463 | | − |
| 464 | | − cancel_delayed_work_sync(&dev->priv.fc_stats.work); |
| 465 | | − destroy_workqueue(dev->priv.fc_stats.wq); |
| 466 | | − dev->priv.fc_stats.wq = NULL; |
| 366 | + unsigned long id; |
| 467 | 367 | |
| 468 | | − tmplist = llist_del_all(&fc_stats->addlist); |
| 469 | | − llist_for_each_entry_safe(counter, tmp, tmplist, addlist) |
| 470 | | − mlx5_fc_release(dev, counter); |
| 368 | + cancel_delayed_work_sync(&fc_stats->work); |
| 369 | + destroy_workqueue(fc_stats->wq); |
| 370 | + fc_stats->wq = NULL; |
| 471 | 371 | |
| 472 | | − list_for_each_entry_safe(counter, tmp, &fc_stats->counters, list) |
| 372 | + xa_for_each(&fc_stats->counters, id, counter) { |
| 373 | + xa_erase(&fc_stats->counters, id); |
| 473 | 374 | mlx5_fc_release(dev, counter); |
| 375 | + } |
| 376 | + xa_destroy(&fc_stats->counters); |
| 474 | 377 | |
| 475 | 378 | mlx5_fc_pool_cleanup(&fc_stats->fc_pool); |
| 476 | | − idr_destroy(&fc_stats->counters_idr); |
| 477 | | − kfree(fc_stats->bulk_query_out); |
| 379 | + kvfree(fc_stats->bulk_query_out); |
| 380 | + kfree(fc_stats); |
| 478 | 381 | } |
| 479 | 382 | |
| 480 | 383 | int mlx5_fc_query(struct mlx5_core_dev *dev, struct mlx5_fc *counter, |
| @@ -518,7 +421,7 @@ void mlx5_fc_queue_stats_work(struct mlx5_core_dev *dev, · 37 unmodified lines |
| 518 | 421 | struct delayed_work *dwork, |
| 519 | 422 | unsigned long delay) |
| 520 | 423 | { |
| 521 | | − struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats; |
| 424 | + struct mlx5_fc_stats *fc_stats = dev->priv.fc_stats; |
| 522 | 425 | |
| 523 | 426 | queue_delayed_work(fc_stats->wq, dwork, delay); |
| 524 | 427 | } |
| @@ -526,7 +429,7 @@ void mlx5_fc_queue_stats_work(struct mlx5_core_dev *dev, · 1 unmodified line |
| 526 | 429 | void mlx5_fc_update_sampling_interval(struct mlx5_core_dev *dev, |
| 527 | 430 | unsigned long interval) |
| 528 | 431 | { |
| 529 | | − struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats; |
| 432 | + struct mlx5_fc_stats *fc_stats = dev->priv.fc_stats; |
| 530 | 433 | |
| 531 | 434 | fc_stats->sampling_interval = min_t(unsigned long, interval, |
| 532 | 435 | fc_stats->sampling_interval); |