netdev_set_tc_queue() (and related helpers/drivers such as netdev_bind_sb_channel_queue(), netdev_reset_tc(), and netdev_unbind_sb_channel()) perform separate 16-bit writes to dev->tc_to_txq[tc].count and dev->tc_to_txq[tc].offset. Furthermore, memset() in netdev_reset_tc() and netdev_unbind_sb_channel() provides no guarantee of performing full 32-bit word stores. Concurrent lockless readers (e.g. skb_tx_hash(), netdev_txq_to_tc(), ixgbe_select_queue(), taprio, mqprio, FPE drivers) can observe torn values where offset and count belong to inconsistent configurations. Redefine struct netdev_tc_txq to embed count and offset inside a union with a u32 combined field, allowing atomic manipulation via READ_ONCE() and WRITE_ONCE(). Update all lockless readers and writers across the kernel to use READ_ONCE() and WRITE_ONCE() on the combined field.
Unified · Split
| @@ -183,13 +183,15 @@ static u32 igc_fpe_map_preempt_tc_to_queue(const struct igc_adapter *adapter, · 182 unmodified lines | ||
| 183 | 183 | u32 i, queue = 0; |
| 184 | 184 | |
| 185 | 185 | for (i = 0; i < dev->num_tc; i++) { |
| 186 | + struct netdev_tc_txq res; | |
| 186 | 187 | u32 offset, count; |
| 187 | 188 | |
| 188 | 189 | if (!(preemptible_tcs & BIT(i))) |
| 189 | 190 | continue; |
| 190 | 191 | |
| 191 | − offset = dev->tc_to_txq[i].offset; | |
| 192 | − count = dev->tc_to_txq[i].count; | |
| 192 | + res.combined = READ_ONCE(dev->tc_to_txq[i].combined); | |
| 193 | + offset = res.offset; | |
| 194 | + count = res.count; | |
| 193 | 195 | queue |= GENMASK(offset + count - 1, offset); |
| 194 | 196 | } |
| 195 | 197 | |
| @@ -9273,10 +9273,11 @@ static u16 ixgbe_select_queue(struct net_device *dev, struct sk_buff *skb, · 9272 unmodified lines | ||
| 9273 | 9273 | if (sb_dev) { |
| 9274 | 9274 | u8 tc = netdev_get_prio_tc_map(dev, skb->priority); |
| 9275 | 9275 | struct net_device *vdev = sb_dev; |
| 9276 | + struct netdev_tc_txq res; | |
| 9276 | 9277 | |
| 9277 | − txq = vdev->tc_to_txq[tc].offset; | |
| 9278 | − txq += reciprocal_scale(skb_get_hash(skb), | |
| 9279 | − vdev->tc_to_txq[tc].count); | |
| 9278 | + res.combined = READ_ONCE(vdev->tc_to_txq[tc].combined); | |
| 9279 | + txq = res.offset; | |
| 9280 | + txq += reciprocal_scale(skb_get_hash(skb), res.count); | |
| 9280 | 9281 | |
| 9281 | 9282 | return txq; |
| 9282 | 9283 | } |
| @@ -3247,7 +3247,7 @@ static int mlx5e_update_tc_and_tx_queues(struct mlx5e_priv *priv) · 3246 unmodified lines | ||
| 3247 | 3247 | old_num_txqs = netdev->real_num_tx_queues; |
| 3248 | 3248 | old_ntc = netdev->num_tc ? : 1; |
| 3249 | 3249 | for (i = 0; i < ARRAY_SIZE(old_tc_to_txq); i++) |
| 3250 | − old_tc_to_txq[i] = netdev->tc_to_txq[i]; | |
| 3250 | + old_tc_to_txq[i].combined = READ_ONCE(netdev->tc_to_txq[i].combined); | |
| 3251 | 3251 | |
| 3252 | 3252 | nch = priv->channels.params.num_channels; |
| 3253 | 3253 | ntc = priv->channels.params.mqprio.num_tc; |
| @@ -439,8 +439,12 @@ int ef4_setup_tc(struct net_device *net_dev, enum tc_setup_type type, · 438 unmodified lines | ||
| 439 | 439 | return 0; |
| 440 | 440 | |
| 441 | 441 | for (tc = 0; tc < num_tc; tc++) { |
| 442 | − net_dev->tc_to_txq[tc].offset = tc * efx->n_tx_channels; | |
| 443 | − net_dev->tc_to_txq[tc].count = efx->n_tx_channels; | |
| 442 | + struct netdev_tc_txq res = { | |
| 443 | + .offset = tc * efx->n_tx_channels, | |
| 444 | + .count = efx->n_tx_channels, | |
| 445 | + }; | |
| 446 | + | |
| 447 | + WRITE_ONCE(net_dev->tc_to_txq[tc].combined, res.combined); | |
| 444 | 448 | } |
| 445 | 449 | |
| 446 | 450 | if (num_tc > net_dev->num_tc) { |
| @@ -380,8 +380,12 @@ int efx_siena_setup_tc(struct net_device *net_dev, enum tc_setup_type type, · 379 unmodified lines | ||
| 380 | 380 | return 0; |
| 381 | 381 | |
| 382 | 382 | for (tc = 0; tc < num_tc; tc++) { |
| 383 | − net_dev->tc_to_txq[tc].offset = tc * efx->n_tx_channels; | |
| 384 | − net_dev->tc_to_txq[tc].count = efx->n_tx_channels; | |
| 383 | + struct netdev_tc_txq res = { | |
| 384 | + .offset = tc * efx->n_tx_channels, | |
| 385 | + .count = efx->n_tx_channels, | |
| 386 | + }; | |
| 387 | + | |
| 388 | + WRITE_ONCE(net_dev->tc_to_txq[tc].combined, res.combined); | |
| 385 | 389 | } |
| 386 | 390 | |
| 387 | 391 | net_dev->num_tc = num_tc; |
| @@ -217,8 +217,11 @@ int dwmac5_fpe_map_preemption_class(struct net_device *ndev, · 216 unmodified lines | ||
| 217 | 217 | * and is direct one-to-one mapping." |
| 218 | 218 | */ |
| 219 | 219 | for (u32 tc = 0; tc < num_tc; tc++) { |
| 220 | − count = ndev->tc_to_txq[tc].count; | |
| 221 | − offset = ndev->tc_to_txq[tc].offset; | |
| 220 | + struct netdev_tc_txq res; | |
| 221 | + | |
| 222 | + res.combined = READ_ONCE(ndev->tc_to_txq[tc].combined); | |
| 223 | + count = res.count; | |
| 224 | + offset = res.offset; | |
| 222 | 225 | |
| 223 | 226 | if (pclass & BIT(tc)) |
| 224 | 227 | preemptible_txqs |= GENMASK(offset + count - 1, offset); |
| @@ -275,8 +278,11 @@ int dwxgmac3_fpe_map_preemption_class(struct net_device *ndev, · 50 unmodified lines | ||
| 275 | 278 | * any of the scheduling algorithms." |
| 276 | 279 | */ |
| 277 | 280 | for (u32 tc = 0; tc < num_tc; tc++) { |
| 278 | − count = ndev->tc_to_txq[tc].count; | |
| 279 | − offset = ndev->tc_to_txq[tc].offset; | |
| 281 | + struct netdev_tc_txq res; | |
| 282 | + | |
| 283 | + res.combined = READ_ONCE(ndev->tc_to_txq[tc].combined); | |
| 284 | + count = res.count; | |
| 285 | + offset = res.offset; | |
| 280 | 286 | |
| 281 | 287 | if (pclass & BIT(tc)) |
| 282 | 288 | preemptible_txqs |= GENMASK(offset + count - 1, offset); |
| @@ -832,8 +832,13 @@ struct xps_dev_maps { · 831 unmodified lines | ||
| 832 | 832 | #define TC_BITMASK 15 |
| 833 | 833 | /* HW offloaded queuing disciplines txq count and offset maps */ |
| 834 | 834 | struct netdev_tc_txq { |
| 835 | − u16 count; | |
| 836 | − u16 offset; | |
| 835 | + union { | |
| 836 | + struct { | |
| 837 | + u16 count; | |
| 838 | + u16 offset; | |
| 839 | + }; | |
| 840 | + u32 combined; | |
| 841 | + }; | |
| 837 | 842 | }; |
| 838 | 843 | |
| 839 | 844 | #if defined(CONFIG_FCOE) || defined(CONFIG_FCOE_MODULE) |
| @@ -2652,11 +2652,13 @@ EXPORT_SYMBOL_GPL(dev_queue_xmit_nit); · 2651 unmodified lines | ||
| 2652 | 2652 | */ |
| 2653 | 2653 | static void netif_setup_tc(struct net_device *dev, unsigned int txq) |
| 2654 | 2654 | { |
| 2655 | + struct netdev_tc_txq res; | |
| 2655 | 2656 | int i; |
| 2656 | − struct netdev_tc_txq *tc = &dev->tc_to_txq[0]; | |
| 2657 | + | |
| 2658 | + res.combined = READ_ONCE(dev->tc_to_txq[0].combined); | |
| 2657 | 2659 | |
| 2658 | 2660 | /* If TC0 is invalidated disable TC mapping */ |
| 2659 | − if (tc->offset + tc->count > txq) { | |
| 2661 | + if (res.offset + res.count > txq) { | |
| 2660 | 2662 | netdev_warn(dev, "Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n"); |
| 2661 | 2663 | dev->num_tc = 0; |
| 2662 | 2664 | return; |
| @@ -2666,8 +2668,8 @@ static void netif_setup_tc(struct net_device *dev, unsigned int txq) · 3 unmodified lines | ||
| 2666 | 2668 | for (i = 1; i < TC_BITMASK + 1; i++) { |
| 2667 | 2669 | int q = netdev_get_prio_tc_map(dev, i); |
| 2668 | 2670 | |
| 2669 | − tc = &dev->tc_to_txq[q]; | |
| 2670 | − if (tc->offset + tc->count > txq) { | |
| 2671 | + res.combined = READ_ONCE(dev->tc_to_txq[q].combined); | |
| 2672 | + if (res.offset + res.count > txq) { | |
| 2671 | 2673 | netdev_warn(dev, "Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n", |
| 2672 | 2674 | i, q); |
| 2673 | 2675 | netdev_set_prio_tc_map(dev, i, 0); |
| @@ -2683,7 +2685,10 @@ int netdev_txq_to_tc(struct net_device *dev, unsigned int txq) · 9 unmodified lines | ||
| 2683 | 2685 | |
| 2684 | 2686 | /* walk through the TCs and see if it falls into any of them */ |
| 2685 | 2687 | for (i = 0; i < TC_MAX_QUEUE; i++, tc++) { |
| 2686 | − if ((txq - tc->offset) < tc->count) | |
| 2688 | + struct netdev_tc_txq res; | |
| 2689 | + | |
| 2690 | + res.combined = READ_ONCE(tc->combined); | |
| 2691 | + if ((txq - res.offset) < res.count) | |
| 2687 | 2692 | return i; |
| 2688 | 2693 | } |
| 2689 | 2694 | |
| @@ -3103,6 +3108,8 @@ static void netdev_unbind_all_sb_channels(struct net_device *dev) · 413 unmodified lines | ||
| 3103 | 3108 | |
| 3104 | 3109 | void netdev_reset_tc(struct net_device *dev) |
| 3105 | 3110 | { |
| 3111 | + int i; | |
| 3112 | + | |
| 3106 | 3113 | #ifdef CONFIG_XPS |
| 3107 | 3114 | netif_reset_xps_queues_gt(dev, 0); |
| 3108 | 3115 | #endif |
| @@ -3110,21 +3117,26 @@ void netdev_reset_tc(struct net_device *dev) · 1 unmodified line | ||
| 3110 | 3117 | |
| 3111 | 3118 | /* Reset TC configuration of device */ |
| 3112 | 3119 | dev->num_tc = 0; |
| 3113 | − memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq)); | |
| 3120 | + for (i = 0; i < TC_MAX_QUEUE; i++) | |
| 3121 | + WRITE_ONCE(dev->tc_to_txq[i].combined, 0); | |
| 3114 | 3122 | memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map)); |
| 3115 | 3123 | } |
| 3116 | 3124 | EXPORT_SYMBOL(netdev_reset_tc); |
| 3117 | 3125 | |
| 3118 | 3126 | int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset) |
| 3119 | 3127 | { |
| 3128 | + struct netdev_tc_txq res = { | |
| 3129 | + .count = count, | |
| 3130 | + .offset = offset, | |
| 3131 | + }; | |
| 3132 | + | |
| 3120 | 3133 | if (tc >= dev->num_tc) |
| 3121 | 3134 | return -EINVAL; |
| 3122 | 3135 | |
| 3123 | 3136 | #ifdef CONFIG_XPS |
| 3124 | 3137 | netif_reset_xps_queues(dev, offset, count); |
| 3125 | 3138 | #endif |
| 3126 | − dev->tc_to_txq[tc].count = count; | |
| 3127 | − dev->tc_to_txq[tc].offset = offset; | |
| 3139 | + WRITE_ONCE(dev->tc_to_txq[tc].combined, res.combined); | |
| 3128 | 3140 | return 0; |
| 3129 | 3141 | } |
| 3130 | 3142 | EXPORT_SYMBOL(netdev_set_tc_queue); |
| @@ -3148,11 +3160,13 @@ void netdev_unbind_sb_channel(struct net_device *dev, · 17 unmodified lines | ||
| 3148 | 3160 | struct net_device *sb_dev) |
| 3149 | 3161 | { |
| 3150 | 3162 | struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues]; |
| 3163 | + int i; | |
| 3151 | 3164 | |
| 3152 | 3165 | #ifdef CONFIG_XPS |
| 3153 | 3166 | netif_reset_xps_queues_gt(sb_dev, 0); |
| 3154 | 3167 | #endif |
| 3155 | − memset(sb_dev->tc_to_txq, 0, sizeof(sb_dev->tc_to_txq)); | |
| 3168 | + for (i = 0; i < TC_MAX_QUEUE; i++) | |
| 3169 | + WRITE_ONCE(sb_dev->tc_to_txq[i].combined, 0); | |
| 3156 | 3170 | memset(sb_dev->prio_tc_map, 0, sizeof(sb_dev->prio_tc_map)); |
| 3157 | 3171 | |
| 3158 | 3172 | while (txq-- != &dev->_tx[0]) { |
| @@ -3175,8 +3189,12 @@ int netdev_bind_sb_channel_queue(struct net_device *dev, · 16 unmodified lines | ||
| 3175 | 3189 | return -EINVAL; |
| 3176 | 3190 | |
| 3177 | 3191 | /* Record the mapping */ |
| 3178 | − sb_dev->tc_to_txq[tc].count = count; | |
| 3179 | − sb_dev->tc_to_txq[tc].offset = offset; | |
| 3192 | + struct netdev_tc_txq res = { | |
| 3193 | + .count = count, | |
| 3194 | + .offset = offset, | |
| 3195 | + }; | |
| 3196 | + | |
| 3197 | + WRITE_ONCE(sb_dev->tc_to_txq[tc].combined, res.combined); | |
| 3180 | 3198 | |
| 3181 | 3199 | /* Provide a way for Tx queue to find the tc_to_txq map or |
| 3182 | 3200 | * XPS map for itself. |
| @@ -3542,9 +3560,11 @@ static u16 skb_tx_hash(const struct net_device *dev, · 359 unmodified lines | ||
| 3542 | 3560 | |
| 3543 | 3561 | if (dev->num_tc) { |
| 3544 | 3562 | u8 tc = netdev_get_prio_tc_map(dev, skb->priority); |
| 3563 | + struct netdev_tc_txq res; | |
| 3545 | 3564 | |
| 3546 | − qoffset = sb_dev->tc_to_txq[tc].offset; | |
| 3547 | − qcount = sb_dev->tc_to_txq[tc].count; | |
| 3565 | + res.combined = READ_ONCE(sb_dev->tc_to_txq[tc].combined); | |
| 3566 | + qoffset = res.offset; | |
| 3567 | + qcount = res.count; | |
| 3548 | 3568 | if (unlikely(!qcount)) { |
| 3549 | 3569 | net_warn_ratelimited("%s: invalid qcount, qoffset %u for tc %u\n", |
| 3550 | 3570 | sb_dev->name, qoffset, tc); |
| @@ -679,12 +679,14 @@ static int mqprio_dump_class_stats(struct Qdisc *sch, unsigned long cl, · 678 unmodified lines | ||
| 679 | 679 | rcu_read_lock(); |
| 680 | 680 | if (cl >= TC_H_MIN_PRIORITY) { |
| 681 | 681 | struct net_device *dev = qdisc_dev(sch); |
| 682 | − struct netdev_tc_txq tc = dev->tc_to_txq[cl & TC_BITMASK]; | |
| 682 | + struct netdev_tc_txq tc; | |
| 683 | 683 | struct gnet_stats_queue qstats = {0}; |
| 684 | 684 | struct gnet_stats_basic_sync bstats; |
| 685 | 685 | u32 qlen = 0; |
| 686 | 686 | int i; |
| 687 | 687 | |
| 688 | + tc.combined = READ_ONCE(dev->tc_to_txq[cl & TC_BITMASK].combined); | |
| 689 | + | |
| 688 | 690 | gnet_stats_basic_sync_init(&bstats); |
| 689 | 691 | |
| 690 | 692 | for (i = tc.offset; i < tc.offset + tc.count; i++) { |
| @@ -108,8 +108,11 @@ void mqprio_qopt_reconstruct(struct net_device *dev, struct tc_mqprio_qopt *qopt · 107 unmodified lines | ||
| 108 | 108 | memcpy(qopt->prio_tc_map, dev->prio_tc_map, sizeof(qopt->prio_tc_map)); |
| 109 | 109 | |
| 110 | 110 | for (tc = 0; tc < num_tc; tc++) { |
| 111 | − qopt->count[tc] = dev->tc_to_txq[tc].count; | |
| 112 | − qopt->offset[tc] = dev->tc_to_txq[tc].offset; | |
| 111 | + struct netdev_tc_txq res; | |
| 112 | + | |
| 113 | + res.combined = READ_ONCE(dev->tc_to_txq[tc].combined); | |
| 114 | + qopt->count[tc] = res.count; | |
| 115 | + qopt->offset[tc] = res.offset; | |
| 113 | 116 | } |
| 114 | 117 | } |
| 115 | 118 | EXPORT_SYMBOL_GPL(mqprio_qopt_reconstruct); |
| @@ -762,12 +762,13 @@ skip_peek_checks: · 761 unmodified lines | ||
| 762 | 762 | |
| 763 | 763 | static void taprio_next_tc_txq(struct net_device *dev, int tc, int *txq) |
| 764 | 764 | { |
| 765 | − int offset = dev->tc_to_txq[tc].offset; | |
| 766 | − int count = dev->tc_to_txq[tc].count; | |
| 765 | + struct netdev_tc_txq res; | |
| 766 | + | |
| 767 | + res.combined = READ_ONCE(dev->tc_to_txq[tc].combined); | |
| 767 | 768 | |
| 768 | 769 | (*txq)++; |
| 769 | − if (*txq == offset + count) | |
| 770 | − *txq = offset; | |
| 770 | + if (*txq == res.offset + res.count) | |
| 771 | + *txq = res.offset; | |
| 771 | 772 | } |
| 772 | 773 | |
| 773 | 774 | /* Prioritize higher traffic classes, and select among TXQs belonging to the |
| @@ -1441,15 +1442,14 @@ static u32 tc_map_to_queue_mask(struct net_device *dev, u32 tc_mask) · 667 unmodified lines | ||
| 1441 | 1442 | u32 i, queue_mask = 0; |
| 1442 | 1443 | |
| 1443 | 1444 | for (i = 0; i < dev->num_tc; i++) { |
| 1444 | − u32 offset, count; | |
| 1445 | + struct netdev_tc_txq res; | |
| 1445 | 1446 | |
| 1446 | 1447 | if (!(tc_mask & BIT(i))) |
| 1447 | 1448 | continue; |
| 1448 | 1449 | |
| 1449 | − offset = dev->tc_to_txq[i].offset; | |
| 1450 | − count = dev->tc_to_txq[i].count; | |
| 1450 | + res.combined = READ_ONCE(dev->tc_to_txq[i].combined); | |
| 1451 | 1451 | |
| 1452 | − queue_mask |= GENMASK(offset + count - 1, offset); | |
| 1452 | + queue_mask |= GENMASK(res.offset + res.count - 1, res.offset); | |
| 1453 | 1453 | } |
| 1454 | 1454 | |
| 1455 | 1455 | return queue_mask; |
| @@ -1802,10 +1802,14 @@ static int taprio_mqprio_cmp(const struct net_device *dev, · 346 unmodified lines | ||
| 1802 | 1802 | if (!mqprio || mqprio->num_tc != dev->num_tc) |
| 1803 | 1803 | return -1; |
| 1804 | 1804 | |
| 1805 | − for (i = 0; i < mqprio->num_tc; i++) | |
| 1806 | − if (dev->tc_to_txq[i].count != mqprio->count[i] || | |
| 1807 | − dev->tc_to_txq[i].offset != mqprio->offset[i]) | |
| 1805 | + for (i = 0; i < mqprio->num_tc; i++) { | |
| 1806 | + struct netdev_tc_txq res; | |
| 1807 | + | |
| 1808 | + res.combined = READ_ONCE(dev->tc_to_txq[i].combined); | |
| 1809 | + if (res.count != mqprio->count[i] || | |
| 1810 | + res.offset != mqprio->offset[i]) | |
| 1808 | 1811 | return -1; |
| 1812 | + } | |
| 1809 | 1813 | |
| 1810 | 1814 | for (i = 0; i <= TC_BITMASK; i++) |
| 1811 | 1815 | if (dev->prio_tc_map[i] != mqprio->prio_tc_map[i]) |