History

Merge branch '10GbE' of git://git.kernel.org/pub/scm/linux/kernel/git/jkirsher/next-queue

Jeff Kirsher says: ==================== 10GbE Intel Wired LAN Driver Updates 2018-01-26 This series contains updates to ixgbe and ixgbevf. Emil updates ixgbevf to match ixgbe functionality, starting with the consolidating of functions that represent logical steps in the receive process so we can later update them more easily. Updated ixgbevf to only synchronize the length of the frame, which will typically be the MTU or smaller. Updated the VF driver to use the length of the packet instead of the DD status bit to determine if a new descriptor is ready to be processed, which saves on reads and we can save time on initialization. Added support for DMA_ATTR_SKIP_CPU_SYNC/WEAK_ORDERING to help improve performance on some platforms. Updated the VF driver to do bulk updates of the page reference count instead of just incrementing it by one reference at a time. Updated the VF driver to only go through the region of the receive ring that was designated to be cleaned up, rather than process the entire ring. Colin Ian King adds the use of ARRAY_SIZE() on various arrays. Miroslav Lichvar fixes an issue where ethtool was reporting timestamping filters unsupported for X550, which is incorrect. Paul adds support for reporting 5G link speed for some devices. Dan Carpenter fixes a typo where && was used when it should have been ||. ====================

David S. Miller committed 8 years ago

Commit 6b2e2822 parents 751c45b 2bafa8f8 files · +271 −179Browse files

Unified · Split

8 files
  1. drivers/net/ethernet/intel/ixgbe/ixgbe_common.c +1 −1
  2. drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c +19 −18
  3. drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +9 −2
  4. drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c +1 −1
  5. drivers/net/ethernet/intel/ixgbevf/ethtool.c +3 −0
  6. drivers/net/ethernet/intel/ixgbevf/ixgbevf.h +13 −3
  7. drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +218 −144
  8. drivers/net/ethernet/intel/ixgbevf/vf.c +7 −10
drivers/net/ethernet/intel/ixgbe/ixgbe_common.c+1−1
@@ -4087,7 +4087,7 @@ void ixgbe_get_oem_prod_version(struct ixgbe_hw *hw, · 4086 unmodified lines
40874087 hw->eeprom.ops.read(hw, NVM_OEM_PROD_VER_PTR, &offset);
40884088
40894089 /* Return is offset to OEM Product Version block is invalid */
4090− if (offset == 0x0 && offset == NVM_INVALID_PTR)
4090+ if (offset == 0x0 || offset == NVM_INVALID_PTR)
40914091 return;
40924092
40934093 /* Read product version block */
drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c+19−18
@@ -3085,26 +3085,9 @@ static int ixgbe_get_ts_info(struct net_device *dev, · 3084 unmodified lines
30853085 case ixgbe_mac_X550EM_x:
30863086 case ixgbe_mac_x550em_a:
30873087 info->rx_filters |= BIT(HWTSTAMP_FILTER_ALL);
3088− /* fallthrough */
3088+ break;
30893089 case ixgbe_mac_X540:
30903090 case ixgbe_mac_82599EB:
3091− info->so_timestamping =
3092− SOF_TIMESTAMPING_TX_SOFTWARE |
3093− SOF_TIMESTAMPING_RX_SOFTWARE |
3094− SOF_TIMESTAMPING_SOFTWARE |
3095− SOF_TIMESTAMPING_TX_HARDWARE |
3096− SOF_TIMESTAMPING_RX_HARDWARE |
3097− SOF_TIMESTAMPING_RAW_HARDWARE;
3098−
3099− if (adapter->ptp_clock)
3100− info->phc_index = ptp_clock_index(adapter->ptp_clock);
3101− else
3102− info->phc_index = -1;
3103−
3104− info->tx_types =
3105− BIT(HWTSTAMP_TX_OFF) |
3106− BIT(HWTSTAMP_TX_ON);
3107−
31083091 info->rx_filters |=
31093092 BIT(HWTSTAMP_FILTER_PTP_V1_L4_SYNC) |
31103093 BIT(HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ) |
@@ -3113,6 +3096,24 @@ static int ixgbe_get_ts_info(struct net_device *dev, · 2 unmodified lines
31133096 default:
31143097 return ethtool_op_get_ts_info(dev, info);
31153098 }
3099+
3100+ info->so_timestamping =
3101+ SOF_TIMESTAMPING_TX_SOFTWARE |
3102+ SOF_TIMESTAMPING_RX_SOFTWARE |
3103+ SOF_TIMESTAMPING_SOFTWARE |
3104+ SOF_TIMESTAMPING_TX_HARDWARE |
3105+ SOF_TIMESTAMPING_RX_HARDWARE |
3106+ SOF_TIMESTAMPING_RAW_HARDWARE;
3107+
3108+ if (adapter->ptp_clock)
3109+ info->phc_index = ptp_clock_index(adapter->ptp_clock);
3110+ else
3111+ info->phc_index = -1;
3112+
3113+ info->tx_types =
3114+ BIT(HWTSTAMP_TX_OFF) |
3115+ BIT(HWTSTAMP_TX_ON);
3116+
31163117 return 0;
31173118 }
31183119
drivers/net/ethernet/intel/ixgbe/ixgbe_main.c+9−2
@@ -4133,11 +4133,15 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, · 4132 unmodified lines
41334133 rxdctl &= ~0x3FFFFF;
41344134 rxdctl |= 0x080420;
41354135 #if (PAGE_SIZE < 8192)
4136− } else {
4136+ /* RXDCTL.RLPML does not work on 82599 */
4137+ } else if (hw->mac.type != ixgbe_mac_82599EB) {
41374138 rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK |
41384139 IXGBE_RXDCTL_RLPML_EN);
41394140
4140− /* Limit the maximum frame size so we don't overrun the skb */
4141+ /* Limit the maximum frame size so we don't overrun the skb.
4142+ * This can happen in SRIOV mode when the MTU of the VF is
4143+ * higher than the MTU of the PF.
4144+ */
41414145 if (ring_uses_build_skb(ring) &&
41424146 !test_bit(__IXGBE_RX_3K_BUFFER, &ring->state))
41434147 rxdctl |= IXGBE_MAX_2K_FRAME_BUILD_SKB |
@@ -7259,6 +7263,9 @@ static void ixgbe_watchdog_link_is_up(struct ixgbe_adapter *adapter) · 3115 unmodified lines
72597263 case IXGBE_LINK_SPEED_10GB_FULL:
72607264 speed_str = "10 Gbps";
72617265 break;
7266+ case IXGBE_LINK_SPEED_5GB_FULL:
7267+ speed_str = "5 Gbps";
7268+ break;
72627269 case IXGBE_LINK_SPEED_2_5GB_FULL:
72637270 speed_str = "2.5 Gbps";
72647271 break;
drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c+1−1
@@ -949,7 +949,7 @@ static s32 ixgbe_checksum_ptr_x550(struct ixgbe_hw *hw, u16 ptr, · 948 unmodified lines
949949 u16 length, bufsz, i, start;
950950 u16 *local_buffer;
951951
952− bufsz = sizeof(buf) / sizeof(buf[0]);
952+ bufsz = ARRAY_SIZE(buf);
953953
954954 /* Read a chunk at the pointer location */
955955 if (!buffer) {
drivers/net/ethernet/intel/ixgbevf/ethtool.c+3−0
@@ -75,6 +75,9 @@ static struct ixgbe_stats ixgbevf_gstrings_stats[] = { · 74 unmodified lines
7575 IXGBEVF_STAT("tx_timeout_count", tx_timeout_count),
7676 IXGBEVF_NETDEV_STAT(multicast),
7777 IXGBEVF_STAT("rx_csum_offload_errors", hw_csum_rx_error),
78+ IXGBEVF_STAT("alloc_rx_page", alloc_rx_page),
79+ IXGBEVF_STAT("alloc_rx_page_failed", alloc_rx_page_failed),
80+ IXGBEVF_STAT("alloc_rx_buff_failed", alloc_rx_buff_failed),
7881 };
7982
8083 #define IXGBEVF_QUEUE_STATS_LEN ( \
drivers/net/ethernet/intel/ixgbevf/ixgbevf.h+13−3
@@ -62,7 +62,12 @@ struct ixgbevf_tx_buffer { · 61 unmodified lines
6262 struct ixgbevf_rx_buffer {
6363 dma_addr_t dma;
6464 struct page *page;
65− unsigned int page_offset;
65+#if (BITS_PER_LONG > 32) || (PAGE_SIZE >= 65536)
66+ __u32 page_offset;
67+#else
68+ __u16 page_offset;
69+#endif
70+ __u16 pagecnt_bias;
6671 };
6772
6873 struct ixgbevf_stats {
@@ -79,6 +84,7 @@ struct ixgbevf_tx_queue_stats { · 10 unmodified lines
7984 struct ixgbevf_rx_queue_stats {
8085 u64 alloc_rx_page_failed;
8186 u64 alloc_rx_buff_failed;
87+ u64 alloc_rx_page;
8288 u64 csum_err;
8389 };
8490
@@ -260,6 +266,9 @@ static inline void ixgbevf_write_tail(struct ixgbevf_ring *ring, u32 value) · 175 unmodified lines
260266 #define MIN_MSIX_Q_VECTORS 1
261267 #define MIN_MSIX_COUNT (MIN_MSIX_Q_VECTORS + NON_Q_VECTORS)
262268
269+#define IXGBEVF_RX_DMA_ATTR \
270+ (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING)
271+
263272 /* board specific private data structure */
264273 struct ixgbevf_adapter {
265274 /* this field must be first, see ixgbevf_process_skb_fields */
@@ -287,8 +296,9 @@ struct ixgbevf_adapter { · 21 unmodified lines
287296 u64 hw_csum_rx_error;
288297 u64 hw_rx_no_dma_resources;
289298 int num_msix_vectors;
290− u32 alloc_rx_page_failed;
291− u32 alloc_rx_buff_failed;
299+ u64 alloc_rx_page_failed;
300+ u64 alloc_rx_buff_failed;
301+ u64 alloc_rx_page;
292302
293303 struct msix_entry *msix_entries;
294304
drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c+218−144
@@ -206,28 +206,6 @@ static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction, · 205 unmodified lines
206206 }
207207 }
208208
209−static void ixgbevf_unmap_and_free_tx_resource(struct ixgbevf_ring *tx_ring,
210− struct ixgbevf_tx_buffer *tx_buffer)
211−{
212− if (tx_buffer->skb) {
213− dev_kfree_skb_any(tx_buffer->skb);
214− if (dma_unmap_len(tx_buffer, len))
215− dma_unmap_single(tx_ring->dev,
216− dma_unmap_addr(tx_buffer, dma),
217− dma_unmap_len(tx_buffer, len),
218− DMA_TO_DEVICE);
219− } else if (dma_unmap_len(tx_buffer, len)) {
220− dma_unmap_page(tx_ring->dev,
221− dma_unmap_addr(tx_buffer, dma),
222− dma_unmap_len(tx_buffer, len),
223− DMA_TO_DEVICE);
224− }
225− tx_buffer->next_to_watch = NULL;
226− tx_buffer->skb = NULL;
227− dma_unmap_len_set(tx_buffer, len, 0);
228− /* tx_buffer must be completely set up in the transmit path */
229−}
230−
231209 static u64 ixgbevf_get_tx_completed(struct ixgbevf_ring *ring)
232210 {
233211 return ring->stats.packets;
@@ -349,7 +327,6 @@ static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector, · 115 unmodified lines
349327 DMA_TO_DEVICE);
350328
351329 /* clear tx_buffer data */
352− tx_buffer->skb = NULL;
353330 dma_unmap_len_set(tx_buffer, len, 0);
354331
355332 /* unmap remaining buffers */
@@ -595,8 +572,8 @@ static bool ixgbevf_alloc_mapped_page(struct ixgbevf_ring *rx_ring, · 239 unmodified lines
595572 }
596573
597574 /* map page for use */
598− dma = dma_map_page(rx_ring->dev, page, 0,
599− PAGE_SIZE, DMA_FROM_DEVICE);
575+ dma = dma_map_page_attrs(rx_ring->dev, page, 0, PAGE_SIZE,
576+ DMA_FROM_DEVICE, IXGBEVF_RX_DMA_ATTR);
600577
601578 /* if mapping failed free memory back to system since
602579 * there isn't much point in holding memory we can't use
@@ -604,13 +581,15 @@ static bool ixgbevf_alloc_mapped_page(struct ixgbevf_ring *rx_ring, · 1 unmodified line
604581 if (dma_mapping_error(rx_ring->dev, dma)) {
605582 __free_page(page);
606583
607− rx_ring->rx_stats.alloc_rx_buff_failed++;
584+ rx_ring->rx_stats.alloc_rx_page_failed++;
608585 return false;
609586 }
610587
611588 bi->dma = dma;
612589 bi->page = page;
613590 bi->page_offset = 0;
591+ bi->pagecnt_bias = 1;
592+ rx_ring->rx_stats.alloc_rx_page++;
614593
615594 return true;
616595 }
@@ -639,6 +618,12 @@ static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring, · 22 unmodified lines
639618 if (!ixgbevf_alloc_mapped_page(rx_ring, bi))
640619 break;
641620
621+ /* sync the buffer for use by the device */
622+ dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
623+ bi->page_offset,
624+ IXGBEVF_RX_BUFSZ,
625+ DMA_FROM_DEVICE);
626+
642627 /* Refresh the desc even if pkt_addr didn't change
643628 * because each write-back erases this info.
644629 */
@@ -653,8 +638,8 @@ static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring, · 8 unmodified lines
653638 i -= rx_ring->count;
654639 }
655640
656− /* clear the hdr_addr for the next_to_use descriptor */
657− rx_desc->read.hdr_addr = 0;
641+ /* clear the length for the next_to_use descriptor */
642+ rx_desc->wb.upper.length = 0;
658643
659644 cleaned_count--;
660645 } while (cleaned_count);
@@ -741,12 +726,7 @@ static void ixgbevf_reuse_rx_page(struct ixgbevf_ring *rx_ring, · 80 unmodified lines
741726 new_buff->page = old_buff->page;
742727 new_buff->dma = old_buff->dma;
743728 new_buff->page_offset = old_buff->page_offset;
744−
745− /* sync the buffer for use by the device */
746− dma_sync_single_range_for_device(rx_ring->dev, new_buff->dma,
747− new_buff->page_offset,
748− IXGBEVF_RX_BUFSZ,
749− DMA_FROM_DEVICE);
729+ new_buff->pagecnt_bias = old_buff->pagecnt_bias;
750730 }
751731
752732 static inline bool ixgbevf_page_is_reserved(struct page *page)
@@ -754,6 +734,45 @@ static inline bool ixgbevf_page_is_reserved(struct page *page) · 1 unmodified line
754734 return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page);
755735 }
756736
737+static bool ixgbevf_can_reuse_rx_page(struct ixgbevf_rx_buffer *rx_buffer,
738+ struct page *page,
739+ const unsigned int truesize)
740+{
741+ unsigned int pagecnt_bias = rx_buffer->pagecnt_bias--;
742+
743+ /* avoid re-using remote pages */
744+ if (unlikely(ixgbevf_page_is_reserved(page)))
745+ return false;
746+
747+#if (PAGE_SIZE < 8192)
748+ /* if we are only owner of page we can reuse it */
749+ if (unlikely(page_ref_count(page) != pagecnt_bias))
750+ return false;
751+
752+ /* flip page offset to other buffer */
753+ rx_buffer->page_offset ^= IXGBEVF_RX_BUFSZ;
754+
755+#else
756+ /* move offset up to the next cache line */
757+ rx_buffer->page_offset += truesize;
758+
759+ if (rx_buffer->page_offset > (PAGE_SIZE - IXGBEVF_RX_BUFSZ))
760+ return false;
761+
762+#endif
763+
764+ /* If we have drained the page fragment pool we need to update
765+ * the pagecnt_bias and page count so that we fully restock the
766+ * number of references the driver holds.
767+ */
768+ if (unlikely(pagecnt_bias == 1)) {
769+ page_ref_add(page, USHRT_MAX);
770+ rx_buffer->pagecnt_bias = USHRT_MAX;
771+ }
772+
773+ return true;
774+}
775+
757776 /**
758777 * ixgbevf_add_rx_frag - Add contents of Rx buffer to sk_buff
759778 * @rx_ring: rx descriptor ring to transact packets on
@@ -771,12 +790,12 @@ static inline bool ixgbevf_page_is_reserved(struct page *page) · 11 unmodified lines
771790 **/
772791 static bool ixgbevf_add_rx_frag(struct ixgbevf_ring *rx_ring,
773792 struct ixgbevf_rx_buffer *rx_buffer,
793+ u16 size,
774794 union ixgbe_adv_rx_desc *rx_desc,
775795 struct sk_buff *skb)
776796 {
777797 struct page *page = rx_buffer->page;
778798 unsigned char *va = page_address(page) + rx_buffer->page_offset;
779− unsigned int size = le16_to_cpu(rx_desc->wb.upper.length);
780799 #if (PAGE_SIZE < 8192)
781800 unsigned int truesize = IXGBEVF_RX_BUFSZ;
782801 #else
@@ -795,7 +814,6 @@ static bool ixgbevf_add_rx_frag(struct ixgbevf_ring *rx_ring, · 12 unmodified lines
795814 return true;
796815
797816 /* this page cannot be reused so discard it */
798− put_page(page);
799817 return false;
800818 }
801819
@@ -815,32 +833,7 @@ add_tail_frag: · 13 unmodified lines
815833 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,
816834 (unsigned long)va & ~PAGE_MASK, size, truesize);
817835
818− /* avoid re-using remote pages */
819− if (unlikely(ixgbevf_page_is_reserved(page)))
820− return false;
821−
822−#if (PAGE_SIZE < 8192)
823− /* if we are only owner of page we can reuse it */
824− if (unlikely(page_count(page) != 1))
825− return false;
826−
827− /* flip page offset to other buffer */
828− rx_buffer->page_offset ^= IXGBEVF_RX_BUFSZ;
829−
830−#else
831− /* move offset up to the next cache line */
832− rx_buffer->page_offset += truesize;
833−
834− if (rx_buffer->page_offset > (PAGE_SIZE - IXGBEVF_RX_BUFSZ))
835− return false;
836−
837−#endif
838− /* Even if we own the page, we are not allowed to use atomic_set()
839− * This would break get_page_unless_zero() users.
840− */
841− page_ref_inc(page);
842−
843− return true;
836+ return ixgbevf_can_reuse_rx_page(rx_buffer, page, truesize);
844837 }
845838
846839 static struct sk_buff *ixgbevf_fetch_rx_buffer(struct ixgbevf_ring *rx_ring,
@@ -849,11 +842,19 @@ static struct sk_buff *ixgbevf_fetch_rx_buffer(struct ixgbevf_ring *rx_ring, · 2 unmodified lines
849842 {
850843 struct ixgbevf_rx_buffer *rx_buffer;
851844 struct page *page;
845+ u16 size = le16_to_cpu(rx_desc->wb.upper.length);
852846
853847 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
854848 page = rx_buffer->page;
855849 prefetchw(page);
856850
851+ /* we are reusing so sync this buffer for CPU use */
852+ dma_sync_single_range_for_cpu(rx_ring->dev,
853+ rx_buffer->dma,
854+ rx_buffer->page_offset,
855+ size,
856+ DMA_FROM_DEVICE);
857+
857858 if (likely(!skb)) {
858859 void *page_addr = page_address(page) +
859860 rx_buffer->page_offset;
@@ -879,21 +880,18 @@ static struct sk_buff *ixgbevf_fetch_rx_buffer(struct ixgbevf_ring *rx_ring, · 19 unmodified lines
879880 prefetchw(skb->data);
880881 }
881882
882− /* we are reusing so sync this buffer for CPU use */
883− dma_sync_single_range_for_cpu(rx_ring->dev,
884− rx_buffer->dma,
885− rx_buffer->page_offset,
886− IXGBEVF_RX_BUFSZ,
887− DMA_FROM_DEVICE);
888−
889883 /* pull page into skb */
890− if (ixgbevf_add_rx_frag(rx_ring, rx_buffer, rx_desc, skb)) {
884+ if (ixgbevf_add_rx_frag(rx_ring, rx_buffer, size, rx_desc, skb)) {
891885 /* hand second half of page back to the ring */
892886 ixgbevf_reuse_rx_page(rx_ring, rx_buffer);
893887 } else {
894− /* we are not reusing the buffer so unmap it */
895− dma_unmap_page(rx_ring->dev, rx_buffer->dma,
896− PAGE_SIZE, DMA_FROM_DEVICE);
888+ /* We are not reusing the buffer so unmap it and free
889+ * any references we are holding to it
890+ */
891+ dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,
892+ PAGE_SIZE, DMA_FROM_DEVICE,
893+ IXGBEVF_RX_DMA_ATTR);
894+ __page_frag_cache_drain(page, rx_buffer->pagecnt_bias);
897895 }
898896
899897 /* clear contents of buffer_info */
@@ -930,7 +928,7 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, · 30 unmodified lines
930928
931929 rx_desc = IXGBEVF_RX_DESC(rx_ring, rx_ring->next_to_clean);
932930
933− if (!ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_DD))
931+ if (!rx_desc->wb.upper.length)
934932 break;
935933
936934 /* This memory barrier is needed to keep us from reading
@@ -943,8 +941,10 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, · 6 unmodified lines
943941 skb = ixgbevf_fetch_rx_buffer(rx_ring, rx_desc, skb);
944942
945943 /* exit if we failed to retrieve a buffer */
946− if (!skb)
944+ if (!skb) {
945+ rx_ring->rx_stats.alloc_rx_buff_failed++;
947946 break;
947+ }
948948
949949 cleaned_count++;
950950
@@ -1553,6 +1553,10 @@ static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter, · 602 unmodified lines
15531553 txdctl |= (1u << 8) | /* HTHRESH = 1 */
15541554 32; /* PTHRESH = 32 */
15551555
1556+ /* reinitialize tx_buffer_info */
1557+ memset(ring->tx_buffer_info, 0,
1558+ sizeof(struct ixgbevf_tx_buffer) * ring->count);
1559+
15561560 clear_bit(__IXGBEVF_HANG_CHECK_ARMED, &ring->state);
15571561
15581562 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), txdctl);
@@ -1721,6 +1725,7 @@ static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, · 162 unmodified lines
17211725 struct ixgbevf_ring *ring)
17221726 {
17231727 struct ixgbe_hw *hw = &adapter->hw;
1728+ union ixgbe_adv_rx_desc *rx_desc;
17241729 u64 rdba = ring->dma;
17251730 u32 rxdctl;
17261731 u8 reg_idx = ring->reg_idx;
@@ -1749,6 +1754,14 @@ static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, · 22 unmodified lines
17491754 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(reg_idx), 0);
17501755 ring->tail = adapter->io_addr + IXGBE_VFRDT(reg_idx);
17511756
1757+ /* initialize rx_buffer_info */
1758+ memset(ring->rx_buffer_info, 0,
1759+ sizeof(struct ixgbevf_rx_buffer) * ring->count);
1760+
1761+ /* initialize Rx descriptor 0 */
1762+ rx_desc = IXGBEVF_RX_DESC(ring, 0);
1763+ rx_desc->wb.upper.length = 0;
1764+
17521765 /* reset ntu and ntc to place SW in sync with hardwdare */
17531766 ring->next_to_clean = 0;
17541767 ring->next_to_use = 0;
@@ -2103,9 +2116,7 @@ void ixgbevf_up(struct ixgbevf_adapter *adapter) · 348 unmodified lines
21032116 **/
21042117 static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring)
21052118 {
2106− struct device *dev = rx_ring->dev;
2107− unsigned long size;
2108− unsigned int i;
2119+ u16 i = rx_ring->next_to_clean;
21092120
21102121 /* Free Rx ring sk_buff */
21112122 if (rx_ring->skb) {
@@ -2113,29 +2124,39 @@ static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring) · 1 unmodified line
21132124 rx_ring->skb = NULL;
21142125 }
21152126
2116− /* ring already cleared, nothing to do */
2117− if (!rx_ring->rx_buffer_info)
2118− return;
2119−
21202127 /* Free all the Rx ring pages */
2121− for (i = 0; i < rx_ring->count; i++) {
2128+ while (i != rx_ring->next_to_alloc) {
21222129 struct ixgbevf_rx_buffer *rx_buffer;
21232130
21242131 rx_buffer = &rx_ring->rx_buffer_info[i];
2125− if (rx_buffer->dma)
2126− dma_unmap_page(dev, rx_buffer->dma,
2127− PAGE_SIZE, DMA_FROM_DEVICE);
2128− rx_buffer->dma = 0;
2129− if (rx_buffer->page)
2130− __free_page(rx_buffer->page);
2131− rx_buffer->page = NULL;
2132− }
21332132
2134− size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count;
2135− memset(rx_ring->rx_buffer_info, 0, size);
2133+ /* Invalidate cache lines that may have been written to by
2134+ * device so that we avoid corrupting memory.
2135+ */
2136+ dma_sync_single_range_for_cpu(rx_ring->dev,
2137+ rx_buffer->dma,
2138+ rx_buffer->page_offset,
2139+ IXGBEVF_RX_BUFSZ,
2140+ DMA_FROM_DEVICE);
2141+
2142+ /* free resources associated with mapping */
2143+ dma_unmap_page_attrs(rx_ring->dev,
2144+ rx_buffer->dma,
2145+ PAGE_SIZE,
2146+ DMA_FROM_DEVICE,
2147+ IXGBEVF_RX_DMA_ATTR);
2148+
2149+ __page_frag_cache_drain(rx_buffer->page,
2150+ rx_buffer->pagecnt_bias);
21362151
2137− /* Zero out the descriptor ring */
2138− memset(rx_ring->desc, 0, rx_ring->size);
2152+ i++;
2153+ if (i == rx_ring->count)
2154+ i = 0;
2155+ }
2156+
2157+ rx_ring->next_to_alloc = 0;
2158+ rx_ring->next_to_clean = 0;
2159+ rx_ring->next_to_use = 0;
21392160 }
21402161
21412162 /**
@@ -2144,23 +2165,57 @@ static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring) · 2 unmodified lines
21442165 **/
21452166 static void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring)
21462167 {
2147− struct ixgbevf_tx_buffer *tx_buffer_info;
2148− unsigned long size;
2149− unsigned int i;
2168+ u16 i = tx_ring->next_to_clean;
2169+ struct ixgbevf_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i];
21502170
2151− if (!tx_ring->tx_buffer_info)
2152− return;
2171+ while (i != tx_ring->next_to_use) {
2172+ union ixgbe_adv_tx_desc *eop_desc, *tx_desc;
21532173
2154− /* Free all the Tx ring sk_buffs */
2155− for (i = 0; i < tx_ring->count; i++) {
2156− tx_buffer_info = &tx_ring->tx_buffer_info[i];
2157− ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
2174+ /* Free all the Tx ring sk_buffs */
2175+ dev_kfree_skb_any(tx_buffer->skb);
2176+
2177+ /* unmap skb header data */
2178+ dma_unmap_single(tx_ring->dev,
2179+ dma_unmap_addr(tx_buffer, dma),
2180+ dma_unmap_len(tx_buffer, len),
2181+ DMA_TO_DEVICE);
2182+
2183+ /* check for eop_desc to determine the end of the packet */
2184+ eop_desc = tx_buffer->next_to_watch;
2185+ tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
2186+
2187+ /* unmap remaining buffers */
2188+ while (tx_desc != eop_desc) {
2189+ tx_buffer++;
2190+ tx_desc++;
2191+ i++;
2192+ if (unlikely(i == tx_ring->count)) {
2193+ i = 0;
2194+ tx_buffer = tx_ring->tx_buffer_info;
2195+ tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
2196+ }
2197+
2198+ /* unmap any remaining paged data */
2199+ if (dma_unmap_len(tx_buffer, len))
2200+ dma_unmap_page(tx_ring->dev,
2201+ dma_unmap_addr(tx_buffer, dma),
2202+ dma_unmap_len(tx_buffer, len),
2203+ DMA_TO_DEVICE);
2204+ }
2205+
2206+ /* move us one more past the eop_desc for start of next pkt */
2207+ tx_buffer++;
2208+ i++;
2209+ if (unlikely(i == tx_ring->count)) {
2210+ i = 0;
2211+ tx_buffer = tx_ring->tx_buffer_info;
2212+ }
21582213 }
21592214
2160− size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
2161− memset(tx_ring->tx_buffer_info, 0, size);
2215+ /* reset next_to_use and next_to_clean */
2216+ tx_ring->next_to_use = 0;
2217+ tx_ring->next_to_clean = 0;
21622218
2163− memset(tx_ring->desc, 0, tx_ring->size);
21642219 }
21652220
21662221 /**
@@ -2712,6 +2767,8 @@ out: · 545 unmodified lines
27122767 void ixgbevf_update_stats(struct ixgbevf_adapter *adapter)
27132768 {
27142769 struct ixgbe_hw *hw = &adapter->hw;
2770+ u64 alloc_rx_page_failed = 0, alloc_rx_buff_failed = 0;
2771+ u64 alloc_rx_page = 0, hw_csum_rx_error = 0;
27152772 int i;
27162773
27172774 if (test_bit(__IXGBEVF_DOWN, &adapter->state) ||
@@ -2732,10 +2789,18 @@ void ixgbevf_update_stats(struct ixgbevf_adapter *adapter) · 14 unmodified lines
27322789 adapter->stats.vfmprc);
27332790
27342791 for (i = 0; i < adapter->num_rx_queues; i++) {
2735− adapter->hw_csum_rx_error +=
2736− adapter->rx_ring[i]->hw_csum_rx_error;
2737− adapter->rx_ring[i]->hw_csum_rx_error = 0;
2792+ struct ixgbevf_ring *rx_ring = adapter->rx_ring[i];
2793+
2794+ hw_csum_rx_error += rx_ring->rx_stats.csum_err;
2795+ alloc_rx_page_failed += rx_ring->rx_stats.alloc_rx_page_failed;
2796+ alloc_rx_buff_failed += rx_ring->rx_stats.alloc_rx_buff_failed;
2797+ alloc_rx_page += rx_ring->rx_stats.alloc_rx_page;
27382798 }
2799+
2800+ adapter->hw_csum_rx_error = hw_csum_rx_error;
2801+ adapter->alloc_rx_page_failed = alloc_rx_page_failed;
2802+ adapter->alloc_rx_buff_failed = alloc_rx_buff_failed;
2803+ adapter->alloc_rx_page = alloc_rx_page;
27392804 }
27402805
27412806 /**
@@ -2980,7 +3045,7 @@ int ixgbevf_setup_tx_resources(struct ixgbevf_ring *tx_ring) · 238 unmodified lines
29803045 int size;
29813046
29823047 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
2983− tx_ring->tx_buffer_info = vzalloc(size);
3048+ tx_ring->tx_buffer_info = vmalloc(size);
29843049 if (!tx_ring->tx_buffer_info)
29853050 goto err;
29863051
@@ -3040,7 +3105,7 @@ int ixgbevf_setup_rx_resources(struct ixgbevf_ring *rx_ring) · 53 unmodified lines
30403105 int size;
30413106
30423107 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count;
3043− rx_ring->rx_buffer_info = vzalloc(size);
3108+ rx_ring->rx_buffer_info = vmalloc(size);
30443109 if (!rx_ring->rx_buffer_info)
30453110 goto err;
30463111
@@ -3482,34 +3547,37 @@ static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring, · 435 unmodified lines
34823547 struct ixgbevf_tx_buffer *first,
34833548 const u8 hdr_len)
34843549 {
3485− dma_addr_t dma;
34863550 struct sk_buff *skb = first->skb;
34873551 struct ixgbevf_tx_buffer *tx_buffer;
34883552 union ixgbe_adv_tx_desc *tx_desc;
3489− struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
3490− unsigned int data_len = skb->data_len;
3491− unsigned int size = skb_headlen(skb);
3492− unsigned int paylen = skb->len - hdr_len;
3553+ struct skb_frag_struct *frag;
3554+ dma_addr_t dma;
3555+ unsigned int data_len, size;
34933556 u32 tx_flags = first->tx_flags;
3494− __le32 cmd_type;
3557+ __le32 cmd_type = ixgbevf_tx_cmd_type(tx_flags);
34953558 u16 i = tx_ring->next_to_use;
34963559
34973560 tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
34983561
3499− ixgbevf_tx_olinfo_status(tx_desc, tx_flags, paylen);
3500− cmd_type = ixgbevf_tx_cmd_type(tx_flags);
3562+ ixgbevf_tx_olinfo_status(tx_desc, tx_flags, skb->len - hdr_len);
3563+
3564+ size = skb_headlen(skb);
3565+ data_len = skb->data_len;
35013566
35023567 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
3503− if (dma_mapping_error(tx_ring->dev, dma))
3504− goto dma_error;
35053568
3506− /* record length, and DMA address */
3507− dma_unmap_len_set(first, len, size);
3508− dma_unmap_addr_set(first, dma, dma);
3569+ tx_buffer = first;
3570+
3571+ for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
3572+ if (dma_mapping_error(tx_ring->dev, dma))
3573+ goto dma_error;
3574+
3575+ /* record length, and DMA address */
3576+ dma_unmap_len_set(tx_buffer, len, size);
3577+ dma_unmap_addr_set(tx_buffer, dma, dma);
35093578
3510− tx_desc->read.buffer_addr = cpu_to_le64(dma);
3579+ tx_desc->read.buffer_addr = cpu_to_le64(dma);
35113580
3512− for (;;) {
35133581 while (unlikely(size > IXGBE_MAX_DATA_PER_TXD)) {
35143582 tx_desc->read.cmd_type_len =
35153583 cmd_type | cpu_to_le32(IXGBE_MAX_DATA_PER_TXD);
@@ -3520,12 +3588,12 @@ static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring, · 4 unmodified lines
35203588 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
35213589 i = 0;
35223590 }
3591+ tx_desc->read.olinfo_status = 0;
35233592
35243593 dma += IXGBE_MAX_DATA_PER_TXD;
35253594 size -= IXGBE_MAX_DATA_PER_TXD;
35263595
35273596 tx_desc->read.buffer_addr = cpu_to_le64(dma);
3528− tx_desc->read.olinfo_status = 0;
35293597 }
35303598
35313599 if (likely(!data_len))
@@ -3539,23 +3607,15 @@ static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring, · 7 unmodified lines
35393607 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
35403608 i = 0;
35413609 }
3610+ tx_desc->read.olinfo_status = 0;
35423611
35433612 size = skb_frag_size(frag);
35443613 data_len -= size;
35453614
35463615 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
35473616 DMA_TO_DEVICE);
3548− if (dma_mapping_error(tx_ring->dev, dma))
3549− goto dma_error;
35503617
35513618 tx_buffer = &tx_ring->tx_buffer_info[i];
3552− dma_unmap_len_set(tx_buffer, len, size);
3553− dma_unmap_addr_set(tx_buffer, dma, dma);
3554−
3555− tx_desc->read.buffer_addr = cpu_to_le64(dma);
3556− tx_desc->read.olinfo_status = 0;
3557−
3558− frag++;
35593619 }
35603620
35613621 /* write last descriptor with RS and EOP bits */
@@ -3589,18 +3649,32 @@ static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring, · 27 unmodified lines
35893649 return;
35903650 dma_error:
35913651 dev_err(tx_ring->dev, "TX DMA map failed\n");
3652+ tx_buffer = &tx_ring->tx_buffer_info[i];
35923653
35933654 /* clear dma mappings for failed tx_buffer_info map */
3594− for (;;) {
3655+ while (tx_buffer != first) {
3656+ if (dma_unmap_len(tx_buffer, len))
3657+ dma_unmap_page(tx_ring->dev,
3658+ dma_unmap_addr(tx_buffer, dma),
3659+ dma_unmap_len(tx_buffer, len),
3660+ DMA_TO_DEVICE);
3661+ dma_unmap_len_set(tx_buffer, len, 0);
3662+
3663+ if (i-- == 0)
3664+ i += tx_ring->count;
35953665 tx_buffer = &tx_ring->tx_buffer_info[i];
3596− ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer);
3597− if (tx_buffer == first)
3598− break;
3599− if (i == 0)
3600− i = tx_ring->count;
3601− i--;
36023666 }
36033667
3668+ if (dma_unmap_len(tx_buffer, len))
3669+ dma_unmap_single(tx_ring->dev,
3670+ dma_unmap_addr(tx_buffer, dma),
3671+ dma_unmap_len(tx_buffer, len),
3672+ DMA_TO_DEVICE);
3673+ dma_unmap_len_set(tx_buffer, len, 0);
3674+
3675+ dev_kfree_skb_any(tx_buffer->skb);
3676+ tx_buffer->skb = NULL;
3677+
36043678 tx_ring->next_to_use = i;
36053679 }
36063680
drivers/net/ethernet/intel/ixgbevf/vf.c+7−10
@@ -286,7 +286,7 @@ static s32 ixgbevf_set_uc_addr_vf(struct ixgbe_hw *hw, u32 index, u8 *addr) · 285 unmodified lines
286286 ether_addr_copy(msg_addr, addr);
287287
288288 ret_val = ixgbevf_write_msg_read_ack(hw, msgbuf, msgbuf,
289− sizeof(msgbuf) / sizeof(u32));
289+ ARRAY_SIZE(msgbuf));
290290 if (!ret_val) {
291291 msgbuf[0] &= ~IXGBE_VT_MSGTYPE_CTS;
292292
@@ -456,8 +456,7 @@ static s32 ixgbevf_set_rar_vf(struct ixgbe_hw *hw, u32 index, u8 *addr, · 163 unmodified lines
456456 ether_addr_copy(msg_addr, addr);
457457
458458 ret_val = ixgbevf_write_msg_read_ack(hw, msgbuf, msgbuf,
459− sizeof(msgbuf) / sizeof(u32));
460−
459+ ARRAY_SIZE(msgbuf));
461460 msgbuf[0] &= ~IXGBE_VT_MSGTYPE_CTS;
462461
463462 /* if nacked the address was rejected, use "perm_addr" */
@@ -574,7 +573,7 @@ static s32 ixgbevf_update_xcast_mode(struct ixgbe_hw *hw, int xcast_mode) · 110 unmodified lines
574573 msgbuf[1] = xcast_mode;
575574
576575 err = ixgbevf_write_msg_read_ack(hw, msgbuf, msgbuf,
577− sizeof(msgbuf) / sizeof(u32));
576+ ARRAY_SIZE(msgbuf));
578577 if (err)
579578 return err;
580579
@@ -614,7 +613,7 @@ static s32 ixgbevf_set_vfta_vf(struct ixgbe_hw *hw, u32 vlan, u32 vind, · 33 unmodified lines
614613 msgbuf[0] |= vlan_on << IXGBE_VT_MSGINFO_SHIFT;
615614
616615 err = ixgbevf_write_msg_read_ack(hw, msgbuf, msgbuf,
617− sizeof(msgbuf) / sizeof(u32));
616+ ARRAY_SIZE(msgbuf));
618617 if (err)
619618 goto mbx_err;
620619
@@ -826,7 +825,7 @@ static s32 ixgbevf_set_rlpml_vf(struct ixgbe_hw *hw, u16 max_size) · 205 unmodified lines
826825 msgbuf[1] = max_size;
827826
828827 ret_val = ixgbevf_write_msg_read_ack(hw, msgbuf, msgbuf,
829− sizeof(msgbuf) / sizeof(u32));
828+ ARRAY_SIZE(msgbuf));
830829 if (ret_val)
831830 return ret_val;
832831 if ((msgbuf[0] & IXGBE_VF_SET_LPE) &&
@@ -872,8 +871,7 @@ static int ixgbevf_negotiate_api_version_vf(struct ixgbe_hw *hw, int api) · 39 unmodified lines
872871 msg[1] = api;
873872 msg[2] = 0;
874873
875− err = ixgbevf_write_msg_read_ack(hw, msg, msg,
876− sizeof(msg) / sizeof(u32));
874+ err = ixgbevf_write_msg_read_ack(hw, msg, msg, ARRAY_SIZE(msg));
877875 if (!err) {
878876 msg[0] &= ~IXGBE_VT_MSGTYPE_CTS;
879877
@@ -924,8 +922,7 @@ int ixgbevf_get_queues(struct ixgbe_hw *hw, unsigned int *num_tcs, · 44 unmodified lines
924922 msg[0] = IXGBE_VF_GET_QUEUE;
925923 msg[1] = msg[2] = msg[3] = msg[4] = 0;
926924
927− err = ixgbevf_write_msg_read_ack(hw, msg, msg,
928− sizeof(msg) / sizeof(u32));
925+ err = ixgbevf_write_msg_read_ack(hw, msg, msg, ARRAY_SIZE(msg));
929926 if (!err) {
930927 msg[0] &= ~IXGBE_VT_MSGTYPE_CTS;
931928