1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2018 Intel Corporation */
4 #include <linux/module.h>
5 #include <linux/types.h>
6 #include <linux/if_vlan.h>
11 #include <linux/pm_runtime.h>
12 #include <net/pkt_sched.h>
20 #define DRV_VERSION "0.0.1-k"
21 #define DRV_SUMMARY "Intel(R) 2.5G Ethernet Linux Driver"
23 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
25 static int debug
= -1;
27 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
28 MODULE_DESCRIPTION(DRV_SUMMARY
);
29 MODULE_LICENSE("GPL v2");
30 MODULE_VERSION(DRV_VERSION
);
31 module_param(debug
, int, 0);
32 MODULE_PARM_DESC(debug
, "Debug level (0=none,...,16=all)");
34 char igc_driver_name
[] = "igc";
35 char igc_driver_version
[] = DRV_VERSION
;
36 static const char igc_driver_string
[] = DRV_SUMMARY
;
37 static const char igc_copyright
[] =
38 "Copyright(c) 2018 Intel Corporation.";
40 static const struct igc_info
*igc_info_tbl
[] = {
41 [board_base
] = &igc_base_info
,
44 static const struct pci_device_id igc_pci_tbl
[] = {
45 { PCI_VDEVICE(INTEL
, IGC_DEV_ID_I225_LM
), board_base
},
46 { PCI_VDEVICE(INTEL
, IGC_DEV_ID_I225_V
), board_base
},
47 { PCI_VDEVICE(INTEL
, IGC_DEV_ID_I225_I
), board_base
},
48 { PCI_VDEVICE(INTEL
, IGC_DEV_ID_I220_V
), board_base
},
49 { PCI_VDEVICE(INTEL
, IGC_DEV_ID_I225_K
), board_base
},
50 { PCI_VDEVICE(INTEL
, IGC_DEV_ID_I225_K2
), board_base
},
51 { PCI_VDEVICE(INTEL
, IGC_DEV_ID_I225_LMVP
), board_base
},
52 { PCI_VDEVICE(INTEL
, IGC_DEV_ID_I225_IT
), board_base
},
53 { PCI_VDEVICE(INTEL
, IGC_DEV_ID_I225_BLANK_NVM
), board_base
},
54 /* required last entry */
58 MODULE_DEVICE_TABLE(pci
, igc_pci_tbl
);
68 * igc_power_down_link - Power down the phy/serdes link
69 * @adapter: address of board private structure
71 static void igc_power_down_link(struct igc_adapter
*adapter
)
73 if (adapter
->hw
.phy
.media_type
== igc_media_type_copper
)
74 igc_power_down_phy_copper_base(&adapter
->hw
);
77 void igc_reset(struct igc_adapter
*adapter
)
79 struct net_device
*dev
= adapter
->netdev
;
80 struct igc_hw
*hw
= &adapter
->hw
;
81 struct igc_fc_info
*fc
= &hw
->fc
;
84 /* Repartition PBA for greater than 9k MTU if required */
87 /* flow control settings
88 * The high water mark must be low enough to fit one full frame
89 * after transmitting the pause frame. As such we must have enough
90 * space to allow for us to complete our current transmit and then
91 * receive the frame that is in progress from the link partner.
93 * - the full Rx FIFO size minus one full Tx plus one full Rx frame
95 hwm
= (pba
<< 10) - (adapter
->max_frame_size
+ MAX_JUMBO_FRAME_SIZE
);
97 fc
->high_water
= hwm
& 0xFFFFFFF0; /* 16-byte granularity */
98 fc
->low_water
= fc
->high_water
- 16;
99 fc
->pause_time
= 0xFFFF;
101 fc
->current_mode
= fc
->requested_mode
;
103 hw
->mac
.ops
.reset_hw(hw
);
105 if (hw
->mac
.ops
.init_hw(hw
))
106 netdev_err(dev
, "Error on hardware initialization\n");
108 if (!netif_running(adapter
->netdev
))
109 igc_power_down_link(adapter
);
111 /* Re-enable PTP, where applicable. */
112 igc_ptp_reset(adapter
);
114 /* Re-enable TSN offloading, where applicable. */
115 igc_tsn_offload_apply(adapter
);
117 igc_get_phy_info(hw
);
121 * igc_power_up_link - Power up the phy link
122 * @adapter: address of board private structure
124 static void igc_power_up_link(struct igc_adapter
*adapter
)
126 igc_reset_phy(&adapter
->hw
);
128 if (adapter
->hw
.phy
.media_type
== igc_media_type_copper
)
129 igc_power_up_phy_copper(&adapter
->hw
);
131 igc_setup_link(&adapter
->hw
);
135 * igc_release_hw_control - release control of the h/w to f/w
136 * @adapter: address of board private structure
138 * igc_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
139 * For ASF and Pass Through versions of f/w this means that the
140 * driver is no longer loaded.
142 static void igc_release_hw_control(struct igc_adapter
*adapter
)
144 struct igc_hw
*hw
= &adapter
->hw
;
147 /* Let firmware take over control of h/w */
148 ctrl_ext
= rd32(IGC_CTRL_EXT
);
150 ctrl_ext
& ~IGC_CTRL_EXT_DRV_LOAD
);
154 * igc_get_hw_control - get control of the h/w from f/w
155 * @adapter: address of board private structure
157 * igc_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
158 * For ASF and Pass Through versions of f/w this means that
159 * the driver is loaded.
161 static void igc_get_hw_control(struct igc_adapter
*adapter
)
163 struct igc_hw
*hw
= &adapter
->hw
;
166 /* Let firmware know the driver has taken over */
167 ctrl_ext
= rd32(IGC_CTRL_EXT
);
169 ctrl_ext
| IGC_CTRL_EXT_DRV_LOAD
);
173 * igc_clean_tx_ring - Free Tx Buffers
174 * @tx_ring: ring to be cleaned
176 static void igc_clean_tx_ring(struct igc_ring
*tx_ring
)
178 u16 i
= tx_ring
->next_to_clean
;
179 struct igc_tx_buffer
*tx_buffer
= &tx_ring
->tx_buffer_info
[i
];
181 while (i
!= tx_ring
->next_to_use
) {
182 union igc_adv_tx_desc
*eop_desc
, *tx_desc
;
184 /* Free all the Tx ring sk_buffs */
185 dev_kfree_skb_any(tx_buffer
->skb
);
187 /* unmap skb header data */
188 dma_unmap_single(tx_ring
->dev
,
189 dma_unmap_addr(tx_buffer
, dma
),
190 dma_unmap_len(tx_buffer
, len
),
193 /* check for eop_desc to determine the end of the packet */
194 eop_desc
= tx_buffer
->next_to_watch
;
195 tx_desc
= IGC_TX_DESC(tx_ring
, i
);
197 /* unmap remaining buffers */
198 while (tx_desc
!= eop_desc
) {
202 if (unlikely(i
== tx_ring
->count
)) {
204 tx_buffer
= tx_ring
->tx_buffer_info
;
205 tx_desc
= IGC_TX_DESC(tx_ring
, 0);
208 /* unmap any remaining paged data */
209 if (dma_unmap_len(tx_buffer
, len
))
210 dma_unmap_page(tx_ring
->dev
,
211 dma_unmap_addr(tx_buffer
, dma
),
212 dma_unmap_len(tx_buffer
, len
),
216 /* move us one more past the eop_desc for start of next pkt */
219 if (unlikely(i
== tx_ring
->count
)) {
221 tx_buffer
= tx_ring
->tx_buffer_info
;
225 /* reset BQL for queue */
226 netdev_tx_reset_queue(txring_txq(tx_ring
));
228 /* reset next_to_use and next_to_clean */
229 tx_ring
->next_to_use
= 0;
230 tx_ring
->next_to_clean
= 0;
234 * igc_free_tx_resources - Free Tx Resources per Queue
235 * @tx_ring: Tx descriptor ring for a specific queue
237 * Free all transmit software resources
239 void igc_free_tx_resources(struct igc_ring
*tx_ring
)
241 igc_clean_tx_ring(tx_ring
);
243 vfree(tx_ring
->tx_buffer_info
);
244 tx_ring
->tx_buffer_info
= NULL
;
246 /* if not set, then don't free */
250 dma_free_coherent(tx_ring
->dev
, tx_ring
->size
,
251 tx_ring
->desc
, tx_ring
->dma
);
253 tx_ring
->desc
= NULL
;
257 * igc_free_all_tx_resources - Free Tx Resources for All Queues
258 * @adapter: board private structure
260 * Free all transmit software resources
262 static void igc_free_all_tx_resources(struct igc_adapter
*adapter
)
266 for (i
= 0; i
< adapter
->num_tx_queues
; i
++)
267 igc_free_tx_resources(adapter
->tx_ring
[i
]);
271 * igc_clean_all_tx_rings - Free Tx Buffers for all queues
272 * @adapter: board private structure
274 static void igc_clean_all_tx_rings(struct igc_adapter
*adapter
)
278 for (i
= 0; i
< adapter
->num_tx_queues
; i
++)
279 if (adapter
->tx_ring
[i
])
280 igc_clean_tx_ring(adapter
->tx_ring
[i
]);
284 * igc_setup_tx_resources - allocate Tx resources (Descriptors)
285 * @tx_ring: tx descriptor ring (for a specific queue) to setup
287 * Return 0 on success, negative on failure
289 int igc_setup_tx_resources(struct igc_ring
*tx_ring
)
291 struct net_device
*ndev
= tx_ring
->netdev
;
292 struct device
*dev
= tx_ring
->dev
;
295 size
= sizeof(struct igc_tx_buffer
) * tx_ring
->count
;
296 tx_ring
->tx_buffer_info
= vzalloc(size
);
297 if (!tx_ring
->tx_buffer_info
)
300 /* round up to nearest 4K */
301 tx_ring
->size
= tx_ring
->count
* sizeof(union igc_adv_tx_desc
);
302 tx_ring
->size
= ALIGN(tx_ring
->size
, 4096);
304 tx_ring
->desc
= dma_alloc_coherent(dev
, tx_ring
->size
,
305 &tx_ring
->dma
, GFP_KERNEL
);
310 tx_ring
->next_to_use
= 0;
311 tx_ring
->next_to_clean
= 0;
316 vfree(tx_ring
->tx_buffer_info
);
317 netdev_err(ndev
, "Unable to allocate memory for Tx descriptor ring\n");
322 * igc_setup_all_tx_resources - wrapper to allocate Tx resources for all queues
323 * @adapter: board private structure
325 * Return 0 on success, negative on failure
327 static int igc_setup_all_tx_resources(struct igc_adapter
*adapter
)
329 struct net_device
*dev
= adapter
->netdev
;
332 for (i
= 0; i
< adapter
->num_tx_queues
; i
++) {
333 err
= igc_setup_tx_resources(adapter
->tx_ring
[i
]);
335 netdev_err(dev
, "Error on Tx queue %u setup\n", i
);
336 for (i
--; i
>= 0; i
--)
337 igc_free_tx_resources(adapter
->tx_ring
[i
]);
346 * igc_clean_rx_ring - Free Rx Buffers per Queue
347 * @rx_ring: ring to free buffers from
349 static void igc_clean_rx_ring(struct igc_ring
*rx_ring
)
351 u16 i
= rx_ring
->next_to_clean
;
353 dev_kfree_skb(rx_ring
->skb
);
356 /* Free all the Rx ring sk_buffs */
357 while (i
!= rx_ring
->next_to_alloc
) {
358 struct igc_rx_buffer
*buffer_info
= &rx_ring
->rx_buffer_info
[i
];
360 /* Invalidate cache lines that may have been written to by
361 * device so that we avoid corrupting memory.
363 dma_sync_single_range_for_cpu(rx_ring
->dev
,
365 buffer_info
->page_offset
,
366 igc_rx_bufsz(rx_ring
),
369 /* free resources associated with mapping */
370 dma_unmap_page_attrs(rx_ring
->dev
,
372 igc_rx_pg_size(rx_ring
),
375 __page_frag_cache_drain(buffer_info
->page
,
376 buffer_info
->pagecnt_bias
);
379 if (i
== rx_ring
->count
)
383 rx_ring
->next_to_alloc
= 0;
384 rx_ring
->next_to_clean
= 0;
385 rx_ring
->next_to_use
= 0;
389 * igc_clean_all_rx_rings - Free Rx Buffers for all queues
390 * @adapter: board private structure
392 static void igc_clean_all_rx_rings(struct igc_adapter
*adapter
)
396 for (i
= 0; i
< adapter
->num_rx_queues
; i
++)
397 if (adapter
->rx_ring
[i
])
398 igc_clean_rx_ring(adapter
->rx_ring
[i
]);
402 * igc_free_rx_resources - Free Rx Resources
403 * @rx_ring: ring to clean the resources from
405 * Free all receive software resources
407 void igc_free_rx_resources(struct igc_ring
*rx_ring
)
409 igc_clean_rx_ring(rx_ring
);
411 vfree(rx_ring
->rx_buffer_info
);
412 rx_ring
->rx_buffer_info
= NULL
;
414 /* if not set, then don't free */
418 dma_free_coherent(rx_ring
->dev
, rx_ring
->size
,
419 rx_ring
->desc
, rx_ring
->dma
);
421 rx_ring
->desc
= NULL
;
425 * igc_free_all_rx_resources - Free Rx Resources for All Queues
426 * @adapter: board private structure
428 * Free all receive software resources
430 static void igc_free_all_rx_resources(struct igc_adapter
*adapter
)
434 for (i
= 0; i
< adapter
->num_rx_queues
; i
++)
435 igc_free_rx_resources(adapter
->rx_ring
[i
]);
439 * igc_setup_rx_resources - allocate Rx resources (Descriptors)
440 * @rx_ring: rx descriptor ring (for a specific queue) to setup
442 * Returns 0 on success, negative on failure
444 int igc_setup_rx_resources(struct igc_ring
*rx_ring
)
446 struct net_device
*ndev
= rx_ring
->netdev
;
447 struct device
*dev
= rx_ring
->dev
;
450 size
= sizeof(struct igc_rx_buffer
) * rx_ring
->count
;
451 rx_ring
->rx_buffer_info
= vzalloc(size
);
452 if (!rx_ring
->rx_buffer_info
)
455 desc_len
= sizeof(union igc_adv_rx_desc
);
457 /* Round up to nearest 4K */
458 rx_ring
->size
= rx_ring
->count
* desc_len
;
459 rx_ring
->size
= ALIGN(rx_ring
->size
, 4096);
461 rx_ring
->desc
= dma_alloc_coherent(dev
, rx_ring
->size
,
462 &rx_ring
->dma
, GFP_KERNEL
);
467 rx_ring
->next_to_alloc
= 0;
468 rx_ring
->next_to_clean
= 0;
469 rx_ring
->next_to_use
= 0;
474 vfree(rx_ring
->rx_buffer_info
);
475 rx_ring
->rx_buffer_info
= NULL
;
476 netdev_err(ndev
, "Unable to allocate memory for Rx descriptor ring\n");
481 * igc_setup_all_rx_resources - wrapper to allocate Rx resources
482 * (Descriptors) for all queues
483 * @adapter: board private structure
485 * Return 0 on success, negative on failure
487 static int igc_setup_all_rx_resources(struct igc_adapter
*adapter
)
489 struct net_device
*dev
= adapter
->netdev
;
492 for (i
= 0; i
< adapter
->num_rx_queues
; i
++) {
493 err
= igc_setup_rx_resources(adapter
->rx_ring
[i
]);
495 netdev_err(dev
, "Error on Rx queue %u setup\n", i
);
496 for (i
--; i
>= 0; i
--)
497 igc_free_rx_resources(adapter
->rx_ring
[i
]);
506 * igc_configure_rx_ring - Configure a receive ring after Reset
507 * @adapter: board private structure
508 * @ring: receive ring to be configured
510 * Configure the Rx unit of the MAC after a reset.
512 static void igc_configure_rx_ring(struct igc_adapter
*adapter
,
513 struct igc_ring
*ring
)
515 struct igc_hw
*hw
= &adapter
->hw
;
516 union igc_adv_rx_desc
*rx_desc
;
517 int reg_idx
= ring
->reg_idx
;
518 u32 srrctl
= 0, rxdctl
= 0;
519 u64 rdba
= ring
->dma
;
521 /* disable the queue */
522 wr32(IGC_RXDCTL(reg_idx
), 0);
524 /* Set DMA base address registers */
525 wr32(IGC_RDBAL(reg_idx
),
526 rdba
& 0x00000000ffffffffULL
);
527 wr32(IGC_RDBAH(reg_idx
), rdba
>> 32);
528 wr32(IGC_RDLEN(reg_idx
),
529 ring
->count
* sizeof(union igc_adv_rx_desc
));
531 /* initialize head and tail */
532 ring
->tail
= adapter
->io_addr
+ IGC_RDT(reg_idx
);
533 wr32(IGC_RDH(reg_idx
), 0);
534 writel(0, ring
->tail
);
536 /* reset next-to- use/clean to place SW in sync with hardware */
537 ring
->next_to_clean
= 0;
538 ring
->next_to_use
= 0;
540 /* set descriptor configuration */
541 srrctl
= IGC_RX_HDR_LEN
<< IGC_SRRCTL_BSIZEHDRSIZE_SHIFT
;
542 if (ring_uses_large_buffer(ring
))
543 srrctl
|= IGC_RXBUFFER_3072
>> IGC_SRRCTL_BSIZEPKT_SHIFT
;
545 srrctl
|= IGC_RXBUFFER_2048
>> IGC_SRRCTL_BSIZEPKT_SHIFT
;
546 srrctl
|= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF
;
548 wr32(IGC_SRRCTL(reg_idx
), srrctl
);
550 rxdctl
|= IGC_RX_PTHRESH
;
551 rxdctl
|= IGC_RX_HTHRESH
<< 8;
552 rxdctl
|= IGC_RX_WTHRESH
<< 16;
554 /* initialize rx_buffer_info */
555 memset(ring
->rx_buffer_info
, 0,
556 sizeof(struct igc_rx_buffer
) * ring
->count
);
558 /* initialize Rx descriptor 0 */
559 rx_desc
= IGC_RX_DESC(ring
, 0);
560 rx_desc
->wb
.upper
.length
= 0;
562 /* enable receive descriptor fetching */
563 rxdctl
|= IGC_RXDCTL_QUEUE_ENABLE
;
565 wr32(IGC_RXDCTL(reg_idx
), rxdctl
);
569 * igc_configure_rx - Configure receive Unit after Reset
570 * @adapter: board private structure
572 * Configure the Rx unit of the MAC after a reset.
574 static void igc_configure_rx(struct igc_adapter
*adapter
)
578 /* Setup the HW Rx Head and Tail Descriptor Pointers and
579 * the Base and Length of the Rx Descriptor Ring
581 for (i
= 0; i
< adapter
->num_rx_queues
; i
++)
582 igc_configure_rx_ring(adapter
, adapter
->rx_ring
[i
]);
586 * igc_configure_tx_ring - Configure transmit ring after Reset
587 * @adapter: board private structure
588 * @ring: tx ring to configure
590 * Configure a transmit ring after a reset.
592 static void igc_configure_tx_ring(struct igc_adapter
*adapter
,
593 struct igc_ring
*ring
)
595 struct igc_hw
*hw
= &adapter
->hw
;
596 int reg_idx
= ring
->reg_idx
;
597 u64 tdba
= ring
->dma
;
600 /* disable the queue */
601 wr32(IGC_TXDCTL(reg_idx
), 0);
605 wr32(IGC_TDLEN(reg_idx
),
606 ring
->count
* sizeof(union igc_adv_tx_desc
));
607 wr32(IGC_TDBAL(reg_idx
),
608 tdba
& 0x00000000ffffffffULL
);
609 wr32(IGC_TDBAH(reg_idx
), tdba
>> 32);
611 ring
->tail
= adapter
->io_addr
+ IGC_TDT(reg_idx
);
612 wr32(IGC_TDH(reg_idx
), 0);
613 writel(0, ring
->tail
);
615 txdctl
|= IGC_TX_PTHRESH
;
616 txdctl
|= IGC_TX_HTHRESH
<< 8;
617 txdctl
|= IGC_TX_WTHRESH
<< 16;
619 txdctl
|= IGC_TXDCTL_QUEUE_ENABLE
;
620 wr32(IGC_TXDCTL(reg_idx
), txdctl
);
624 * igc_configure_tx - Configure transmit Unit after Reset
625 * @adapter: board private structure
627 * Configure the Tx unit of the MAC after a reset.
629 static void igc_configure_tx(struct igc_adapter
*adapter
)
633 for (i
= 0; i
< adapter
->num_tx_queues
; i
++)
634 igc_configure_tx_ring(adapter
, adapter
->tx_ring
[i
]);
638 * igc_setup_mrqc - configure the multiple receive queue control registers
639 * @adapter: Board private structure
641 static void igc_setup_mrqc(struct igc_adapter
*adapter
)
643 struct igc_hw
*hw
= &adapter
->hw
;
644 u32 j
, num_rx_queues
;
648 netdev_rss_key_fill(rss_key
, sizeof(rss_key
));
649 for (j
= 0; j
< 10; j
++)
650 wr32(IGC_RSSRK(j
), rss_key
[j
]);
652 num_rx_queues
= adapter
->rss_queues
;
654 if (adapter
->rss_indir_tbl_init
!= num_rx_queues
) {
655 for (j
= 0; j
< IGC_RETA_SIZE
; j
++)
656 adapter
->rss_indir_tbl
[j
] =
657 (j
* num_rx_queues
) / IGC_RETA_SIZE
;
658 adapter
->rss_indir_tbl_init
= num_rx_queues
;
660 igc_write_rss_indir_tbl(adapter
);
662 /* Disable raw packet checksumming so that RSS hash is placed in
663 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
664 * offloads as they are enabled by default
666 rxcsum
= rd32(IGC_RXCSUM
);
667 rxcsum
|= IGC_RXCSUM_PCSD
;
669 /* Enable Receive Checksum Offload for SCTP */
670 rxcsum
|= IGC_RXCSUM_CRCOFL
;
672 /* Don't need to set TUOFL or IPOFL, they default to 1 */
673 wr32(IGC_RXCSUM
, rxcsum
);
675 /* Generate RSS hash based on packet types, TCP/UDP
676 * port numbers and/or IPv4/v6 src and dst addresses
678 mrqc
= IGC_MRQC_RSS_FIELD_IPV4
|
679 IGC_MRQC_RSS_FIELD_IPV4_TCP
|
680 IGC_MRQC_RSS_FIELD_IPV6
|
681 IGC_MRQC_RSS_FIELD_IPV6_TCP
|
682 IGC_MRQC_RSS_FIELD_IPV6_TCP_EX
;
684 if (adapter
->flags
& IGC_FLAG_RSS_FIELD_IPV4_UDP
)
685 mrqc
|= IGC_MRQC_RSS_FIELD_IPV4_UDP
;
686 if (adapter
->flags
& IGC_FLAG_RSS_FIELD_IPV6_UDP
)
687 mrqc
|= IGC_MRQC_RSS_FIELD_IPV6_UDP
;
689 mrqc
|= IGC_MRQC_ENABLE_RSS_MQ
;
691 wr32(IGC_MRQC
, mrqc
);
695 * igc_setup_rctl - configure the receive control registers
696 * @adapter: Board private structure
698 static void igc_setup_rctl(struct igc_adapter
*adapter
)
700 struct igc_hw
*hw
= &adapter
->hw
;
703 rctl
= rd32(IGC_RCTL
);
705 rctl
&= ~(3 << IGC_RCTL_MO_SHIFT
);
706 rctl
&= ~(IGC_RCTL_LBM_TCVR
| IGC_RCTL_LBM_MAC
);
708 rctl
|= IGC_RCTL_EN
| IGC_RCTL_BAM
| IGC_RCTL_RDMTS_HALF
|
709 (hw
->mac
.mc_filter_type
<< IGC_RCTL_MO_SHIFT
);
711 /* enable stripping of CRC. Newer features require
712 * that the HW strips the CRC.
714 rctl
|= IGC_RCTL_SECRC
;
716 /* disable store bad packets and clear size bits. */
717 rctl
&= ~(IGC_RCTL_SBP
| IGC_RCTL_SZ_256
);
719 /* enable LPE to allow for reception of jumbo frames */
720 rctl
|= IGC_RCTL_LPE
;
722 /* disable queue 0 to prevent tail write w/o re-config */
723 wr32(IGC_RXDCTL(0), 0);
725 /* This is useful for sniffing bad packets. */
726 if (adapter
->netdev
->features
& NETIF_F_RXALL
) {
727 /* UPE and MPE will be handled by normal PROMISC logic
730 rctl
|= (IGC_RCTL_SBP
| /* Receive bad packets */
731 IGC_RCTL_BAM
| /* RX All Bcast Pkts */
732 IGC_RCTL_PMCF
); /* RX All MAC Ctrl Pkts */
734 rctl
&= ~(IGC_RCTL_DPF
| /* Allow filtered pause */
735 IGC_RCTL_CFIEN
); /* Disable VLAN CFIEN Filter */
738 wr32(IGC_RCTL
, rctl
);
742 * igc_setup_tctl - configure the transmit control registers
743 * @adapter: Board private structure
745 static void igc_setup_tctl(struct igc_adapter
*adapter
)
747 struct igc_hw
*hw
= &adapter
->hw
;
750 /* disable queue 0 which icould be enabled by default */
751 wr32(IGC_TXDCTL(0), 0);
753 /* Program the Transmit Control Register */
754 tctl
= rd32(IGC_TCTL
);
755 tctl
&= ~IGC_TCTL_CT
;
756 tctl
|= IGC_TCTL_PSP
| IGC_TCTL_RTLC
|
757 (IGC_COLLISION_THRESHOLD
<< IGC_CT_SHIFT
);
759 /* Enable transmits */
762 wr32(IGC_TCTL
, tctl
);
766 * igc_set_mac_filter_hw() - Set MAC address filter in hardware
767 * @adapter: Pointer to adapter where the filter should be set
768 * @index: Filter index
769 * @type: MAC address filter type (source or destination)
771 * @queue: If non-negative, queue assignment feature is enabled and frames
772 * matching the filter are enqueued onto 'queue'. Otherwise, queue
773 * assignment is disabled.
775 static void igc_set_mac_filter_hw(struct igc_adapter
*adapter
, int index
,
776 enum igc_mac_filter_type type
,
777 const u8
*addr
, int queue
)
779 struct net_device
*dev
= adapter
->netdev
;
780 struct igc_hw
*hw
= &adapter
->hw
;
783 if (WARN_ON(index
>= hw
->mac
.rar_entry_count
))
786 ral
= le32_to_cpup((__le32
*)(addr
));
787 rah
= le16_to_cpup((__le16
*)(addr
+ 4));
789 if (type
== IGC_MAC_FILTER_TYPE_SRC
) {
790 rah
&= ~IGC_RAH_ASEL_MASK
;
791 rah
|= IGC_RAH_ASEL_SRC_ADDR
;
795 rah
&= ~IGC_RAH_QSEL_MASK
;
796 rah
|= (queue
<< IGC_RAH_QSEL_SHIFT
);
797 rah
|= IGC_RAH_QSEL_ENABLE
;
802 wr32(IGC_RAL(index
), ral
);
803 wr32(IGC_RAH(index
), rah
);
805 netdev_dbg(dev
, "MAC address filter set in HW: index %d", index
);
809 * igc_clear_mac_filter_hw() - Clear MAC address filter in hardware
810 * @adapter: Pointer to adapter where the filter should be cleared
811 * @index: Filter index
813 static void igc_clear_mac_filter_hw(struct igc_adapter
*adapter
, int index
)
815 struct net_device
*dev
= adapter
->netdev
;
816 struct igc_hw
*hw
= &adapter
->hw
;
818 if (WARN_ON(index
>= hw
->mac
.rar_entry_count
))
821 wr32(IGC_RAL(index
), 0);
822 wr32(IGC_RAH(index
), 0);
824 netdev_dbg(dev
, "MAC address filter cleared in HW: index %d", index
);
827 /* Set default MAC address for the PF in the first RAR entry */
828 static void igc_set_default_mac_filter(struct igc_adapter
*adapter
)
830 struct net_device
*dev
= adapter
->netdev
;
831 u8
*addr
= adapter
->hw
.mac
.addr
;
833 netdev_dbg(dev
, "Set default MAC address filter: address %pM", addr
);
835 igc_set_mac_filter_hw(adapter
, 0, IGC_MAC_FILTER_TYPE_DST
, addr
, -1);
839 * igc_set_mac - Change the Ethernet Address of the NIC
840 * @netdev: network interface device structure
841 * @p: pointer to an address structure
843 * Returns 0 on success, negative on failure
845 static int igc_set_mac(struct net_device
*netdev
, void *p
)
847 struct igc_adapter
*adapter
= netdev_priv(netdev
);
848 struct igc_hw
*hw
= &adapter
->hw
;
849 struct sockaddr
*addr
= p
;
851 if (!is_valid_ether_addr(addr
->sa_data
))
852 return -EADDRNOTAVAIL
;
854 memcpy(netdev
->dev_addr
, addr
->sa_data
, netdev
->addr_len
);
855 memcpy(hw
->mac
.addr
, addr
->sa_data
, netdev
->addr_len
);
857 /* set the correct pool for the new PF MAC address in entry 0 */
858 igc_set_default_mac_filter(adapter
);
864 * igc_write_mc_addr_list - write multicast addresses to MTA
865 * @netdev: network interface device structure
867 * Writes multicast address list to the MTA hash table.
868 * Returns: -ENOMEM on failure
869 * 0 on no addresses written
870 * X on writing X addresses to MTA
872 static int igc_write_mc_addr_list(struct net_device
*netdev
)
874 struct igc_adapter
*adapter
= netdev_priv(netdev
);
875 struct igc_hw
*hw
= &adapter
->hw
;
876 struct netdev_hw_addr
*ha
;
880 if (netdev_mc_empty(netdev
)) {
881 /* nothing to program, so clear mc list */
882 igc_update_mc_addr_list(hw
, NULL
, 0);
886 mta_list
= kcalloc(netdev_mc_count(netdev
), 6, GFP_ATOMIC
);
890 /* The shared function expects a packed array of only addresses. */
892 netdev_for_each_mc_addr(ha
, netdev
)
893 memcpy(mta_list
+ (i
++ * ETH_ALEN
), ha
->addr
, ETH_ALEN
);
895 igc_update_mc_addr_list(hw
, mta_list
, i
);
898 return netdev_mc_count(netdev
);
901 static __le32
igc_tx_launchtime(struct igc_adapter
*adapter
, ktime_t txtime
)
903 ktime_t cycle_time
= adapter
->cycle_time
;
904 ktime_t base_time
= adapter
->base_time
;
907 /* FIXME: when using ETF together with taprio, we may have a
908 * case where 'delta' is larger than the cycle_time, this may
909 * cause problems if we don't read the current value of
910 * IGC_BASET, as the value writen into the launchtime
911 * descriptor field may be misinterpreted.
913 div_s64_rem(ktime_sub_ns(txtime
, base_time
), cycle_time
, &launchtime
);
915 return cpu_to_le32(launchtime
);
918 static void igc_tx_ctxtdesc(struct igc_ring
*tx_ring
,
919 struct igc_tx_buffer
*first
,
920 u32 vlan_macip_lens
, u32 type_tucmd
,
923 struct igc_adv_tx_context_desc
*context_desc
;
924 u16 i
= tx_ring
->next_to_use
;
926 context_desc
= IGC_TX_CTXTDESC(tx_ring
, i
);
929 tx_ring
->next_to_use
= (i
< tx_ring
->count
) ? i
: 0;
931 /* set bits to identify this as an advanced context descriptor */
932 type_tucmd
|= IGC_TXD_CMD_DEXT
| IGC_ADVTXD_DTYP_CTXT
;
934 /* For i225, context index must be unique per ring. */
935 if (test_bit(IGC_RING_FLAG_TX_CTX_IDX
, &tx_ring
->flags
))
936 mss_l4len_idx
|= tx_ring
->reg_idx
<< 4;
938 context_desc
->vlan_macip_lens
= cpu_to_le32(vlan_macip_lens
);
939 context_desc
->type_tucmd_mlhl
= cpu_to_le32(type_tucmd
);
940 context_desc
->mss_l4len_idx
= cpu_to_le32(mss_l4len_idx
);
942 /* We assume there is always a valid Tx time available. Invalid times
943 * should have been handled by the upper layers.
945 if (tx_ring
->launchtime_enable
) {
946 struct igc_adapter
*adapter
= netdev_priv(tx_ring
->netdev
);
947 ktime_t txtime
= first
->skb
->tstamp
;
949 first
->skb
->tstamp
= ktime_set(0, 0);
950 context_desc
->launch_time
= igc_tx_launchtime(adapter
,
953 context_desc
->launch_time
= 0;
957 static inline bool igc_ipv6_csum_is_sctp(struct sk_buff
*skb
)
959 unsigned int offset
= 0;
961 ipv6_find_hdr(skb
, &offset
, IPPROTO_SCTP
, NULL
, NULL
);
963 return offset
== skb_checksum_start_offset(skb
);
966 static void igc_tx_csum(struct igc_ring
*tx_ring
, struct igc_tx_buffer
*first
)
968 struct sk_buff
*skb
= first
->skb
;
969 u32 vlan_macip_lens
= 0;
972 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
) {
974 if (!(first
->tx_flags
& IGC_TX_FLAGS_VLAN
) &&
975 !tx_ring
->launchtime_enable
)
980 switch (skb
->csum_offset
) {
981 case offsetof(struct tcphdr
, check
):
982 type_tucmd
= IGC_ADVTXD_TUCMD_L4T_TCP
;
984 case offsetof(struct udphdr
, check
):
986 case offsetof(struct sctphdr
, checksum
):
987 /* validate that this is actually an SCTP request */
988 if ((first
->protocol
== htons(ETH_P_IP
) &&
989 (ip_hdr(skb
)->protocol
== IPPROTO_SCTP
)) ||
990 (first
->protocol
== htons(ETH_P_IPV6
) &&
991 igc_ipv6_csum_is_sctp(skb
))) {
992 type_tucmd
= IGC_ADVTXD_TUCMD_L4T_SCTP
;
997 skb_checksum_help(skb
);
1001 /* update TX checksum flag */
1002 first
->tx_flags
|= IGC_TX_FLAGS_CSUM
;
1003 vlan_macip_lens
= skb_checksum_start_offset(skb
) -
1004 skb_network_offset(skb
);
1006 vlan_macip_lens
|= skb_network_offset(skb
) << IGC_ADVTXD_MACLEN_SHIFT
;
1007 vlan_macip_lens
|= first
->tx_flags
& IGC_TX_FLAGS_VLAN_MASK
;
1009 igc_tx_ctxtdesc(tx_ring
, first
, vlan_macip_lens
, type_tucmd
, 0);
1012 static int __igc_maybe_stop_tx(struct igc_ring
*tx_ring
, const u16 size
)
1014 struct net_device
*netdev
= tx_ring
->netdev
;
1016 netif_stop_subqueue(netdev
, tx_ring
->queue_index
);
1018 /* memory barriier comment */
1021 /* We need to check again in a case another CPU has just
1022 * made room available.
1024 if (igc_desc_unused(tx_ring
) < size
)
1028 netif_wake_subqueue(netdev
, tx_ring
->queue_index
);
1030 u64_stats_update_begin(&tx_ring
->tx_syncp2
);
1031 tx_ring
->tx_stats
.restart_queue2
++;
1032 u64_stats_update_end(&tx_ring
->tx_syncp2
);
1037 static inline int igc_maybe_stop_tx(struct igc_ring
*tx_ring
, const u16 size
)
1039 if (igc_desc_unused(tx_ring
) >= size
)
1041 return __igc_maybe_stop_tx(tx_ring
, size
);
1044 #define IGC_SET_FLAG(_input, _flag, _result) \
1045 (((_flag) <= (_result)) ? \
1046 ((u32)((_input) & (_flag)) * ((_result) / (_flag))) : \
1047 ((u32)((_input) & (_flag)) / ((_flag) / (_result))))
1049 static u32
igc_tx_cmd_type(struct sk_buff
*skb
, u32 tx_flags
)
1051 /* set type for advanced descriptor with frame checksum insertion */
1052 u32 cmd_type
= IGC_ADVTXD_DTYP_DATA
|
1053 IGC_ADVTXD_DCMD_DEXT
|
1054 IGC_ADVTXD_DCMD_IFCS
;
1056 /* set segmentation bits for TSO */
1057 cmd_type
|= IGC_SET_FLAG(tx_flags
, IGC_TX_FLAGS_TSO
,
1058 (IGC_ADVTXD_DCMD_TSE
));
1060 /* set timestamp bit if present */
1061 cmd_type
|= IGC_SET_FLAG(tx_flags
, IGC_TX_FLAGS_TSTAMP
,
1062 (IGC_ADVTXD_MAC_TSTAMP
));
1067 static void igc_tx_olinfo_status(struct igc_ring
*tx_ring
,
1068 union igc_adv_tx_desc
*tx_desc
,
1069 u32 tx_flags
, unsigned int paylen
)
1071 u32 olinfo_status
= paylen
<< IGC_ADVTXD_PAYLEN_SHIFT
;
1073 /* insert L4 checksum */
1074 olinfo_status
|= (tx_flags
& IGC_TX_FLAGS_CSUM
) *
1075 ((IGC_TXD_POPTS_TXSM
<< 8) /
1078 /* insert IPv4 checksum */
1079 olinfo_status
|= (tx_flags
& IGC_TX_FLAGS_IPV4
) *
1080 (((IGC_TXD_POPTS_IXSM
<< 8)) /
1083 tx_desc
->read
.olinfo_status
= cpu_to_le32(olinfo_status
);
1086 static int igc_tx_map(struct igc_ring
*tx_ring
,
1087 struct igc_tx_buffer
*first
,
1090 struct sk_buff
*skb
= first
->skb
;
1091 struct igc_tx_buffer
*tx_buffer
;
1092 union igc_adv_tx_desc
*tx_desc
;
1093 u32 tx_flags
= first
->tx_flags
;
1095 u16 i
= tx_ring
->next_to_use
;
1096 unsigned int data_len
, size
;
1098 u32 cmd_type
= igc_tx_cmd_type(skb
, tx_flags
);
1100 tx_desc
= IGC_TX_DESC(tx_ring
, i
);
1102 igc_tx_olinfo_status(tx_ring
, tx_desc
, tx_flags
, skb
->len
- hdr_len
);
1104 size
= skb_headlen(skb
);
1105 data_len
= skb
->data_len
;
1107 dma
= dma_map_single(tx_ring
->dev
, skb
->data
, size
, DMA_TO_DEVICE
);
1111 for (frag
= &skb_shinfo(skb
)->frags
[0];; frag
++) {
1112 if (dma_mapping_error(tx_ring
->dev
, dma
))
1115 /* record length, and DMA address */
1116 dma_unmap_len_set(tx_buffer
, len
, size
);
1117 dma_unmap_addr_set(tx_buffer
, dma
, dma
);
1119 tx_desc
->read
.buffer_addr
= cpu_to_le64(dma
);
1121 while (unlikely(size
> IGC_MAX_DATA_PER_TXD
)) {
1122 tx_desc
->read
.cmd_type_len
=
1123 cpu_to_le32(cmd_type
^ IGC_MAX_DATA_PER_TXD
);
1127 if (i
== tx_ring
->count
) {
1128 tx_desc
= IGC_TX_DESC(tx_ring
, 0);
1131 tx_desc
->read
.olinfo_status
= 0;
1133 dma
+= IGC_MAX_DATA_PER_TXD
;
1134 size
-= IGC_MAX_DATA_PER_TXD
;
1136 tx_desc
->read
.buffer_addr
= cpu_to_le64(dma
);
1139 if (likely(!data_len
))
1142 tx_desc
->read
.cmd_type_len
= cpu_to_le32(cmd_type
^ size
);
1146 if (i
== tx_ring
->count
) {
1147 tx_desc
= IGC_TX_DESC(tx_ring
, 0);
1150 tx_desc
->read
.olinfo_status
= 0;
1152 size
= skb_frag_size(frag
);
1155 dma
= skb_frag_dma_map(tx_ring
->dev
, frag
, 0,
1156 size
, DMA_TO_DEVICE
);
1158 tx_buffer
= &tx_ring
->tx_buffer_info
[i
];
1161 /* write last descriptor with RS and EOP bits */
1162 cmd_type
|= size
| IGC_TXD_DCMD
;
1163 tx_desc
->read
.cmd_type_len
= cpu_to_le32(cmd_type
);
1165 netdev_tx_sent_queue(txring_txq(tx_ring
), first
->bytecount
);
1167 /* set the timestamp */
1168 first
->time_stamp
= jiffies
;
1170 skb_tx_timestamp(skb
);
1172 /* Force memory writes to complete before letting h/w know there
1173 * are new descriptors to fetch. (Only applicable for weak-ordered
1174 * memory model archs, such as IA-64).
1176 * We also need this memory barrier to make certain all of the
1177 * status bits have been updated before next_to_watch is written.
1181 /* set next_to_watch value indicating a packet is present */
1182 first
->next_to_watch
= tx_desc
;
1185 if (i
== tx_ring
->count
)
1188 tx_ring
->next_to_use
= i
;
1190 /* Make sure there is space in the ring for the next send. */
1191 igc_maybe_stop_tx(tx_ring
, DESC_NEEDED
);
1193 if (netif_xmit_stopped(txring_txq(tx_ring
)) || !netdev_xmit_more()) {
1194 writel(i
, tx_ring
->tail
);
1199 netdev_err(tx_ring
->netdev
, "TX DMA map failed\n");
1200 tx_buffer
= &tx_ring
->tx_buffer_info
[i
];
1202 /* clear dma mappings for failed tx_buffer_info map */
1203 while (tx_buffer
!= first
) {
1204 if (dma_unmap_len(tx_buffer
, len
))
1205 dma_unmap_page(tx_ring
->dev
,
1206 dma_unmap_addr(tx_buffer
, dma
),
1207 dma_unmap_len(tx_buffer
, len
),
1209 dma_unmap_len_set(tx_buffer
, len
, 0);
1212 i
+= tx_ring
->count
;
1213 tx_buffer
= &tx_ring
->tx_buffer_info
[i
];
1216 if (dma_unmap_len(tx_buffer
, len
))
1217 dma_unmap_single(tx_ring
->dev
,
1218 dma_unmap_addr(tx_buffer
, dma
),
1219 dma_unmap_len(tx_buffer
, len
),
1221 dma_unmap_len_set(tx_buffer
, len
, 0);
1223 dev_kfree_skb_any(tx_buffer
->skb
);
1224 tx_buffer
->skb
= NULL
;
1226 tx_ring
->next_to_use
= i
;
1231 static int igc_tso(struct igc_ring
*tx_ring
,
1232 struct igc_tx_buffer
*first
,
1235 u32 vlan_macip_lens
, type_tucmd
, mss_l4len_idx
;
1236 struct sk_buff
*skb
= first
->skb
;
1247 u32 paylen
, l4_offset
;
1250 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
)
1253 if (!skb_is_gso(skb
))
1256 err
= skb_cow_head(skb
, 0);
1260 ip
.hdr
= skb_network_header(skb
);
1261 l4
.hdr
= skb_checksum_start(skb
);
1263 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
1264 type_tucmd
= IGC_ADVTXD_TUCMD_L4T_TCP
;
1266 /* initialize outer IP header fields */
1267 if (ip
.v4
->version
== 4) {
1268 unsigned char *csum_start
= skb_checksum_start(skb
);
1269 unsigned char *trans_start
= ip
.hdr
+ (ip
.v4
->ihl
* 4);
1271 /* IP header will have to cancel out any data that
1272 * is not a part of the outer IP header
1274 ip
.v4
->check
= csum_fold(csum_partial(trans_start
,
1275 csum_start
- trans_start
,
1277 type_tucmd
|= IGC_ADVTXD_TUCMD_IPV4
;
1280 first
->tx_flags
|= IGC_TX_FLAGS_TSO
|
1284 ip
.v6
->payload_len
= 0;
1285 first
->tx_flags
|= IGC_TX_FLAGS_TSO
|
1289 /* determine offset of inner transport header */
1290 l4_offset
= l4
.hdr
- skb
->data
;
1292 /* remove payload length from inner checksum */
1293 paylen
= skb
->len
- l4_offset
;
1294 if (type_tucmd
& IGC_ADVTXD_TUCMD_L4T_TCP
) {
1295 /* compute length of segmentation header */
1296 *hdr_len
= (l4
.tcp
->doff
* 4) + l4_offset
;
1297 csum_replace_by_diff(&l4
.tcp
->check
,
1298 (__force __wsum
)htonl(paylen
));
1300 /* compute length of segmentation header */
1301 *hdr_len
= sizeof(*l4
.udp
) + l4_offset
;
1302 csum_replace_by_diff(&l4
.udp
->check
,
1303 (__force __wsum
)htonl(paylen
));
1306 /* update gso size and bytecount with header size */
1307 first
->gso_segs
= skb_shinfo(skb
)->gso_segs
;
1308 first
->bytecount
+= (first
->gso_segs
- 1) * *hdr_len
;
1311 mss_l4len_idx
= (*hdr_len
- l4_offset
) << IGC_ADVTXD_L4LEN_SHIFT
;
1312 mss_l4len_idx
|= skb_shinfo(skb
)->gso_size
<< IGC_ADVTXD_MSS_SHIFT
;
1314 /* VLAN MACLEN IPLEN */
1315 vlan_macip_lens
= l4
.hdr
- ip
.hdr
;
1316 vlan_macip_lens
|= (ip
.hdr
- skb
->data
) << IGC_ADVTXD_MACLEN_SHIFT
;
1317 vlan_macip_lens
|= first
->tx_flags
& IGC_TX_FLAGS_VLAN_MASK
;
1319 igc_tx_ctxtdesc(tx_ring
, first
, vlan_macip_lens
,
1320 type_tucmd
, mss_l4len_idx
);
1325 static netdev_tx_t
igc_xmit_frame_ring(struct sk_buff
*skb
,
1326 struct igc_ring
*tx_ring
)
1328 u16 count
= TXD_USE_COUNT(skb_headlen(skb
));
1329 __be16 protocol
= vlan_get_protocol(skb
);
1330 struct igc_tx_buffer
*first
;
1336 /* need: 1 descriptor per page * PAGE_SIZE/IGC_MAX_DATA_PER_TXD,
1337 * + 1 desc for skb_headlen/IGC_MAX_DATA_PER_TXD,
1338 * + 2 desc gap to keep tail from touching head,
1339 * + 1 desc for context descriptor,
1340 * otherwise try next time
1342 for (f
= 0; f
< skb_shinfo(skb
)->nr_frags
; f
++)
1343 count
+= TXD_USE_COUNT(skb_frag_size(
1344 &skb_shinfo(skb
)->frags
[f
]));
1346 if (igc_maybe_stop_tx(tx_ring
, count
+ 3)) {
1347 /* this is a hard error */
1348 return NETDEV_TX_BUSY
;
1351 /* record the location of the first descriptor for this packet */
1352 first
= &tx_ring
->tx_buffer_info
[tx_ring
->next_to_use
];
1354 first
->bytecount
= skb
->len
;
1355 first
->gso_segs
= 1;
1357 if (unlikely(skb_shinfo(skb
)->tx_flags
& SKBTX_HW_TSTAMP
)) {
1358 struct igc_adapter
*adapter
= netdev_priv(tx_ring
->netdev
);
1360 /* FIXME: add support for retrieving timestamps from
1361 * the other timer registers before skipping the
1362 * timestamping request.
1364 if (adapter
->tstamp_config
.tx_type
== HWTSTAMP_TX_ON
&&
1365 !test_and_set_bit_lock(__IGC_PTP_TX_IN_PROGRESS
,
1367 skb_shinfo(skb
)->tx_flags
|= SKBTX_IN_PROGRESS
;
1368 tx_flags
|= IGC_TX_FLAGS_TSTAMP
;
1370 adapter
->ptp_tx_skb
= skb_get(skb
);
1371 adapter
->ptp_tx_start
= jiffies
;
1373 adapter
->tx_hwtstamp_skipped
++;
1377 /* record initial flags and protocol */
1378 first
->tx_flags
= tx_flags
;
1379 first
->protocol
= protocol
;
1381 tso
= igc_tso(tx_ring
, first
, &hdr_len
);
1385 igc_tx_csum(tx_ring
, first
);
1387 igc_tx_map(tx_ring
, first
, hdr_len
);
1389 return NETDEV_TX_OK
;
1392 dev_kfree_skb_any(first
->skb
);
1395 return NETDEV_TX_OK
;
1398 static inline struct igc_ring
*igc_tx_queue_mapping(struct igc_adapter
*adapter
,
1399 struct sk_buff
*skb
)
1401 unsigned int r_idx
= skb
->queue_mapping
;
1403 if (r_idx
>= adapter
->num_tx_queues
)
1404 r_idx
= r_idx
% adapter
->num_tx_queues
;
1406 return adapter
->tx_ring
[r_idx
];
1409 static netdev_tx_t
igc_xmit_frame(struct sk_buff
*skb
,
1410 struct net_device
*netdev
)
1412 struct igc_adapter
*adapter
= netdev_priv(netdev
);
1414 /* The minimum packet size with TCTL.PSP set is 17 so pad the skb
1415 * in order to meet this minimum size requirement.
1417 if (skb
->len
< 17) {
1418 if (skb_padto(skb
, 17))
1419 return NETDEV_TX_OK
;
1423 return igc_xmit_frame_ring(skb
, igc_tx_queue_mapping(adapter
, skb
));
1426 static void igc_rx_checksum(struct igc_ring
*ring
,
1427 union igc_adv_rx_desc
*rx_desc
,
1428 struct sk_buff
*skb
)
1430 skb_checksum_none_assert(skb
);
1432 /* Ignore Checksum bit is set */
1433 if (igc_test_staterr(rx_desc
, IGC_RXD_STAT_IXSM
))
1436 /* Rx checksum disabled via ethtool */
1437 if (!(ring
->netdev
->features
& NETIF_F_RXCSUM
))
1440 /* TCP/UDP checksum error bit is set */
1441 if (igc_test_staterr(rx_desc
,
1442 IGC_RXDEXT_STATERR_TCPE
|
1443 IGC_RXDEXT_STATERR_IPE
)) {
1444 /* work around errata with sctp packets where the TCPE aka
1445 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
1446 * packets (aka let the stack check the crc32c)
1448 if (!(skb
->len
== 60 &&
1449 test_bit(IGC_RING_FLAG_RX_SCTP_CSUM
, &ring
->flags
))) {
1450 u64_stats_update_begin(&ring
->rx_syncp
);
1451 ring
->rx_stats
.csum_err
++;
1452 u64_stats_update_end(&ring
->rx_syncp
);
1454 /* let the stack verify checksum errors */
1457 /* It must be a TCP or UDP packet with a valid checksum */
1458 if (igc_test_staterr(rx_desc
, IGC_RXD_STAT_TCPCS
|
1459 IGC_RXD_STAT_UDPCS
))
1460 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
1462 netdev_dbg(ring
->netdev
, "cksum success: bits %08X\n",
1463 le32_to_cpu(rx_desc
->wb
.upper
.status_error
));
1466 static inline void igc_rx_hash(struct igc_ring
*ring
,
1467 union igc_adv_rx_desc
*rx_desc
,
1468 struct sk_buff
*skb
)
1470 if (ring
->netdev
->features
& NETIF_F_RXHASH
)
1472 le32_to_cpu(rx_desc
->wb
.lower
.hi_dword
.rss
),
1477 * igc_process_skb_fields - Populate skb header fields from Rx descriptor
1478 * @rx_ring: rx descriptor ring packet is being transacted on
1479 * @rx_desc: pointer to the EOP Rx descriptor
1480 * @skb: pointer to current skb being populated
1482 * This function checks the ring, descriptor, and packet information in
1483 * order to populate the hash, checksum, VLAN, timestamp, protocol, and
1484 * other fields within the skb.
1486 static void igc_process_skb_fields(struct igc_ring
*rx_ring
,
1487 union igc_adv_rx_desc
*rx_desc
,
1488 struct sk_buff
*skb
)
1490 igc_rx_hash(rx_ring
, rx_desc
, skb
);
1492 igc_rx_checksum(rx_ring
, rx_desc
, skb
);
1494 if (igc_test_staterr(rx_desc
, IGC_RXDADV_STAT_TS
) &&
1495 !igc_test_staterr(rx_desc
, IGC_RXDADV_STAT_TSIP
))
1496 igc_ptp_rx_rgtstamp(rx_ring
->q_vector
, skb
);
1498 skb_record_rx_queue(skb
, rx_ring
->queue_index
);
1500 skb
->protocol
= eth_type_trans(skb
, rx_ring
->netdev
);
1503 static struct igc_rx_buffer
*igc_get_rx_buffer(struct igc_ring
*rx_ring
,
1504 const unsigned int size
)
1506 struct igc_rx_buffer
*rx_buffer
;
1508 rx_buffer
= &rx_ring
->rx_buffer_info
[rx_ring
->next_to_clean
];
1509 prefetchw(rx_buffer
->page
);
1511 /* we are reusing so sync this buffer for CPU use */
1512 dma_sync_single_range_for_cpu(rx_ring
->dev
,
1514 rx_buffer
->page_offset
,
1518 rx_buffer
->pagecnt_bias
--;
1524 * igc_add_rx_frag - Add contents of Rx buffer to sk_buff
1525 * @rx_ring: rx descriptor ring to transact packets on
1526 * @rx_buffer: buffer containing page to add
1527 * @skb: sk_buff to place the data into
1528 * @size: size of buffer to be added
1530 * This function will add the data contained in rx_buffer->page to the skb.
1532 static void igc_add_rx_frag(struct igc_ring
*rx_ring
,
1533 struct igc_rx_buffer
*rx_buffer
,
1534 struct sk_buff
*skb
,
1537 #if (PAGE_SIZE < 8192)
1538 unsigned int truesize
= igc_rx_pg_size(rx_ring
) / 2;
1540 skb_add_rx_frag(skb
, skb_shinfo(skb
)->nr_frags
, rx_buffer
->page
,
1541 rx_buffer
->page_offset
, size
, truesize
);
1542 rx_buffer
->page_offset
^= truesize
;
1544 unsigned int truesize
= ring_uses_build_skb(rx_ring
) ?
1545 SKB_DATA_ALIGN(IGC_SKB_PAD
+ size
) :
1546 SKB_DATA_ALIGN(size
);
1547 skb_add_rx_frag(skb
, skb_shinfo(skb
)->nr_frags
, rx_buffer
->page
,
1548 rx_buffer
->page_offset
, size
, truesize
);
1549 rx_buffer
->page_offset
+= truesize
;
1553 static struct sk_buff
*igc_build_skb(struct igc_ring
*rx_ring
,
1554 struct igc_rx_buffer
*rx_buffer
,
1555 union igc_adv_rx_desc
*rx_desc
,
1558 void *va
= page_address(rx_buffer
->page
) + rx_buffer
->page_offset
;
1559 #if (PAGE_SIZE < 8192)
1560 unsigned int truesize
= igc_rx_pg_size(rx_ring
) / 2;
1562 unsigned int truesize
= SKB_DATA_ALIGN(sizeof(struct skb_shared_info
)) +
1563 SKB_DATA_ALIGN(IGC_SKB_PAD
+ size
);
1565 struct sk_buff
*skb
;
1567 /* prefetch first cache line of first page */
1569 #if L1_CACHE_BYTES < 128
1570 prefetch(va
+ L1_CACHE_BYTES
);
1573 /* build an skb around the page buffer */
1574 skb
= build_skb(va
- IGC_SKB_PAD
, truesize
);
1578 /* update pointers within the skb to store the data */
1579 skb_reserve(skb
, IGC_SKB_PAD
);
1580 __skb_put(skb
, size
);
1582 /* update buffer offset */
1583 #if (PAGE_SIZE < 8192)
1584 rx_buffer
->page_offset
^= truesize
;
1586 rx_buffer
->page_offset
+= truesize
;
1592 static struct sk_buff
*igc_construct_skb(struct igc_ring
*rx_ring
,
1593 struct igc_rx_buffer
*rx_buffer
,
1594 union igc_adv_rx_desc
*rx_desc
,
1597 void *va
= page_address(rx_buffer
->page
) + rx_buffer
->page_offset
;
1598 #if (PAGE_SIZE < 8192)
1599 unsigned int truesize
= igc_rx_pg_size(rx_ring
) / 2;
1601 unsigned int truesize
= SKB_DATA_ALIGN(size
);
1603 unsigned int headlen
;
1604 struct sk_buff
*skb
;
1606 /* prefetch first cache line of first page */
1608 #if L1_CACHE_BYTES < 128
1609 prefetch(va
+ L1_CACHE_BYTES
);
1612 /* allocate a skb to store the frags */
1613 skb
= napi_alloc_skb(&rx_ring
->q_vector
->napi
, IGC_RX_HDR_LEN
);
1617 if (unlikely(igc_test_staterr(rx_desc
, IGC_RXDADV_STAT_TSIP
))) {
1618 igc_ptp_rx_pktstamp(rx_ring
->q_vector
, va
, skb
);
1619 va
+= IGC_TS_HDR_LEN
;
1620 size
-= IGC_TS_HDR_LEN
;
1623 /* Determine available headroom for copy */
1625 if (headlen
> IGC_RX_HDR_LEN
)
1626 headlen
= eth_get_headlen(skb
->dev
, va
, IGC_RX_HDR_LEN
);
1628 /* align pull length to size of long to optimize memcpy performance */
1629 memcpy(__skb_put(skb
, headlen
), va
, ALIGN(headlen
, sizeof(long)));
1631 /* update all of the pointers */
1634 skb_add_rx_frag(skb
, 0, rx_buffer
->page
,
1635 (va
+ headlen
) - page_address(rx_buffer
->page
),
1637 #if (PAGE_SIZE < 8192)
1638 rx_buffer
->page_offset
^= truesize
;
1640 rx_buffer
->page_offset
+= truesize
;
1643 rx_buffer
->pagecnt_bias
++;
1650 * igc_reuse_rx_page - page flip buffer and store it back on the ring
1651 * @rx_ring: rx descriptor ring to store buffers on
1652 * @old_buff: donor buffer to have page reused
1654 * Synchronizes page for reuse by the adapter
1656 static void igc_reuse_rx_page(struct igc_ring
*rx_ring
,
1657 struct igc_rx_buffer
*old_buff
)
1659 u16 nta
= rx_ring
->next_to_alloc
;
1660 struct igc_rx_buffer
*new_buff
;
1662 new_buff
= &rx_ring
->rx_buffer_info
[nta
];
1664 /* update, and store next to alloc */
1666 rx_ring
->next_to_alloc
= (nta
< rx_ring
->count
) ? nta
: 0;
1668 /* Transfer page from old buffer to new buffer.
1669 * Move each member individually to avoid possible store
1670 * forwarding stalls.
1672 new_buff
->dma
= old_buff
->dma
;
1673 new_buff
->page
= old_buff
->page
;
1674 new_buff
->page_offset
= old_buff
->page_offset
;
1675 new_buff
->pagecnt_bias
= old_buff
->pagecnt_bias
;
1678 static inline bool igc_page_is_reserved(struct page
*page
)
1680 return (page_to_nid(page
) != numa_mem_id()) || page_is_pfmemalloc(page
);
1683 static bool igc_can_reuse_rx_page(struct igc_rx_buffer
*rx_buffer
)
1685 unsigned int pagecnt_bias
= rx_buffer
->pagecnt_bias
;
1686 struct page
*page
= rx_buffer
->page
;
1688 /* avoid re-using remote pages */
1689 if (unlikely(igc_page_is_reserved(page
)))
1692 #if (PAGE_SIZE < 8192)
1693 /* if we are only owner of page we can reuse it */
1694 if (unlikely((page_ref_count(page
) - pagecnt_bias
) > 1))
1697 #define IGC_LAST_OFFSET \
1698 (SKB_WITH_OVERHEAD(PAGE_SIZE) - IGC_RXBUFFER_2048)
1700 if (rx_buffer
->page_offset
> IGC_LAST_OFFSET
)
1704 /* If we have drained the page fragment pool we need to update
1705 * the pagecnt_bias and page count so that we fully restock the
1706 * number of references the driver holds.
1708 if (unlikely(!pagecnt_bias
)) {
1709 page_ref_add(page
, USHRT_MAX
);
1710 rx_buffer
->pagecnt_bias
= USHRT_MAX
;
1717 * igc_is_non_eop - process handling of non-EOP buffers
1718 * @rx_ring: Rx ring being processed
1719 * @rx_desc: Rx descriptor for current buffer
1721 * This function updates next to clean. If the buffer is an EOP buffer
1722 * this function exits returning false, otherwise it will place the
1723 * sk_buff in the next buffer to be chained and return true indicating
1724 * that this is in fact a non-EOP buffer.
1726 static bool igc_is_non_eop(struct igc_ring
*rx_ring
,
1727 union igc_adv_rx_desc
*rx_desc
)
1729 u32 ntc
= rx_ring
->next_to_clean
+ 1;
1731 /* fetch, update, and store next to clean */
1732 ntc
= (ntc
< rx_ring
->count
) ? ntc
: 0;
1733 rx_ring
->next_to_clean
= ntc
;
1735 prefetch(IGC_RX_DESC(rx_ring
, ntc
));
1737 if (likely(igc_test_staterr(rx_desc
, IGC_RXD_STAT_EOP
)))
1744 * igc_cleanup_headers - Correct corrupted or empty headers
1745 * @rx_ring: rx descriptor ring packet is being transacted on
1746 * @rx_desc: pointer to the EOP Rx descriptor
1747 * @skb: pointer to current skb being fixed
1749 * Address the case where we are pulling data in on pages only
1750 * and as such no data is present in the skb header.
1752 * In addition if skb is not at least 60 bytes we need to pad it so that
1753 * it is large enough to qualify as a valid Ethernet frame.
1755 * Returns true if an error was encountered and skb was freed.
1757 static bool igc_cleanup_headers(struct igc_ring
*rx_ring
,
1758 union igc_adv_rx_desc
*rx_desc
,
1759 struct sk_buff
*skb
)
1761 if (unlikely((igc_test_staterr(rx_desc
,
1762 IGC_RXDEXT_ERR_FRAME_ERR_MASK
)))) {
1763 struct net_device
*netdev
= rx_ring
->netdev
;
1765 if (!(netdev
->features
& NETIF_F_RXALL
)) {
1766 dev_kfree_skb_any(skb
);
1771 /* if eth_skb_pad returns an error the skb was freed */
1772 if (eth_skb_pad(skb
))
1778 static void igc_put_rx_buffer(struct igc_ring
*rx_ring
,
1779 struct igc_rx_buffer
*rx_buffer
)
1781 if (igc_can_reuse_rx_page(rx_buffer
)) {
1782 /* hand second half of page back to the ring */
1783 igc_reuse_rx_page(rx_ring
, rx_buffer
);
1785 /* We are not reusing the buffer so unmap it and free
1786 * any references we are holding to it
1788 dma_unmap_page_attrs(rx_ring
->dev
, rx_buffer
->dma
,
1789 igc_rx_pg_size(rx_ring
), DMA_FROM_DEVICE
,
1791 __page_frag_cache_drain(rx_buffer
->page
,
1792 rx_buffer
->pagecnt_bias
);
1795 /* clear contents of rx_buffer */
1796 rx_buffer
->page
= NULL
;
1799 static inline unsigned int igc_rx_offset(struct igc_ring
*rx_ring
)
1801 return ring_uses_build_skb(rx_ring
) ? IGC_SKB_PAD
: 0;
1804 static bool igc_alloc_mapped_page(struct igc_ring
*rx_ring
,
1805 struct igc_rx_buffer
*bi
)
1807 struct page
*page
= bi
->page
;
1810 /* since we are recycling buffers we should seldom need to alloc */
1814 /* alloc new page for storage */
1815 page
= dev_alloc_pages(igc_rx_pg_order(rx_ring
));
1816 if (unlikely(!page
)) {
1817 rx_ring
->rx_stats
.alloc_failed
++;
1821 /* map page for use */
1822 dma
= dma_map_page_attrs(rx_ring
->dev
, page
, 0,
1823 igc_rx_pg_size(rx_ring
),
1827 /* if mapping failed free memory back to system since
1828 * there isn't much point in holding memory we can't use
1830 if (dma_mapping_error(rx_ring
->dev
, dma
)) {
1833 rx_ring
->rx_stats
.alloc_failed
++;
1839 bi
->page_offset
= igc_rx_offset(rx_ring
);
1840 bi
->pagecnt_bias
= 1;
1846 * igc_alloc_rx_buffers - Replace used receive buffers; packet split
1847 * @rx_ring: rx descriptor ring
1848 * @cleaned_count: number of buffers to clean
1850 static void igc_alloc_rx_buffers(struct igc_ring
*rx_ring
, u16 cleaned_count
)
1852 union igc_adv_rx_desc
*rx_desc
;
1853 u16 i
= rx_ring
->next_to_use
;
1854 struct igc_rx_buffer
*bi
;
1861 rx_desc
= IGC_RX_DESC(rx_ring
, i
);
1862 bi
= &rx_ring
->rx_buffer_info
[i
];
1863 i
-= rx_ring
->count
;
1865 bufsz
= igc_rx_bufsz(rx_ring
);
1868 if (!igc_alloc_mapped_page(rx_ring
, bi
))
1871 /* sync the buffer for use by the device */
1872 dma_sync_single_range_for_device(rx_ring
->dev
, bi
->dma
,
1873 bi
->page_offset
, bufsz
,
1876 /* Refresh the desc even if buffer_addrs didn't change
1877 * because each write-back erases this info.
1879 rx_desc
->read
.pkt_addr
= cpu_to_le64(bi
->dma
+ bi
->page_offset
);
1885 rx_desc
= IGC_RX_DESC(rx_ring
, 0);
1886 bi
= rx_ring
->rx_buffer_info
;
1887 i
-= rx_ring
->count
;
1890 /* clear the length for the next_to_use descriptor */
1891 rx_desc
->wb
.upper
.length
= 0;
1894 } while (cleaned_count
);
1896 i
+= rx_ring
->count
;
1898 if (rx_ring
->next_to_use
!= i
) {
1899 /* record the next descriptor to use */
1900 rx_ring
->next_to_use
= i
;
1902 /* update next to alloc since we have filled the ring */
1903 rx_ring
->next_to_alloc
= i
;
1905 /* Force memory writes to complete before letting h/w
1906 * know there are new descriptors to fetch. (Only
1907 * applicable for weak-ordered memory model archs,
1911 writel(i
, rx_ring
->tail
);
1915 static int igc_clean_rx_irq(struct igc_q_vector
*q_vector
, const int budget
)
1917 unsigned int total_bytes
= 0, total_packets
= 0;
1918 struct igc_ring
*rx_ring
= q_vector
->rx
.ring
;
1919 struct sk_buff
*skb
= rx_ring
->skb
;
1920 u16 cleaned_count
= igc_desc_unused(rx_ring
);
1922 while (likely(total_packets
< budget
)) {
1923 union igc_adv_rx_desc
*rx_desc
;
1924 struct igc_rx_buffer
*rx_buffer
;
1927 /* return some buffers to hardware, one at a time is too slow */
1928 if (cleaned_count
>= IGC_RX_BUFFER_WRITE
) {
1929 igc_alloc_rx_buffers(rx_ring
, cleaned_count
);
1933 rx_desc
= IGC_RX_DESC(rx_ring
, rx_ring
->next_to_clean
);
1934 size
= le16_to_cpu(rx_desc
->wb
.upper
.length
);
1938 /* This memory barrier is needed to keep us from reading
1939 * any other fields out of the rx_desc until we know the
1940 * descriptor has been written back
1944 rx_buffer
= igc_get_rx_buffer(rx_ring
, size
);
1946 /* retrieve a buffer from the ring */
1948 igc_add_rx_frag(rx_ring
, rx_buffer
, skb
, size
);
1949 else if (ring_uses_build_skb(rx_ring
))
1950 skb
= igc_build_skb(rx_ring
, rx_buffer
, rx_desc
, size
);
1952 skb
= igc_construct_skb(rx_ring
, rx_buffer
,
1955 /* exit if we failed to retrieve a buffer */
1957 rx_ring
->rx_stats
.alloc_failed
++;
1958 rx_buffer
->pagecnt_bias
++;
1962 igc_put_rx_buffer(rx_ring
, rx_buffer
);
1965 /* fetch next buffer in frame if non-eop */
1966 if (igc_is_non_eop(rx_ring
, rx_desc
))
1969 /* verify the packet layout is correct */
1970 if (igc_cleanup_headers(rx_ring
, rx_desc
, skb
)) {
1975 /* probably a little skewed due to removing CRC */
1976 total_bytes
+= skb
->len
;
1978 /* populate checksum, timestamp, VLAN, and protocol */
1979 igc_process_skb_fields(rx_ring
, rx_desc
, skb
);
1981 napi_gro_receive(&q_vector
->napi
, skb
);
1983 /* reset skb pointer */
1986 /* update budget accounting */
1990 /* place incomplete frames back on ring for completion */
1993 u64_stats_update_begin(&rx_ring
->rx_syncp
);
1994 rx_ring
->rx_stats
.packets
+= total_packets
;
1995 rx_ring
->rx_stats
.bytes
+= total_bytes
;
1996 u64_stats_update_end(&rx_ring
->rx_syncp
);
1997 q_vector
->rx
.total_packets
+= total_packets
;
1998 q_vector
->rx
.total_bytes
+= total_bytes
;
2001 igc_alloc_rx_buffers(rx_ring
, cleaned_count
);
2003 return total_packets
;
2007 * igc_clean_tx_irq - Reclaim resources after transmit completes
2008 * @q_vector: pointer to q_vector containing needed info
2009 * @napi_budget: Used to determine if we are in netpoll
2011 * returns true if ring is completely cleaned
2013 static bool igc_clean_tx_irq(struct igc_q_vector
*q_vector
, int napi_budget
)
2015 struct igc_adapter
*adapter
= q_vector
->adapter
;
2016 unsigned int total_bytes
= 0, total_packets
= 0;
2017 unsigned int budget
= q_vector
->tx
.work_limit
;
2018 struct igc_ring
*tx_ring
= q_vector
->tx
.ring
;
2019 unsigned int i
= tx_ring
->next_to_clean
;
2020 struct igc_tx_buffer
*tx_buffer
;
2021 union igc_adv_tx_desc
*tx_desc
;
2023 if (test_bit(__IGC_DOWN
, &adapter
->state
))
2026 tx_buffer
= &tx_ring
->tx_buffer_info
[i
];
2027 tx_desc
= IGC_TX_DESC(tx_ring
, i
);
2028 i
-= tx_ring
->count
;
2031 union igc_adv_tx_desc
*eop_desc
= tx_buffer
->next_to_watch
;
2033 /* if next_to_watch is not set then there is no work pending */
2037 /* prevent any other reads prior to eop_desc */
2040 /* if DD is not set pending work has not been completed */
2041 if (!(eop_desc
->wb
.status
& cpu_to_le32(IGC_TXD_STAT_DD
)))
2044 /* clear next_to_watch to prevent false hangs */
2045 tx_buffer
->next_to_watch
= NULL
;
2047 /* update the statistics for this packet */
2048 total_bytes
+= tx_buffer
->bytecount
;
2049 total_packets
+= tx_buffer
->gso_segs
;
2052 napi_consume_skb(tx_buffer
->skb
, napi_budget
);
2054 /* unmap skb header data */
2055 dma_unmap_single(tx_ring
->dev
,
2056 dma_unmap_addr(tx_buffer
, dma
),
2057 dma_unmap_len(tx_buffer
, len
),
2060 /* clear tx_buffer data */
2061 dma_unmap_len_set(tx_buffer
, len
, 0);
2063 /* clear last DMA location and unmap remaining buffers */
2064 while (tx_desc
!= eop_desc
) {
2069 i
-= tx_ring
->count
;
2070 tx_buffer
= tx_ring
->tx_buffer_info
;
2071 tx_desc
= IGC_TX_DESC(tx_ring
, 0);
2074 /* unmap any remaining paged data */
2075 if (dma_unmap_len(tx_buffer
, len
)) {
2076 dma_unmap_page(tx_ring
->dev
,
2077 dma_unmap_addr(tx_buffer
, dma
),
2078 dma_unmap_len(tx_buffer
, len
),
2080 dma_unmap_len_set(tx_buffer
, len
, 0);
2084 /* move us one more past the eop_desc for start of next pkt */
2089 i
-= tx_ring
->count
;
2090 tx_buffer
= tx_ring
->tx_buffer_info
;
2091 tx_desc
= IGC_TX_DESC(tx_ring
, 0);
2094 /* issue prefetch for next Tx descriptor */
2097 /* update budget accounting */
2099 } while (likely(budget
));
2101 netdev_tx_completed_queue(txring_txq(tx_ring
),
2102 total_packets
, total_bytes
);
2104 i
+= tx_ring
->count
;
2105 tx_ring
->next_to_clean
= i
;
2106 u64_stats_update_begin(&tx_ring
->tx_syncp
);
2107 tx_ring
->tx_stats
.bytes
+= total_bytes
;
2108 tx_ring
->tx_stats
.packets
+= total_packets
;
2109 u64_stats_update_end(&tx_ring
->tx_syncp
);
2110 q_vector
->tx
.total_bytes
+= total_bytes
;
2111 q_vector
->tx
.total_packets
+= total_packets
;
2113 if (test_bit(IGC_RING_FLAG_TX_DETECT_HANG
, &tx_ring
->flags
)) {
2114 struct igc_hw
*hw
= &adapter
->hw
;
2116 /* Detect a transmit hang in hardware, this serializes the
2117 * check with the clearing of time_stamp and movement of i
2119 clear_bit(IGC_RING_FLAG_TX_DETECT_HANG
, &tx_ring
->flags
);
2120 if (tx_buffer
->next_to_watch
&&
2121 time_after(jiffies
, tx_buffer
->time_stamp
+
2122 (adapter
->tx_timeout_factor
* HZ
)) &&
2123 !(rd32(IGC_STATUS
) & IGC_STATUS_TXOFF
)) {
2124 /* detected Tx unit hang */
2125 netdev_err(tx_ring
->netdev
,
2126 "Detected Tx Unit Hang\n"
2130 " next_to_use <%x>\n"
2131 " next_to_clean <%x>\n"
2132 "buffer_info[next_to_clean]\n"
2133 " time_stamp <%lx>\n"
2134 " next_to_watch <%p>\n"
2136 " desc.status <%x>\n",
2137 tx_ring
->queue_index
,
2138 rd32(IGC_TDH(tx_ring
->reg_idx
)),
2139 readl(tx_ring
->tail
),
2140 tx_ring
->next_to_use
,
2141 tx_ring
->next_to_clean
,
2142 tx_buffer
->time_stamp
,
2143 tx_buffer
->next_to_watch
,
2145 tx_buffer
->next_to_watch
->wb
.status
);
2146 netif_stop_subqueue(tx_ring
->netdev
,
2147 tx_ring
->queue_index
);
2149 /* we are about to reset, no point in enabling stuff */
2154 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
2155 if (unlikely(total_packets
&&
2156 netif_carrier_ok(tx_ring
->netdev
) &&
2157 igc_desc_unused(tx_ring
) >= TX_WAKE_THRESHOLD
)) {
2158 /* Make sure that anybody stopping the queue after this
2159 * sees the new next_to_clean.
2162 if (__netif_subqueue_stopped(tx_ring
->netdev
,
2163 tx_ring
->queue_index
) &&
2164 !(test_bit(__IGC_DOWN
, &adapter
->state
))) {
2165 netif_wake_subqueue(tx_ring
->netdev
,
2166 tx_ring
->queue_index
);
2168 u64_stats_update_begin(&tx_ring
->tx_syncp
);
2169 tx_ring
->tx_stats
.restart_queue
++;
2170 u64_stats_update_end(&tx_ring
->tx_syncp
);
2177 static int igc_find_mac_filter(struct igc_adapter
*adapter
,
2178 enum igc_mac_filter_type type
, const u8
*addr
)
2180 struct igc_hw
*hw
= &adapter
->hw
;
2181 int max_entries
= hw
->mac
.rar_entry_count
;
2185 for (i
= 0; i
< max_entries
; i
++) {
2186 ral
= rd32(IGC_RAL(i
));
2187 rah
= rd32(IGC_RAH(i
));
2189 if (!(rah
& IGC_RAH_AV
))
2191 if (!!(rah
& IGC_RAH_ASEL_SRC_ADDR
) != type
)
2193 if ((rah
& IGC_RAH_RAH_MASK
) !=
2194 le16_to_cpup((__le16
*)(addr
+ 4)))
2196 if (ral
!= le32_to_cpup((__le32
*)(addr
)))
2205 static int igc_get_avail_mac_filter_slot(struct igc_adapter
*adapter
)
2207 struct igc_hw
*hw
= &adapter
->hw
;
2208 int max_entries
= hw
->mac
.rar_entry_count
;
2212 for (i
= 0; i
< max_entries
; i
++) {
2213 rah
= rd32(IGC_RAH(i
));
2215 if (!(rah
& IGC_RAH_AV
))
2223 * igc_add_mac_filter() - Add MAC address filter
2224 * @adapter: Pointer to adapter where the filter should be added
2225 * @type: MAC address filter type (source or destination)
2226 * @addr: MAC address
2227 * @queue: If non-negative, queue assignment feature is enabled and frames
2228 * matching the filter are enqueued onto 'queue'. Otherwise, queue
2229 * assignment is disabled.
2231 * Return: 0 in case of success, negative errno code otherwise.
2233 static int igc_add_mac_filter(struct igc_adapter
*adapter
,
2234 enum igc_mac_filter_type type
, const u8
*addr
,
2237 struct net_device
*dev
= adapter
->netdev
;
2240 index
= igc_find_mac_filter(adapter
, type
, addr
);
2244 index
= igc_get_avail_mac_filter_slot(adapter
);
2248 netdev_dbg(dev
, "Add MAC address filter: index %d type %s address %pM queue %d\n",
2249 index
, type
== IGC_MAC_FILTER_TYPE_DST
? "dst" : "src",
2253 igc_set_mac_filter_hw(adapter
, index
, type
, addr
, queue
);
2258 * igc_del_mac_filter() - Delete MAC address filter
2259 * @adapter: Pointer to adapter where the filter should be deleted from
2260 * @type: MAC address filter type (source or destination)
2261 * @addr: MAC address
2263 static void igc_del_mac_filter(struct igc_adapter
*adapter
,
2264 enum igc_mac_filter_type type
, const u8
*addr
)
2266 struct net_device
*dev
= adapter
->netdev
;
2269 index
= igc_find_mac_filter(adapter
, type
, addr
);
2274 /* If this is the default filter, we don't actually delete it.
2275 * We just reset to its default value i.e. disable queue
2278 netdev_dbg(dev
, "Disable default MAC filter queue assignment");
2280 igc_set_mac_filter_hw(adapter
, 0, type
, addr
, -1);
2282 netdev_dbg(dev
, "Delete MAC address filter: index %d type %s address %pM\n",
2284 type
== IGC_MAC_FILTER_TYPE_DST
? "dst" : "src",
2287 igc_clear_mac_filter_hw(adapter
, index
);
2292 * igc_add_vlan_prio_filter() - Add VLAN priority filter
2293 * @adapter: Pointer to adapter where the filter should be added
2294 * @prio: VLAN priority value
2295 * @queue: Queue number which matching frames are assigned to
2297 * Return: 0 in case of success, negative errno code otherwise.
2299 static int igc_add_vlan_prio_filter(struct igc_adapter
*adapter
, int prio
,
2302 struct net_device
*dev
= adapter
->netdev
;
2303 struct igc_hw
*hw
= &adapter
->hw
;
2306 vlanpqf
= rd32(IGC_VLANPQF
);
2308 if (vlanpqf
& IGC_VLANPQF_VALID(prio
)) {
2309 netdev_dbg(dev
, "VLAN priority filter already in use\n");
2313 vlanpqf
|= IGC_VLANPQF_QSEL(prio
, queue
);
2314 vlanpqf
|= IGC_VLANPQF_VALID(prio
);
2316 wr32(IGC_VLANPQF
, vlanpqf
);
2318 netdev_dbg(dev
, "Add VLAN priority filter: prio %d queue %d\n",
2324 * igc_del_vlan_prio_filter() - Delete VLAN priority filter
2325 * @adapter: Pointer to adapter where the filter should be deleted from
2326 * @prio: VLAN priority value
2328 static void igc_del_vlan_prio_filter(struct igc_adapter
*adapter
, int prio
)
2330 struct igc_hw
*hw
= &adapter
->hw
;
2333 vlanpqf
= rd32(IGC_VLANPQF
);
2335 vlanpqf
&= ~IGC_VLANPQF_VALID(prio
);
2336 vlanpqf
&= ~IGC_VLANPQF_QSEL(prio
, IGC_VLANPQF_QUEUE_MASK
);
2338 wr32(IGC_VLANPQF
, vlanpqf
);
2340 netdev_dbg(adapter
->netdev
, "Delete VLAN priority filter: prio %d\n",
2344 static int igc_get_avail_etype_filter_slot(struct igc_adapter
*adapter
)
2346 struct igc_hw
*hw
= &adapter
->hw
;
2349 for (i
= 0; i
< MAX_ETYPE_FILTER
; i
++) {
2350 u32 etqf
= rd32(IGC_ETQF(i
));
2352 if (!(etqf
& IGC_ETQF_FILTER_ENABLE
))
2360 * igc_add_etype_filter() - Add ethertype filter
2361 * @adapter: Pointer to adapter where the filter should be added
2362 * @etype: Ethertype value
2363 * @queue: If non-negative, queue assignment feature is enabled and frames
2364 * matching the filter are enqueued onto 'queue'. Otherwise, queue
2365 * assignment is disabled.
2367 * Return: 0 in case of success, negative errno code otherwise.
2369 static int igc_add_etype_filter(struct igc_adapter
*adapter
, u16 etype
,
2372 struct igc_hw
*hw
= &adapter
->hw
;
2376 index
= igc_get_avail_etype_filter_slot(adapter
);
2380 etqf
= rd32(IGC_ETQF(index
));
2382 etqf
&= ~IGC_ETQF_ETYPE_MASK
;
2386 etqf
&= ~IGC_ETQF_QUEUE_MASK
;
2387 etqf
|= (queue
<< IGC_ETQF_QUEUE_SHIFT
);
2388 etqf
|= IGC_ETQF_QUEUE_ENABLE
;
2391 etqf
|= IGC_ETQF_FILTER_ENABLE
;
2393 wr32(IGC_ETQF(index
), etqf
);
2395 netdev_dbg(adapter
->netdev
, "Add ethertype filter: etype %04x queue %d\n",
2400 static int igc_find_etype_filter(struct igc_adapter
*adapter
, u16 etype
)
2402 struct igc_hw
*hw
= &adapter
->hw
;
2405 for (i
= 0; i
< MAX_ETYPE_FILTER
; i
++) {
2406 u32 etqf
= rd32(IGC_ETQF(i
));
2408 if ((etqf
& IGC_ETQF_ETYPE_MASK
) == etype
)
2416 * igc_del_etype_filter() - Delete ethertype filter
2417 * @adapter: Pointer to adapter where the filter should be deleted from
2418 * @etype: Ethertype value
2420 static void igc_del_etype_filter(struct igc_adapter
*adapter
, u16 etype
)
2422 struct igc_hw
*hw
= &adapter
->hw
;
2425 index
= igc_find_etype_filter(adapter
, etype
);
2429 wr32(IGC_ETQF(index
), 0);
2431 netdev_dbg(adapter
->netdev
, "Delete ethertype filter: etype %04x\n",
2435 static int igc_enable_nfc_rule(struct igc_adapter
*adapter
,
2436 const struct igc_nfc_rule
*rule
)
2440 if (rule
->filter
.match_flags
& IGC_FILTER_FLAG_ETHER_TYPE
) {
2441 err
= igc_add_etype_filter(adapter
, rule
->filter
.etype
,
2447 if (rule
->filter
.match_flags
& IGC_FILTER_FLAG_SRC_MAC_ADDR
) {
2448 err
= igc_add_mac_filter(adapter
, IGC_MAC_FILTER_TYPE_SRC
,
2449 rule
->filter
.src_addr
, rule
->action
);
2454 if (rule
->filter
.match_flags
& IGC_FILTER_FLAG_DST_MAC_ADDR
) {
2455 err
= igc_add_mac_filter(adapter
, IGC_MAC_FILTER_TYPE_DST
,
2456 rule
->filter
.dst_addr
, rule
->action
);
2461 if (rule
->filter
.match_flags
& IGC_FILTER_FLAG_VLAN_TCI
) {
2462 int prio
= (rule
->filter
.vlan_tci
& VLAN_PRIO_MASK
) >>
2465 err
= igc_add_vlan_prio_filter(adapter
, prio
, rule
->action
);
2473 static void igc_disable_nfc_rule(struct igc_adapter
*adapter
,
2474 const struct igc_nfc_rule
*rule
)
2476 if (rule
->filter
.match_flags
& IGC_FILTER_FLAG_ETHER_TYPE
)
2477 igc_del_etype_filter(adapter
, rule
->filter
.etype
);
2479 if (rule
->filter
.match_flags
& IGC_FILTER_FLAG_VLAN_TCI
) {
2480 int prio
= (rule
->filter
.vlan_tci
& VLAN_PRIO_MASK
) >>
2483 igc_del_vlan_prio_filter(adapter
, prio
);
2486 if (rule
->filter
.match_flags
& IGC_FILTER_FLAG_SRC_MAC_ADDR
)
2487 igc_del_mac_filter(adapter
, IGC_MAC_FILTER_TYPE_SRC
,
2488 rule
->filter
.src_addr
);
2490 if (rule
->filter
.match_flags
& IGC_FILTER_FLAG_DST_MAC_ADDR
)
2491 igc_del_mac_filter(adapter
, IGC_MAC_FILTER_TYPE_DST
,
2492 rule
->filter
.dst_addr
);
2496 * igc_get_nfc_rule() - Get NFC rule
2497 * @adapter: Pointer to adapter
2498 * @location: Rule location
2500 * Context: Expects adapter->nfc_rule_lock to be held by caller.
2502 * Return: Pointer to NFC rule at @location. If not found, NULL.
2504 struct igc_nfc_rule
*igc_get_nfc_rule(struct igc_adapter
*adapter
,
2507 struct igc_nfc_rule
*rule
;
2509 list_for_each_entry(rule
, &adapter
->nfc_rule_list
, list
) {
2510 if (rule
->location
== location
)
2512 if (rule
->location
> location
)
2520 * igc_del_nfc_rule() - Delete NFC rule
2521 * @adapter: Pointer to adapter
2522 * @rule: Pointer to rule to be deleted
2524 * Disable NFC rule in hardware and delete it from adapter.
2526 * Context: Expects adapter->nfc_rule_lock to be held by caller.
2528 void igc_del_nfc_rule(struct igc_adapter
*adapter
, struct igc_nfc_rule
*rule
)
2530 igc_disable_nfc_rule(adapter
, rule
);
2532 list_del(&rule
->list
);
2533 adapter
->nfc_rule_count
--;
2538 static void igc_flush_nfc_rules(struct igc_adapter
*adapter
)
2540 struct igc_nfc_rule
*rule
, *tmp
;
2542 mutex_lock(&adapter
->nfc_rule_lock
);
2544 list_for_each_entry_safe(rule
, tmp
, &adapter
->nfc_rule_list
, list
)
2545 igc_del_nfc_rule(adapter
, rule
);
2547 mutex_unlock(&adapter
->nfc_rule_lock
);
2551 * igc_add_nfc_rule() - Add NFC rule
2552 * @adapter: Pointer to adapter
2553 * @rule: Pointer to rule to be added
2555 * Enable NFC rule in hardware and add it to adapter.
2557 * Context: Expects adapter->nfc_rule_lock to be held by caller.
2559 * Return: 0 on success, negative errno on failure.
2561 int igc_add_nfc_rule(struct igc_adapter
*adapter
, struct igc_nfc_rule
*rule
)
2563 struct igc_nfc_rule
*pred
, *cur
;
2566 err
= igc_enable_nfc_rule(adapter
, rule
);
2571 list_for_each_entry(cur
, &adapter
->nfc_rule_list
, list
) {
2572 if (cur
->location
>= rule
->location
)
2577 list_add(&rule
->list
, pred
? &pred
->list
: &adapter
->nfc_rule_list
);
2578 adapter
->nfc_rule_count
++;
2582 static void igc_restore_nfc_rules(struct igc_adapter
*adapter
)
2584 struct igc_nfc_rule
*rule
;
2586 mutex_lock(&adapter
->nfc_rule_lock
);
2588 list_for_each_entry_reverse(rule
, &adapter
->nfc_rule_list
, list
)
2589 igc_enable_nfc_rule(adapter
, rule
);
2591 mutex_unlock(&adapter
->nfc_rule_lock
);
2594 static int igc_uc_sync(struct net_device
*netdev
, const unsigned char *addr
)
2596 struct igc_adapter
*adapter
= netdev_priv(netdev
);
2598 return igc_add_mac_filter(adapter
, IGC_MAC_FILTER_TYPE_DST
, addr
, -1);
2601 static int igc_uc_unsync(struct net_device
*netdev
, const unsigned char *addr
)
2603 struct igc_adapter
*adapter
= netdev_priv(netdev
);
2605 igc_del_mac_filter(adapter
, IGC_MAC_FILTER_TYPE_DST
, addr
);
2610 * igc_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
2611 * @netdev: network interface device structure
2613 * The set_rx_mode entry point is called whenever the unicast or multicast
2614 * address lists or the network interface flags are updated. This routine is
2615 * responsible for configuring the hardware for proper unicast, multicast,
2616 * promiscuous mode, and all-multi behavior.
2618 static void igc_set_rx_mode(struct net_device
*netdev
)
2620 struct igc_adapter
*adapter
= netdev_priv(netdev
);
2621 struct igc_hw
*hw
= &adapter
->hw
;
2622 u32 rctl
= 0, rlpml
= MAX_JUMBO_FRAME_SIZE
;
2625 /* Check for Promiscuous and All Multicast modes */
2626 if (netdev
->flags
& IFF_PROMISC
) {
2627 rctl
|= IGC_RCTL_UPE
| IGC_RCTL_MPE
;
2629 if (netdev
->flags
& IFF_ALLMULTI
) {
2630 rctl
|= IGC_RCTL_MPE
;
2632 /* Write addresses to the MTA, if the attempt fails
2633 * then we should just turn on promiscuous mode so
2634 * that we can at least receive multicast traffic
2636 count
= igc_write_mc_addr_list(netdev
);
2638 rctl
|= IGC_RCTL_MPE
;
2642 /* Write addresses to available RAR registers, if there is not
2643 * sufficient space to store all the addresses then enable
2644 * unicast promiscuous mode
2646 if (__dev_uc_sync(netdev
, igc_uc_sync
, igc_uc_unsync
))
2647 rctl
|= IGC_RCTL_UPE
;
2649 /* update state of unicast and multicast */
2650 rctl
|= rd32(IGC_RCTL
) & ~(IGC_RCTL_UPE
| IGC_RCTL_MPE
);
2651 wr32(IGC_RCTL
, rctl
);
2653 #if (PAGE_SIZE < 8192)
2654 if (adapter
->max_frame_size
<= IGC_MAX_FRAME_BUILD_SKB
)
2655 rlpml
= IGC_MAX_FRAME_BUILD_SKB
;
2657 wr32(IGC_RLPML
, rlpml
);
2661 * igc_configure - configure the hardware for RX and TX
2662 * @adapter: private board structure
2664 static void igc_configure(struct igc_adapter
*adapter
)
2666 struct net_device
*netdev
= adapter
->netdev
;
2669 igc_get_hw_control(adapter
);
2670 igc_set_rx_mode(netdev
);
2672 igc_setup_tctl(adapter
);
2673 igc_setup_mrqc(adapter
);
2674 igc_setup_rctl(adapter
);
2676 igc_set_default_mac_filter(adapter
);
2677 igc_restore_nfc_rules(adapter
);
2679 igc_configure_tx(adapter
);
2680 igc_configure_rx(adapter
);
2682 igc_rx_fifo_flush_base(&adapter
->hw
);
2684 /* call igc_desc_unused which always leaves
2685 * at least 1 descriptor unused to make sure
2686 * next_to_use != next_to_clean
2688 for (i
= 0; i
< adapter
->num_rx_queues
; i
++) {
2689 struct igc_ring
*ring
= adapter
->rx_ring
[i
];
2691 igc_alloc_rx_buffers(ring
, igc_desc_unused(ring
));
2696 * igc_write_ivar - configure ivar for given MSI-X vector
2697 * @hw: pointer to the HW structure
2698 * @msix_vector: vector number we are allocating to a given ring
2699 * @index: row index of IVAR register to write within IVAR table
2700 * @offset: column offset of in IVAR, should be multiple of 8
2702 * The IVAR table consists of 2 columns,
2703 * each containing an cause allocation for an Rx and Tx ring, and a
2704 * variable number of rows depending on the number of queues supported.
2706 static void igc_write_ivar(struct igc_hw
*hw
, int msix_vector
,
2707 int index
, int offset
)
2709 u32 ivar
= array_rd32(IGC_IVAR0
, index
);
2711 /* clear any bits that are currently set */
2712 ivar
&= ~((u32
)0xFF << offset
);
2714 /* write vector and valid bit */
2715 ivar
|= (msix_vector
| IGC_IVAR_VALID
) << offset
;
2717 array_wr32(IGC_IVAR0
, index
, ivar
);
2720 static void igc_assign_vector(struct igc_q_vector
*q_vector
, int msix_vector
)
2722 struct igc_adapter
*adapter
= q_vector
->adapter
;
2723 struct igc_hw
*hw
= &adapter
->hw
;
2724 int rx_queue
= IGC_N0_QUEUE
;
2725 int tx_queue
= IGC_N0_QUEUE
;
2727 if (q_vector
->rx
.ring
)
2728 rx_queue
= q_vector
->rx
.ring
->reg_idx
;
2729 if (q_vector
->tx
.ring
)
2730 tx_queue
= q_vector
->tx
.ring
->reg_idx
;
2732 switch (hw
->mac
.type
) {
2734 if (rx_queue
> IGC_N0_QUEUE
)
2735 igc_write_ivar(hw
, msix_vector
,
2737 (rx_queue
& 0x1) << 4);
2738 if (tx_queue
> IGC_N0_QUEUE
)
2739 igc_write_ivar(hw
, msix_vector
,
2741 ((tx_queue
& 0x1) << 4) + 8);
2742 q_vector
->eims_value
= BIT(msix_vector
);
2745 WARN_ONCE(hw
->mac
.type
!= igc_i225
, "Wrong MAC type\n");
2749 /* add q_vector eims value to global eims_enable_mask */
2750 adapter
->eims_enable_mask
|= q_vector
->eims_value
;
2752 /* configure q_vector to set itr on first interrupt */
2753 q_vector
->set_itr
= 1;
2757 * igc_configure_msix - Configure MSI-X hardware
2758 * @adapter: Pointer to adapter structure
2760 * igc_configure_msix sets up the hardware to properly
2761 * generate MSI-X interrupts.
2763 static void igc_configure_msix(struct igc_adapter
*adapter
)
2765 struct igc_hw
*hw
= &adapter
->hw
;
2769 adapter
->eims_enable_mask
= 0;
2771 /* set vector for other causes, i.e. link changes */
2772 switch (hw
->mac
.type
) {
2774 /* Turn on MSI-X capability first, or our settings
2775 * won't stick. And it will take days to debug.
2777 wr32(IGC_GPIE
, IGC_GPIE_MSIX_MODE
|
2778 IGC_GPIE_PBA
| IGC_GPIE_EIAME
|
2781 /* enable msix_other interrupt */
2782 adapter
->eims_other
= BIT(vector
);
2783 tmp
= (vector
++ | IGC_IVAR_VALID
) << 8;
2785 wr32(IGC_IVAR_MISC
, tmp
);
2788 /* do nothing, since nothing else supports MSI-X */
2790 } /* switch (hw->mac.type) */
2792 adapter
->eims_enable_mask
|= adapter
->eims_other
;
2794 for (i
= 0; i
< adapter
->num_q_vectors
; i
++)
2795 igc_assign_vector(adapter
->q_vector
[i
], vector
++);
2801 * igc_irq_enable - Enable default interrupt generation settings
2802 * @adapter: board private structure
2804 static void igc_irq_enable(struct igc_adapter
*adapter
)
2806 struct igc_hw
*hw
= &adapter
->hw
;
2808 if (adapter
->msix_entries
) {
2809 u32 ims
= IGC_IMS_LSC
| IGC_IMS_DOUTSYNC
| IGC_IMS_DRSTA
;
2810 u32 regval
= rd32(IGC_EIAC
);
2812 wr32(IGC_EIAC
, regval
| adapter
->eims_enable_mask
);
2813 regval
= rd32(IGC_EIAM
);
2814 wr32(IGC_EIAM
, regval
| adapter
->eims_enable_mask
);
2815 wr32(IGC_EIMS
, adapter
->eims_enable_mask
);
2818 wr32(IGC_IMS
, IMS_ENABLE_MASK
| IGC_IMS_DRSTA
);
2819 wr32(IGC_IAM
, IMS_ENABLE_MASK
| IGC_IMS_DRSTA
);
2824 * igc_irq_disable - Mask off interrupt generation on the NIC
2825 * @adapter: board private structure
2827 static void igc_irq_disable(struct igc_adapter
*adapter
)
2829 struct igc_hw
*hw
= &adapter
->hw
;
2831 if (adapter
->msix_entries
) {
2832 u32 regval
= rd32(IGC_EIAM
);
2834 wr32(IGC_EIAM
, regval
& ~adapter
->eims_enable_mask
);
2835 wr32(IGC_EIMC
, adapter
->eims_enable_mask
);
2836 regval
= rd32(IGC_EIAC
);
2837 wr32(IGC_EIAC
, regval
& ~adapter
->eims_enable_mask
);
2844 if (adapter
->msix_entries
) {
2847 synchronize_irq(adapter
->msix_entries
[vector
++].vector
);
2849 for (i
= 0; i
< adapter
->num_q_vectors
; i
++)
2850 synchronize_irq(adapter
->msix_entries
[vector
++].vector
);
2852 synchronize_irq(adapter
->pdev
->irq
);
2856 void igc_set_flag_queue_pairs(struct igc_adapter
*adapter
,
2857 const u32 max_rss_queues
)
2859 /* Determine if we need to pair queues. */
2860 /* If rss_queues > half of max_rss_queues, pair the queues in
2861 * order to conserve interrupts due to limited supply.
2863 if (adapter
->rss_queues
> (max_rss_queues
/ 2))
2864 adapter
->flags
|= IGC_FLAG_QUEUE_PAIRS
;
2866 adapter
->flags
&= ~IGC_FLAG_QUEUE_PAIRS
;
2869 unsigned int igc_get_max_rss_queues(struct igc_adapter
*adapter
)
2871 return IGC_MAX_RX_QUEUES
;
2874 static void igc_init_queue_configuration(struct igc_adapter
*adapter
)
2878 max_rss_queues
= igc_get_max_rss_queues(adapter
);
2879 adapter
->rss_queues
= min_t(u32
, max_rss_queues
, num_online_cpus());
2881 igc_set_flag_queue_pairs(adapter
, max_rss_queues
);
2885 * igc_reset_q_vector - Reset config for interrupt vector
2886 * @adapter: board private structure to initialize
2887 * @v_idx: Index of vector to be reset
2889 * If NAPI is enabled it will delete any references to the
2890 * NAPI struct. This is preparation for igc_free_q_vector.
2892 static void igc_reset_q_vector(struct igc_adapter
*adapter
, int v_idx
)
2894 struct igc_q_vector
*q_vector
= adapter
->q_vector
[v_idx
];
2896 /* if we're coming from igc_set_interrupt_capability, the vectors are
2902 if (q_vector
->tx
.ring
)
2903 adapter
->tx_ring
[q_vector
->tx
.ring
->queue_index
] = NULL
;
2905 if (q_vector
->rx
.ring
)
2906 adapter
->rx_ring
[q_vector
->rx
.ring
->queue_index
] = NULL
;
2908 netif_napi_del(&q_vector
->napi
);
2912 * igc_free_q_vector - Free memory allocated for specific interrupt vector
2913 * @adapter: board private structure to initialize
2914 * @v_idx: Index of vector to be freed
2916 * This function frees the memory allocated to the q_vector.
2918 static void igc_free_q_vector(struct igc_adapter
*adapter
, int v_idx
)
2920 struct igc_q_vector
*q_vector
= adapter
->q_vector
[v_idx
];
2922 adapter
->q_vector
[v_idx
] = NULL
;
2924 /* igc_get_stats64() might access the rings on this vector,
2925 * we must wait a grace period before freeing it.
2928 kfree_rcu(q_vector
, rcu
);
2932 * igc_free_q_vectors - Free memory allocated for interrupt vectors
2933 * @adapter: board private structure to initialize
2935 * This function frees the memory allocated to the q_vectors. In addition if
2936 * NAPI is enabled it will delete any references to the NAPI struct prior
2937 * to freeing the q_vector.
2939 static void igc_free_q_vectors(struct igc_adapter
*adapter
)
2941 int v_idx
= adapter
->num_q_vectors
;
2943 adapter
->num_tx_queues
= 0;
2944 adapter
->num_rx_queues
= 0;
2945 adapter
->num_q_vectors
= 0;
2948 igc_reset_q_vector(adapter
, v_idx
);
2949 igc_free_q_vector(adapter
, v_idx
);
2954 * igc_update_itr - update the dynamic ITR value based on statistics
2955 * @q_vector: pointer to q_vector
2956 * @ring_container: ring info to update the itr for
2958 * Stores a new ITR value based on packets and byte
2959 * counts during the last interrupt. The advantage of per interrupt
2960 * computation is faster updates and more accurate ITR for the current
2961 * traffic pattern. Constants in this function were computed
2962 * based on theoretical maximum wire speed and thresholds were set based
2963 * on testing data as well as attempting to minimize response time
2964 * while increasing bulk throughput.
2965 * NOTE: These calculations are only valid when operating in a single-
2966 * queue environment.
2968 static void igc_update_itr(struct igc_q_vector
*q_vector
,
2969 struct igc_ring_container
*ring_container
)
2971 unsigned int packets
= ring_container
->total_packets
;
2972 unsigned int bytes
= ring_container
->total_bytes
;
2973 u8 itrval
= ring_container
->itr
;
2975 /* no packets, exit with status unchanged */
2980 case lowest_latency
:
2981 /* handle TSO and jumbo frames */
2982 if (bytes
/ packets
> 8000)
2983 itrval
= bulk_latency
;
2984 else if ((packets
< 5) && (bytes
> 512))
2985 itrval
= low_latency
;
2987 case low_latency
: /* 50 usec aka 20000 ints/s */
2988 if (bytes
> 10000) {
2989 /* this if handles the TSO accounting */
2990 if (bytes
/ packets
> 8000)
2991 itrval
= bulk_latency
;
2992 else if ((packets
< 10) || ((bytes
/ packets
) > 1200))
2993 itrval
= bulk_latency
;
2994 else if ((packets
> 35))
2995 itrval
= lowest_latency
;
2996 } else if (bytes
/ packets
> 2000) {
2997 itrval
= bulk_latency
;
2998 } else if (packets
<= 2 && bytes
< 512) {
2999 itrval
= lowest_latency
;
3002 case bulk_latency
: /* 250 usec aka 4000 ints/s */
3003 if (bytes
> 25000) {
3005 itrval
= low_latency
;
3006 } else if (bytes
< 1500) {
3007 itrval
= low_latency
;
3012 /* clear work counters since we have the values we need */
3013 ring_container
->total_bytes
= 0;
3014 ring_container
->total_packets
= 0;
3016 /* write updated itr to ring container */
3017 ring_container
->itr
= itrval
;
3020 static void igc_set_itr(struct igc_q_vector
*q_vector
)
3022 struct igc_adapter
*adapter
= q_vector
->adapter
;
3023 u32 new_itr
= q_vector
->itr_val
;
3026 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3027 switch (adapter
->link_speed
) {
3031 new_itr
= IGC_4K_ITR
;
3037 igc_update_itr(q_vector
, &q_vector
->tx
);
3038 igc_update_itr(q_vector
, &q_vector
->rx
);
3040 current_itr
= max(q_vector
->rx
.itr
, q_vector
->tx
.itr
);
3042 /* conservative mode (itr 3) eliminates the lowest_latency setting */
3043 if (current_itr
== lowest_latency
&&
3044 ((q_vector
->rx
.ring
&& adapter
->rx_itr_setting
== 3) ||
3045 (!q_vector
->rx
.ring
&& adapter
->tx_itr_setting
== 3)))
3046 current_itr
= low_latency
;
3048 switch (current_itr
) {
3049 /* counts and packets in update_itr are dependent on these numbers */
3050 case lowest_latency
:
3051 new_itr
= IGC_70K_ITR
; /* 70,000 ints/sec */
3054 new_itr
= IGC_20K_ITR
; /* 20,000 ints/sec */
3057 new_itr
= IGC_4K_ITR
; /* 4,000 ints/sec */
3064 if (new_itr
!= q_vector
->itr_val
) {
3065 /* this attempts to bias the interrupt rate towards Bulk
3066 * by adding intermediate steps when interrupt rate is
3069 new_itr
= new_itr
> q_vector
->itr_val
?
3070 max((new_itr
* q_vector
->itr_val
) /
3071 (new_itr
+ (q_vector
->itr_val
>> 2)),
3073 /* Don't write the value here; it resets the adapter's
3074 * internal timer, and causes us to delay far longer than
3075 * we should between interrupts. Instead, we write the ITR
3076 * value at the beginning of the next interrupt so the timing
3077 * ends up being correct.
3079 q_vector
->itr_val
= new_itr
;
3080 q_vector
->set_itr
= 1;
3084 static void igc_reset_interrupt_capability(struct igc_adapter
*adapter
)
3086 int v_idx
= adapter
->num_q_vectors
;
3088 if (adapter
->msix_entries
) {
3089 pci_disable_msix(adapter
->pdev
);
3090 kfree(adapter
->msix_entries
);
3091 adapter
->msix_entries
= NULL
;
3092 } else if (adapter
->flags
& IGC_FLAG_HAS_MSI
) {
3093 pci_disable_msi(adapter
->pdev
);
3097 igc_reset_q_vector(adapter
, v_idx
);
3101 * igc_set_interrupt_capability - set MSI or MSI-X if supported
3102 * @adapter: Pointer to adapter structure
3103 * @msix: boolean value for MSI-X capability
3105 * Attempt to configure interrupts using the best available
3106 * capabilities of the hardware and kernel.
3108 static void igc_set_interrupt_capability(struct igc_adapter
*adapter
,
3116 adapter
->flags
|= IGC_FLAG_HAS_MSIX
;
3118 /* Number of supported queues. */
3119 adapter
->num_rx_queues
= adapter
->rss_queues
;
3121 adapter
->num_tx_queues
= adapter
->rss_queues
;
3123 /* start with one vector for every Rx queue */
3124 numvecs
= adapter
->num_rx_queues
;
3126 /* if Tx handler is separate add 1 for every Tx queue */
3127 if (!(adapter
->flags
& IGC_FLAG_QUEUE_PAIRS
))
3128 numvecs
+= adapter
->num_tx_queues
;
3130 /* store the number of vectors reserved for queues */
3131 adapter
->num_q_vectors
= numvecs
;
3133 /* add 1 vector for link status interrupts */
3136 adapter
->msix_entries
= kcalloc(numvecs
, sizeof(struct msix_entry
),
3139 if (!adapter
->msix_entries
)
3142 /* populate entry values */
3143 for (i
= 0; i
< numvecs
; i
++)
3144 adapter
->msix_entries
[i
].entry
= i
;
3146 err
= pci_enable_msix_range(adapter
->pdev
,
3147 adapter
->msix_entries
,
3153 kfree(adapter
->msix_entries
);
3154 adapter
->msix_entries
= NULL
;
3156 igc_reset_interrupt_capability(adapter
);
3159 adapter
->flags
&= ~IGC_FLAG_HAS_MSIX
;
3161 adapter
->rss_queues
= 1;
3162 adapter
->flags
|= IGC_FLAG_QUEUE_PAIRS
;
3163 adapter
->num_rx_queues
= 1;
3164 adapter
->num_tx_queues
= 1;
3165 adapter
->num_q_vectors
= 1;
3166 if (!pci_enable_msi(adapter
->pdev
))
3167 adapter
->flags
|= IGC_FLAG_HAS_MSI
;
3171 * igc_update_ring_itr - update the dynamic ITR value based on packet size
3172 * @q_vector: pointer to q_vector
3174 * Stores a new ITR value based on strictly on packet size. This
3175 * algorithm is less sophisticated than that used in igc_update_itr,
3176 * due to the difficulty of synchronizing statistics across multiple
3177 * receive rings. The divisors and thresholds used by this function
3178 * were determined based on theoretical maximum wire speed and testing
3179 * data, in order to minimize response time while increasing bulk
3181 * NOTE: This function is called only when operating in a multiqueue
3182 * receive environment.
3184 static void igc_update_ring_itr(struct igc_q_vector
*q_vector
)
3186 struct igc_adapter
*adapter
= q_vector
->adapter
;
3187 int new_val
= q_vector
->itr_val
;
3188 int avg_wire_size
= 0;
3189 unsigned int packets
;
3191 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3192 * ints/sec - ITR timer value of 120 ticks.
3194 switch (adapter
->link_speed
) {
3197 new_val
= IGC_4K_ITR
;
3203 packets
= q_vector
->rx
.total_packets
;
3205 avg_wire_size
= q_vector
->rx
.total_bytes
/ packets
;
3207 packets
= q_vector
->tx
.total_packets
;
3209 avg_wire_size
= max_t(u32
, avg_wire_size
,
3210 q_vector
->tx
.total_bytes
/ packets
);
3212 /* if avg_wire_size isn't set no work was done */
3216 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3217 avg_wire_size
+= 24;
3219 /* Don't starve jumbo frames */
3220 avg_wire_size
= min(avg_wire_size
, 3000);
3222 /* Give a little boost to mid-size frames */
3223 if (avg_wire_size
> 300 && avg_wire_size
< 1200)
3224 new_val
= avg_wire_size
/ 3;
3226 new_val
= avg_wire_size
/ 2;
3228 /* conservative mode (itr 3) eliminates the lowest_latency setting */
3229 if (new_val
< IGC_20K_ITR
&&
3230 ((q_vector
->rx
.ring
&& adapter
->rx_itr_setting
== 3) ||
3231 (!q_vector
->rx
.ring
&& adapter
->tx_itr_setting
== 3)))
3232 new_val
= IGC_20K_ITR
;
3235 if (new_val
!= q_vector
->itr_val
) {
3236 q_vector
->itr_val
= new_val
;
3237 q_vector
->set_itr
= 1;
3240 q_vector
->rx
.total_bytes
= 0;
3241 q_vector
->rx
.total_packets
= 0;
3242 q_vector
->tx
.total_bytes
= 0;
3243 q_vector
->tx
.total_packets
= 0;
3246 static void igc_ring_irq_enable(struct igc_q_vector
*q_vector
)
3248 struct igc_adapter
*adapter
= q_vector
->adapter
;
3249 struct igc_hw
*hw
= &adapter
->hw
;
3251 if ((q_vector
->rx
.ring
&& (adapter
->rx_itr_setting
& 3)) ||
3252 (!q_vector
->rx
.ring
&& (adapter
->tx_itr_setting
& 3))) {
3253 if (adapter
->num_q_vectors
== 1)
3254 igc_set_itr(q_vector
);
3256 igc_update_ring_itr(q_vector
);
3259 if (!test_bit(__IGC_DOWN
, &adapter
->state
)) {
3260 if (adapter
->msix_entries
)
3261 wr32(IGC_EIMS
, q_vector
->eims_value
);
3263 igc_irq_enable(adapter
);
3267 static void igc_add_ring(struct igc_ring
*ring
,
3268 struct igc_ring_container
*head
)
3275 * igc_cache_ring_register - Descriptor ring to register mapping
3276 * @adapter: board private structure to initialize
3278 * Once we know the feature-set enabled for the device, we'll cache
3279 * the register offset the descriptor ring is assigned to.
3281 static void igc_cache_ring_register(struct igc_adapter
*adapter
)
3285 switch (adapter
->hw
.mac
.type
) {
3289 for (; i
< adapter
->num_rx_queues
; i
++)
3290 adapter
->rx_ring
[i
]->reg_idx
= i
;
3291 for (; j
< adapter
->num_tx_queues
; j
++)
3292 adapter
->tx_ring
[j
]->reg_idx
= j
;
3298 * igc_poll - NAPI Rx polling callback
3299 * @napi: napi polling structure
3300 * @budget: count of how many packets we should handle
3302 static int igc_poll(struct napi_struct
*napi
, int budget
)
3304 struct igc_q_vector
*q_vector
= container_of(napi
,
3305 struct igc_q_vector
,
3307 bool clean_complete
= true;
3310 if (q_vector
->tx
.ring
)
3311 clean_complete
= igc_clean_tx_irq(q_vector
, budget
);
3313 if (q_vector
->rx
.ring
) {
3314 int cleaned
= igc_clean_rx_irq(q_vector
, budget
);
3316 work_done
+= cleaned
;
3317 if (cleaned
>= budget
)
3318 clean_complete
= false;
3321 /* If all work not completed, return budget and keep polling */
3322 if (!clean_complete
)
3325 /* Exit the polling mode, but don't re-enable interrupts if stack might
3326 * poll us due to busy-polling
3328 if (likely(napi_complete_done(napi
, work_done
)))
3329 igc_ring_irq_enable(q_vector
);
3331 return min(work_done
, budget
- 1);
3335 * igc_alloc_q_vector - Allocate memory for a single interrupt vector
3336 * @adapter: board private structure to initialize
3337 * @v_count: q_vectors allocated on adapter, used for ring interleaving
3338 * @v_idx: index of vector in adapter struct
3339 * @txr_count: total number of Tx rings to allocate
3340 * @txr_idx: index of first Tx ring to allocate
3341 * @rxr_count: total number of Rx rings to allocate
3342 * @rxr_idx: index of first Rx ring to allocate
3344 * We allocate one q_vector. If allocation fails we return -ENOMEM.
3346 static int igc_alloc_q_vector(struct igc_adapter
*adapter
,
3347 unsigned int v_count
, unsigned int v_idx
,
3348 unsigned int txr_count
, unsigned int txr_idx
,
3349 unsigned int rxr_count
, unsigned int rxr_idx
)
3351 struct igc_q_vector
*q_vector
;
3352 struct igc_ring
*ring
;
3355 /* igc only supports 1 Tx and/or 1 Rx queue per vector */
3356 if (txr_count
> 1 || rxr_count
> 1)
3359 ring_count
= txr_count
+ rxr_count
;
3361 /* allocate q_vector and rings */
3362 q_vector
= adapter
->q_vector
[v_idx
];
3364 q_vector
= kzalloc(struct_size(q_vector
, ring
, ring_count
),
3367 memset(q_vector
, 0, struct_size(q_vector
, ring
, ring_count
));
3371 /* initialize NAPI */
3372 netif_napi_add(adapter
->netdev
, &q_vector
->napi
,
3375 /* tie q_vector and adapter together */
3376 adapter
->q_vector
[v_idx
] = q_vector
;
3377 q_vector
->adapter
= adapter
;
3379 /* initialize work limits */
3380 q_vector
->tx
.work_limit
= adapter
->tx_work_limit
;
3382 /* initialize ITR configuration */
3383 q_vector
->itr_register
= adapter
->io_addr
+ IGC_EITR(0);
3384 q_vector
->itr_val
= IGC_START_ITR
;
3386 /* initialize pointer to rings */
3387 ring
= q_vector
->ring
;
3389 /* initialize ITR */
3391 /* rx or rx/tx vector */
3392 if (!adapter
->rx_itr_setting
|| adapter
->rx_itr_setting
> 3)
3393 q_vector
->itr_val
= adapter
->rx_itr_setting
;
3395 /* tx only vector */
3396 if (!adapter
->tx_itr_setting
|| adapter
->tx_itr_setting
> 3)
3397 q_vector
->itr_val
= adapter
->tx_itr_setting
;
3401 /* assign generic ring traits */
3402 ring
->dev
= &adapter
->pdev
->dev
;
3403 ring
->netdev
= adapter
->netdev
;
3405 /* configure backlink on ring */
3406 ring
->q_vector
= q_vector
;
3408 /* update q_vector Tx values */
3409 igc_add_ring(ring
, &q_vector
->tx
);
3411 /* apply Tx specific ring traits */
3412 ring
->count
= adapter
->tx_ring_count
;
3413 ring
->queue_index
= txr_idx
;
3415 /* assign ring to adapter */
3416 adapter
->tx_ring
[txr_idx
] = ring
;
3418 /* push pointer to next ring */
3423 /* assign generic ring traits */
3424 ring
->dev
= &adapter
->pdev
->dev
;
3425 ring
->netdev
= adapter
->netdev
;
3427 /* configure backlink on ring */
3428 ring
->q_vector
= q_vector
;
3430 /* update q_vector Rx values */
3431 igc_add_ring(ring
, &q_vector
->rx
);
3433 /* apply Rx specific ring traits */
3434 ring
->count
= adapter
->rx_ring_count
;
3435 ring
->queue_index
= rxr_idx
;
3437 /* assign ring to adapter */
3438 adapter
->rx_ring
[rxr_idx
] = ring
;
3445 * igc_alloc_q_vectors - Allocate memory for interrupt vectors
3446 * @adapter: board private structure to initialize
3448 * We allocate one q_vector per queue interrupt. If allocation fails we
3451 static int igc_alloc_q_vectors(struct igc_adapter
*adapter
)
3453 int rxr_remaining
= adapter
->num_rx_queues
;
3454 int txr_remaining
= adapter
->num_tx_queues
;
3455 int rxr_idx
= 0, txr_idx
= 0, v_idx
= 0;
3456 int q_vectors
= adapter
->num_q_vectors
;
3459 if (q_vectors
>= (rxr_remaining
+ txr_remaining
)) {
3460 for (; rxr_remaining
; v_idx
++) {
3461 err
= igc_alloc_q_vector(adapter
, q_vectors
, v_idx
,
3467 /* update counts and index */
3473 for (; v_idx
< q_vectors
; v_idx
++) {
3474 int rqpv
= DIV_ROUND_UP(rxr_remaining
, q_vectors
- v_idx
);
3475 int tqpv
= DIV_ROUND_UP(txr_remaining
, q_vectors
- v_idx
);
3477 err
= igc_alloc_q_vector(adapter
, q_vectors
, v_idx
,
3478 tqpv
, txr_idx
, rqpv
, rxr_idx
);
3483 /* update counts and index */
3484 rxr_remaining
-= rqpv
;
3485 txr_remaining
-= tqpv
;
3493 adapter
->num_tx_queues
= 0;
3494 adapter
->num_rx_queues
= 0;
3495 adapter
->num_q_vectors
= 0;
3498 igc_free_q_vector(adapter
, v_idx
);
3504 * igc_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
3505 * @adapter: Pointer to adapter structure
3506 * @msix: boolean for MSI-X capability
3508 * This function initializes the interrupts and allocates all of the queues.
3510 static int igc_init_interrupt_scheme(struct igc_adapter
*adapter
, bool msix
)
3512 struct net_device
*dev
= adapter
->netdev
;
3515 igc_set_interrupt_capability(adapter
, msix
);
3517 err
= igc_alloc_q_vectors(adapter
);
3519 netdev_err(dev
, "Unable to allocate memory for vectors\n");
3520 goto err_alloc_q_vectors
;
3523 igc_cache_ring_register(adapter
);
3527 err_alloc_q_vectors
:
3528 igc_reset_interrupt_capability(adapter
);
3533 * igc_sw_init - Initialize general software structures (struct igc_adapter)
3534 * @adapter: board private structure to initialize
3536 * igc_sw_init initializes the Adapter private data structure.
3537 * Fields are initialized based on PCI device information and
3538 * OS network device settings (MTU size).
3540 static int igc_sw_init(struct igc_adapter
*adapter
)
3542 struct net_device
*netdev
= adapter
->netdev
;
3543 struct pci_dev
*pdev
= adapter
->pdev
;
3544 struct igc_hw
*hw
= &adapter
->hw
;
3546 pci_read_config_word(pdev
, PCI_COMMAND
, &hw
->bus
.pci_cmd_word
);
3548 /* set default ring sizes */
3549 adapter
->tx_ring_count
= IGC_DEFAULT_TXD
;
3550 adapter
->rx_ring_count
= IGC_DEFAULT_RXD
;
3552 /* set default ITR values */
3553 adapter
->rx_itr_setting
= IGC_DEFAULT_ITR
;
3554 adapter
->tx_itr_setting
= IGC_DEFAULT_ITR
;
3556 /* set default work limits */
3557 adapter
->tx_work_limit
= IGC_DEFAULT_TX_WORK
;
3559 /* adjust max frame to be at least the size of a standard frame */
3560 adapter
->max_frame_size
= netdev
->mtu
+ ETH_HLEN
+ ETH_FCS_LEN
+
3562 adapter
->min_frame_size
= ETH_ZLEN
+ ETH_FCS_LEN
;
3564 mutex_init(&adapter
->nfc_rule_lock
);
3565 INIT_LIST_HEAD(&adapter
->nfc_rule_list
);
3566 adapter
->nfc_rule_count
= 0;
3568 spin_lock_init(&adapter
->stats64_lock
);
3569 /* Assume MSI-X interrupts, will be checked during IRQ allocation */
3570 adapter
->flags
|= IGC_FLAG_HAS_MSIX
;
3572 igc_init_queue_configuration(adapter
);
3574 /* This call may decrease the number of queues */
3575 if (igc_init_interrupt_scheme(adapter
, true)) {
3576 netdev_err(netdev
, "Unable to allocate memory for queues\n");
3580 /* Explicitly disable IRQ since the NIC can be in any state. */
3581 igc_irq_disable(adapter
);
3583 set_bit(__IGC_DOWN
, &adapter
->state
);
3589 * igc_up - Open the interface and prepare it to handle traffic
3590 * @adapter: board private structure
3592 void igc_up(struct igc_adapter
*adapter
)
3594 struct igc_hw
*hw
= &adapter
->hw
;
3597 /* hardware has been reset, we need to reload some things */
3598 igc_configure(adapter
);
3600 clear_bit(__IGC_DOWN
, &adapter
->state
);
3602 for (i
= 0; i
< adapter
->num_q_vectors
; i
++)
3603 napi_enable(&adapter
->q_vector
[i
]->napi
);
3605 if (adapter
->msix_entries
)
3606 igc_configure_msix(adapter
);
3608 igc_assign_vector(adapter
->q_vector
[0], 0);
3610 /* Clear any pending interrupts. */
3612 igc_irq_enable(adapter
);
3614 netif_tx_start_all_queues(adapter
->netdev
);
3616 /* start the watchdog. */
3617 hw
->mac
.get_link_status
= 1;
3618 schedule_work(&adapter
->watchdog_task
);
3622 * igc_update_stats - Update the board statistics counters
3623 * @adapter: board private structure
3625 void igc_update_stats(struct igc_adapter
*adapter
)
3627 struct rtnl_link_stats64
*net_stats
= &adapter
->stats64
;
3628 struct pci_dev
*pdev
= adapter
->pdev
;
3629 struct igc_hw
*hw
= &adapter
->hw
;
3630 u64 _bytes
, _packets
;
3636 /* Prevent stats update while adapter is being reset, or if the pci
3637 * connection is down.
3639 if (adapter
->link_speed
== 0)
3641 if (pci_channel_offline(pdev
))
3648 for (i
= 0; i
< adapter
->num_rx_queues
; i
++) {
3649 struct igc_ring
*ring
= adapter
->rx_ring
[i
];
3650 u32 rqdpc
= rd32(IGC_RQDPC(i
));
3652 if (hw
->mac
.type
>= igc_i225
)
3653 wr32(IGC_RQDPC(i
), 0);
3656 ring
->rx_stats
.drops
+= rqdpc
;
3657 net_stats
->rx_fifo_errors
+= rqdpc
;
3661 start
= u64_stats_fetch_begin_irq(&ring
->rx_syncp
);
3662 _bytes
= ring
->rx_stats
.bytes
;
3663 _packets
= ring
->rx_stats
.packets
;
3664 } while (u64_stats_fetch_retry_irq(&ring
->rx_syncp
, start
));
3666 packets
+= _packets
;
3669 net_stats
->rx_bytes
= bytes
;
3670 net_stats
->rx_packets
= packets
;
3674 for (i
= 0; i
< adapter
->num_tx_queues
; i
++) {
3675 struct igc_ring
*ring
= adapter
->tx_ring
[i
];
3678 start
= u64_stats_fetch_begin_irq(&ring
->tx_syncp
);
3679 _bytes
= ring
->tx_stats
.bytes
;
3680 _packets
= ring
->tx_stats
.packets
;
3681 } while (u64_stats_fetch_retry_irq(&ring
->tx_syncp
, start
));
3683 packets
+= _packets
;
3685 net_stats
->tx_bytes
= bytes
;
3686 net_stats
->tx_packets
= packets
;
3689 /* read stats registers */
3690 adapter
->stats
.crcerrs
+= rd32(IGC_CRCERRS
);
3691 adapter
->stats
.gprc
+= rd32(IGC_GPRC
);
3692 adapter
->stats
.gorc
+= rd32(IGC_GORCL
);
3693 rd32(IGC_GORCH
); /* clear GORCL */
3694 adapter
->stats
.bprc
+= rd32(IGC_BPRC
);
3695 adapter
->stats
.mprc
+= rd32(IGC_MPRC
);
3696 adapter
->stats
.roc
+= rd32(IGC_ROC
);
3698 adapter
->stats
.prc64
+= rd32(IGC_PRC64
);
3699 adapter
->stats
.prc127
+= rd32(IGC_PRC127
);
3700 adapter
->stats
.prc255
+= rd32(IGC_PRC255
);
3701 adapter
->stats
.prc511
+= rd32(IGC_PRC511
);
3702 adapter
->stats
.prc1023
+= rd32(IGC_PRC1023
);
3703 adapter
->stats
.prc1522
+= rd32(IGC_PRC1522
);
3705 mpc
= rd32(IGC_MPC
);
3706 adapter
->stats
.mpc
+= mpc
;
3707 net_stats
->rx_fifo_errors
+= mpc
;
3708 adapter
->stats
.scc
+= rd32(IGC_SCC
);
3709 adapter
->stats
.ecol
+= rd32(IGC_ECOL
);
3710 adapter
->stats
.mcc
+= rd32(IGC_MCC
);
3711 adapter
->stats
.latecol
+= rd32(IGC_LATECOL
);
3712 adapter
->stats
.dc
+= rd32(IGC_DC
);
3713 adapter
->stats
.rlec
+= rd32(IGC_RLEC
);
3714 adapter
->stats
.xonrxc
+= rd32(IGC_XONRXC
);
3715 adapter
->stats
.xontxc
+= rd32(IGC_XONTXC
);
3716 adapter
->stats
.xoffrxc
+= rd32(IGC_XOFFRXC
);
3717 adapter
->stats
.xofftxc
+= rd32(IGC_XOFFTXC
);
3718 adapter
->stats
.fcruc
+= rd32(IGC_FCRUC
);
3719 adapter
->stats
.gptc
+= rd32(IGC_GPTC
);
3720 adapter
->stats
.gotc
+= rd32(IGC_GOTCL
);
3721 rd32(IGC_GOTCH
); /* clear GOTCL */
3722 adapter
->stats
.rnbc
+= rd32(IGC_RNBC
);
3723 adapter
->stats
.ruc
+= rd32(IGC_RUC
);
3724 adapter
->stats
.rfc
+= rd32(IGC_RFC
);
3725 adapter
->stats
.rjc
+= rd32(IGC_RJC
);
3726 adapter
->stats
.tor
+= rd32(IGC_TORH
);
3727 adapter
->stats
.tot
+= rd32(IGC_TOTH
);
3728 adapter
->stats
.tpr
+= rd32(IGC_TPR
);
3730 adapter
->stats
.ptc64
+= rd32(IGC_PTC64
);
3731 adapter
->stats
.ptc127
+= rd32(IGC_PTC127
);
3732 adapter
->stats
.ptc255
+= rd32(IGC_PTC255
);
3733 adapter
->stats
.ptc511
+= rd32(IGC_PTC511
);
3734 adapter
->stats
.ptc1023
+= rd32(IGC_PTC1023
);
3735 adapter
->stats
.ptc1522
+= rd32(IGC_PTC1522
);
3737 adapter
->stats
.mptc
+= rd32(IGC_MPTC
);
3738 adapter
->stats
.bptc
+= rd32(IGC_BPTC
);
3740 adapter
->stats
.tpt
+= rd32(IGC_TPT
);
3741 adapter
->stats
.colc
+= rd32(IGC_COLC
);
3742 adapter
->stats
.colc
+= rd32(IGC_RERC
);
3744 adapter
->stats
.algnerrc
+= rd32(IGC_ALGNERRC
);
3746 adapter
->stats
.tsctc
+= rd32(IGC_TSCTC
);
3747 adapter
->stats
.tsctfc
+= rd32(IGC_TSCTFC
);
3749 adapter
->stats
.iac
+= rd32(IGC_IAC
);
3750 adapter
->stats
.icrxoc
+= rd32(IGC_ICRXOC
);
3751 adapter
->stats
.icrxptc
+= rd32(IGC_ICRXPTC
);
3752 adapter
->stats
.icrxatc
+= rd32(IGC_ICRXATC
);
3753 adapter
->stats
.ictxptc
+= rd32(IGC_ICTXPTC
);
3754 adapter
->stats
.ictxatc
+= rd32(IGC_ICTXATC
);
3755 adapter
->stats
.ictxqec
+= rd32(IGC_ICTXQEC
);
3756 adapter
->stats
.ictxqmtc
+= rd32(IGC_ICTXQMTC
);
3757 adapter
->stats
.icrxdmtc
+= rd32(IGC_ICRXDMTC
);
3759 /* Fill out the OS statistics structure */
3760 net_stats
->multicast
= adapter
->stats
.mprc
;
3761 net_stats
->collisions
= adapter
->stats
.colc
;
3765 /* RLEC on some newer hardware can be incorrect so build
3766 * our own version based on RUC and ROC
3768 net_stats
->rx_errors
= adapter
->stats
.rxerrc
+
3769 adapter
->stats
.crcerrs
+ adapter
->stats
.algnerrc
+
3770 adapter
->stats
.ruc
+ adapter
->stats
.roc
+
3771 adapter
->stats
.cexterr
;
3772 net_stats
->rx_length_errors
= adapter
->stats
.ruc
+
3774 net_stats
->rx_crc_errors
= adapter
->stats
.crcerrs
;
3775 net_stats
->rx_frame_errors
= adapter
->stats
.algnerrc
;
3776 net_stats
->rx_missed_errors
= adapter
->stats
.mpc
;
3779 net_stats
->tx_errors
= adapter
->stats
.ecol
+
3780 adapter
->stats
.latecol
;
3781 net_stats
->tx_aborted_errors
= adapter
->stats
.ecol
;
3782 net_stats
->tx_window_errors
= adapter
->stats
.latecol
;
3783 net_stats
->tx_carrier_errors
= adapter
->stats
.tncrs
;
3785 /* Tx Dropped needs to be maintained elsewhere */
3787 /* Management Stats */
3788 adapter
->stats
.mgptc
+= rd32(IGC_MGTPTC
);
3789 adapter
->stats
.mgprc
+= rd32(IGC_MGTPRC
);
3790 adapter
->stats
.mgpdc
+= rd32(IGC_MGTPDC
);
3794 * igc_down - Close the interface
3795 * @adapter: board private structure
3797 void igc_down(struct igc_adapter
*adapter
)
3799 struct net_device
*netdev
= adapter
->netdev
;
3800 struct igc_hw
*hw
= &adapter
->hw
;
3804 set_bit(__IGC_DOWN
, &adapter
->state
);
3806 /* disable receives in the hardware */
3807 rctl
= rd32(IGC_RCTL
);
3808 wr32(IGC_RCTL
, rctl
& ~IGC_RCTL_EN
);
3809 /* flush and sleep below */
3811 /* set trans_start so we don't get spurious watchdogs during reset */
3812 netif_trans_update(netdev
);
3814 netif_carrier_off(netdev
);
3815 netif_tx_stop_all_queues(netdev
);
3817 /* disable transmits in the hardware */
3818 tctl
= rd32(IGC_TCTL
);
3819 tctl
&= ~IGC_TCTL_EN
;
3820 wr32(IGC_TCTL
, tctl
);
3821 /* flush both disables and wait for them to finish */
3823 usleep_range(10000, 20000);
3825 igc_irq_disable(adapter
);
3827 adapter
->flags
&= ~IGC_FLAG_NEED_LINK_UPDATE
;
3829 for (i
= 0; i
< adapter
->num_q_vectors
; i
++) {
3830 if (adapter
->q_vector
[i
]) {
3831 napi_synchronize(&adapter
->q_vector
[i
]->napi
);
3832 napi_disable(&adapter
->q_vector
[i
]->napi
);
3836 del_timer_sync(&adapter
->watchdog_timer
);
3837 del_timer_sync(&adapter
->phy_info_timer
);
3839 /* record the stats before reset*/
3840 spin_lock(&adapter
->stats64_lock
);
3841 igc_update_stats(adapter
);
3842 spin_unlock(&adapter
->stats64_lock
);
3844 adapter
->link_speed
= 0;
3845 adapter
->link_duplex
= 0;
3847 if (!pci_channel_offline(adapter
->pdev
))
3850 /* clear VLAN promisc flag so VFTA will be updated if necessary */
3851 adapter
->flags
&= ~IGC_FLAG_VLAN_PROMISC
;
3853 igc_clean_all_tx_rings(adapter
);
3854 igc_clean_all_rx_rings(adapter
);
3857 void igc_reinit_locked(struct igc_adapter
*adapter
)
3859 WARN_ON(in_interrupt());
3860 while (test_and_set_bit(__IGC_RESETTING
, &adapter
->state
))
3861 usleep_range(1000, 2000);
3864 clear_bit(__IGC_RESETTING
, &adapter
->state
);
3867 static void igc_reset_task(struct work_struct
*work
)
3869 struct igc_adapter
*adapter
;
3871 adapter
= container_of(work
, struct igc_adapter
, reset_task
);
3873 igc_rings_dump(adapter
);
3874 igc_regs_dump(adapter
);
3875 netdev_err(adapter
->netdev
, "Reset adapter\n");
3876 igc_reinit_locked(adapter
);
3880 * igc_change_mtu - Change the Maximum Transfer Unit
3881 * @netdev: network interface device structure
3882 * @new_mtu: new value for maximum frame size
3884 * Returns 0 on success, negative on failure
3886 static int igc_change_mtu(struct net_device
*netdev
, int new_mtu
)
3888 int max_frame
= new_mtu
+ ETH_HLEN
+ ETH_FCS_LEN
+ VLAN_HLEN
;
3889 struct igc_adapter
*adapter
= netdev_priv(netdev
);
3891 /* adjust max frame to be at least the size of a standard frame */
3892 if (max_frame
< (ETH_FRAME_LEN
+ ETH_FCS_LEN
))
3893 max_frame
= ETH_FRAME_LEN
+ ETH_FCS_LEN
;
3895 while (test_and_set_bit(__IGC_RESETTING
, &adapter
->state
))
3896 usleep_range(1000, 2000);
3898 /* igc_down has a dependency on max_frame_size */
3899 adapter
->max_frame_size
= max_frame
;
3901 if (netif_running(netdev
))
3904 netdev_dbg(netdev
, "changing MTU from %d to %d\n", netdev
->mtu
, new_mtu
);
3905 netdev
->mtu
= new_mtu
;
3907 if (netif_running(netdev
))
3912 clear_bit(__IGC_RESETTING
, &adapter
->state
);
3918 * igc_get_stats - Get System Network Statistics
3919 * @netdev: network interface device structure
3921 * Returns the address of the device statistics structure.
3922 * The statistics are updated here and also from the timer callback.
3924 static struct net_device_stats
*igc_get_stats(struct net_device
*netdev
)
3926 struct igc_adapter
*adapter
= netdev_priv(netdev
);
3928 if (!test_bit(__IGC_RESETTING
, &adapter
->state
))
3929 igc_update_stats(adapter
);
3931 /* only return the current stats */
3932 return &netdev
->stats
;
3935 static netdev_features_t
igc_fix_features(struct net_device
*netdev
,
3936 netdev_features_t features
)
3938 /* Since there is no support for separate Rx/Tx vlan accel
3939 * enable/disable make sure Tx flag is always in same state as Rx.
3941 if (features
& NETIF_F_HW_VLAN_CTAG_RX
)
3942 features
|= NETIF_F_HW_VLAN_CTAG_TX
;
3944 features
&= ~NETIF_F_HW_VLAN_CTAG_TX
;
3949 static int igc_set_features(struct net_device
*netdev
,
3950 netdev_features_t features
)
3952 netdev_features_t changed
= netdev
->features
^ features
;
3953 struct igc_adapter
*adapter
= netdev_priv(netdev
);
3955 /* Add VLAN support */
3956 if (!(changed
& (NETIF_F_RXALL
| NETIF_F_NTUPLE
)))
3959 if (!(features
& NETIF_F_NTUPLE
))
3960 igc_flush_nfc_rules(adapter
);
3962 netdev
->features
= features
;
3964 if (netif_running(netdev
))
3965 igc_reinit_locked(adapter
);
3972 static netdev_features_t
3973 igc_features_check(struct sk_buff
*skb
, struct net_device
*dev
,
3974 netdev_features_t features
)
3976 unsigned int network_hdr_len
, mac_hdr_len
;
3978 /* Make certain the headers can be described by a context descriptor */
3979 mac_hdr_len
= skb_network_header(skb
) - skb
->data
;
3980 if (unlikely(mac_hdr_len
> IGC_MAX_MAC_HDR_LEN
))
3981 return features
& ~(NETIF_F_HW_CSUM
|
3983 NETIF_F_HW_VLAN_CTAG_TX
|
3987 network_hdr_len
= skb_checksum_start(skb
) - skb_network_header(skb
);
3988 if (unlikely(network_hdr_len
> IGC_MAX_NETWORK_HDR_LEN
))
3989 return features
& ~(NETIF_F_HW_CSUM
|
3994 /* We can only support IPv4 TSO in tunnels if we can mangle the
3995 * inner IP ID field, so strip TSO if MANGLEID is not supported.
3997 if (skb
->encapsulation
&& !(features
& NETIF_F_TSO_MANGLEID
))
3998 features
&= ~NETIF_F_TSO
;
4003 static void igc_tsync_interrupt(struct igc_adapter
*adapter
)
4005 struct igc_hw
*hw
= &adapter
->hw
;
4006 u32 tsicr
= rd32(IGC_TSICR
);
4009 if (tsicr
& IGC_TSICR_TXTS
) {
4010 /* retrieve hardware timestamp */
4011 schedule_work(&adapter
->ptp_tx_work
);
4012 ack
|= IGC_TSICR_TXTS
;
4015 /* acknowledge the interrupts */
4016 wr32(IGC_TSICR
, ack
);
4020 * igc_msix_other - msix other interrupt handler
4021 * @irq: interrupt number
4022 * @data: pointer to a q_vector
4024 static irqreturn_t
igc_msix_other(int irq
, void *data
)
4026 struct igc_adapter
*adapter
= data
;
4027 struct igc_hw
*hw
= &adapter
->hw
;
4028 u32 icr
= rd32(IGC_ICR
);
4030 /* reading ICR causes bit 31 of EICR to be cleared */
4031 if (icr
& IGC_ICR_DRSTA
)
4032 schedule_work(&adapter
->reset_task
);
4034 if (icr
& IGC_ICR_DOUTSYNC
) {
4035 /* HW is reporting DMA is out of sync */
4036 adapter
->stats
.doosync
++;
4039 if (icr
& IGC_ICR_LSC
) {
4040 hw
->mac
.get_link_status
= 1;
4041 /* guard against interrupt when we're going down */
4042 if (!test_bit(__IGC_DOWN
, &adapter
->state
))
4043 mod_timer(&adapter
->watchdog_timer
, jiffies
+ 1);
4046 if (icr
& IGC_ICR_TS
)
4047 igc_tsync_interrupt(adapter
);
4049 wr32(IGC_EIMS
, adapter
->eims_other
);
4054 static void igc_write_itr(struct igc_q_vector
*q_vector
)
4056 u32 itr_val
= q_vector
->itr_val
& IGC_QVECTOR_MASK
;
4058 if (!q_vector
->set_itr
)
4062 itr_val
= IGC_ITR_VAL_MASK
;
4064 itr_val
|= IGC_EITR_CNT_IGNR
;
4066 writel(itr_val
, q_vector
->itr_register
);
4067 q_vector
->set_itr
= 0;
4070 static irqreturn_t
igc_msix_ring(int irq
, void *data
)
4072 struct igc_q_vector
*q_vector
= data
;
4074 /* Write the ITR value calculated from the previous interrupt. */
4075 igc_write_itr(q_vector
);
4077 napi_schedule(&q_vector
->napi
);
4083 * igc_request_msix - Initialize MSI-X interrupts
4084 * @adapter: Pointer to adapter structure
4086 * igc_request_msix allocates MSI-X vectors and requests interrupts from the
4089 static int igc_request_msix(struct igc_adapter
*adapter
)
4091 int i
= 0, err
= 0, vector
= 0, free_vector
= 0;
4092 struct net_device
*netdev
= adapter
->netdev
;
4094 err
= request_irq(adapter
->msix_entries
[vector
].vector
,
4095 &igc_msix_other
, 0, netdev
->name
, adapter
);
4099 for (i
= 0; i
< adapter
->num_q_vectors
; i
++) {
4100 struct igc_q_vector
*q_vector
= adapter
->q_vector
[i
];
4104 q_vector
->itr_register
= adapter
->io_addr
+ IGC_EITR(vector
);
4106 if (q_vector
->rx
.ring
&& q_vector
->tx
.ring
)
4107 sprintf(q_vector
->name
, "%s-TxRx-%u", netdev
->name
,
4108 q_vector
->rx
.ring
->queue_index
);
4109 else if (q_vector
->tx
.ring
)
4110 sprintf(q_vector
->name
, "%s-tx-%u", netdev
->name
,
4111 q_vector
->tx
.ring
->queue_index
);
4112 else if (q_vector
->rx
.ring
)
4113 sprintf(q_vector
->name
, "%s-rx-%u", netdev
->name
,
4114 q_vector
->rx
.ring
->queue_index
);
4116 sprintf(q_vector
->name
, "%s-unused", netdev
->name
);
4118 err
= request_irq(adapter
->msix_entries
[vector
].vector
,
4119 igc_msix_ring
, 0, q_vector
->name
,
4125 igc_configure_msix(adapter
);
4129 /* free already assigned IRQs */
4130 free_irq(adapter
->msix_entries
[free_vector
++].vector
, adapter
);
4133 for (i
= 0; i
< vector
; i
++) {
4134 free_irq(adapter
->msix_entries
[free_vector
++].vector
,
4135 adapter
->q_vector
[i
]);
4142 * igc_clear_interrupt_scheme - reset the device to a state of no interrupts
4143 * @adapter: Pointer to adapter structure
4145 * This function resets the device so that it has 0 rx queues, tx queues, and
4146 * MSI-X interrupts allocated.
4148 static void igc_clear_interrupt_scheme(struct igc_adapter
*adapter
)
4150 igc_free_q_vectors(adapter
);
4151 igc_reset_interrupt_capability(adapter
);
4154 /* Need to wait a few seconds after link up to get diagnostic information from
4157 static void igc_update_phy_info(struct timer_list
*t
)
4159 struct igc_adapter
*adapter
= from_timer(adapter
, t
, phy_info_timer
);
4161 igc_get_phy_info(&adapter
->hw
);
4165 * igc_has_link - check shared code for link and determine up/down
4166 * @adapter: pointer to driver private info
4168 bool igc_has_link(struct igc_adapter
*adapter
)
4170 struct igc_hw
*hw
= &adapter
->hw
;
4171 bool link_active
= false;
4173 /* get_link_status is set on LSC (link status) interrupt or
4174 * rx sequence error interrupt. get_link_status will stay
4175 * false until the igc_check_for_link establishes link
4176 * for copper adapters ONLY
4178 switch (hw
->phy
.media_type
) {
4179 case igc_media_type_copper
:
4180 if (!hw
->mac
.get_link_status
)
4182 hw
->mac
.ops
.check_for_link(hw
);
4183 link_active
= !hw
->mac
.get_link_status
;
4186 case igc_media_type_unknown
:
4190 if (hw
->mac
.type
== igc_i225
&&
4191 hw
->phy
.id
== I225_I_PHY_ID
) {
4192 if (!netif_carrier_ok(adapter
->netdev
)) {
4193 adapter
->flags
&= ~IGC_FLAG_NEED_LINK_UPDATE
;
4194 } else if (!(adapter
->flags
& IGC_FLAG_NEED_LINK_UPDATE
)) {
4195 adapter
->flags
|= IGC_FLAG_NEED_LINK_UPDATE
;
4196 adapter
->link_check_timeout
= jiffies
;
4204 * igc_watchdog - Timer Call-back
4205 * @t: timer for the watchdog
4207 static void igc_watchdog(struct timer_list
*t
)
4209 struct igc_adapter
*adapter
= from_timer(adapter
, t
, watchdog_timer
);
4210 /* Do the rest outside of interrupt context */
4211 schedule_work(&adapter
->watchdog_task
);
4214 static void igc_watchdog_task(struct work_struct
*work
)
4216 struct igc_adapter
*adapter
= container_of(work
,
4219 struct net_device
*netdev
= adapter
->netdev
;
4220 struct igc_hw
*hw
= &adapter
->hw
;
4221 struct igc_phy_info
*phy
= &hw
->phy
;
4222 u16 phy_data
, retry_count
= 20;
4226 link
= igc_has_link(adapter
);
4228 if (adapter
->flags
& IGC_FLAG_NEED_LINK_UPDATE
) {
4229 if (time_after(jiffies
, (adapter
->link_check_timeout
+ HZ
)))
4230 adapter
->flags
&= ~IGC_FLAG_NEED_LINK_UPDATE
;
4236 /* Cancel scheduled suspend requests. */
4237 pm_runtime_resume(netdev
->dev
.parent
);
4239 if (!netif_carrier_ok(netdev
)) {
4242 hw
->mac
.ops
.get_speed_and_duplex(hw
,
4243 &adapter
->link_speed
,
4244 &adapter
->link_duplex
);
4246 ctrl
= rd32(IGC_CTRL
);
4247 /* Link status message must follow this format */
4249 "NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n",
4250 adapter
->link_speed
,
4251 adapter
->link_duplex
== FULL_DUPLEX
?
4253 (ctrl
& IGC_CTRL_TFCE
) &&
4254 (ctrl
& IGC_CTRL_RFCE
) ? "RX/TX" :
4255 (ctrl
& IGC_CTRL_RFCE
) ? "RX" :
4256 (ctrl
& IGC_CTRL_TFCE
) ? "TX" : "None");
4258 /* check if SmartSpeed worked */
4259 igc_check_downshift(hw
);
4260 if (phy
->speed_downgraded
)
4261 netdev_warn(netdev
, "Link Speed was downgraded by SmartSpeed\n");
4263 /* adjust timeout factor according to speed/duplex */
4264 adapter
->tx_timeout_factor
= 1;
4265 switch (adapter
->link_speed
) {
4267 adapter
->tx_timeout_factor
= 14;
4270 /* maybe add some timeout factor ? */
4274 if (adapter
->link_speed
!= SPEED_1000
)
4277 /* wait for Remote receiver status OK */
4279 if (!igc_read_phy_reg(hw
, PHY_1000T_STATUS
,
4281 if (!(phy_data
& SR_1000T_REMOTE_RX_STATUS
) &&
4285 goto retry_read_status
;
4286 } else if (!retry_count
) {
4287 netdev_err(netdev
, "exceed max 2 second\n");
4290 netdev_err(netdev
, "read 1000Base-T Status Reg\n");
4293 netif_carrier_on(netdev
);
4295 /* link state has changed, schedule phy info update */
4296 if (!test_bit(__IGC_DOWN
, &adapter
->state
))
4297 mod_timer(&adapter
->phy_info_timer
,
4298 round_jiffies(jiffies
+ 2 * HZ
));
4301 if (netif_carrier_ok(netdev
)) {
4302 adapter
->link_speed
= 0;
4303 adapter
->link_duplex
= 0;
4305 /* Links status message must follow this format */
4306 netdev_info(netdev
, "NIC Link is Down\n");
4307 netif_carrier_off(netdev
);
4309 /* link state has changed, schedule phy info update */
4310 if (!test_bit(__IGC_DOWN
, &adapter
->state
))
4311 mod_timer(&adapter
->phy_info_timer
,
4312 round_jiffies(jiffies
+ 2 * HZ
));
4314 /* link is down, time to check for alternate media */
4315 if (adapter
->flags
& IGC_FLAG_MAS_ENABLE
) {
4316 if (adapter
->flags
& IGC_FLAG_MEDIA_RESET
) {
4317 schedule_work(&adapter
->reset_task
);
4318 /* return immediately */
4322 pm_schedule_suspend(netdev
->dev
.parent
,
4325 /* also check for alternate media here */
4326 } else if (!netif_carrier_ok(netdev
) &&
4327 (adapter
->flags
& IGC_FLAG_MAS_ENABLE
)) {
4328 if (adapter
->flags
& IGC_FLAG_MEDIA_RESET
) {
4329 schedule_work(&adapter
->reset_task
);
4330 /* return immediately */
4336 spin_lock(&adapter
->stats64_lock
);
4337 igc_update_stats(adapter
);
4338 spin_unlock(&adapter
->stats64_lock
);
4340 for (i
= 0; i
< adapter
->num_tx_queues
; i
++) {
4341 struct igc_ring
*tx_ring
= adapter
->tx_ring
[i
];
4343 if (!netif_carrier_ok(netdev
)) {
4344 /* We've lost link, so the controller stops DMA,
4345 * but we've got queued Tx work that's never going
4346 * to get done, so reset controller to flush Tx.
4347 * (Do the reset outside of interrupt context).
4349 if (igc_desc_unused(tx_ring
) + 1 < tx_ring
->count
) {
4350 adapter
->tx_timeout_count
++;
4351 schedule_work(&adapter
->reset_task
);
4352 /* return immediately since reset is imminent */
4357 /* Force detection of hung controller every watchdog period */
4358 set_bit(IGC_RING_FLAG_TX_DETECT_HANG
, &tx_ring
->flags
);
4361 /* Cause software interrupt to ensure Rx ring is cleaned */
4362 if (adapter
->flags
& IGC_FLAG_HAS_MSIX
) {
4365 for (i
= 0; i
< adapter
->num_q_vectors
; i
++)
4366 eics
|= adapter
->q_vector
[i
]->eims_value
;
4367 wr32(IGC_EICS
, eics
);
4369 wr32(IGC_ICS
, IGC_ICS_RXDMT0
);
4372 igc_ptp_tx_hang(adapter
);
4374 /* Reset the timer */
4375 if (!test_bit(__IGC_DOWN
, &adapter
->state
)) {
4376 if (adapter
->flags
& IGC_FLAG_NEED_LINK_UPDATE
)
4377 mod_timer(&adapter
->watchdog_timer
,
4378 round_jiffies(jiffies
+ HZ
));
4380 mod_timer(&adapter
->watchdog_timer
,
4381 round_jiffies(jiffies
+ 2 * HZ
));
4386 * igc_intr_msi - Interrupt Handler
4387 * @irq: interrupt number
4388 * @data: pointer to a network interface device structure
4390 static irqreturn_t
igc_intr_msi(int irq
, void *data
)
4392 struct igc_adapter
*adapter
= data
;
4393 struct igc_q_vector
*q_vector
= adapter
->q_vector
[0];
4394 struct igc_hw
*hw
= &adapter
->hw
;
4395 /* read ICR disables interrupts using IAM */
4396 u32 icr
= rd32(IGC_ICR
);
4398 igc_write_itr(q_vector
);
4400 if (icr
& IGC_ICR_DRSTA
)
4401 schedule_work(&adapter
->reset_task
);
4403 if (icr
& IGC_ICR_DOUTSYNC
) {
4404 /* HW is reporting DMA is out of sync */
4405 adapter
->stats
.doosync
++;
4408 if (icr
& (IGC_ICR_RXSEQ
| IGC_ICR_LSC
)) {
4409 hw
->mac
.get_link_status
= 1;
4410 if (!test_bit(__IGC_DOWN
, &adapter
->state
))
4411 mod_timer(&adapter
->watchdog_timer
, jiffies
+ 1);
4414 napi_schedule(&q_vector
->napi
);
4420 * igc_intr - Legacy Interrupt Handler
4421 * @irq: interrupt number
4422 * @data: pointer to a network interface device structure
4424 static irqreturn_t
igc_intr(int irq
, void *data
)
4426 struct igc_adapter
*adapter
= data
;
4427 struct igc_q_vector
*q_vector
= adapter
->q_vector
[0];
4428 struct igc_hw
*hw
= &adapter
->hw
;
4429 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
4430 * need for the IMC write
4432 u32 icr
= rd32(IGC_ICR
);
4434 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
4435 * not set, then the adapter didn't send an interrupt
4437 if (!(icr
& IGC_ICR_INT_ASSERTED
))
4440 igc_write_itr(q_vector
);
4442 if (icr
& IGC_ICR_DRSTA
)
4443 schedule_work(&adapter
->reset_task
);
4445 if (icr
& IGC_ICR_DOUTSYNC
) {
4446 /* HW is reporting DMA is out of sync */
4447 adapter
->stats
.doosync
++;
4450 if (icr
& (IGC_ICR_RXSEQ
| IGC_ICR_LSC
)) {
4451 hw
->mac
.get_link_status
= 1;
4452 /* guard against interrupt when we're going down */
4453 if (!test_bit(__IGC_DOWN
, &adapter
->state
))
4454 mod_timer(&adapter
->watchdog_timer
, jiffies
+ 1);
4457 napi_schedule(&q_vector
->napi
);
4462 static void igc_free_irq(struct igc_adapter
*adapter
)
4464 if (adapter
->msix_entries
) {
4467 free_irq(adapter
->msix_entries
[vector
++].vector
, adapter
);
4469 for (i
= 0; i
< adapter
->num_q_vectors
; i
++)
4470 free_irq(adapter
->msix_entries
[vector
++].vector
,
4471 adapter
->q_vector
[i
]);
4473 free_irq(adapter
->pdev
->irq
, adapter
);
4478 * igc_request_irq - initialize interrupts
4479 * @adapter: Pointer to adapter structure
4481 * Attempts to configure interrupts using the best available
4482 * capabilities of the hardware and kernel.
4484 static int igc_request_irq(struct igc_adapter
*adapter
)
4486 struct net_device
*netdev
= adapter
->netdev
;
4487 struct pci_dev
*pdev
= adapter
->pdev
;
4490 if (adapter
->flags
& IGC_FLAG_HAS_MSIX
) {
4491 err
= igc_request_msix(adapter
);
4494 /* fall back to MSI */
4495 igc_free_all_tx_resources(adapter
);
4496 igc_free_all_rx_resources(adapter
);
4498 igc_clear_interrupt_scheme(adapter
);
4499 err
= igc_init_interrupt_scheme(adapter
, false);
4502 igc_setup_all_tx_resources(adapter
);
4503 igc_setup_all_rx_resources(adapter
);
4504 igc_configure(adapter
);
4507 igc_assign_vector(adapter
->q_vector
[0], 0);
4509 if (adapter
->flags
& IGC_FLAG_HAS_MSI
) {
4510 err
= request_irq(pdev
->irq
, &igc_intr_msi
, 0,
4511 netdev
->name
, adapter
);
4515 /* fall back to legacy interrupts */
4516 igc_reset_interrupt_capability(adapter
);
4517 adapter
->flags
&= ~IGC_FLAG_HAS_MSI
;
4520 err
= request_irq(pdev
->irq
, &igc_intr
, IRQF_SHARED
,
4521 netdev
->name
, adapter
);
4524 netdev_err(netdev
, "Error %d getting interrupt\n", err
);
4531 * __igc_open - Called when a network interface is made active
4532 * @netdev: network interface device structure
4533 * @resuming: boolean indicating if the device is resuming
4535 * Returns 0 on success, negative value on failure
4537 * The open entry point is called when a network interface is made
4538 * active by the system (IFF_UP). At this point all resources needed
4539 * for transmit and receive operations are allocated, the interrupt
4540 * handler is registered with the OS, the watchdog timer is started,
4541 * and the stack is notified that the interface is ready.
4543 static int __igc_open(struct net_device
*netdev
, bool resuming
)
4545 struct igc_adapter
*adapter
= netdev_priv(netdev
);
4546 struct pci_dev
*pdev
= adapter
->pdev
;
4547 struct igc_hw
*hw
= &adapter
->hw
;
4551 /* disallow open during test */
4553 if (test_bit(__IGC_TESTING
, &adapter
->state
)) {
4559 pm_runtime_get_sync(&pdev
->dev
);
4561 netif_carrier_off(netdev
);
4563 /* allocate transmit descriptors */
4564 err
= igc_setup_all_tx_resources(adapter
);
4568 /* allocate receive descriptors */
4569 err
= igc_setup_all_rx_resources(adapter
);
4573 igc_power_up_link(adapter
);
4575 igc_configure(adapter
);
4577 err
= igc_request_irq(adapter
);
4581 /* Notify the stack of the actual queue counts. */
4582 err
= netif_set_real_num_tx_queues(netdev
, adapter
->num_tx_queues
);
4584 goto err_set_queues
;
4586 err
= netif_set_real_num_rx_queues(netdev
, adapter
->num_rx_queues
);
4588 goto err_set_queues
;
4590 clear_bit(__IGC_DOWN
, &adapter
->state
);
4592 for (i
= 0; i
< adapter
->num_q_vectors
; i
++)
4593 napi_enable(&adapter
->q_vector
[i
]->napi
);
4595 /* Clear any pending interrupts. */
4597 igc_irq_enable(adapter
);
4600 pm_runtime_put(&pdev
->dev
);
4602 netif_tx_start_all_queues(netdev
);
4604 /* start the watchdog. */
4605 hw
->mac
.get_link_status
= 1;
4606 schedule_work(&adapter
->watchdog_task
);
4611 igc_free_irq(adapter
);
4613 igc_release_hw_control(adapter
);
4614 igc_power_down_link(adapter
);
4615 igc_free_all_rx_resources(adapter
);
4617 igc_free_all_tx_resources(adapter
);
4621 pm_runtime_put(&pdev
->dev
);
4626 int igc_open(struct net_device
*netdev
)
4628 return __igc_open(netdev
, false);
4632 * __igc_close - Disables a network interface
4633 * @netdev: network interface device structure
4634 * @suspending: boolean indicating the device is suspending
4636 * Returns 0, this is not allowed to fail
4638 * The close entry point is called when an interface is de-activated
4639 * by the OS. The hardware is still under the driver's control, but
4640 * needs to be disabled. A global MAC reset is issued to stop the
4641 * hardware, and all transmit and receive resources are freed.
4643 static int __igc_close(struct net_device
*netdev
, bool suspending
)
4645 struct igc_adapter
*adapter
= netdev_priv(netdev
);
4646 struct pci_dev
*pdev
= adapter
->pdev
;
4648 WARN_ON(test_bit(__IGC_RESETTING
, &adapter
->state
));
4651 pm_runtime_get_sync(&pdev
->dev
);
4655 igc_release_hw_control(adapter
);
4657 igc_free_irq(adapter
);
4659 igc_free_all_tx_resources(adapter
);
4660 igc_free_all_rx_resources(adapter
);
4663 pm_runtime_put_sync(&pdev
->dev
);
4668 int igc_close(struct net_device
*netdev
)
4670 if (netif_device_present(netdev
) || netdev
->dismantle
)
4671 return __igc_close(netdev
, false);
4676 * igc_ioctl - Access the hwtstamp interface
4677 * @netdev: network interface device structure
4678 * @ifreq: interface request data
4679 * @cmd: ioctl command
4681 static int igc_ioctl(struct net_device
*netdev
, struct ifreq
*ifr
, int cmd
)
4685 return igc_ptp_get_ts_config(netdev
, ifr
);
4687 return igc_ptp_set_ts_config(netdev
, ifr
);
4693 static int igc_save_launchtime_params(struct igc_adapter
*adapter
, int queue
,
4696 struct igc_ring
*ring
;
4699 if (queue
< 0 || queue
>= adapter
->num_tx_queues
)
4702 ring
= adapter
->tx_ring
[queue
];
4703 ring
->launchtime_enable
= enable
;
4705 if (adapter
->base_time
)
4708 adapter
->cycle_time
= NSEC_PER_SEC
;
4710 for (i
= 0; i
< adapter
->num_tx_queues
; i
++) {
4711 ring
= adapter
->tx_ring
[i
];
4712 ring
->start_time
= 0;
4713 ring
->end_time
= NSEC_PER_SEC
;
4719 static bool validate_schedule(const struct tc_taprio_qopt_offload
*qopt
)
4721 int queue_uses
[IGC_MAX_TX_QUEUES
] = { };
4724 if (qopt
->cycle_time_extension
)
4727 for (n
= 0; n
< qopt
->num_entries
; n
++) {
4728 const struct tc_taprio_sched_entry
*e
;
4731 e
= &qopt
->entries
[n
];
4733 /* i225 only supports "global" frame preemption
4736 if (e
->command
!= TC_TAPRIO_CMD_SET_GATES
)
4739 for (i
= 0; i
< IGC_MAX_TX_QUEUES
; i
++) {
4740 if (e
->gate_mask
& BIT(i
))
4743 if (queue_uses
[i
] > 1)
4751 static int igc_tsn_enable_launchtime(struct igc_adapter
*adapter
,
4752 struct tc_etf_qopt_offload
*qopt
)
4754 struct igc_hw
*hw
= &adapter
->hw
;
4757 if (hw
->mac
.type
!= igc_i225
)
4760 err
= igc_save_launchtime_params(adapter
, qopt
->queue
, qopt
->enable
);
4764 return igc_tsn_offload_apply(adapter
);
4767 static int igc_save_qbv_schedule(struct igc_adapter
*adapter
,
4768 struct tc_taprio_qopt_offload
*qopt
)
4770 u32 start_time
= 0, end_time
= 0;
4773 if (!qopt
->enable
) {
4774 adapter
->base_time
= 0;
4778 if (adapter
->base_time
)
4781 if (!validate_schedule(qopt
))
4784 adapter
->cycle_time
= qopt
->cycle_time
;
4785 adapter
->base_time
= qopt
->base_time
;
4787 /* FIXME: be a little smarter about cases when the gate for a
4788 * queue stays open for more than one entry.
4790 for (n
= 0; n
< qopt
->num_entries
; n
++) {
4791 struct tc_taprio_sched_entry
*e
= &qopt
->entries
[n
];
4794 end_time
+= e
->interval
;
4796 for (i
= 0; i
< IGC_MAX_TX_QUEUES
; i
++) {
4797 struct igc_ring
*ring
= adapter
->tx_ring
[i
];
4799 if (!(e
->gate_mask
& BIT(i
)))
4802 ring
->start_time
= start_time
;
4803 ring
->end_time
= end_time
;
4806 start_time
+= e
->interval
;
4812 static int igc_tsn_enable_qbv_scheduling(struct igc_adapter
*adapter
,
4813 struct tc_taprio_qopt_offload
*qopt
)
4815 struct igc_hw
*hw
= &adapter
->hw
;
4818 if (hw
->mac
.type
!= igc_i225
)
4821 err
= igc_save_qbv_schedule(adapter
, qopt
);
4825 return igc_tsn_offload_apply(adapter
);
4828 static int igc_setup_tc(struct net_device
*dev
, enum tc_setup_type type
,
4831 struct igc_adapter
*adapter
= netdev_priv(dev
);
4834 case TC_SETUP_QDISC_TAPRIO
:
4835 return igc_tsn_enable_qbv_scheduling(adapter
, type_data
);
4837 case TC_SETUP_QDISC_ETF
:
4838 return igc_tsn_enable_launchtime(adapter
, type_data
);
4845 static const struct net_device_ops igc_netdev_ops
= {
4846 .ndo_open
= igc_open
,
4847 .ndo_stop
= igc_close
,
4848 .ndo_start_xmit
= igc_xmit_frame
,
4849 .ndo_set_rx_mode
= igc_set_rx_mode
,
4850 .ndo_set_mac_address
= igc_set_mac
,
4851 .ndo_change_mtu
= igc_change_mtu
,
4852 .ndo_get_stats
= igc_get_stats
,
4853 .ndo_fix_features
= igc_fix_features
,
4854 .ndo_set_features
= igc_set_features
,
4855 .ndo_features_check
= igc_features_check
,
4856 .ndo_do_ioctl
= igc_ioctl
,
4857 .ndo_setup_tc
= igc_setup_tc
,
4860 /* PCIe configuration access */
4861 void igc_read_pci_cfg(struct igc_hw
*hw
, u32 reg
, u16
*value
)
4863 struct igc_adapter
*adapter
= hw
->back
;
4865 pci_read_config_word(adapter
->pdev
, reg
, value
);
4868 void igc_write_pci_cfg(struct igc_hw
*hw
, u32 reg
, u16
*value
)
4870 struct igc_adapter
*adapter
= hw
->back
;
4872 pci_write_config_word(adapter
->pdev
, reg
, *value
);
4875 s32
igc_read_pcie_cap_reg(struct igc_hw
*hw
, u32 reg
, u16
*value
)
4877 struct igc_adapter
*adapter
= hw
->back
;
4879 if (!pci_is_pcie(adapter
->pdev
))
4880 return -IGC_ERR_CONFIG
;
4882 pcie_capability_read_word(adapter
->pdev
, reg
, value
);
4887 s32
igc_write_pcie_cap_reg(struct igc_hw
*hw
, u32 reg
, u16
*value
)
4889 struct igc_adapter
*adapter
= hw
->back
;
4891 if (!pci_is_pcie(adapter
->pdev
))
4892 return -IGC_ERR_CONFIG
;
4894 pcie_capability_write_word(adapter
->pdev
, reg
, *value
);
4899 u32
igc_rd32(struct igc_hw
*hw
, u32 reg
)
4901 struct igc_adapter
*igc
= container_of(hw
, struct igc_adapter
, hw
);
4902 u8 __iomem
*hw_addr
= READ_ONCE(hw
->hw_addr
);
4905 value
= readl(&hw_addr
[reg
]);
4907 /* reads should not return all F's */
4908 if (!(~value
) && (!reg
|| !(~readl(hw_addr
)))) {
4909 struct net_device
*netdev
= igc
->netdev
;
4912 netif_device_detach(netdev
);
4913 netdev_err(netdev
, "PCIe link lost, device now detached\n");
4914 WARN(pci_device_is_present(igc
->pdev
),
4915 "igc: Failed to read reg 0x%x!\n", reg
);
4921 int igc_set_spd_dplx(struct igc_adapter
*adapter
, u32 spd
, u8 dplx
)
4923 struct igc_mac_info
*mac
= &adapter
->hw
.mac
;
4927 /* Make sure dplx is at most 1 bit and lsb of speed is not set
4928 * for the switch() below to work
4930 if ((spd
& 1) || (dplx
& ~1))
4933 switch (spd
+ dplx
) {
4934 case SPEED_10
+ DUPLEX_HALF
:
4935 mac
->forced_speed_duplex
= ADVERTISE_10_HALF
;
4937 case SPEED_10
+ DUPLEX_FULL
:
4938 mac
->forced_speed_duplex
= ADVERTISE_10_FULL
;
4940 case SPEED_100
+ DUPLEX_HALF
:
4941 mac
->forced_speed_duplex
= ADVERTISE_100_HALF
;
4943 case SPEED_100
+ DUPLEX_FULL
:
4944 mac
->forced_speed_duplex
= ADVERTISE_100_FULL
;
4946 case SPEED_1000
+ DUPLEX_FULL
:
4948 adapter
->hw
.phy
.autoneg_advertised
= ADVERTISE_1000_FULL
;
4950 case SPEED_1000
+ DUPLEX_HALF
: /* not supported */
4952 case SPEED_2500
+ DUPLEX_FULL
:
4954 adapter
->hw
.phy
.autoneg_advertised
= ADVERTISE_2500_FULL
;
4956 case SPEED_2500
+ DUPLEX_HALF
: /* not supported */
4961 /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */
4962 adapter
->hw
.phy
.mdix
= AUTO_ALL_MODES
;
4967 netdev_err(adapter
->netdev
, "Unsupported Speed/Duplex configuration\n");
4972 * igc_probe - Device Initialization Routine
4973 * @pdev: PCI device information struct
4974 * @ent: entry in igc_pci_tbl
4976 * Returns 0 on success, negative on failure
4978 * igc_probe initializes an adapter identified by a pci_dev structure.
4979 * The OS initialization, configuring the adapter private structure,
4980 * and a hardware reset occur.
4982 static int igc_probe(struct pci_dev
*pdev
,
4983 const struct pci_device_id
*ent
)
4985 struct igc_adapter
*adapter
;
4986 struct net_device
*netdev
;
4988 const struct igc_info
*ei
= igc_info_tbl
[ent
->driver_data
];
4989 int err
, pci_using_dac
;
4991 err
= pci_enable_device_mem(pdev
);
4996 err
= dma_set_mask_and_coherent(&pdev
->dev
, DMA_BIT_MASK(64));
5000 err
= dma_set_mask_and_coherent(&pdev
->dev
, DMA_BIT_MASK(32));
5003 "No usable DMA configuration, aborting\n");
5008 err
= pci_request_mem_regions(pdev
, igc_driver_name
);
5012 pci_enable_pcie_error_reporting(pdev
);
5014 pci_set_master(pdev
);
5017 netdev
= alloc_etherdev_mq(sizeof(struct igc_adapter
),
5021 goto err_alloc_etherdev
;
5023 SET_NETDEV_DEV(netdev
, &pdev
->dev
);
5025 pci_set_drvdata(pdev
, netdev
);
5026 adapter
= netdev_priv(netdev
);
5027 adapter
->netdev
= netdev
;
5028 adapter
->pdev
= pdev
;
5031 adapter
->port_num
= hw
->bus
.func
;
5032 adapter
->msg_enable
= netif_msg_init(debug
, DEFAULT_MSG_ENABLE
);
5034 err
= pci_save_state(pdev
);
5039 adapter
->io_addr
= ioremap(pci_resource_start(pdev
, 0),
5040 pci_resource_len(pdev
, 0));
5041 if (!adapter
->io_addr
)
5044 /* hw->hw_addr can be zeroed, so use adapter->io_addr for unmap */
5045 hw
->hw_addr
= adapter
->io_addr
;
5047 netdev
->netdev_ops
= &igc_netdev_ops
;
5048 igc_ethtool_set_ops(netdev
);
5049 netdev
->watchdog_timeo
= 5 * HZ
;
5051 netdev
->mem_start
= pci_resource_start(pdev
, 0);
5052 netdev
->mem_end
= pci_resource_end(pdev
, 0);
5054 /* PCI config space info */
5055 hw
->vendor_id
= pdev
->vendor
;
5056 hw
->device_id
= pdev
->device
;
5057 hw
->revision_id
= pdev
->revision
;
5058 hw
->subsystem_vendor_id
= pdev
->subsystem_vendor
;
5059 hw
->subsystem_device_id
= pdev
->subsystem_device
;
5061 /* Copy the default MAC and PHY function pointers */
5062 memcpy(&hw
->mac
.ops
, ei
->mac_ops
, sizeof(hw
->mac
.ops
));
5063 memcpy(&hw
->phy
.ops
, ei
->phy_ops
, sizeof(hw
->phy
.ops
));
5065 /* Initialize skew-specific constants */
5066 err
= ei
->get_invariants(hw
);
5070 /* Add supported features to the features list*/
5071 netdev
->features
|= NETIF_F_SG
;
5072 netdev
->features
|= NETIF_F_TSO
;
5073 netdev
->features
|= NETIF_F_TSO6
;
5074 netdev
->features
|= NETIF_F_TSO_ECN
;
5075 netdev
->features
|= NETIF_F_RXCSUM
;
5076 netdev
->features
|= NETIF_F_HW_CSUM
;
5077 netdev
->features
|= NETIF_F_SCTP_CRC
;
5078 netdev
->features
|= NETIF_F_HW_TC
;
5080 #define IGC_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \
5081 NETIF_F_GSO_GRE_CSUM | \
5082 NETIF_F_GSO_IPXIP4 | \
5083 NETIF_F_GSO_IPXIP6 | \
5084 NETIF_F_GSO_UDP_TUNNEL | \
5085 NETIF_F_GSO_UDP_TUNNEL_CSUM)
5087 netdev
->gso_partial_features
= IGC_GSO_PARTIAL_FEATURES
;
5088 netdev
->features
|= NETIF_F_GSO_PARTIAL
| IGC_GSO_PARTIAL_FEATURES
;
5090 /* setup the private structure */
5091 err
= igc_sw_init(adapter
);
5095 /* copy netdev features into list of user selectable features */
5096 netdev
->hw_features
|= NETIF_F_NTUPLE
;
5097 netdev
->hw_features
|= netdev
->features
;
5100 netdev
->features
|= NETIF_F_HIGHDMA
;
5102 /* MTU range: 68 - 9216 */
5103 netdev
->min_mtu
= ETH_MIN_MTU
;
5104 netdev
->max_mtu
= MAX_STD_JUMBO_FRAME_SIZE
;
5106 /* before reading the NVM, reset the controller to put the device in a
5107 * known good starting state
5109 hw
->mac
.ops
.reset_hw(hw
);
5111 if (igc_get_flash_presence_i225(hw
)) {
5112 if (hw
->nvm
.ops
.validate(hw
) < 0) {
5113 dev_err(&pdev
->dev
, "The NVM Checksum Is Not Valid\n");
5119 if (eth_platform_get_mac_address(&pdev
->dev
, hw
->mac
.addr
)) {
5120 /* copy the MAC address out of the NVM */
5121 if (hw
->mac
.ops
.read_mac_addr(hw
))
5122 dev_err(&pdev
->dev
, "NVM Read Error\n");
5125 memcpy(netdev
->dev_addr
, hw
->mac
.addr
, netdev
->addr_len
);
5127 if (!is_valid_ether_addr(netdev
->dev_addr
)) {
5128 dev_err(&pdev
->dev
, "Invalid MAC Address\n");
5133 /* configure RXPBSIZE and TXPBSIZE */
5134 wr32(IGC_RXPBS
, I225_RXPBSIZE_DEFAULT
);
5135 wr32(IGC_TXPBS
, I225_TXPBSIZE_DEFAULT
);
5137 timer_setup(&adapter
->watchdog_timer
, igc_watchdog
, 0);
5138 timer_setup(&adapter
->phy_info_timer
, igc_update_phy_info
, 0);
5140 INIT_WORK(&adapter
->reset_task
, igc_reset_task
);
5141 INIT_WORK(&adapter
->watchdog_task
, igc_watchdog_task
);
5143 /* Initialize link properties that are user-changeable */
5144 adapter
->fc_autoneg
= true;
5145 hw
->mac
.autoneg
= true;
5146 hw
->phy
.autoneg_advertised
= 0xaf;
5148 hw
->fc
.requested_mode
= igc_fc_default
;
5149 hw
->fc
.current_mode
= igc_fc_default
;
5151 /* By default, support wake on port A */
5152 adapter
->flags
|= IGC_FLAG_WOL_SUPPORTED
;
5154 /* initialize the wol settings based on the eeprom settings */
5155 if (adapter
->flags
& IGC_FLAG_WOL_SUPPORTED
)
5156 adapter
->wol
|= IGC_WUFC_MAG
;
5158 device_set_wakeup_enable(&adapter
->pdev
->dev
,
5159 adapter
->flags
& IGC_FLAG_WOL_SUPPORTED
);
5161 /* reset the hardware with the new settings */
5164 /* let the f/w know that the h/w is now under the control of the
5167 igc_get_hw_control(adapter
);
5169 strncpy(netdev
->name
, "eth%d", IFNAMSIZ
);
5170 err
= register_netdev(netdev
);
5174 /* carrier off reporting is important to ethtool even BEFORE open */
5175 netif_carrier_off(netdev
);
5177 /* do hw tstamp init after resetting */
5178 igc_ptp_init(adapter
);
5180 /* Check if Media Autosense is enabled */
5183 /* print pcie link status and MAC address */
5184 pcie_print_link_status(pdev
);
5185 netdev_info(netdev
, "MAC: %pM\n", netdev
->dev_addr
);
5187 dev_pm_set_driver_flags(&pdev
->dev
, DPM_FLAG_NO_DIRECT_COMPLETE
);
5189 pm_runtime_put_noidle(&pdev
->dev
);
5194 igc_release_hw_control(adapter
);
5196 if (!igc_check_reset_block(hw
))
5199 igc_clear_interrupt_scheme(adapter
);
5200 iounmap(adapter
->io_addr
);
5202 free_netdev(netdev
);
5204 pci_release_mem_regions(pdev
);
5207 pci_disable_device(pdev
);
5212 * igc_remove - Device Removal Routine
5213 * @pdev: PCI device information struct
5215 * igc_remove is called by the PCI subsystem to alert the driver
5216 * that it should release a PCI device. This could be caused by a
5217 * Hot-Plug event, or because the driver is going to be removed from
5220 static void igc_remove(struct pci_dev
*pdev
)
5222 struct net_device
*netdev
= pci_get_drvdata(pdev
);
5223 struct igc_adapter
*adapter
= netdev_priv(netdev
);
5225 pm_runtime_get_noresume(&pdev
->dev
);
5227 igc_flush_nfc_rules(adapter
);
5229 igc_ptp_stop(adapter
);
5231 set_bit(__IGC_DOWN
, &adapter
->state
);
5233 del_timer_sync(&adapter
->watchdog_timer
);
5234 del_timer_sync(&adapter
->phy_info_timer
);
5236 cancel_work_sync(&adapter
->reset_task
);
5237 cancel_work_sync(&adapter
->watchdog_task
);
5239 /* Release control of h/w to f/w. If f/w is AMT enabled, this
5240 * would have already happened in close and is redundant.
5242 igc_release_hw_control(adapter
);
5243 unregister_netdev(netdev
);
5245 igc_clear_interrupt_scheme(adapter
);
5246 pci_iounmap(pdev
, adapter
->io_addr
);
5247 pci_release_mem_regions(pdev
);
5249 free_netdev(netdev
);
5251 pci_disable_pcie_error_reporting(pdev
);
5253 pci_disable_device(pdev
);
5256 static int __igc_shutdown(struct pci_dev
*pdev
, bool *enable_wake
,
5259 struct net_device
*netdev
= pci_get_drvdata(pdev
);
5260 struct igc_adapter
*adapter
= netdev_priv(netdev
);
5261 u32 wufc
= runtime
? IGC_WUFC_LNKC
: adapter
->wol
;
5262 struct igc_hw
*hw
= &adapter
->hw
;
5263 u32 ctrl
, rctl
, status
;
5267 netif_device_detach(netdev
);
5269 if (netif_running(netdev
))
5270 __igc_close(netdev
, true);
5272 igc_ptp_suspend(adapter
);
5274 igc_clear_interrupt_scheme(adapter
);
5277 status
= rd32(IGC_STATUS
);
5278 if (status
& IGC_STATUS_LU
)
5279 wufc
&= ~IGC_WUFC_LNKC
;
5282 igc_setup_rctl(adapter
);
5283 igc_set_rx_mode(netdev
);
5285 /* turn on all-multi mode if wake on multicast is enabled */
5286 if (wufc
& IGC_WUFC_MC
) {
5287 rctl
= rd32(IGC_RCTL
);
5288 rctl
|= IGC_RCTL_MPE
;
5289 wr32(IGC_RCTL
, rctl
);
5292 ctrl
= rd32(IGC_CTRL
);
5293 ctrl
|= IGC_CTRL_ADVD3WUC
;
5294 wr32(IGC_CTRL
, ctrl
);
5296 /* Allow time for pending master requests to run */
5297 igc_disable_pcie_master(hw
);
5299 wr32(IGC_WUC
, IGC_WUC_PME_EN
);
5300 wr32(IGC_WUFC
, wufc
);
5306 wake
= wufc
|| adapter
->en_mng_pt
;
5308 igc_power_down_link(adapter
);
5310 igc_power_up_link(adapter
);
5313 *enable_wake
= wake
;
5315 /* Release control of h/w to f/w. If f/w is AMT enabled, this
5316 * would have already happened in close and is redundant.
5318 igc_release_hw_control(adapter
);
5320 pci_disable_device(pdev
);
5326 static int __maybe_unused
igc_runtime_suspend(struct device
*dev
)
5328 return __igc_shutdown(to_pci_dev(dev
), NULL
, 1);
5331 static void igc_deliver_wake_packet(struct net_device
*netdev
)
5333 struct igc_adapter
*adapter
= netdev_priv(netdev
);
5334 struct igc_hw
*hw
= &adapter
->hw
;
5335 struct sk_buff
*skb
;
5338 wupl
= rd32(IGC_WUPL
) & IGC_WUPL_MASK
;
5340 /* WUPM stores only the first 128 bytes of the wake packet.
5341 * Read the packet only if we have the whole thing.
5343 if (wupl
== 0 || wupl
> IGC_WUPM_BYTES
)
5346 skb
= netdev_alloc_skb_ip_align(netdev
, IGC_WUPM_BYTES
);
5352 /* Ensure reads are 32-bit aligned */
5353 wupl
= roundup(wupl
, 4);
5355 memcpy_fromio(skb
->data
, hw
->hw_addr
+ IGC_WUPM_REG(0), wupl
);
5357 skb
->protocol
= eth_type_trans(skb
, netdev
);
5361 static int __maybe_unused
igc_resume(struct device
*dev
)
5363 struct pci_dev
*pdev
= to_pci_dev(dev
);
5364 struct net_device
*netdev
= pci_get_drvdata(pdev
);
5365 struct igc_adapter
*adapter
= netdev_priv(netdev
);
5366 struct igc_hw
*hw
= &adapter
->hw
;
5369 pci_set_power_state(pdev
, PCI_D0
);
5370 pci_restore_state(pdev
);
5371 pci_save_state(pdev
);
5373 if (!pci_device_is_present(pdev
))
5375 err
= pci_enable_device_mem(pdev
);
5377 netdev_err(netdev
, "Cannot enable PCI device from suspend\n");
5380 pci_set_master(pdev
);
5382 pci_enable_wake(pdev
, PCI_D3hot
, 0);
5383 pci_enable_wake(pdev
, PCI_D3cold
, 0);
5385 if (igc_init_interrupt_scheme(adapter
, true)) {
5386 netdev_err(netdev
, "Unable to allocate memory for queues\n");
5392 /* let the f/w know that the h/w is now under the control of the
5395 igc_get_hw_control(adapter
);
5397 val
= rd32(IGC_WUS
);
5398 if (val
& WAKE_PKT_WUS
)
5399 igc_deliver_wake_packet(netdev
);
5404 if (!err
&& netif_running(netdev
))
5405 err
= __igc_open(netdev
, true);
5408 netif_device_attach(netdev
);
5414 static int __maybe_unused
igc_runtime_resume(struct device
*dev
)
5416 return igc_resume(dev
);
5419 static int __maybe_unused
igc_suspend(struct device
*dev
)
5421 return __igc_shutdown(to_pci_dev(dev
), NULL
, 0);
5424 static int __maybe_unused
igc_runtime_idle(struct device
*dev
)
5426 struct net_device
*netdev
= dev_get_drvdata(dev
);
5427 struct igc_adapter
*adapter
= netdev_priv(netdev
);
5429 if (!igc_has_link(adapter
))
5430 pm_schedule_suspend(dev
, MSEC_PER_SEC
* 5);
5434 #endif /* CONFIG_PM */
5436 static void igc_shutdown(struct pci_dev
*pdev
)
5440 __igc_shutdown(pdev
, &wake
, 0);
5442 if (system_state
== SYSTEM_POWER_OFF
) {
5443 pci_wake_from_d3(pdev
, wake
);
5444 pci_set_power_state(pdev
, PCI_D3hot
);
5449 * igc_io_error_detected - called when PCI error is detected
5450 * @pdev: Pointer to PCI device
5451 * @state: The current PCI connection state
5453 * This function is called after a PCI bus error affecting
5454 * this device has been detected.
5456 static pci_ers_result_t
igc_io_error_detected(struct pci_dev
*pdev
,
5457 pci_channel_state_t state
)
5459 struct net_device
*netdev
= pci_get_drvdata(pdev
);
5460 struct igc_adapter
*adapter
= netdev_priv(netdev
);
5462 netif_device_detach(netdev
);
5464 if (state
== pci_channel_io_perm_failure
)
5465 return PCI_ERS_RESULT_DISCONNECT
;
5467 if (netif_running(netdev
))
5469 pci_disable_device(pdev
);
5471 /* Request a slot reset. */
5472 return PCI_ERS_RESULT_NEED_RESET
;
5476 * igc_io_slot_reset - called after the PCI bus has been reset.
5477 * @pdev: Pointer to PCI device
5479 * Restart the card from scratch, as if from a cold-boot. Implementation
5480 * resembles the first-half of the igc_resume routine.
5482 static pci_ers_result_t
igc_io_slot_reset(struct pci_dev
*pdev
)
5484 struct net_device
*netdev
= pci_get_drvdata(pdev
);
5485 struct igc_adapter
*adapter
= netdev_priv(netdev
);
5486 struct igc_hw
*hw
= &adapter
->hw
;
5487 pci_ers_result_t result
;
5489 if (pci_enable_device_mem(pdev
)) {
5490 netdev_err(netdev
, "Could not re-enable PCI device after reset\n");
5491 result
= PCI_ERS_RESULT_DISCONNECT
;
5493 pci_set_master(pdev
);
5494 pci_restore_state(pdev
);
5495 pci_save_state(pdev
);
5497 pci_enable_wake(pdev
, PCI_D3hot
, 0);
5498 pci_enable_wake(pdev
, PCI_D3cold
, 0);
5500 /* In case of PCI error, adapter loses its HW address
5501 * so we should re-assign it here.
5503 hw
->hw_addr
= adapter
->io_addr
;
5507 result
= PCI_ERS_RESULT_RECOVERED
;
5514 * igc_io_resume - called when traffic can start to flow again.
5515 * @pdev: Pointer to PCI device
5517 * This callback is called when the error recovery driver tells us that
5518 * its OK to resume normal operation. Implementation resembles the
5519 * second-half of the igc_resume routine.
5521 static void igc_io_resume(struct pci_dev
*pdev
)
5523 struct net_device
*netdev
= pci_get_drvdata(pdev
);
5524 struct igc_adapter
*adapter
= netdev_priv(netdev
);
5527 if (netif_running(netdev
)) {
5528 if (igc_open(netdev
)) {
5529 netdev_err(netdev
, "igc_open failed after reset\n");
5534 netif_device_attach(netdev
);
5536 /* let the f/w know that the h/w is now under the control of the
5539 igc_get_hw_control(adapter
);
5543 static const struct pci_error_handlers igc_err_handler
= {
5544 .error_detected
= igc_io_error_detected
,
5545 .slot_reset
= igc_io_slot_reset
,
5546 .resume
= igc_io_resume
,
5550 static const struct dev_pm_ops igc_pm_ops
= {
5551 SET_SYSTEM_SLEEP_PM_OPS(igc_suspend
, igc_resume
)
5552 SET_RUNTIME_PM_OPS(igc_runtime_suspend
, igc_runtime_resume
,
5557 static struct pci_driver igc_driver
= {
5558 .name
= igc_driver_name
,
5559 .id_table
= igc_pci_tbl
,
5561 .remove
= igc_remove
,
5563 .driver
.pm
= &igc_pm_ops
,
5565 .shutdown
= igc_shutdown
,
5566 .err_handler
= &igc_err_handler
,
5570 * igc_reinit_queues - return error
5571 * @adapter: pointer to adapter structure
5573 int igc_reinit_queues(struct igc_adapter
*adapter
)
5575 struct net_device
*netdev
= adapter
->netdev
;
5578 if (netif_running(netdev
))
5581 igc_reset_interrupt_capability(adapter
);
5583 if (igc_init_interrupt_scheme(adapter
, true)) {
5584 netdev_err(netdev
, "Unable to allocate memory for queues\n");
5588 if (netif_running(netdev
))
5589 err
= igc_open(netdev
);
5595 * igc_get_hw_dev - return device
5596 * @hw: pointer to hardware structure
5598 * used by hardware layer to print debugging information
5600 struct net_device
*igc_get_hw_dev(struct igc_hw
*hw
)
5602 struct igc_adapter
*adapter
= hw
->back
;
5604 return adapter
->netdev
;
5608 * igc_init_module - Driver Registration Routine
5610 * igc_init_module is the first routine called when the driver is
5611 * loaded. All it does is register with the PCI subsystem.
5613 static int __init
igc_init_module(void)
5617 pr_info("%s - version %s\n",
5618 igc_driver_string
, igc_driver_version
);
5620 pr_info("%s\n", igc_copyright
);
5622 ret
= pci_register_driver(&igc_driver
);
5626 module_init(igc_init_module
);
5629 * igc_exit_module - Driver Exit Cleanup Routine
5631 * igc_exit_module is called just before the driver is removed
5634 static void __exit
igc_exit_module(void)
5636 pci_unregister_driver(&igc_driver
);
5639 module_exit(igc_exit_module
);