target/linux/bmips/files/drivers/net/ethernet/broadcom/bcm6368-enetsw.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * BCM6368 Ethernet Switch Controller Driver
   4  *
   5  * Copyright (C) 2021 Álvaro Fernández Rojas <noltari@gmail.com>
   6  * Copyright (C) 2015 Jonas Gorski <jonas.gorski@gmail.com>
   7  * Copyright (C) 2008 Maxime Bizon <mbizon@freebox.fr>
   8  */
   9
  10 #include <linux/clk.h>
  11 #include <linux/delay.h>
  12 #include <linux/dma-mapping.h>
  13 #include <linux/err.h>
  14 #include <linux/ethtool.h>
  15 #include <linux/if_vlan.h>
  16 #include <linux/interrupt.h>
  17 #include <linux/of_clk.h>
  18 #include <linux/of_net.h>
  19 #include <linux/platform_device.h>
  20 #include <linux/pm_domain.h>
  21 #include <linux/pm_runtime.h>
  22 #include <linux/reset.h>
  23
  24 /* MTU */
  25 #define ENETSW_TAG_SIZE                 (6 + VLAN_HLEN)
  26 #define ENETSW_MTU_OVERHEAD             (VLAN_ETH_HLEN + VLAN_HLEN + \
  27                                          ENETSW_TAG_SIZE)
  28 #define ENETSW_FRAG_SIZE(x)             (SKB_DATA_ALIGN(NET_SKB_PAD + x + \
  29                                          SKB_DATA_ALIGN(sizeof(struct skb_shared_info))))
  30
  31 /* default number of descriptor */
  32 #define ENETSW_DEF_RX_DESC              64
  33 #define ENETSW_DEF_TX_DESC              32
  34 #define ENETSW_DEF_CPY_BREAK            128
  35
  36 /* maximum burst len for dma (4 bytes unit) */
  37 #define ENETSW_DMA_MAXBURST             8
  38
  39 /* DMA channels */
  40 #define DMA_CHAN_WIDTH                  0x10
  41
  42 /* Controller Configuration Register */
  43 #define DMA_CFG_REG                     0x0
  44 #define DMA_CFG_EN_SHIFT                0
  45 #define DMA_CFG_EN_MASK                 (1 << DMA_CFG_EN_SHIFT)
  46 #define DMA_CFG_FLOWCH_MASK(x)          (1 << ((x >> 1) + 1))
  47
  48 /* Flow Control Descriptor Low Threshold register */
  49 #define DMA_FLOWCL_REG(x)               (0x4 + (x) * 6)
  50
  51 /* Flow Control Descriptor High Threshold register */
  52 #define DMA_FLOWCH_REG(x)               (0x8 + (x) * 6)
  53
  54 /* Flow Control Descriptor Buffer Alloca Threshold register */
  55 #define DMA_BUFALLOC_REG(x)             (0xc + (x) * 6)
  56 #define DMA_BUFALLOC_FORCE_SHIFT        31
  57 #define DMA_BUFALLOC_FORCE_MASK         (1 << DMA_BUFALLOC_FORCE_SHIFT)
  58
  59 /* Channel Configuration register */
  60 #define DMAC_CHANCFG_REG                0x0
  61 #define DMAC_CHANCFG_EN_SHIFT           0
  62 #define DMAC_CHANCFG_EN_MASK            (1 << DMAC_CHANCFG_EN_SHIFT)
  63 #define DMAC_CHANCFG_PKTHALT_SHIFT      1
  64 #define DMAC_CHANCFG_PKTHALT_MASK       (1 << DMAC_CHANCFG_PKTHALT_SHIFT)
  65 #define DMAC_CHANCFG_BUFHALT_SHIFT      2
  66 #define DMAC_CHANCFG_BUFHALT_MASK       (1 << DMAC_CHANCFG_BUFHALT_SHIFT)
  67 #define DMAC_CHANCFG_CHAINING_SHIFT     2
  68 #define DMAC_CHANCFG_CHAINING_MASK      (1 << DMAC_CHANCFG_CHAINING_SHIFT)
  69 #define DMAC_CHANCFG_WRAP_EN_SHIFT      3
  70 #define DMAC_CHANCFG_WRAP_EN_MASK       (1 << DMAC_CHANCFG_WRAP_EN_SHIFT)
  71 #define DMAC_CHANCFG_FLOWC_EN_SHIFT     4
  72 #define DMAC_CHANCFG_FLOWC_EN_MASK      (1 << DMAC_CHANCFG_FLOWC_EN_SHIFT)
  73
  74 /* Interrupt Control/Status register */
  75 #define DMAC_IR_REG                     0x4
  76 #define DMAC_IR_BUFDONE_MASK            (1 << 0)
  77 #define DMAC_IR_PKTDONE_MASK            (1 << 1)
  78 #define DMAC_IR_NOTOWNER_MASK           (1 << 2)
  79
  80 /* Interrupt Mask register */
  81 #define DMAC_IRMASK_REG                 0x8
  82
  83 /* Maximum Burst Length */
  84 #define DMAC_MAXBURST_REG               0xc
  85
  86 /* Ring Start Address register */
  87 #define DMAS_RSTART_REG                 0x0
  88
  89 /* State Ram Word 2 */
  90 #define DMAS_SRAM2_REG                  0x4
  91
  92 /* State Ram Word 3 */
  93 #define DMAS_SRAM3_REG                  0x8
  94
  95 /* State Ram Word 4 */
  96 #define DMAS_SRAM4_REG                  0xc
  97
  98 struct bcm6368_enetsw_desc {
  99         u32 len_stat;
 100         u32 address;
 101 };
 102
 103 /* control */
 104 #define DMADESC_LENGTH_SHIFT            16
 105 #define DMADESC_LENGTH_MASK             (0xfff << DMADESC_LENGTH_SHIFT)
 106 #define DMADESC_OWNER_MASK              (1 << 15)
 107 #define DMADESC_EOP_MASK                (1 << 14)
 108 #define DMADESC_SOP_MASK                (1 << 13)
 109 #define DMADESC_ESOP_MASK               (DMADESC_EOP_MASK | DMADESC_SOP_MASK)
 110 #define DMADESC_WRAP_MASK               (1 << 12)
 111 #define DMADESC_USB_NOZERO_MASK         (1 << 1)
 112 #define DMADESC_USB_ZERO_MASK           (1 << 0)
 113
 114 /* status */
 115 #define DMADESC_UNDER_MASK              (1 << 9)
 116 #define DMADESC_APPEND_CRC              (1 << 8)
 117 #define DMADESC_OVSIZE_MASK             (1 << 4)
 118 #define DMADESC_RXER_MASK               (1 << 2)
 119 #define DMADESC_CRC_MASK                (1 << 1)
 120 #define DMADESC_OV_MASK                 (1 << 0)
 121 #define DMADESC_ERR_MASK                (DMADESC_UNDER_MASK | \
 122                                          DMADESC_OVSIZE_MASK | \
 123                                          DMADESC_RXER_MASK | \
 124                                          DMADESC_CRC_MASK | \
 125                                          DMADESC_OV_MASK)
 126
 127 struct bcm6368_enetsw {
 128         void __iomem *dma_base;
 129         void __iomem *dma_chan;
 130         void __iomem *dma_sram;
 131
 132         struct device **pm;
 133         struct device_link **link_pm;
 134         int num_pms;
 135
 136         struct clk **clock;
 137         unsigned int num_clocks;
 138
 139         struct reset_control **reset;
 140         unsigned int num_resets;
 141
 142         int copybreak;
 143
 144         int irq_rx;
 145         int irq_tx;
 146
 147         /* hw view of rx & tx dma ring */
 148         dma_addr_t rx_desc_dma;
 149         dma_addr_t tx_desc_dma;
 150
 151         /* allocated size (in bytes) for rx & tx dma ring */
 152         unsigned int rx_desc_alloc_size;
 153         unsigned int tx_desc_alloc_size;
 154
 155         struct napi_struct napi;
 156
 157         /* dma channel id for rx */
 158         int rx_chan;
 159
 160         /* number of dma desc in rx ring */
 161         int rx_ring_size;
 162
 163         /* cpu view of rx dma ring */
 164         struct bcm6368_enetsw_desc *rx_desc_cpu;
 165
 166         /* current number of armed descriptor given to hardware for rx */
 167         int rx_desc_count;
 168
 169         /* next rx descriptor to fetch from hardware */
 170         int rx_curr_desc;
 171
 172         /* next dirty rx descriptor to refill */
 173         int rx_dirty_desc;
 174
 175         /* size of allocated rx buffer */
 176         unsigned int rx_buf_size;
 177
 178         /* size of allocated rx frag */
 179         unsigned int rx_frag_size;
 180
 181         /* list of buffer given to hw for rx */
 182         unsigned char **rx_buf;
 183
 184         /* used when rx buffer allocation failed, so we defer rx queue
 185          * refill */
 186         struct timer_list rx_timeout;
 187
 188         /* lock rx_timeout against rx normal operation */
 189         spinlock_t rx_lock;
 190
 191         /* dma channel id for tx */
 192         int tx_chan;
 193
 194         /* number of dma desc in tx ring */
 195         int tx_ring_size;
 196
 197         /* cpu view of rx dma ring */
 198         struct bcm6368_enetsw_desc *tx_desc_cpu;
 199
 200         /* number of available descriptor for tx */
 201         int tx_desc_count;
 202
 203         /* next tx descriptor avaiable */
 204         int tx_curr_desc;
 205
 206         /* next dirty tx descriptor to reclaim */
 207         int tx_dirty_desc;
 208
 209         /* list of skb given to hw for tx */
 210         struct sk_buff **tx_skb;
 211
 212         /* lock used by tx reclaim and xmit */
 213         spinlock_t tx_lock;
 214
 215         /* network device reference */
 216         struct net_device *net_dev;
 217
 218         /* platform device reference */
 219         struct platform_device *pdev;
 220 };
 221
 222 static inline void dma_writel(struct bcm6368_enetsw *priv, u32 val, u32 off)
 223 {
 224         __raw_writel(val, priv->dma_base + off);
 225 }
 226
 227 static inline u32 dma_readl(struct bcm6368_enetsw *priv, u32 off, int chan)
 228 {
 229         return __raw_readl(priv->dma_chan + off + chan * DMA_CHAN_WIDTH);
 230 }
 231
 232 static inline void dmac_writel(struct bcm6368_enetsw *priv, u32 val, u32 off,
 233                                int chan)
 234 {
 235         __raw_writel(val, priv->dma_chan + off + chan * DMA_CHAN_WIDTH);
 236 }
 237
 238 static inline void dmas_writel(struct bcm6368_enetsw *priv, u32 val,
 239                                     u32 off, int chan)
 240 {
 241         __raw_writel(val, priv->dma_sram + off + chan * DMA_CHAN_WIDTH);
 242 }
 243
 244 /*
 245  * refill rx queue
 246  */
 247 static int bcm6368_enetsw_refill_rx(struct net_device *ndev, bool napi_mode)
 248 {
 249         struct bcm6368_enetsw *priv = netdev_priv(ndev);
 250         struct platform_device *pdev = priv->pdev;
 251         struct device *dev = &pdev->dev;
 252
 253         while (priv->rx_desc_count < priv->rx_ring_size) {
 254                 struct bcm6368_enetsw_desc *desc;
 255                 int desc_idx;
 256                 u32 len_stat;
 257
 258                 desc_idx = priv->rx_dirty_desc;
 259                 desc = &priv->rx_desc_cpu[desc_idx];
 260
 261                 if (!priv->rx_buf[desc_idx]) {
 262                         unsigned char *buf;
 263                         dma_addr_t p;
 264
 265                         if (likely(napi_mode))
 266                                 buf = napi_alloc_frag(priv->rx_frag_size);
 267                         else
 268                                 buf = netdev_alloc_frag(priv->rx_frag_size);
 269
 270                         if (unlikely(!buf))
 271                                 break;
 272
 273                         p = dma_map_single(dev, buf + NET_SKB_PAD,
 274                                            priv->rx_buf_size, DMA_FROM_DEVICE);
 275                         if (unlikely(dma_mapping_error(dev, p))) {
 276                                 skb_free_frag(buf);
 277                                 break;
 278                         }
 279
 280                         priv->rx_buf[desc_idx] = buf;
 281                         desc->address = p;
 282                 }
 283
 284                 len_stat = priv->rx_buf_size << DMADESC_LENGTH_SHIFT;
 285                 len_stat |= DMADESC_OWNER_MASK;
 286                 if (priv->rx_dirty_desc == priv->rx_ring_size - 1) {
 287                         len_stat |= DMADESC_WRAP_MASK;
 288                         priv->rx_dirty_desc = 0;
 289                 } else {
 290                         priv->rx_dirty_desc++;
 291                 }
 292                 wmb();
 293                 desc->len_stat = len_stat;
 294
 295                 priv->rx_desc_count++;
 296
 297                 /* tell dma engine we allocated one buffer */
 298                 dma_writel(priv, 1, DMA_BUFALLOC_REG(priv->rx_chan));
 299         }
 300
 301         /* If rx ring is still empty, set a timer to try allocating
 302          * again at a later time. */
 303         if (priv->rx_desc_count == 0 && netif_running(ndev)) {
 304                 dev_warn(dev, "unable to refill rx ring\n");
 305                 priv->rx_timeout.expires = jiffies + HZ;
 306                 add_timer(&priv->rx_timeout);
 307         }
 308
 309         return 0;
 310 }
 311
 312 /*
 313  * timer callback to defer refill rx queue in case we're OOM
 314  */
 315 static void bcm6368_enetsw_refill_rx_timer(struct timer_list *t)
 316 {
 317         struct bcm6368_enetsw *priv = from_timer(priv, t, rx_timeout);
 318         struct net_device *ndev = priv->net_dev;
 319
 320         spin_lock(&priv->rx_lock);
 321         bcm6368_enetsw_refill_rx(ndev, false);
 322         spin_unlock(&priv->rx_lock);
 323 }
 324
 325 /*
 326  * extract packet from rx queue
 327  */
 328 static int bcm6368_enetsw_receive_queue(struct net_device *ndev, int budget)
 329 {
 330         struct bcm6368_enetsw *priv = netdev_priv(ndev);
 331         struct platform_device *pdev = priv->pdev;
 332         struct device *dev = &pdev->dev;
 333         struct list_head rx_list;
 334         struct sk_buff *skb;
 335         int processed = 0;
 336
 337         INIT_LIST_HEAD(&rx_list);
 338
 339         /* don't scan ring further than number of refilled
 340          * descriptor */
 341         if (budget > priv->rx_desc_count)
 342                 budget = priv->rx_desc_count;
 343
 344         do {
 345                 struct bcm6368_enetsw_desc *desc;
 346                 unsigned int frag_size;
 347                 unsigned char *buf;
 348                 int desc_idx;
 349                 u32 len_stat;
 350                 unsigned int len;
 351
 352                 desc_idx = priv->rx_curr_desc;
 353                 desc = &priv->rx_desc_cpu[desc_idx];
 354
 355                 /* make sure we actually read the descriptor status at
 356                  * each loop */
 357                 rmb();
 358
 359                 len_stat = desc->len_stat;
 360
 361                 /* break if dma ownership belongs to hw */
 362                 if (len_stat & DMADESC_OWNER_MASK)
 363                         break;
 364
 365                 processed++;
 366                 priv->rx_curr_desc++;
 367                 if (priv->rx_curr_desc == priv->rx_ring_size)
 368                         priv->rx_curr_desc = 0;
 369
 370                 /* if the packet does not have start of packet _and_
 371                  * end of packet flag set, then just recycle it */
 372                 if ((len_stat & DMADESC_ESOP_MASK) != DMADESC_ESOP_MASK) {
 373                         ndev->stats.rx_dropped++;
 374                         continue;
 375                 }
 376
 377                 /* valid packet */
 378                 buf = priv->rx_buf[desc_idx];
 379                 len = (len_stat & DMADESC_LENGTH_MASK)
 380                       >> DMADESC_LENGTH_SHIFT;
 381                 /* don't include FCS */
 382                 len -= 4;
 383
 384                 if (len < priv->copybreak) {
 385                         unsigned int nfrag_size = ENETSW_FRAG_SIZE(len);
 386                         unsigned char *nbuf = napi_alloc_frag(nfrag_size);
 387
 388                         if (unlikely(!nbuf)) {
 389                                 /* forget packet, just rearm desc */
 390                                 ndev->stats.rx_dropped++;
 391                                 continue;
 392                         }
 393
 394                         dma_sync_single_for_cpu(dev, desc->address,
 395                                                 len, DMA_FROM_DEVICE);
 396                         memcpy(nbuf + NET_SKB_PAD, buf + NET_SKB_PAD, len);
 397                         dma_sync_single_for_device(dev, desc->address,
 398                                                    len, DMA_FROM_DEVICE);
 399                         buf = nbuf;
 400                         frag_size = nfrag_size;
 401                 } else {
 402                         dma_unmap_single(dev, desc->address,
 403                                          priv->rx_buf_size, DMA_FROM_DEVICE);
 404                         priv->rx_buf[desc_idx] = NULL;
 405                         frag_size = priv->rx_frag_size;
 406                 }
 407
 408                 skb = napi_build_skb(buf, frag_size);
 409                 if (unlikely(!skb)) {
 410                         skb_free_frag(buf);
 411                         ndev->stats.rx_dropped++;
 412                         continue;
 413                 }
 414
 415                 skb_reserve(skb, NET_SKB_PAD);
 416                 skb_put(skb, len);
 417                 ndev->stats.rx_packets++;
 418                 ndev->stats.rx_bytes += len;
 419                 list_add_tail(&skb->list, &rx_list);
 420         } while (processed < budget);
 421
 422         list_for_each_entry(skb, &rx_list, list)
 423                 skb->protocol = eth_type_trans(skb, ndev);
 424         netif_receive_skb_list(&rx_list);
 425         priv->rx_desc_count -= processed;
 426
 427         if (processed || !priv->rx_desc_count) {
 428                 bcm6368_enetsw_refill_rx(ndev, true);
 429
 430                 /* kick rx dma */
 431                 dmac_writel(priv, DMAC_CHANCFG_EN_MASK,
 432                             DMAC_CHANCFG_REG, priv->rx_chan);
 433         }
 434
 435         return processed;
 436 }
 437
 438 /*
 439  * try to or force reclaim of transmitted buffers
 440  */
 441 static int bcm6368_enetsw_tx_reclaim(struct net_device *ndev, int force,
 442                                      int budget)
 443 {
 444         struct bcm6368_enetsw *priv = netdev_priv(ndev);
 445         struct platform_device *pdev = priv->pdev;
 446         struct device *dev = &pdev->dev;
 447         unsigned int bytes = 0;
 448         int released = 0;
 449
 450         while (priv->tx_desc_count < priv->tx_ring_size) {
 451                 struct bcm6368_enetsw_desc *desc;
 452                 struct sk_buff *skb;
 453
 454                 /* We run in a bh and fight against start_xmit, which
 455                  * is called with bh disabled */
 456                 spin_lock(&priv->tx_lock);
 457
 458                 desc = &priv->tx_desc_cpu[priv->tx_dirty_desc];
 459
 460                 if (!force && (desc->len_stat & DMADESC_OWNER_MASK)) {
 461                         spin_unlock(&priv->tx_lock);
 462                         break;
 463                 }
 464
 465                 /* ensure other field of the descriptor were not read
 466                  * before we checked ownership */
 467                 rmb();
 468
 469                 skb = priv->tx_skb[priv->tx_dirty_desc];
 470                 priv->tx_skb[priv->tx_dirty_desc] = NULL;
 471                 dma_unmap_single(dev, desc->address, skb->len,
 472                                  DMA_TO_DEVICE);
 473
 474                 priv->tx_dirty_desc++;
 475                 if (priv->tx_dirty_desc == priv->tx_ring_size)
 476                         priv->tx_dirty_desc = 0;
 477                 priv->tx_desc_count++;
 478
 479                 spin_unlock(&priv->tx_lock);
 480
 481                 if (desc->len_stat & DMADESC_UNDER_MASK)
 482                         ndev->stats.tx_errors++;
 483
 484                 bytes += skb->len;
 485                 napi_consume_skb(skb, budget);
 486                 released++;
 487         }
 488
 489         netdev_completed_queue(ndev, released, bytes);
 490
 491         if (netif_queue_stopped(ndev) && released)
 492                 netif_wake_queue(ndev);
 493
 494         return released;
 495 }
 496
 497 /*
 498  * poll func, called by network core
 499  */
 500 static int bcm6368_enetsw_poll(struct napi_struct *napi, int budget)
 501 {
 502         struct bcm6368_enetsw *priv = container_of(napi, struct bcm6368_enetsw, napi);
 503         struct net_device *ndev = priv->net_dev;
 504         int rx_work_done;
 505
 506         /* ack interrupts */
 507         dmac_writel(priv, DMAC_IR_PKTDONE_MASK,
 508                          DMAC_IR_REG, priv->rx_chan);
 509         dmac_writel(priv, DMAC_IR_PKTDONE_MASK,
 510                          DMAC_IR_REG, priv->tx_chan);
 511
 512         /* reclaim sent skb */
 513         bcm6368_enetsw_tx_reclaim(ndev, 0, budget);
 514
 515         spin_lock(&priv->rx_lock);
 516         rx_work_done = bcm6368_enetsw_receive_queue(ndev, budget);
 517         spin_unlock(&priv->rx_lock);
 518
 519         if (rx_work_done >= budget) {
 520                 /* rx queue is not yet empty/clean */
 521                 return rx_work_done;
 522         }
 523
 524         /* no more packet in rx/tx queue, remove device from poll
 525          * queue */
 526         napi_complete_done(napi, rx_work_done);
 527
 528         /* restore rx/tx interrupt */
 529         dmac_writel(priv, DMAC_IR_PKTDONE_MASK,
 530                     DMAC_IRMASK_REG, priv->rx_chan);
 531         dmac_writel(priv, DMAC_IR_PKTDONE_MASK,
 532                     DMAC_IRMASK_REG, priv->tx_chan);
 533
 534         return rx_work_done;
 535 }
 536
 537 /*
 538  * rx/tx dma interrupt handler
 539  */
 540 static irqreturn_t bcm6368_enetsw_isr_dma(int irq, void *dev_id)
 541 {
 542         struct net_device *ndev = dev_id;
 543         struct bcm6368_enetsw *priv = netdev_priv(ndev);
 544
 545         /* mask rx/tx interrupts */
 546         dmac_writel(priv, 0, DMAC_IRMASK_REG, priv->rx_chan);
 547         dmac_writel(priv, 0, DMAC_IRMASK_REG, priv->tx_chan);
 548
 549         napi_schedule(&priv->napi);
 550
 551         return IRQ_HANDLED;
 552 }
 553
 554 /*
 555  * tx request callback
 556  */
 557 static netdev_tx_t
 558 bcm6368_enetsw_start_xmit(struct sk_buff *skb, struct net_device *ndev)
 559 {
 560         struct bcm6368_enetsw *priv = netdev_priv(ndev);
 561         struct platform_device *pdev = priv->pdev;
 562         struct device *dev = &pdev->dev;
 563         struct bcm6368_enetsw_desc *desc;
 564         u32 len_stat;
 565         netdev_tx_t ret;
 566         dma_addr_t p;
 567
 568         /* lock against tx reclaim */
 569         spin_lock(&priv->tx_lock);
 570
 571         /* make sure the tx hw queue is not full, should not happen
 572          * since we stop queue before it's the case */
 573         if (unlikely(!priv->tx_desc_count)) {
 574                 netif_stop_queue(ndev);
 575                 dev_err(dev, "xmit called with no tx desc available?\n");
 576                 ret = NETDEV_TX_BUSY;
 577                 goto out_unlock;
 578         }
 579
 580         /* pad small packets */
 581         if (skb->len < (ETH_ZLEN + ETH_FCS_LEN)) {
 582                 int needed = (ETH_ZLEN + ETH_FCS_LEN) - skb->len;
 583                 char *data;
 584
 585                 if (unlikely(skb_tailroom(skb) < needed)) {
 586                         struct sk_buff *nskb;
 587
 588                         nskb = skb_copy_expand(skb, 0, needed, GFP_ATOMIC);
 589                         if (!nskb) {
 590                                 ret = NETDEV_TX_BUSY;
 591                                 goto out_unlock;
 592                         }
 593
 594                         dev_kfree_skb(skb);
 595                         skb = nskb;
 596                 }
 597                 data = skb_put_zero(skb, needed);
 598         }
 599
 600         /* fill descriptor */
 601         p = dma_map_single(dev, skb->data, skb->len, DMA_TO_DEVICE);
 602         if (unlikely(dma_mapping_error(dev, p))) {
 603                 dev_kfree_skb(skb);
 604                 ret = NETDEV_TX_OK;
 605                 goto out_unlock;
 606         }
 607
 608         /* point to the next available desc */
 609         desc = &priv->tx_desc_cpu[priv->tx_curr_desc];
 610         priv->tx_skb[priv->tx_curr_desc] = skb;
 611         desc->address = p;
 612
 613         len_stat = (skb->len << DMADESC_LENGTH_SHIFT) & DMADESC_LENGTH_MASK;
 614         len_stat |= DMADESC_ESOP_MASK | DMADESC_APPEND_CRC |
 615                     DMADESC_OWNER_MASK;
 616
 617         priv->tx_curr_desc++;
 618         if (priv->tx_curr_desc == priv->tx_ring_size) {
 619                 priv->tx_curr_desc = 0;
 620                 len_stat |= DMADESC_WRAP_MASK;
 621         }
 622         priv->tx_desc_count--;
 623
 624         /* dma might be already polling, make sure we update desc
 625          * fields in correct order */
 626         wmb();
 627         desc->len_stat = len_stat;
 628         wmb();
 629
 630         netdev_sent_queue(ndev, skb->len);
 631
 632         /* kick tx dma */
 633         dmac_writel(priv, DMAC_CHANCFG_EN_MASK, DMAC_CHANCFG_REG,
 634                     priv->tx_chan);
 635
 636         /* stop queue if no more desc available */
 637         if (!priv->tx_desc_count)
 638                 netif_stop_queue(ndev);
 639
 640         ndev->stats.tx_bytes += skb->len;
 641         ndev->stats.tx_packets++;
 642         ret = NETDEV_TX_OK;
 643
 644 out_unlock:
 645         spin_unlock(&priv->tx_lock);
 646         return ret;
 647 }
 648
 649 /*
 650  * disable dma in given channel
 651  */
 652 static void bcm6368_enetsw_disable_dma(struct bcm6368_enetsw *priv, int chan)
 653 {
 654         int limit = 1000;
 655
 656         dmac_writel(priv, 0, DMAC_CHANCFG_REG, chan);
 657
 658         do {
 659                 u32 val;
 660
 661                 val = dma_readl(priv, DMAC_CHANCFG_REG, chan);
 662                 if (!(val & DMAC_CHANCFG_EN_MASK))
 663                         break;
 664
 665                 udelay(1);
 666         } while (limit--);
 667 }
 668
 669 static int bcm6368_enetsw_open(struct net_device *ndev)
 670 {
 671         struct bcm6368_enetsw *priv = netdev_priv(ndev);
 672         struct platform_device *pdev = priv->pdev;
 673         struct device *dev = &pdev->dev;
 674         int i, ret;
 675         unsigned int size;
 676         void *p;
 677         u32 val;
 678
 679         /* mask all interrupts and request them */
 680         dmac_writel(priv, 0, DMAC_IRMASK_REG, priv->rx_chan);
 681         dmac_writel(priv, 0, DMAC_IRMASK_REG, priv->tx_chan);
 682
 683         ret = request_irq(priv->irq_rx, bcm6368_enetsw_isr_dma,
 684                           0, ndev->name, ndev);
 685         if (ret)
 686                 goto out_freeirq;
 687
 688         if (priv->irq_tx != -1) {
 689                 ret = request_irq(priv->irq_tx, bcm6368_enetsw_isr_dma,
 690                                   0, ndev->name, ndev);
 691                 if (ret)
 692                         goto out_freeirq_rx;
 693         }
 694
 695         /* allocate rx dma ring */
 696         size = priv->rx_ring_size * sizeof(struct bcm6368_enetsw_desc);
 697         p = dma_alloc_coherent(dev, size, &priv->rx_desc_dma, GFP_KERNEL);
 698         if (!p) {
 699                 dev_err(dev, "cannot allocate rx ring %u\n", size);
 700                 ret = -ENOMEM;
 701                 goto out_freeirq_tx;
 702         }
 703
 704         memset(p, 0, size);
 705         priv->rx_desc_alloc_size = size;
 706         priv->rx_desc_cpu = p;
 707
 708         /* allocate tx dma ring */
 709         size = priv->tx_ring_size * sizeof(struct bcm6368_enetsw_desc);
 710         p = dma_alloc_coherent(dev, size, &priv->tx_desc_dma, GFP_KERNEL);
 711         if (!p) {
 712                 dev_err(dev, "cannot allocate tx ring\n");
 713                 ret = -ENOMEM;
 714                 goto out_free_rx_ring;
 715         }
 716
 717         memset(p, 0, size);
 718         priv->tx_desc_alloc_size = size;
 719         priv->tx_desc_cpu = p;
 720
 721         priv->tx_skb = kzalloc(sizeof(struct sk_buff *) * priv->tx_ring_size,
 722                                GFP_KERNEL);
 723         if (!priv->tx_skb) {
 724                 dev_err(dev, "cannot allocate tx skb queue\n");
 725                 ret = -ENOMEM;
 726                 goto out_free_tx_ring;
 727         }
 728
 729         priv->tx_desc_count = priv->tx_ring_size;
 730         priv->tx_dirty_desc = 0;
 731         priv->tx_curr_desc = 0;
 732         spin_lock_init(&priv->tx_lock);
 733
 734         /* init & fill rx ring with buffers */
 735         priv->rx_buf = kzalloc(sizeof(unsigned char *) * priv->rx_ring_size,
 736                                GFP_KERNEL);
 737         if (!priv->rx_buf) {
 738                 dev_err(dev, "cannot allocate rx buffer queue\n");
 739                 ret = -ENOMEM;
 740                 goto out_free_tx_skb;
 741         }
 742
 743         priv->rx_desc_count = 0;
 744         priv->rx_dirty_desc = 0;
 745         priv->rx_curr_desc = 0;
 746
 747         /* initialize flow control buffer allocation */
 748         dma_writel(priv, DMA_BUFALLOC_FORCE_MASK | 0,
 749                    DMA_BUFALLOC_REG(priv->rx_chan));
 750
 751         if (bcm6368_enetsw_refill_rx(ndev, false)) {
 752                 dev_err(dev, "cannot allocate rx buffer queue\n");
 753                 ret = -ENOMEM;
 754                 goto out;
 755         }
 756
 757         /* write rx & tx ring addresses */
 758         dmas_writel(priv, priv->rx_desc_dma,
 759                     DMAS_RSTART_REG, priv->rx_chan);
 760         dmas_writel(priv, priv->tx_desc_dma,
 761                     DMAS_RSTART_REG, priv->tx_chan);
 762
 763         /* clear remaining state ram for rx & tx channel */
 764         dmas_writel(priv, 0, DMAS_SRAM2_REG, priv->rx_chan);
 765         dmas_writel(priv, 0, DMAS_SRAM2_REG, priv->tx_chan);
 766         dmas_writel(priv, 0, DMAS_SRAM3_REG, priv->rx_chan);
 767         dmas_writel(priv, 0, DMAS_SRAM3_REG, priv->tx_chan);
 768         dmas_writel(priv, 0, DMAS_SRAM4_REG, priv->rx_chan);
 769         dmas_writel(priv, 0, DMAS_SRAM4_REG, priv->tx_chan);
 770
 771         /* set dma maximum burst len */
 772         dmac_writel(priv, ENETSW_DMA_MAXBURST,
 773                     DMAC_MAXBURST_REG, priv->rx_chan);
 774         dmac_writel(priv, ENETSW_DMA_MAXBURST,
 775                     DMAC_MAXBURST_REG, priv->tx_chan);
 776
 777         /* set flow control low/high threshold to 1/3 / 2/3 */
 778         val = priv->rx_ring_size / 3;
 779         dma_writel(priv, val, DMA_FLOWCL_REG(priv->rx_chan));
 780         val = (priv->rx_ring_size * 2) / 3;
 781         dma_writel(priv, val, DMA_FLOWCH_REG(priv->rx_chan));
 782
 783         /* all set, enable mac and interrupts, start dma engine and
 784          * kick rx dma channel
 785          */
 786         wmb();
 787         dma_writel(priv, DMA_CFG_EN_MASK, DMA_CFG_REG);
 788         dmac_writel(priv, DMAC_CHANCFG_EN_MASK,
 789                     DMAC_CHANCFG_REG, priv->rx_chan);
 790
 791         /* watch "packet transferred" interrupt in rx and tx */
 792         dmac_writel(priv, DMAC_IR_PKTDONE_MASK,
 793                     DMAC_IR_REG, priv->rx_chan);
 794         dmac_writel(priv, DMAC_IR_PKTDONE_MASK,
 795                     DMAC_IR_REG, priv->tx_chan);
 796
 797         /* make sure we enable napi before rx interrupt  */
 798         napi_enable(&priv->napi);
 799
 800         dmac_writel(priv, DMAC_IR_PKTDONE_MASK,
 801                     DMAC_IRMASK_REG, priv->rx_chan);
 802         dmac_writel(priv, DMAC_IR_PKTDONE_MASK,
 803                     DMAC_IRMASK_REG, priv->tx_chan);
 804
 805         netif_carrier_on(ndev);
 806         netif_start_queue(ndev);
 807
 808         return 0;
 809
 810 out:
 811         for (i = 0; i < priv->rx_ring_size; i++) {
 812                 struct bcm6368_enetsw_desc *desc;
 813
 814                 if (!priv->rx_buf[i])
 815                         continue;
 816
 817                 desc = &priv->rx_desc_cpu[i];
 818                 dma_unmap_single(dev, desc->address, priv->rx_buf_size,
 819                                  DMA_FROM_DEVICE);
 820                 skb_free_frag(priv->rx_buf[i]);
 821         }
 822         kfree(priv->rx_buf);
 823
 824 out_free_tx_skb:
 825         kfree(priv->tx_skb);
 826
 827 out_free_tx_ring:
 828         dma_free_coherent(dev, priv->tx_desc_alloc_size,
 829                           priv->tx_desc_cpu, priv->tx_desc_dma);
 830
 831 out_free_rx_ring:
 832         dma_free_coherent(dev, priv->rx_desc_alloc_size,
 833                           priv->rx_desc_cpu, priv->rx_desc_dma);
 834
 835 out_freeirq_tx:
 836         if (priv->irq_tx != -1)
 837                 free_irq(priv->irq_tx, ndev);
 838
 839 out_freeirq_rx:
 840         free_irq(priv->irq_rx, ndev);
 841
 842 out_freeirq:
 843         return ret;
 844 }
 845
 846 static int bcm6368_enetsw_stop(struct net_device *ndev)
 847 {
 848         struct bcm6368_enetsw *priv = netdev_priv(ndev);
 849         struct platform_device *pdev = priv->pdev;
 850         struct device *dev = &pdev->dev;
 851         int i;
 852
 853         netif_stop_queue(ndev);
 854         napi_disable(&priv->napi);
 855         del_timer_sync(&priv->rx_timeout);
 856
 857         /* mask all interrupts */
 858         dmac_writel(priv, 0, DMAC_IRMASK_REG, priv->rx_chan);
 859         dmac_writel(priv, 0, DMAC_IRMASK_REG, priv->tx_chan);
 860
 861         /* disable dma & mac */
 862         bcm6368_enetsw_disable_dma(priv, priv->tx_chan);
 863         bcm6368_enetsw_disable_dma(priv, priv->rx_chan);
 864
 865         /* force reclaim of all tx buffers */
 866         bcm6368_enetsw_tx_reclaim(ndev, 1, 0);
 867
 868         /* free the rx buffer ring */
 869         for (i = 0; i < priv->rx_ring_size; i++) {
 870                 struct bcm6368_enetsw_desc *desc;
 871
 872                 if (!priv->rx_buf[i])
 873                         continue;
 874
 875                 desc = &priv->rx_desc_cpu[i];
 876                 dma_unmap_single_attrs(dev, desc->address, priv->rx_buf_size,
 877                                        DMA_FROM_DEVICE,
 878                                        DMA_ATTR_SKIP_CPU_SYNC);
 879                 skb_free_frag(priv->rx_buf[i]);
 880         }
 881
 882         /* free remaining allocated memory */
 883         kfree(priv->rx_buf);
 884         kfree(priv->tx_skb);
 885         dma_free_coherent(dev, priv->rx_desc_alloc_size,
 886                           priv->rx_desc_cpu, priv->rx_desc_dma);
 887         dma_free_coherent(dev, priv->tx_desc_alloc_size,
 888                           priv->tx_desc_cpu, priv->tx_desc_dma);
 889         if (priv->irq_tx != -1)
 890                 free_irq(priv->irq_tx, ndev);
 891         free_irq(priv->irq_rx, ndev);
 892
 893         netdev_reset_queue(ndev);
 894
 895         return 0;
 896 }
 897
 898 static const struct net_device_ops bcm6368_enetsw_ops = {
 899         .ndo_open = bcm6368_enetsw_open,
 900         .ndo_stop = bcm6368_enetsw_stop,
 901         .ndo_start_xmit = bcm6368_enetsw_start_xmit,
 902 };
 903
 904 static int bcm6368_enetsw_probe(struct platform_device *pdev)
 905 {
 906         struct bcm6368_enetsw *priv;
 907         struct device *dev = &pdev->dev;
 908         struct device_node *node = dev->of_node;
 909         struct net_device *ndev;
 910         struct resource *res;
 911         unsigned i;
 912         int ret;
 913
 914         ndev = alloc_etherdev(sizeof(*priv));
 915         if (!ndev)
 916                 return -ENOMEM;
 917
 918         priv = netdev_priv(ndev);
 919
 920         priv->num_pms = of_count_phandle_with_args(node, "power-domains",
 921                                                    "#power-domain-cells");
 922         if (priv->num_pms > 1) {
 923                 priv->pm = devm_kcalloc(dev, priv->num_pms,
 924                                         sizeof(struct device *), GFP_KERNEL);
 925                 if (!priv->pm)
 926                         return -ENOMEM;
 927
 928                 priv->link_pm = devm_kcalloc(dev, priv->num_pms,
 929                                              sizeof(struct device_link *),
 930                                              GFP_KERNEL);
 931                 if (!priv->link_pm)
 932                         return -ENOMEM;
 933
 934                 for (i = 0; i < priv->num_pms; i++) {
 935                         priv->pm[i] = genpd_dev_pm_attach_by_id(dev, i);
 936                         if (IS_ERR(priv->pm[i])) {
 937                                 dev_err(dev, "error getting pm %d\n", i);
 938                                 return -EINVAL;
 939                         }
 940
 941                         priv->link_pm[i] = device_link_add(dev, priv->pm[i],
 942                                 DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME |
 943                                 DL_FLAG_RPM_ACTIVE);
 944                 }
 945         }
 946
 947         pm_runtime_enable(dev);
 948         pm_runtime_no_callbacks(dev);
 949         ret = pm_runtime_get_sync(dev);
 950         if (ret < 0) {
 951                 pm_runtime_disable(dev);
 952                 dev_info(dev, "PM prober defer: ret=%d\n", ret);
 953                 return -EPROBE_DEFER;
 954         }
 955
 956         res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dma");
 957         priv->dma_base = devm_ioremap_resource(dev, res);
 958         if (IS_ERR(priv->dma_base))
 959                 return PTR_ERR(priv->dma_base);
 960
 961         res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
 962                                            "dma-channels");
 963         priv->dma_chan = devm_ioremap_resource(dev, res);
 964         if (IS_ERR(priv->dma_chan))
 965                 return PTR_ERR(priv->dma_chan);
 966
 967         res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dma-sram");
 968         priv->dma_sram = devm_ioremap_resource(dev, res);
 969         if (IS_ERR(priv->dma_sram))
 970                 return PTR_ERR(priv->dma_sram);
 971
 972         priv->irq_rx = platform_get_irq_byname(pdev, "rx");
 973         if (!priv->irq_rx)
 974                 return -ENODEV;
 975
 976         priv->irq_tx = platform_get_irq_byname(pdev, "tx");
 977         if (!priv->irq_tx)
 978                 return -ENODEV;
 979         else if (priv->irq_tx < 0)
 980                 priv->irq_tx = -1;
 981
 982         if (device_property_read_u32(dev, "dma-rx", &priv->rx_chan))
 983                 return -ENODEV;
 984
 985         if (device_property_read_u32(dev, "dma-tx", &priv->tx_chan))
 986                 return -ENODEV;
 987
 988         priv->rx_ring_size = ENETSW_DEF_RX_DESC;
 989         priv->tx_ring_size = ENETSW_DEF_TX_DESC;
 990         priv->copybreak = ENETSW_DEF_CPY_BREAK;
 991
 992         of_get_mac_address(node, ndev->dev_addr);
 993         if (is_valid_ether_addr(ndev->dev_addr)) {
 994                 dev_info(dev, "mtd mac %pM\n", ndev->dev_addr);
 995         } else {
 996                 random_ether_addr(ndev->dev_addr);
 997                 dev_info(dev, "random mac %pM\n", ndev->dev_addr);
 998         }
 999
1000         priv->rx_buf_size = ALIGN(ndev->mtu + ENETSW_MTU_OVERHEAD,
1001                                   ENETSW_DMA_MAXBURST * 4);
1002
1003         priv->rx_frag_size = ENETSW_FRAG_SIZE(priv->rx_buf_size);
1004
1005         priv->num_clocks = of_clk_get_parent_count(node);
1006         if (priv->num_clocks) {
1007                 priv->clock = devm_kcalloc(dev, priv->num_clocks,
1008                                            sizeof(struct clk *), GFP_KERNEL);
1009                 if (!priv->clock)
1010                         return -ENOMEM;
1011         }
1012         for (i = 0; i < priv->num_clocks; i++) {
1013                 priv->clock[i] = of_clk_get(node, i);
1014                 if (IS_ERR(priv->clock[i])) {
1015                         dev_err(dev, "error getting clock %d\n", i);
1016                         return -EINVAL;
1017                 }
1018
1019                 ret = clk_prepare_enable(priv->clock[i]);
1020                 if (ret) {
1021                         dev_err(dev, "error enabling clock %d\n", i);
1022                         return ret;
1023                 }
1024         }
1025
1026         priv->num_resets = of_count_phandle_with_args(node, "resets",
1027                                                       "#reset-cells");
1028         if (priv->num_resets) {
1029                 priv->reset = devm_kcalloc(dev, priv->num_resets,
1030                                            sizeof(struct reset_control *),
1031                                            GFP_KERNEL);
1032                 if (!priv->reset)
1033                         return -ENOMEM;
1034         }
1035         for (i = 0; i < priv->num_resets; i++) {
1036                 priv->reset[i] = devm_reset_control_get_by_index(dev, i);
1037                 if (IS_ERR(priv->reset[i])) {
1038                         dev_err(dev, "error getting reset %d\n", i);
1039                         return -EINVAL;
1040                 }
1041
1042                 ret = reset_control_reset(priv->reset[i]);
1043                 if (ret) {
1044                         dev_err(dev, "error performing reset %d\n", i);
1045                         return ret;
1046                 }
1047         }
1048
1049         spin_lock_init(&priv->rx_lock);
1050
1051         timer_setup(&priv->rx_timeout, bcm6368_enetsw_refill_rx_timer, 0);
1052
1053         /* register netdevice */
1054         ndev->netdev_ops = &bcm6368_enetsw_ops;
1055         ndev->min_mtu = ETH_ZLEN;
1056         ndev->mtu = ETH_DATA_LEN + ENETSW_TAG_SIZE;
1057         ndev->max_mtu = ETH_DATA_LEN + ENETSW_TAG_SIZE;
1058         netif_napi_add(ndev, &priv->napi, bcm6368_enetsw_poll, 16);
1059         SET_NETDEV_DEV(ndev, dev);
1060
1061         ret = register_netdev(ndev);
1062         if (ret)
1063                 goto out_disable_clk;
1064
1065         netif_carrier_off(ndev);
1066         platform_set_drvdata(pdev, ndev);
1067         priv->pdev = pdev;
1068         priv->net_dev = ndev;
1069
1070         return 0;
1071
1072 out_disable_clk:
1073         for (i = 0; i < priv->num_resets; i++)
1074                 reset_control_assert(priv->reset[i]);
1075
1076         for (i = 0; i < priv->num_clocks; i++)
1077                 clk_disable_unprepare(priv->clock[i]);
1078
1079         return ret;
1080 }
1081
1082 static int bcm6368_enetsw_remove(struct platform_device *pdev)
1083 {
1084         struct device *dev = &pdev->dev;
1085         struct net_device *ndev = platform_get_drvdata(pdev);
1086         struct bcm6368_enetsw *priv = netdev_priv(ndev);
1087         unsigned int i;
1088
1089         unregister_netdev(ndev);
1090
1091         pm_runtime_put_sync(dev);
1092         for (i = 0; priv->pm && i < priv->num_pms; i++) {
1093                 dev_pm_domain_detach(priv->pm[i], true);
1094                 device_link_del(priv->link_pm[i]);
1095         }
1096
1097         for (i = 0; i < priv->num_resets; i++)
1098                 reset_control_assert(priv->reset[i]);
1099
1100         for (i = 0; i < priv->num_clocks; i++)
1101                 clk_disable_unprepare(priv->clock[i]);
1102
1103         free_netdev(ndev);
1104
1105         return 0;
1106 }
1107
1108 static const struct of_device_id bcm6368_enetsw_of_match[] = {
1109         { .compatible = "brcm,bcm6318-enetsw", },
1110         { .compatible = "brcm,bcm6328-enetsw", },
1111         { .compatible = "brcm,bcm6362-enetsw", },
1112         { .compatible = "brcm,bcm6368-enetsw", },
1113         { .compatible = "brcm,bcm63268-enetsw", },
1114         { /* sentinel */ }
1115 };
1116 MODULE_DEVICE_TABLE(of, bcm6368_enetsw_of_match);
1117
1118 static struct platform_driver bcm6368_enetsw_driver = {
1119         .driver = {
1120                 .name = "bcm6368-enetsw",
1121                 .of_match_table = of_match_ptr(bcm6368_enetsw_of_match),
1122         },
1123         .probe  = bcm6368_enetsw_probe,
1124         .remove = bcm6368_enetsw_remove,
1125 };
1126 module_platform_driver(bcm6368_enetsw_driver);