target/linux/bmips/files/drivers/net/ethernet/broadcom/bcm6368-enetsw.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * BCM6368 Ethernet Switch Controller Driver
   4  *
   5  * Copyright (C) 2021 Álvaro Fernández Rojas <noltari@gmail.com>
   6  * Copyright (C) 2015 Jonas Gorski <jonas.gorski@gmail.com>
   7  * Copyright (C) 2008 Maxime Bizon <mbizon@freebox.fr>
   8  */
   9
  10 #include <linux/clk.h>
  11 #include <linux/delay.h>
  12 #include <linux/dma-mapping.h>
  13 #include <linux/err.h>
  14 #include <linux/ethtool.h>
  15 #include <linux/if_vlan.h>
  16 #include <linux/interrupt.h>
  17 #include <linux/of_clk.h>
  18 #include <linux/of_net.h>
  19 #include <linux/platform_device.h>
  20 #include <linux/pm_domain.h>
  21 #include <linux/pm_runtime.h>
  22 #include <linux/reset.h>
  23
  24 /* MTU */
  25 #define ENETSW_TAG_SIZE                 6
  26 #define ENETSW_MTU_OVERHEAD             (VLAN_ETH_HLEN + VLAN_HLEN + \
  27                                          ENETSW_TAG_SIZE)
  28 #define ENETSW_FRAG_SIZE(x)             (SKB_DATA_ALIGN(NET_SKB_PAD + x + \
  29                                          SKB_DATA_ALIGN(sizeof(struct skb_shared_info))))
  30
  31 /* default number of descriptor */
  32 #define ENETSW_DEF_RX_DESC              64
  33 #define ENETSW_DEF_TX_DESC              32
  34 #define ENETSW_DEF_CPY_BREAK            128
  35
  36 /* maximum burst len for dma (4 bytes unit) */
  37 #define ENETSW_DMA_MAXBURST             8
  38
  39 /* DMA channels */
  40 #define DMA_CHAN_WIDTH                  0x10
  41
  42 /* Controller Configuration Register */
  43 #define DMA_CFG_REG                     0x0
  44 #define DMA_CFG_EN_SHIFT                0
  45 #define DMA_CFG_EN_MASK                 (1 << DMA_CFG_EN_SHIFT)
  46 #define DMA_CFG_FLOWCH_MASK(x)          (1 << ((x >> 1) + 1))
  47
  48 /* Flow Control Descriptor Low Threshold register */
  49 #define DMA_FLOWCL_REG(x)               (0x4 + (x) * 6)
  50
  51 /* Flow Control Descriptor High Threshold register */
  52 #define DMA_FLOWCH_REG(x)               (0x8 + (x) * 6)
  53
  54 /* Flow Control Descriptor Buffer Alloca Threshold register */
  55 #define DMA_BUFALLOC_REG(x)             (0xc + (x) * 6)
  56 #define DMA_BUFALLOC_FORCE_SHIFT        31
  57 #define DMA_BUFALLOC_FORCE_MASK         (1 << DMA_BUFALLOC_FORCE_SHIFT)
  58
  59 /* Channel Configuration register */
  60 #define DMAC_CHANCFG_REG                0x0
  61 #define DMAC_CHANCFG_EN_SHIFT           0
  62 #define DMAC_CHANCFG_EN_MASK            (1 << DMAC_CHANCFG_EN_SHIFT)
  63 #define DMAC_CHANCFG_PKTHALT_SHIFT      1
  64 #define DMAC_CHANCFG_PKTHALT_MASK       (1 << DMAC_CHANCFG_PKTHALT_SHIFT)
  65 #define DMAC_CHANCFG_BUFHALT_SHIFT      2
  66 #define DMAC_CHANCFG_BUFHALT_MASK       (1 << DMAC_CHANCFG_BUFHALT_SHIFT)
  67 #define DMAC_CHANCFG_CHAINING_SHIFT     2
  68 #define DMAC_CHANCFG_CHAINING_MASK      (1 << DMAC_CHANCFG_CHAINING_SHIFT)
  69 #define DMAC_CHANCFG_WRAP_EN_SHIFT      3
  70 #define DMAC_CHANCFG_WRAP_EN_MASK       (1 << DMAC_CHANCFG_WRAP_EN_SHIFT)
  71 #define DMAC_CHANCFG_FLOWC_EN_SHIFT     4
  72 #define DMAC_CHANCFG_FLOWC_EN_MASK      (1 << DMAC_CHANCFG_FLOWC_EN_SHIFT)
  73
  74 /* Interrupt Control/Status register */
  75 #define DMAC_IR_REG                     0x4
  76 #define DMAC_IR_BUFDONE_MASK            (1 << 0)
  77 #define DMAC_IR_PKTDONE_MASK            (1 << 1)
  78 #define DMAC_IR_NOTOWNER_MASK           (1 << 2)
  79
  80 /* Interrupt Mask register */
  81 #define DMAC_IRMASK_REG                 0x8
  82
  83 /* Maximum Burst Length */
  84 #define DMAC_MAXBURST_REG               0xc
  85
  86 /* Ring Start Address register */
  87 #define DMAS_RSTART_REG                 0x0
  88
  89 /* State Ram Word 2 */
  90 #define DMAS_SRAM2_REG                  0x4
  91
  92 /* State Ram Word 3 */
  93 #define DMAS_SRAM3_REG                  0x8
  94
  95 /* State Ram Word 4 */
  96 #define DMAS_SRAM4_REG                  0xc
  97
  98 struct bcm6368_enetsw_desc {
  99         u32 len_stat;
 100         u32 address;
 101 };
 102
 103 /* control */
 104 #define DMADESC_LENGTH_SHIFT            16
 105 #define DMADESC_LENGTH_MASK             (0xfff << DMADESC_LENGTH_SHIFT)
 106 #define DMADESC_OWNER_MASK              (1 << 15)
 107 #define DMADESC_EOP_MASK                (1 << 14)
 108 #define DMADESC_SOP_MASK                (1 << 13)
 109 #define DMADESC_ESOP_MASK               (DMADESC_EOP_MASK | DMADESC_SOP_MASK)
 110 #define DMADESC_WRAP_MASK               (1 << 12)
 111 #define DMADESC_USB_NOZERO_MASK         (1 << 1)
 112 #define DMADESC_USB_ZERO_MASK           (1 << 0)
 113
 114 /* status */
 115 #define DMADESC_UNDER_MASK              (1 << 9)
 116 #define DMADESC_APPEND_CRC              (1 << 8)
 117 #define DMADESC_OVSIZE_MASK             (1 << 4)
 118 #define DMADESC_RXER_MASK               (1 << 2)
 119 #define DMADESC_CRC_MASK                (1 << 1)
 120 #define DMADESC_OV_MASK                 (1 << 0)
 121 #define DMADESC_ERR_MASK                (DMADESC_UNDER_MASK | \
 122                                          DMADESC_OVSIZE_MASK | \
 123                                          DMADESC_RXER_MASK | \
 124                                          DMADESC_CRC_MASK | \
 125                                          DMADESC_OV_MASK)
 126
 127 struct bcm6368_enetsw {
 128         void __iomem *dma_base;
 129         void __iomem *dma_chan;
 130         void __iomem *dma_sram;
 131
 132         struct device **pm;
 133         struct device_link **link_pm;
 134         int num_pms;
 135
 136         struct clk **clock;
 137         unsigned int num_clocks;
 138
 139         struct reset_control **reset;
 140         unsigned int num_resets;
 141
 142         int copybreak;
 143
 144         int irq_rx;
 145         int irq_tx;
 146
 147         /* hw view of rx & tx dma ring */
 148         dma_addr_t rx_desc_dma;
 149         dma_addr_t tx_desc_dma;
 150
 151         /* allocated size (in bytes) for rx & tx dma ring */
 152         unsigned int rx_desc_alloc_size;
 153         unsigned int tx_desc_alloc_size;
 154
 155         struct napi_struct napi;
 156
 157         /* dma channel id for rx */
 158         int rx_chan;
 159
 160         /* number of dma desc in rx ring */
 161         int rx_ring_size;
 162
 163         /* cpu view of rx dma ring */
 164         struct bcm6368_enetsw_desc *rx_desc_cpu;
 165
 166         /* current number of armed descriptor given to hardware for rx */
 167         int rx_desc_count;
 168
 169         /* next rx descriptor to fetch from hardware */
 170         int rx_curr_desc;
 171
 172         /* next dirty rx descriptor to refill */
 173         int rx_dirty_desc;
 174
 175         /* size of allocated rx buffer */
 176         unsigned int rx_buf_size;
 177
 178         /* size of allocated rx frag */
 179         unsigned int rx_frag_size;
 180
 181         /* list of buffer given to hw for rx */
 182         unsigned char **rx_buf;
 183
 184         /* used when rx buffer allocation failed, so we defer rx queue
 185          * refill */
 186         struct timer_list rx_timeout;
 187
 188         /* lock rx_timeout against rx normal operation */
 189         spinlock_t rx_lock;
 190
 191         /* dma channel id for tx */
 192         int tx_chan;
 193
 194         /* number of dma desc in tx ring */
 195         int tx_ring_size;
 196
 197         /* maximum dma burst size */
 198         int dma_maxburst;
 199
 200         /* cpu view of rx dma ring */
 201         struct bcm6368_enetsw_desc *tx_desc_cpu;
 202
 203         /* number of available descriptor for tx */
 204         int tx_desc_count;
 205
 206         /* next tx descriptor avaiable */
 207         int tx_curr_desc;
 208
 209         /* next dirty tx descriptor to reclaim */
 210         int tx_dirty_desc;
 211
 212         /* list of skb given to hw for tx */
 213         struct sk_buff **tx_skb;
 214
 215         /* lock used by tx reclaim and xmit */
 216         spinlock_t tx_lock;
 217
 218         /* network device reference */
 219         struct net_device *net_dev;
 220
 221         /* platform device reference */
 222         struct platform_device *pdev;
 223
 224         /* dma channel enable mask */
 225         u32 dma_chan_en_mask;
 226
 227         /* dma channel interrupt mask */
 228         u32 dma_chan_int_mask;
 229
 230         /* dma channel width */
 231         unsigned int dma_chan_width;
 232 };
 233
 234 static inline void dma_writel(struct bcm6368_enetsw *priv, u32 val, u32 off)
 235 {
 236         __raw_writel(val, priv->dma_base + off);
 237 }
 238
 239 static inline u32 dma_readl(struct bcm6368_enetsw *priv, u32 off, int chan)
 240 {
 241         return __raw_readl(priv->dma_chan + off + chan * priv->dma_chan_width);
 242 }
 243
 244 static inline void dmac_writel(struct bcm6368_enetsw *priv, u32 val,
 245                                     u32 off, int chan)
 246 {
 247         __raw_writel(val, priv->dma_chan + off + chan * priv->dma_chan_width);
 248 }
 249
 250 static inline void dmas_writel(struct bcm6368_enetsw *priv, u32 val,
 251                                     u32 off, int chan)
 252 {
 253         __raw_writel(val, priv->dma_sram + off + chan * priv->dma_chan_width);
 254 }
 255
 256 /*
 257  * refill rx queue
 258  */
 259 static int bcm6368_enetsw_refill_rx(struct net_device *dev, bool napi_mode)
 260 {
 261         struct bcm6368_enetsw *priv = netdev_priv(dev);
 262
 263         while (priv->rx_desc_count < priv->rx_ring_size) {
 264                 struct bcm6368_enetsw_desc *desc;
 265                 int desc_idx;
 266                 u32 len_stat;
 267
 268                 desc_idx = priv->rx_dirty_desc;
 269                 desc = &priv->rx_desc_cpu[desc_idx];
 270
 271                 if (!priv->rx_buf[desc_idx]) {
 272                         unsigned char *buf;
 273
 274                         if (likely(napi_mode))
 275                                 buf = napi_alloc_frag(priv->rx_frag_size);
 276                         else
 277                                 buf = netdev_alloc_frag(priv->rx_frag_size);
 278
 279                         if (unlikely(!buf))
 280                                 break;
 281
 282                         priv->rx_buf[desc_idx] = buf;
 283                         desc->address = dma_map_single(&priv->pdev->dev,
 284                                                        buf + NET_SKB_PAD,
 285                                                        priv->rx_buf_size,
 286                                                        DMA_FROM_DEVICE);
 287                 }
 288
 289                 len_stat = priv->rx_buf_size << DMADESC_LENGTH_SHIFT;
 290                 len_stat |= DMADESC_OWNER_MASK;
 291                 if (priv->rx_dirty_desc == priv->rx_ring_size - 1) {
 292                         len_stat |= DMADESC_WRAP_MASK;
 293                         priv->rx_dirty_desc = 0;
 294                 } else {
 295                         priv->rx_dirty_desc++;
 296                 }
 297                 wmb();
 298                 desc->len_stat = len_stat;
 299
 300                 priv->rx_desc_count++;
 301
 302                 /* tell dma engine we allocated one buffer */
 303                 dma_writel(priv, 1, DMA_BUFALLOC_REG(priv->rx_chan));
 304         }
 305
 306         /* If rx ring is still empty, set a timer to try allocating
 307          * again at a later time. */
 308         if (priv->rx_desc_count == 0 && netif_running(dev)) {
 309                 dev_warn(&priv->pdev->dev, "unable to refill rx ring\n");
 310                 priv->rx_timeout.expires = jiffies + HZ;
 311                 add_timer(&priv->rx_timeout);
 312         }
 313
 314         return 0;
 315 }
 316
 317 /*
 318  * timer callback to defer refill rx queue in case we're OOM
 319  */
 320 static void bcm6368_enetsw_refill_rx_timer(struct timer_list *t)
 321 {
 322         struct bcm6368_enetsw *priv = from_timer(priv, t, rx_timeout);
 323         struct net_device *dev = priv->net_dev;
 324
 325         spin_lock(&priv->rx_lock);
 326         bcm6368_enetsw_refill_rx(dev, false);
 327         spin_unlock(&priv->rx_lock);
 328 }
 329
 330 /*
 331  * extract packet from rx queue
 332  */
 333 static int bcm6368_enetsw_receive_queue(struct net_device *dev, int budget)
 334 {
 335         struct bcm6368_enetsw *priv = netdev_priv(dev);
 336         struct device *kdev = &priv->pdev->dev;
 337         int processed = 0;
 338
 339         /* don't scan ring further than number of refilled
 340          * descriptor */
 341         if (budget > priv->rx_desc_count)
 342                 budget = priv->rx_desc_count;
 343
 344         do {
 345                 struct bcm6368_enetsw_desc *desc;
 346                 unsigned int frag_size;
 347                 struct sk_buff *skb;
 348                 unsigned char *buf;
 349                 int desc_idx;
 350                 u32 len_stat;
 351                 unsigned int len;
 352
 353                 desc_idx = priv->rx_curr_desc;
 354                 desc = &priv->rx_desc_cpu[desc_idx];
 355
 356                 /* make sure we actually read the descriptor status at
 357                  * each loop */
 358                 rmb();
 359
 360                 len_stat = desc->len_stat;
 361
 362                 /* break if dma ownership belongs to hw */
 363                 if (len_stat & DMADESC_OWNER_MASK)
 364                         break;
 365
 366                 processed++;
 367                 priv->rx_curr_desc++;
 368                 if (priv->rx_curr_desc == priv->rx_ring_size)
 369                         priv->rx_curr_desc = 0;
 370                 priv->rx_desc_count--;
 371
 372                 /* if the packet does not have start of packet _and_
 373                  * end of packet flag set, then just recycle it */
 374                 if ((len_stat & DMADESC_ESOP_MASK) != DMADESC_ESOP_MASK) {
 375                         dev->stats.rx_dropped++;
 376                         continue;
 377                 }
 378
 379                 /* valid packet */
 380                 buf = priv->rx_buf[desc_idx];
 381                 len = (len_stat & DMADESC_LENGTH_MASK)
 382                       >> DMADESC_LENGTH_SHIFT;
 383                 /* don't include FCS */
 384                 len -= 4;
 385
 386                 if (len < priv->copybreak) {
 387                         unsigned int nfrag_size = ENETSW_FRAG_SIZE(len);
 388                         unsigned char *nbuf = napi_alloc_frag(nfrag_size);
 389
 390                         if (unlikely(!nbuf)) {
 391                                 /* forget packet, just rearm desc */
 392                                 dev->stats.rx_dropped++;
 393                                 continue;
 394                         }
 395
 396                         dma_sync_single_for_cpu(kdev, desc->address,
 397                                                 len, DMA_FROM_DEVICE);
 398                         memcpy(nbuf + NET_SKB_PAD, buf + NET_SKB_PAD, len);
 399                         dma_sync_single_for_device(kdev, desc->address,
 400                                                    len, DMA_FROM_DEVICE);
 401                         buf = nbuf;
 402                         frag_size = nfrag_size;
 403                 } else {
 404                         dma_unmap_single(kdev, desc->address,
 405                                          priv->rx_buf_size, DMA_FROM_DEVICE);
 406                         priv->rx_buf[desc_idx] = NULL;
 407                         frag_size = priv->rx_frag_size;
 408                 }
 409
 410                 skb = build_skb(buf, frag_size);
 411                 if (unlikely(!skb)) {
 412                         skb_free_frag(buf);
 413                         dev->stats.rx_dropped++;
 414                         continue;
 415                 }
 416
 417                 skb_reserve(skb, NET_SKB_PAD);
 418                 skb_put(skb, len);
 419                 skb->protocol = eth_type_trans(skb, dev);
 420                 dev->stats.rx_packets++;
 421                 dev->stats.rx_bytes += len;
 422                 netif_receive_skb(skb);
 423         } while (--budget > 0);
 424
 425         if (processed || !priv->rx_desc_count) {
 426                 bcm6368_enetsw_refill_rx(dev, true);
 427
 428                 /* kick rx dma */
 429                 dmac_writel(priv, priv->dma_chan_en_mask,
 430                             DMAC_CHANCFG_REG, priv->rx_chan);
 431         }
 432
 433         return processed;
 434 }
 435
 436 /*
 437  * try to or force reclaim of transmitted buffers
 438  */
 439 static int bcm6368_enetsw_tx_reclaim(struct net_device *dev, int force)
 440 {
 441         struct bcm6368_enetsw *priv = netdev_priv(dev);
 442         int released = 0;
 443
 444         while (priv->tx_desc_count < priv->tx_ring_size) {
 445                 struct bcm6368_enetsw_desc *desc;
 446                 struct sk_buff *skb;
 447
 448                 /* We run in a bh and fight against start_xmit, which
 449                  * is called with bh disabled */
 450                 spin_lock(&priv->tx_lock);
 451
 452                 desc = &priv->tx_desc_cpu[priv->tx_dirty_desc];
 453
 454                 if (!force && (desc->len_stat & DMADESC_OWNER_MASK)) {
 455                         spin_unlock(&priv->tx_lock);
 456                         break;
 457                 }
 458
 459                 /* ensure other field of the descriptor were not read
 460                  * before we checked ownership */
 461                 rmb();
 462
 463                 skb = priv->tx_skb[priv->tx_dirty_desc];
 464                 priv->tx_skb[priv->tx_dirty_desc] = NULL;
 465                 dma_unmap_single(&priv->pdev->dev, desc->address, skb->len,
 466                                  DMA_TO_DEVICE);
 467
 468                 priv->tx_dirty_desc++;
 469                 if (priv->tx_dirty_desc == priv->tx_ring_size)
 470                         priv->tx_dirty_desc = 0;
 471                 priv->tx_desc_count++;
 472
 473                 spin_unlock(&priv->tx_lock);
 474
 475                 if (desc->len_stat & DMADESC_UNDER_MASK)
 476                         dev->stats.tx_errors++;
 477
 478                 dev_kfree_skb(skb);
 479                 released++;
 480         }
 481
 482         if (netif_queue_stopped(dev) && released)
 483                 netif_wake_queue(dev);
 484
 485         return released;
 486 }
 487
 488 /*
 489  * poll func, called by network core
 490  */
 491 static int bcm6368_enetsw_poll(struct napi_struct *napi, int budget)
 492 {
 493         struct bcm6368_enetsw *priv = container_of(napi, struct bcm6368_enetsw, napi);
 494         struct net_device *dev = priv->net_dev;
 495         int rx_work_done;
 496
 497         /* ack interrupts */
 498         dmac_writel(priv, priv->dma_chan_int_mask,
 499                          DMAC_IR_REG, priv->rx_chan);
 500         dmac_writel(priv, priv->dma_chan_int_mask,
 501                          DMAC_IR_REG, priv->tx_chan);
 502
 503         /* reclaim sent skb */
 504         bcm6368_enetsw_tx_reclaim(dev, 0);
 505
 506         spin_lock(&priv->rx_lock);
 507         rx_work_done = bcm6368_enetsw_receive_queue(dev, budget);
 508         spin_unlock(&priv->rx_lock);
 509
 510         if (rx_work_done >= budget) {
 511                 /* rx queue is not yet empty/clean */
 512                 return rx_work_done;
 513         }
 514
 515         /* no more packet in rx/tx queue, remove device from poll
 516          * queue */
 517         napi_complete_done(napi, rx_work_done);
 518
 519         /* restore rx/tx interrupt */
 520         dmac_writel(priv, priv->dma_chan_int_mask,
 521                          DMAC_IRMASK_REG, priv->rx_chan);
 522         dmac_writel(priv, priv->dma_chan_int_mask,
 523                          DMAC_IRMASK_REG, priv->tx_chan);
 524
 525         return rx_work_done;
 526 }
 527
 528 /*
 529  * rx/tx dma interrupt handler
 530  */
 531 static irqreturn_t bcm6368_enetsw_isr_dma(int irq, void *dev_id)
 532 {
 533         struct net_device *dev = dev_id;
 534         struct bcm6368_enetsw *priv = netdev_priv(dev);
 535
 536         /* mask rx/tx interrupts */
 537         dmac_writel(priv, 0, DMAC_IRMASK_REG, priv->rx_chan);
 538         dmac_writel(priv, 0, DMAC_IRMASK_REG, priv->tx_chan);
 539
 540         napi_schedule(&priv->napi);
 541
 542         return IRQ_HANDLED;
 543 }
 544
 545 /*
 546  * tx request callback
 547  */
 548 static netdev_tx_t
 549 bcm6368_enetsw_start_xmit(struct sk_buff *skb, struct net_device *dev)
 550 {
 551         struct bcm6368_enetsw *priv = netdev_priv(dev);
 552         struct bcm6368_enetsw_desc *desc;
 553         u32 len_stat;
 554         netdev_tx_t ret;
 555
 556         /* lock against tx reclaim */
 557         spin_lock(&priv->tx_lock);
 558
 559         /* make sure the tx hw queue is not full, should not happen
 560          * since we stop queue before it's the case */
 561         if (unlikely(!priv->tx_desc_count)) {
 562                 netif_stop_queue(dev);
 563                 dev_err(&priv->pdev->dev, "xmit called with no tx desc "
 564                         "available?\n");
 565                 ret = NETDEV_TX_BUSY;
 566                 goto out_unlock;
 567         }
 568
 569         /* pad small packets */
 570         if (skb->len < (ETH_ZLEN + ETH_FCS_LEN)) {
 571                 int needed = (ETH_ZLEN + ETH_FCS_LEN) - skb->len;
 572                 char *data;
 573
 574                 if (unlikely(skb_tailroom(skb) < needed)) {
 575                         struct sk_buff *nskb;
 576
 577                         nskb = skb_copy_expand(skb, 0, needed, GFP_ATOMIC);
 578                         if (!nskb) {
 579                                 ret = NETDEV_TX_BUSY;
 580                                 goto out_unlock;
 581                         }
 582
 583                         dev_kfree_skb(skb);
 584                         skb = nskb;
 585                 }
 586                 data = skb_put_zero(skb, needed);
 587         }
 588
 589         /* point to the next available desc */
 590         desc = &priv->tx_desc_cpu[priv->tx_curr_desc];
 591         priv->tx_skb[priv->tx_curr_desc] = skb;
 592
 593         /* fill descriptor */
 594         desc->address = dma_map_single(&priv->pdev->dev, skb->data, skb->len,
 595                                        DMA_TO_DEVICE);
 596
 597         len_stat = (skb->len << DMADESC_LENGTH_SHIFT) & DMADESC_LENGTH_MASK;
 598         len_stat |= DMADESC_ESOP_MASK | DMADESC_APPEND_CRC |
 599                     DMADESC_OWNER_MASK;
 600
 601         priv->tx_curr_desc++;
 602         if (priv->tx_curr_desc == priv->tx_ring_size) {
 603                 priv->tx_curr_desc = 0;
 604                 len_stat |= DMADESC_WRAP_MASK;
 605         }
 606         priv->tx_desc_count--;
 607
 608         /* dma might be already polling, make sure we update desc
 609          * fields in correct order */
 610         wmb();
 611         desc->len_stat = len_stat;
 612         wmb();
 613
 614         /* kick tx dma */
 615         dmac_writel(priv, priv->dma_chan_en_mask, DMAC_CHANCFG_REG,
 616                     priv->tx_chan);
 617
 618         /* stop queue if no more desc available */
 619         if (!priv->tx_desc_count)
 620                 netif_stop_queue(dev);
 621
 622         dev->stats.tx_bytes += skb->len;
 623         dev->stats.tx_packets++;
 624         ret = NETDEV_TX_OK;
 625
 626 out_unlock:
 627         spin_unlock(&priv->tx_lock);
 628         return ret;
 629 }
 630
 631 /*
 632  * disable dma in given channel
 633  */
 634 static void bcm6368_enetsw_disable_dma(struct bcm6368_enetsw *priv, int chan)
 635 {
 636         int limit = 1000;
 637
 638         dmac_writel(priv, 0, DMAC_CHANCFG_REG, chan);
 639
 640         do {
 641                 u32 val;
 642
 643                 val = dma_readl(priv, DMAC_CHANCFG_REG, chan);
 644                 if (!(val & DMAC_CHANCFG_EN_MASK))
 645                         break;
 646
 647                 udelay(1);
 648         } while (limit--);
 649 }
 650
 651 static int bcm6368_enetsw_open(struct net_device *dev)
 652 {
 653         struct bcm6368_enetsw *priv = netdev_priv(dev);
 654         struct device *kdev = &priv->pdev->dev;
 655         int i, ret;
 656         unsigned int size;
 657         void *p;
 658         u32 val;
 659
 660         /* mask all interrupts and request them */
 661         dmac_writel(priv, 0, DMAC_IRMASK_REG, priv->rx_chan);
 662         dmac_writel(priv, 0, DMAC_IRMASK_REG, priv->tx_chan);
 663
 664         ret = request_irq(priv->irq_rx, bcm6368_enetsw_isr_dma,
 665                           0, dev->name, dev);
 666         if (ret)
 667                 goto out_freeirq;
 668
 669         if (priv->irq_tx != -1) {
 670                 ret = request_irq(priv->irq_tx, bcm6368_enetsw_isr_dma,
 671                                   0, dev->name, dev);
 672                 if (ret)
 673                         goto out_freeirq_rx;
 674         }
 675
 676         /* allocate rx dma ring */
 677         size = priv->rx_ring_size * sizeof(struct bcm6368_enetsw_desc);
 678         p = dma_alloc_coherent(kdev, size, &priv->rx_desc_dma, GFP_KERNEL);
 679         if (!p) {
 680                 dev_err(kdev, "cannot allocate rx ring %u\n", size);
 681                 ret = -ENOMEM;
 682                 goto out_freeirq_tx;
 683         }
 684
 685         memset(p, 0, size);
 686         priv->rx_desc_alloc_size = size;
 687         priv->rx_desc_cpu = p;
 688
 689         /* allocate tx dma ring */
 690         size = priv->tx_ring_size * sizeof(struct bcm6368_enetsw_desc);
 691         p = dma_alloc_coherent(kdev, size, &priv->tx_desc_dma, GFP_KERNEL);
 692         if (!p) {
 693                 dev_err(kdev, "cannot allocate tx ring\n");
 694                 ret = -ENOMEM;
 695                 goto out_free_rx_ring;
 696         }
 697
 698         memset(p, 0, size);
 699         priv->tx_desc_alloc_size = size;
 700         priv->tx_desc_cpu = p;
 701
 702         priv->tx_skb = kzalloc(sizeof(struct sk_buff *) * priv->tx_ring_size,
 703                                GFP_KERNEL);
 704         if (!priv->tx_skb) {
 705                 dev_err(kdev, "cannot allocate tx skb queue\n");
 706                 ret = -ENOMEM;
 707                 goto out_free_tx_ring;
 708         }
 709
 710         priv->tx_desc_count = priv->tx_ring_size;
 711         priv->tx_dirty_desc = 0;
 712         priv->tx_curr_desc = 0;
 713         spin_lock_init(&priv->tx_lock);
 714
 715         /* init & fill rx ring with buffers */
 716         priv->rx_buf = kzalloc(sizeof(unsigned char *) * priv->rx_ring_size,
 717                                GFP_KERNEL);
 718         if (!priv->rx_buf) {
 719                 dev_err(kdev, "cannot allocate rx buffer queue\n");
 720                 ret = -ENOMEM;
 721                 goto out_free_tx_skb;
 722         }
 723
 724         priv->rx_desc_count = 0;
 725         priv->rx_dirty_desc = 0;
 726         priv->rx_curr_desc = 0;
 727
 728         /* initialize flow control buffer allocation */
 729         dma_writel(priv, DMA_BUFALLOC_FORCE_MASK | 0,
 730                    DMA_BUFALLOC_REG(priv->rx_chan));
 731
 732         if (bcm6368_enetsw_refill_rx(dev, false)) {
 733                 dev_err(kdev, "cannot allocate rx buffer queue\n");
 734                 ret = -ENOMEM;
 735                 goto out;
 736         }
 737
 738         /* write rx & tx ring addresses */
 739         dmas_writel(priv, priv->rx_desc_dma,
 740                     DMAS_RSTART_REG, priv->rx_chan);
 741         dmas_writel(priv, priv->tx_desc_dma,
 742                     DMAS_RSTART_REG, priv->tx_chan);
 743
 744         /* clear remaining state ram for rx & tx channel */
 745         dmas_writel(priv, 0, DMAS_SRAM2_REG, priv->rx_chan);
 746         dmas_writel(priv, 0, DMAS_SRAM2_REG, priv->tx_chan);
 747         dmas_writel(priv, 0, DMAS_SRAM3_REG, priv->rx_chan);
 748         dmas_writel(priv, 0, DMAS_SRAM3_REG, priv->tx_chan);
 749         dmas_writel(priv, 0, DMAS_SRAM4_REG, priv->rx_chan);
 750         dmas_writel(priv, 0, DMAS_SRAM4_REG, priv->tx_chan);
 751
 752         /* set dma maximum burst len */
 753         dmac_writel(priv, priv->dma_maxburst,
 754                     DMAC_MAXBURST_REG, priv->rx_chan);
 755         dmac_writel(priv, priv->dma_maxburst,
 756                     DMAC_MAXBURST_REG, priv->tx_chan);
 757
 758         /* set flow control low/high threshold to 1/3 / 2/3 */
 759         val = priv->rx_ring_size / 3;
 760         dma_writel(priv, val, DMA_FLOWCL_REG(priv->rx_chan));
 761         val = (priv->rx_ring_size * 2) / 3;
 762         dma_writel(priv, val, DMA_FLOWCH_REG(priv->rx_chan));
 763
 764         /* all set, enable mac and interrupts, start dma engine and
 765          * kick rx dma channel
 766          */
 767         wmb();
 768         dma_writel(priv, DMA_CFG_EN_MASK, DMA_CFG_REG);
 769         dmac_writel(priv, DMAC_CHANCFG_EN_MASK,
 770                     DMAC_CHANCFG_REG, priv->rx_chan);
 771
 772         /* watch "packet transferred" interrupt in rx and tx */
 773         dmac_writel(priv, DMAC_IR_PKTDONE_MASK,
 774                     DMAC_IR_REG, priv->rx_chan);
 775         dmac_writel(priv, DMAC_IR_PKTDONE_MASK,
 776                     DMAC_IR_REG, priv->tx_chan);
 777
 778         /* make sure we enable napi before rx interrupt  */
 779         napi_enable(&priv->napi);
 780
 781         dmac_writel(priv, DMAC_IR_PKTDONE_MASK,
 782                     DMAC_IRMASK_REG, priv->rx_chan);
 783         dmac_writel(priv, DMAC_IR_PKTDONE_MASK,
 784                     DMAC_IRMASK_REG, priv->tx_chan);
 785
 786         netif_carrier_on(dev);
 787         netif_start_queue(dev);
 788
 789         return 0;
 790
 791 out:
 792         for (i = 0; i < priv->rx_ring_size; i++) {
 793                 struct bcm6368_enetsw_desc *desc;
 794
 795                 if (!priv->rx_buf[i])
 796                         continue;
 797
 798                 desc = &priv->rx_desc_cpu[i];
 799                 dma_unmap_single(kdev, desc->address, priv->rx_buf_size,
 800                                  DMA_FROM_DEVICE);
 801                 skb_free_frag(priv->rx_buf[i]);
 802         }
 803         kfree(priv->rx_buf);
 804
 805 out_free_tx_skb:
 806         kfree(priv->tx_skb);
 807
 808 out_free_tx_ring:
 809         dma_free_coherent(kdev, priv->tx_desc_alloc_size,
 810                           priv->tx_desc_cpu, priv->tx_desc_dma);
 811
 812 out_free_rx_ring:
 813         dma_free_coherent(kdev, priv->rx_desc_alloc_size,
 814                           priv->rx_desc_cpu, priv->rx_desc_dma);
 815
 816 out_freeirq_tx:
 817         if (priv->irq_tx != -1)
 818                 free_irq(priv->irq_tx, dev);
 819
 820 out_freeirq_rx:
 821         free_irq(priv->irq_rx, dev);
 822
 823 out_freeirq:
 824         return ret;
 825 }
 826
 827 static int bcm6368_enetsw_stop(struct net_device *dev)
 828 {
 829         struct bcm6368_enetsw *priv = netdev_priv(dev);
 830         struct device *kdev = &priv->pdev->dev;
 831         int i;
 832
 833         netif_stop_queue(dev);
 834         napi_disable(&priv->napi);
 835         del_timer_sync(&priv->rx_timeout);
 836
 837         /* mask all interrupts */
 838         dmac_writel(priv, 0, DMAC_IRMASK_REG, priv->rx_chan);
 839         dmac_writel(priv, 0, DMAC_IRMASK_REG, priv->tx_chan);
 840
 841         /* disable dma & mac */
 842         bcm6368_enetsw_disable_dma(priv, priv->tx_chan);
 843         bcm6368_enetsw_disable_dma(priv, priv->rx_chan);
 844
 845         /* force reclaim of all tx buffers */
 846         bcm6368_enetsw_tx_reclaim(dev, 1);
 847
 848         /* free the rx buffer ring */
 849         for (i = 0; i < priv->rx_ring_size; i++) {
 850                 struct bcm6368_enetsw_desc *desc;
 851
 852                 if (!priv->rx_buf[i])
 853                         continue;
 854
 855                 desc = &priv->rx_desc_cpu[i];
 856                 dma_unmap_single_attrs(kdev, desc->address, priv->rx_buf_size,
 857                                        DMA_FROM_DEVICE,
 858                                        DMA_ATTR_SKIP_CPU_SYNC);
 859                 skb_free_frag(priv->rx_buf[i]);
 860         }
 861
 862         /* free remaining allocated memory */
 863         kfree(priv->rx_buf);
 864         kfree(priv->tx_skb);
 865         dma_free_coherent(kdev, priv->rx_desc_alloc_size,
 866                           priv->rx_desc_cpu, priv->rx_desc_dma);
 867         dma_free_coherent(kdev, priv->tx_desc_alloc_size,
 868                           priv->tx_desc_cpu, priv->tx_desc_dma);
 869         if (priv->irq_tx != -1)
 870                 free_irq(priv->irq_tx, dev);
 871         free_irq(priv->irq_rx, dev);
 872
 873         return 0;
 874 }
 875
 876 static const struct net_device_ops bcm6368_enetsw_ops = {
 877         .ndo_open = bcm6368_enetsw_open,
 878         .ndo_stop = bcm6368_enetsw_stop,
 879         .ndo_start_xmit = bcm6368_enetsw_start_xmit,
 880 };
 881
 882 static int bcm6368_enetsw_probe(struct platform_device *pdev)
 883 {
 884         struct bcm6368_enetsw *priv;
 885         struct device *dev = &pdev->dev;
 886         struct device_node *node = dev->of_node;
 887         struct net_device *ndev;
 888         struct resource *res;
 889         unsigned i;
 890         int ret;
 891
 892         ndev = alloc_etherdev(sizeof(*priv));
 893         if (!ndev)
 894                 return -ENOMEM;
 895
 896         priv = netdev_priv(ndev);
 897
 898         priv->num_pms = of_count_phandle_with_args(node, "power-domains",
 899                                                    "#power-domain-cells");
 900         if (priv->num_pms > 1) {
 901                 priv->pm = devm_kcalloc(dev, priv->num_pms,
 902                                         sizeof(struct device *), GFP_KERNEL);
 903                 if (!priv->pm)
 904                         return -ENOMEM;
 905
 906                 priv->link_pm = devm_kcalloc(dev, priv->num_pms,
 907                                              sizeof(struct device_link *),
 908                                              GFP_KERNEL);
 909                 if (!priv->link_pm)
 910                         return -ENOMEM;
 911
 912                 for (i = 0; i < priv->num_pms; i++) {
 913                         priv->pm[i] = genpd_dev_pm_attach_by_id(dev, i);
 914                         if (IS_ERR(priv->pm[i])) {
 915                                 dev_err(dev, "error getting pm %d\n", i);
 916                                 return -EINVAL;
 917                         }
 918
 919                         priv->link_pm[i] = device_link_add(dev, priv->pm[i],
 920                                 DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME |
 921                                 DL_FLAG_RPM_ACTIVE);
 922                 }
 923         }
 924
 925         pm_runtime_enable(dev);
 926         pm_runtime_no_callbacks(dev);
 927         ret = pm_runtime_get_sync(dev);
 928         if (ret < 0) {
 929                 pm_runtime_disable(dev);
 930                 dev_info(dev, "PM prober defer: ret=%d\n", ret);
 931                 return -EPROBE_DEFER;
 932         }
 933
 934         res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dma");
 935         priv->dma_base = devm_ioremap_resource(dev, res);
 936         if (IS_ERR(priv->dma_base))
 937                 return PTR_ERR(priv->dma_base);
 938
 939         res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
 940                                            "dma-channels");
 941         priv->dma_chan = devm_ioremap_resource(dev, res);
 942         if (IS_ERR(priv->dma_chan))
 943                 return PTR_ERR(priv->dma_chan);
 944
 945         res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dma-sram");
 946         priv->dma_sram = devm_ioremap_resource(dev, res);
 947         if (IS_ERR(priv->dma_sram))
 948                 return PTR_ERR(priv->dma_sram);
 949
 950         priv->irq_rx = platform_get_irq_byname(pdev, "rx");
 951         if (!priv->irq_rx)
 952                 return -ENODEV;
 953
 954         priv->irq_tx = platform_get_irq_byname(pdev, "tx");
 955         if (!priv->irq_tx)
 956                 return -ENODEV;
 957         else if (priv->irq_tx < 0)
 958                 priv->irq_tx = -1;
 959
 960         if (device_property_read_u32(dev, "dma-rx", &priv->rx_chan))
 961                 return -ENODEV;
 962
 963         if (device_property_read_u32(dev, "dma-tx", &priv->tx_chan))
 964                 return -ENODEV;
 965
 966         priv->rx_ring_size = ENETSW_DEF_RX_DESC;
 967         priv->tx_ring_size = ENETSW_DEF_TX_DESC;
 968
 969         priv->dma_maxburst = ENETSW_DMA_MAXBURST;
 970
 971         priv->copybreak = ENETSW_DEF_CPY_BREAK;
 972
 973         priv->dma_chan_en_mask = DMAC_CHANCFG_EN_MASK;
 974         priv->dma_chan_int_mask = DMAC_IR_PKTDONE_MASK;
 975         priv->dma_chan_width = DMA_CHAN_WIDTH;
 976
 977         of_get_mac_address(node, ndev->dev_addr);
 978         if (is_valid_ether_addr(ndev->dev_addr)) {
 979                 dev_info(dev, "mtd mac %pM\n", ndev->dev_addr);
 980         } else {
 981                 random_ether_addr(ndev->dev_addr);
 982                 dev_info(dev, "random mac %pM\n", ndev->dev_addr);
 983         }
 984
 985         priv->rx_buf_size = ALIGN(ndev->mtu + ENETSW_MTU_OVERHEAD,
 986                                   priv->dma_maxburst * 4);
 987
 988         priv->rx_frag_size = ENETSW_FRAG_SIZE(priv->rx_buf_size);
 989
 990         priv->num_clocks = of_clk_get_parent_count(node);
 991         if (priv->num_clocks) {
 992                 priv->clock = devm_kcalloc(dev, priv->num_clocks,
 993                                            sizeof(struct clk *), GFP_KERNEL);
 994                 if (!priv->clock)
 995                         return -ENOMEM;
 996         }
 997         for (i = 0; i < priv->num_clocks; i++) {
 998                 priv->clock[i] = of_clk_get(node, i);
 999                 if (IS_ERR(priv->clock[i])) {
1000                         dev_err(dev, "error getting clock %d\n", i);
1001                         return -EINVAL;
1002                 }
1003
1004                 ret = clk_prepare_enable(priv->clock[i]);
1005                 if (ret) {
1006                         dev_err(dev, "error enabling clock %d\n", i);
1007                         return ret;
1008                 }
1009         }
1010
1011         priv->num_resets = of_count_phandle_with_args(node, "resets",
1012                                                       "#reset-cells");
1013         if (priv->num_resets) {
1014                 priv->reset = devm_kcalloc(dev, priv->num_resets,
1015                                            sizeof(struct reset_control *),
1016                                            GFP_KERNEL);
1017                 if (!priv->reset)
1018                         return -ENOMEM;
1019         }
1020         for (i = 0; i < priv->num_resets; i++) {
1021                 priv->reset[i] = devm_reset_control_get_by_index(dev, i);
1022                 if (IS_ERR(priv->reset[i])) {
1023                         dev_err(dev, "error getting reset %d\n", i);
1024                         return -EINVAL;
1025                 }
1026
1027                 ret = reset_control_reset(priv->reset[i]);
1028                 if (ret) {
1029                         dev_err(dev, "error performing reset %d\n", i);
1030                         return ret;
1031                 }
1032         }
1033
1034         spin_lock_init(&priv->rx_lock);
1035
1036         timer_setup(&priv->rx_timeout, bcm6368_enetsw_refill_rx_timer, 0);
1037
1038         /* register netdevice */
1039         ndev->netdev_ops = &bcm6368_enetsw_ops;
1040         ndev->min_mtu = ETH_ZLEN;
1041         ndev->mtu = ETH_DATA_LEN + ENETSW_TAG_SIZE;
1042         ndev->max_mtu = ETH_DATA_LEN + ENETSW_TAG_SIZE;
1043         netif_napi_add(ndev, &priv->napi, bcm6368_enetsw_poll, 16);
1044         SET_NETDEV_DEV(ndev, dev);
1045
1046         ret = register_netdev(ndev);
1047         if (ret)
1048                 goto out_disable_clk;
1049
1050         netif_carrier_off(ndev);
1051         platform_set_drvdata(pdev, ndev);
1052         priv->pdev = pdev;
1053         priv->net_dev = ndev;
1054
1055         return 0;
1056
1057 out_disable_clk:
1058         for (i = 0; i < priv->num_resets; i++)
1059                 reset_control_assert(priv->reset[i]);
1060
1061         for (i = 0; i < priv->num_clocks; i++)
1062                 clk_disable_unprepare(priv->clock[i]);
1063
1064         return ret;
1065 }
1066
1067 static int bcm6368_enetsw_remove(struct platform_device *pdev)
1068 {
1069         struct device *dev = &pdev->dev;
1070         struct net_device *ndev = platform_get_drvdata(pdev);
1071         struct bcm6368_enetsw *priv = netdev_priv(ndev);
1072         unsigned int i;
1073
1074         unregister_netdev(ndev);
1075
1076         pm_runtime_put_sync(dev);
1077         for (i = 0; priv->pm && i < priv->num_pms; i++) {
1078                 dev_pm_domain_detach(priv->pm[i], true);
1079                 device_link_del(priv->link_pm[i]);
1080         }
1081
1082         for (i = 0; i < priv->num_resets; i++)
1083                 reset_control_assert(priv->reset[i]);
1084
1085         for (i = 0; i < priv->num_clocks; i++)
1086                 clk_disable_unprepare(priv->clock[i]);
1087
1088         free_netdev(ndev);
1089
1090         return 0;
1091 }
1092
1093 static const struct of_device_id bcm6368_enetsw_of_match[] = {
1094         { .compatible = "brcm,bcm6318-enetsw", },
1095         { .compatible = "brcm,bcm6328-enetsw", },
1096         { .compatible = "brcm,bcm6362-enetsw", },
1097         { .compatible = "brcm,bcm6368-enetsw", },
1098         { .compatible = "brcm,bcm63268-enetsw", },
1099         { /* sentinel */ }
1100 };
1101 MODULE_DEVICE_TABLE(of, bcm6368_enetsw_of_match);
1102
1103 static struct platform_driver bcm6368_enetsw_driver = {
1104         .driver = {
1105                 .name = "bcm6368-enetsw",
1106                 .of_match_table = of_match_ptr(bcm6368_enetsw_of_match),
1107         },
1108         .probe  = bcm6368_enetsw_probe,
1109         .remove = bcm6368_enetsw_remove,
1110 };
1111 module_platform_driver(bcm6368_enetsw_driver);