drivers/net/fs_enet/fs_enet-main.c

   1 /*
   2  * Combined Ethernet driver for Motorola MPC8xx and MPC82xx.
   3  *
   4  * Copyright (c) 2003 Intracom S.A.
   5  *  by Pantelis Antoniou <panto@intracom.gr>
   6  *
   7  * 2005 (c) MontaVista Software, Inc.
   8  * Vitaly Bordug <vbordug@ru.mvista.com>
   9  *
  10  * Heavily based on original FEC driver by Dan Malek <dan@embeddededge.com>
  11  * and modifications by Joakim Tjernlund <joakim.tjernlund@lumentis.se>
  12  *
  13  * This file is licensed under the terms of the GNU General Public License
  14  * version 2. This program is licensed "as is" without any warranty of any
  15  * kind, whether express or implied.
  16  */
  17
  18 #include <linux/module.h>
  19 #include <linux/kernel.h>
  20 #include <linux/types.h>
  21 #include <linux/string.h>
  22 #include <linux/ptrace.h>
  23 #include <linux/errno.h>
  24 #include <linux/ioport.h>
  25 #include <linux/slab.h>
  26 #include <linux/interrupt.h>
  27 #include <linux/init.h>
  28 #include <linux/delay.h>
  29 #include <linux/netdevice.h>
  30 #include <linux/etherdevice.h>
  31 #include <linux/skbuff.h>
  32 #include <linux/spinlock.h>
  33 #include <linux/mii.h>
  34 #include <linux/ethtool.h>
  35 #include <linux/bitops.h>
  36 #include <linux/fs.h>
  37 #include <linux/platform_device.h>
  38 #include <linux/phy.h>
  39 #include <linux/of.h>
  40 #include <linux/of_mdio.h>
  41 #include <linux/of_platform.h>
  42 #include <linux/of_gpio.h>
  43
  44 #include <linux/vmalloc.h>
  45 #include <asm/pgtable.h>
  46 #include <asm/irq.h>
  47 #include <asm/uaccess.h>
  48
  49 #include "fs_enet.h"
  50
  51 /*************************************************/
  52
  53 MODULE_AUTHOR("Pantelis Antoniou <panto@intracom.gr>");
  54 MODULE_DESCRIPTION("Freescale Ethernet Driver");
  55 MODULE_LICENSE("GPL");
  56 MODULE_VERSION(DRV_MODULE_VERSION);
  57
  58 static int fs_enet_debug = -1; /* -1 == use FS_ENET_DEF_MSG_ENABLE as value */
  59 module_param(fs_enet_debug, int, 0);
  60 MODULE_PARM_DESC(fs_enet_debug,
  61                  "Freescale bitmapped debugging message enable value");
  62
  63 #ifdef CONFIG_NET_POLL_CONTROLLER
  64 static void fs_enet_netpoll(struct net_device *dev);
  65 #endif
  66
  67 static void fs_set_multicast_list(struct net_device *dev)
  68 {
  69         struct fs_enet_private *fep = netdev_priv(dev);
  70
  71         (*fep->ops->set_multicast_list)(dev);
  72 }
  73
  74 static void skb_align(struct sk_buff *skb, int align)
  75 {
  76         int off = ((unsigned long)skb->data) & (align - 1);
  77
  78         if (off)
  79                 skb_reserve(skb, align - off);
  80 }
  81
  82 /* NAPI receive function */
  83 static int fs_enet_rx_napi(struct napi_struct *napi, int budget)
  84 {
  85         struct fs_enet_private *fep = container_of(napi, struct fs_enet_private, napi);
  86         struct net_device *dev = fep->ndev;
  87         const struct fs_platform_info *fpi = fep->fpi;
  88         cbd_t __iomem *bdp;
  89         struct sk_buff *skb, *skbn, *skbt;
  90         int received = 0;
  91         u16 pkt_len, sc;
  92         int curidx;
  93
  94         /*
  95          * First, grab all of the stats for the incoming packet.
  96          * These get messed up if we get called due to a busy condition.
  97          */
  98         bdp = fep->cur_rx;
  99
 100         /* clear RX status bits for napi*/
 101         (*fep->ops->napi_clear_rx_event)(dev);
 102
 103         while (((sc = CBDR_SC(bdp)) & BD_ENET_RX_EMPTY) == 0) {
 104                 curidx = bdp - fep->rx_bd_base;
 105
 106                 /*
 107                  * Since we have allocated space to hold a complete frame,
 108                  * the last indicator should be set.
 109                  */
 110                 if ((sc & BD_ENET_RX_LAST) == 0)
 111                         dev_warn(fep->dev, "rcv is not +last\n");
 112
 113                 /*
 114                  * Check for errors.
 115                  */
 116                 if (sc & (BD_ENET_RX_LG | BD_ENET_RX_SH | BD_ENET_RX_CL |
 117                           BD_ENET_RX_NO | BD_ENET_RX_CR | BD_ENET_RX_OV)) {
 118                         fep->stats.rx_errors++;
 119                         /* Frame too long or too short. */
 120                         if (sc & (BD_ENET_RX_LG | BD_ENET_RX_SH))
 121                                 fep->stats.rx_length_errors++;
 122                         /* Frame alignment */
 123                         if (sc & (BD_ENET_RX_NO | BD_ENET_RX_CL))
 124                                 fep->stats.rx_frame_errors++;
 125                         /* CRC Error */
 126                         if (sc & BD_ENET_RX_CR)
 127                                 fep->stats.rx_crc_errors++;
 128                         /* FIFO overrun */
 129                         if (sc & BD_ENET_RX_OV)
 130                                 fep->stats.rx_crc_errors++;
 131
 132                         skb = fep->rx_skbuff[curidx];
 133
 134                         dma_unmap_single(fep->dev, CBDR_BUFADDR(bdp),
 135                                 L1_CACHE_ALIGN(PKT_MAXBUF_SIZE),
 136                                 DMA_FROM_DEVICE);
 137
 138                         skbn = skb;
 139
 140                 } else {
 141                         skb = fep->rx_skbuff[curidx];
 142
 143                         dma_unmap_single(fep->dev, CBDR_BUFADDR(bdp),
 144                                 L1_CACHE_ALIGN(PKT_MAXBUF_SIZE),
 145                                 DMA_FROM_DEVICE);
 146
 147                         /*
 148                          * Process the incoming frame.
 149                          */
 150                         fep->stats.rx_packets++;
 151                         pkt_len = CBDR_DATLEN(bdp) - 4; /* remove CRC */
 152                         fep->stats.rx_bytes += pkt_len + 4;
 153
 154                         if (pkt_len <= fpi->rx_copybreak) {
 155                                 /* +2 to make IP header L1 cache aligned */
 156                                 skbn = dev_alloc_skb(pkt_len + 2);
 157                                 if (skbn != NULL) {
 158                                         skb_reserve(skbn, 2);   /* align IP header */
 159                                         skb_copy_from_linear_data(skb,
 160                                                       skbn->data, pkt_len);
 161                                         /* swap */
 162                                         skbt = skb;
 163                                         skb = skbn;
 164                                         skbn = skbt;
 165                                 }
 166                         } else {
 167                                 skbn = dev_alloc_skb(ENET_RX_FRSIZE);
 168
 169                                 if (skbn)
 170                                         skb_align(skbn, ENET_RX_ALIGN);
 171                         }
 172
 173                         if (skbn != NULL) {
 174                                 skb_put(skb, pkt_len);  /* Make room */
 175                                 skb->protocol = eth_type_trans(skb, dev);
 176                                 received++;
 177                                 netif_receive_skb(skb);
 178                         } else {
 179                                 dev_warn(fep->dev,
 180                                          "Memory squeeze, dropping packet.\n");
 181                                 fep->stats.rx_dropped++;
 182                                 skbn = skb;
 183                         }
 184                 }
 185
 186                 fep->rx_skbuff[curidx] = skbn;
 187                 CBDW_BUFADDR(bdp, dma_map_single(fep->dev, skbn->data,
 188                              L1_CACHE_ALIGN(PKT_MAXBUF_SIZE),
 189                              DMA_FROM_DEVICE));
 190                 CBDW_DATLEN(bdp, 0);
 191                 CBDW_SC(bdp, (sc & ~BD_ENET_RX_STATS) | BD_ENET_RX_EMPTY);
 192
 193                 /*
 194                  * Update BD pointer to next entry.
 195                  */
 196                 if ((sc & BD_ENET_RX_WRAP) == 0)
 197                         bdp++;
 198                 else
 199                         bdp = fep->rx_bd_base;
 200
 201                 (*fep->ops->rx_bd_done)(dev);
 202
 203                 if (received >= budget)
 204                         break;
 205         }
 206
 207         fep->cur_rx = bdp;
 208
 209         if (received < budget) {
 210                 /* done */
 211                 napi_complete(napi);
 212                 (*fep->ops->napi_enable_rx)(dev);
 213         }
 214         return received;
 215 }
 216
 217 /* non NAPI receive function */
 218 static int fs_enet_rx_non_napi(struct net_device *dev)
 219 {
 220         struct fs_enet_private *fep = netdev_priv(dev);
 221         const struct fs_platform_info *fpi = fep->fpi;
 222         cbd_t __iomem *bdp;
 223         struct sk_buff *skb, *skbn, *skbt;
 224         int received = 0;
 225         u16 pkt_len, sc;
 226         int curidx;
 227         /*
 228          * First, grab all of the stats for the incoming packet.
 229          * These get messed up if we get called due to a busy condition.
 230          */
 231         bdp = fep->cur_rx;
 232
 233         while (((sc = CBDR_SC(bdp)) & BD_ENET_RX_EMPTY) == 0) {
 234
 235                 curidx = bdp - fep->rx_bd_base;
 236
 237                 /*
 238                  * Since we have allocated space to hold a complete frame,
 239                  * the last indicator should be set.
 240                  */
 241                 if ((sc & BD_ENET_RX_LAST) == 0)
 242                         dev_warn(fep->dev, "rcv is not +last\n");
 243
 244                 /*
 245                  * Check for errors.
 246                  */
 247                 if (sc & (BD_ENET_RX_LG | BD_ENET_RX_SH | BD_ENET_RX_CL |
 248                           BD_ENET_RX_NO | BD_ENET_RX_CR | BD_ENET_RX_OV)) {
 249                         fep->stats.rx_errors++;
 250                         /* Frame too long or too short. */
 251                         if (sc & (BD_ENET_RX_LG | BD_ENET_RX_SH))
 252                                 fep->stats.rx_length_errors++;
 253                         /* Frame alignment */
 254                         if (sc & (BD_ENET_RX_NO | BD_ENET_RX_CL))
 255                                 fep->stats.rx_frame_errors++;
 256                         /* CRC Error */
 257                         if (sc & BD_ENET_RX_CR)
 258                                 fep->stats.rx_crc_errors++;
 259                         /* FIFO overrun */
 260                         if (sc & BD_ENET_RX_OV)
 261                                 fep->stats.rx_crc_errors++;
 262
 263                         skb = fep->rx_skbuff[curidx];
 264
 265                         dma_unmap_single(fep->dev, CBDR_BUFADDR(bdp),
 266                                 L1_CACHE_ALIGN(PKT_MAXBUF_SIZE),
 267                                 DMA_FROM_DEVICE);
 268
 269                         skbn = skb;
 270
 271                 } else {
 272
 273                         skb = fep->rx_skbuff[curidx];
 274
 275                         dma_unmap_single(fep->dev, CBDR_BUFADDR(bdp),
 276                                 L1_CACHE_ALIGN(PKT_MAXBUF_SIZE),
 277                                 DMA_FROM_DEVICE);
 278
 279                         /*
 280                          * Process the incoming frame.
 281                          */
 282                         fep->stats.rx_packets++;
 283                         pkt_len = CBDR_DATLEN(bdp) - 4; /* remove CRC */
 284                         fep->stats.rx_bytes += pkt_len + 4;
 285
 286                         if (pkt_len <= fpi->rx_copybreak) {
 287                                 /* +2 to make IP header L1 cache aligned */
 288                                 skbn = dev_alloc_skb(pkt_len + 2);
 289                                 if (skbn != NULL) {
 290                                         skb_reserve(skbn, 2);   /* align IP header */
 291                                         skb_copy_from_linear_data(skb,
 292                                                       skbn->data, pkt_len);
 293                                         /* swap */
 294                                         skbt = skb;
 295                                         skb = skbn;
 296                                         skbn = skbt;
 297                                 }
 298                         } else {
 299                                 skbn = dev_alloc_skb(ENET_RX_FRSIZE);
 300
 301                                 if (skbn)
 302                                         skb_align(skbn, ENET_RX_ALIGN);
 303                         }
 304
 305                         if (skbn != NULL) {
 306                                 skb_put(skb, pkt_len);  /* Make room */
 307                                 skb->protocol = eth_type_trans(skb, dev);
 308                                 received++;
 309                                 netif_rx(skb);
 310                         } else {
 311                                 dev_warn(fep->dev,
 312                                          "Memory squeeze, dropping packet.\n");
 313                                 fep->stats.rx_dropped++;
 314                                 skbn = skb;
 315                         }
 316                 }
 317
 318                 fep->rx_skbuff[curidx] = skbn;
 319                 CBDW_BUFADDR(bdp, dma_map_single(fep->dev, skbn->data,
 320                              L1_CACHE_ALIGN(PKT_MAXBUF_SIZE),
 321                              DMA_FROM_DEVICE));
 322                 CBDW_DATLEN(bdp, 0);
 323                 CBDW_SC(bdp, (sc & ~BD_ENET_RX_STATS) | BD_ENET_RX_EMPTY);
 324
 325                 /*
 326                  * Update BD pointer to next entry.
 327                  */
 328                 if ((sc & BD_ENET_RX_WRAP) == 0)
 329                         bdp++;
 330                 else
 331                         bdp = fep->rx_bd_base;
 332
 333                 (*fep->ops->rx_bd_done)(dev);
 334         }
 335
 336         fep->cur_rx = bdp;
 337
 338         return 0;
 339 }
 340
 341 static void fs_enet_tx(struct net_device *dev)
 342 {
 343         struct fs_enet_private *fep = netdev_priv(dev);
 344         cbd_t __iomem *bdp;
 345         struct sk_buff *skb;
 346         int dirtyidx, do_wake, do_restart;
 347         u16 sc;
 348
 349         spin_lock(&fep->tx_lock);
 350         bdp = fep->dirty_tx;
 351
 352         do_wake = do_restart = 0;
 353         while (((sc = CBDR_SC(bdp)) & BD_ENET_TX_READY) == 0) {
 354                 dirtyidx = bdp - fep->tx_bd_base;
 355
 356                 if (fep->tx_free == fep->tx_ring)
 357                         break;
 358
 359                 skb = fep->tx_skbuff[dirtyidx];
 360
 361                 /*
 362                  * Check for errors.
 363                  */
 364                 if (sc & (BD_ENET_TX_HB | BD_ENET_TX_LC |
 365                           BD_ENET_TX_RL | BD_ENET_TX_UN | BD_ENET_TX_CSL)) {
 366
 367                         if (sc & BD_ENET_TX_HB) /* No heartbeat */
 368                                 fep->stats.tx_heartbeat_errors++;
 369                         if (sc & BD_ENET_TX_LC) /* Late collision */
 370                                 fep->stats.tx_window_errors++;
 371                         if (sc & BD_ENET_TX_RL) /* Retrans limit */
 372                                 fep->stats.tx_aborted_errors++;
 373                         if (sc & BD_ENET_TX_UN) /* Underrun */
 374                                 fep->stats.tx_fifo_errors++;
 375                         if (sc & BD_ENET_TX_CSL)        /* Carrier lost */
 376                                 fep->stats.tx_carrier_errors++;
 377
 378                         if (sc & (BD_ENET_TX_LC | BD_ENET_TX_RL | BD_ENET_TX_UN)) {
 379                                 fep->stats.tx_errors++;
 380                                 do_restart = 1;
 381                         }
 382                 } else
 383                         fep->stats.tx_packets++;
 384
 385                 if (sc & BD_ENET_TX_READY) {
 386                         dev_warn(fep->dev,
 387                                  "HEY! Enet xmit interrupt and TX_READY.\n");
 388                 }
 389
 390                 /*
 391                  * Deferred means some collisions occurred during transmit,
 392                  * but we eventually sent the packet OK.
 393                  */
 394                 if (sc & BD_ENET_TX_DEF)
 395                         fep->stats.collisions++;
 396
 397                 /* unmap */
 398                 dma_unmap_single(fep->dev, CBDR_BUFADDR(bdp),
 399                                 skb->len, DMA_TO_DEVICE);
 400
 401                 /*
 402                  * Free the sk buffer associated with this last transmit.
 403                  */
 404                 dev_kfree_skb_irq(skb);
 405                 fep->tx_skbuff[dirtyidx] = NULL;
 406
 407                 /*
 408                  * Update pointer to next buffer descriptor to be transmitted.
 409                  */
 410                 if ((sc & BD_ENET_TX_WRAP) == 0)
 411                         bdp++;
 412                 else
 413                         bdp = fep->tx_bd_base;
 414
 415                 /*
 416                  * Since we have freed up a buffer, the ring is no longer
 417                  * full.
 418                  */
 419                 if (!fep->tx_free++)
 420                         do_wake = 1;
 421         }
 422
 423         fep->dirty_tx = bdp;
 424
 425         if (do_restart)
 426                 (*fep->ops->tx_restart)(dev);
 427
 428         spin_unlock(&fep->tx_lock);
 429
 430         if (do_wake)
 431                 netif_wake_queue(dev);
 432 }
 433
 434 /*
 435  * The interrupt handler.
 436  * This is called from the MPC core interrupt.
 437  */
 438 static irqreturn_t
 439 fs_enet_interrupt(int irq, void *dev_id)
 440 {
 441         struct net_device *dev = dev_id;
 442         struct fs_enet_private *fep;
 443         const struct fs_platform_info *fpi;
 444         u32 int_events;
 445         u32 int_clr_events;
 446         int nr, napi_ok;
 447         int handled;
 448
 449         fep = netdev_priv(dev);
 450         fpi = fep->fpi;
 451
 452         nr = 0;
 453         while ((int_events = (*fep->ops->get_int_events)(dev)) != 0) {
 454                 nr++;
 455
 456                 int_clr_events = int_events;
 457                 if (fpi->use_napi)
 458                         int_clr_events &= ~fep->ev_napi_rx;
 459
 460                 (*fep->ops->clear_int_events)(dev, int_clr_events);
 461
 462                 if (int_events & fep->ev_err)
 463                         (*fep->ops->ev_error)(dev, int_events);
 464
 465                 if (int_events & fep->ev_rx) {
 466                         if (!fpi->use_napi)
 467                                 fs_enet_rx_non_napi(dev);
 468                         else {
 469                                 napi_ok = napi_schedule_prep(&fep->napi);
 470
 471                                 (*fep->ops->napi_disable_rx)(dev);
 472                                 (*fep->ops->clear_int_events)(dev, fep->ev_napi_rx);
 473
 474                                 /* NOTE: it is possible for FCCs in NAPI mode    */
 475                                 /* to submit a spurious interrupt while in poll  */
 476                                 if (napi_ok)
 477                                         __napi_schedule(&fep->napi);
 478                         }
 479                 }
 480
 481                 if (int_events & fep->ev_tx)
 482                         fs_enet_tx(dev);
 483         }
 484
 485         handled = nr > 0;
 486         return IRQ_RETVAL(handled);
 487 }
 488
 489 void fs_init_bds(struct net_device *dev)
 490 {
 491         struct fs_enet_private *fep = netdev_priv(dev);
 492         cbd_t __iomem *bdp;
 493         struct sk_buff *skb;
 494         int i;
 495
 496         fs_cleanup_bds(dev);
 497
 498         fep->dirty_tx = fep->cur_tx = fep->tx_bd_base;
 499         fep->tx_free = fep->tx_ring;
 500         fep->cur_rx = fep->rx_bd_base;
 501
 502         /*
 503          * Initialize the receive buffer descriptors.
 504          */
 505         for (i = 0, bdp = fep->rx_bd_base; i < fep->rx_ring; i++, bdp++) {
 506                 skb = dev_alloc_skb(ENET_RX_FRSIZE);
 507                 if (skb == NULL) {
 508                         dev_warn(fep->dev,
 509                                  "Memory squeeze, unable to allocate skb\n");
 510                         break;
 511                 }
 512                 skb_align(skb, ENET_RX_ALIGN);
 513                 fep->rx_skbuff[i] = skb;
 514                 CBDW_BUFADDR(bdp,
 515                         dma_map_single(fep->dev, skb->data,
 516                                 L1_CACHE_ALIGN(PKT_MAXBUF_SIZE),
 517                                 DMA_FROM_DEVICE));
 518                 CBDW_DATLEN(bdp, 0);    /* zero */
 519                 CBDW_SC(bdp, BD_ENET_RX_EMPTY |
 520                         ((i < fep->rx_ring - 1) ? 0 : BD_SC_WRAP));
 521         }
 522         /*
 523          * if we failed, fillup remainder
 524          */
 525         for (; i < fep->rx_ring; i++, bdp++) {
 526                 fep->rx_skbuff[i] = NULL;
 527                 CBDW_SC(bdp, (i < fep->rx_ring - 1) ? 0 : BD_SC_WRAP);
 528         }
 529
 530         /*
 531          * ...and the same for transmit.
 532          */
 533         for (i = 0, bdp = fep->tx_bd_base; i < fep->tx_ring; i++, bdp++) {
 534                 fep->tx_skbuff[i] = NULL;
 535                 CBDW_BUFADDR(bdp, 0);
 536                 CBDW_DATLEN(bdp, 0);
 537                 CBDW_SC(bdp, (i < fep->tx_ring - 1) ? 0 : BD_SC_WRAP);
 538         }
 539 }
 540
 541 void fs_cleanup_bds(struct net_device *dev)
 542 {
 543         struct fs_enet_private *fep = netdev_priv(dev);
 544         struct sk_buff *skb;
 545         cbd_t __iomem *bdp;
 546         int i;
 547
 548         /*
 549          * Reset SKB transmit buffers.
 550          */
 551         for (i = 0, bdp = fep->tx_bd_base; i < fep->tx_ring; i++, bdp++) {
 552                 if ((skb = fep->tx_skbuff[i]) == NULL)
 553                         continue;
 554
 555                 /* unmap */
 556                 dma_unmap_single(fep->dev, CBDR_BUFADDR(bdp),
 557                                 skb->len, DMA_TO_DEVICE);
 558
 559                 fep->tx_skbuff[i] = NULL;
 560                 dev_kfree_skb(skb);
 561         }
 562
 563         /*
 564          * Reset SKB receive buffers
 565          */
 566         for (i = 0, bdp = fep->rx_bd_base; i < fep->rx_ring; i++, bdp++) {
 567                 if ((skb = fep->rx_skbuff[i]) == NULL)
 568                         continue;
 569
 570                 /* unmap */
 571                 dma_unmap_single(fep->dev, CBDR_BUFADDR(bdp),
 572                         L1_CACHE_ALIGN(PKT_MAXBUF_SIZE),
 573                         DMA_FROM_DEVICE);
 574
 575                 fep->rx_skbuff[i] = NULL;
 576
 577                 dev_kfree_skb(skb);
 578         }
 579 }
 580
 581 /**********************************************************************************/
 582
 583 #ifdef CONFIG_FS_ENET_MPC5121_FEC
 584 /*
 585  * MPC5121 FEC requeries 4-byte alignment for TX data buffer!
 586  */
 587 static struct sk_buff *tx_skb_align_workaround(struct net_device *dev,
 588                                                struct sk_buff *skb)
 589 {
 590         struct sk_buff *new_skb;
 591         struct fs_enet_private *fep = netdev_priv(dev);
 592
 593         /* Alloc new skb */
 594         new_skb = dev_alloc_skb(skb->len + 4);
 595         if (!new_skb) {
 596                 if (net_ratelimit()) {
 597                         dev_warn(fep->dev,
 598                                  "Memory squeeze, dropping tx packet.\n");
 599                 }
 600                 return NULL;
 601         }
 602
 603         /* Make sure new skb is properly aligned */
 604         skb_align(new_skb, 4);
 605
 606         /* Copy data to new skb ... */
 607         skb_copy_from_linear_data(skb, new_skb->data, skb->len);
 608         skb_put(new_skb, skb->len);
 609
 610         /* ... and free an old one */
 611         dev_kfree_skb_any(skb);
 612
 613         return new_skb;
 614 }
 615 #endif
 616
 617 static int fs_enet_start_xmit(struct sk_buff *skb, struct net_device *dev)
 618 {
 619         struct fs_enet_private *fep = netdev_priv(dev);
 620         cbd_t __iomem *bdp;
 621         int curidx;
 622         u16 sc;
 623         unsigned long flags;
 624
 625 #ifdef CONFIG_FS_ENET_MPC5121_FEC
 626         if (((unsigned long)skb->data) & 0x3) {
 627                 skb = tx_skb_align_workaround(dev, skb);
 628                 if (!skb) {
 629                         /*
 630                          * We have lost packet due to memory allocation error
 631                          * in tx_skb_align_workaround(). Hopefully original
 632                          * skb is still valid, so try transmit it later.
 633                          */
 634                         return NETDEV_TX_BUSY;
 635                 }
 636         }
 637 #endif
 638         spin_lock_irqsave(&fep->tx_lock, flags);
 639
 640         /*
 641          * Fill in a Tx ring entry
 642          */
 643         bdp = fep->cur_tx;
 644
 645         if (!fep->tx_free || (CBDR_SC(bdp) & BD_ENET_TX_READY)) {
 646                 netif_stop_queue(dev);
 647                 spin_unlock_irqrestore(&fep->tx_lock, flags);
 648
 649                 /*
 650                  * Ooops.  All transmit buffers are full.  Bail out.
 651                  * This should not happen, since the tx queue should be stopped.
 652                  */
 653                 dev_warn(fep->dev, "tx queue full!.\n");
 654                 return NETDEV_TX_BUSY;
 655         }
 656
 657         curidx = bdp - fep->tx_bd_base;
 658         /*
 659          * Clear all of the status flags.
 660          */
 661         CBDC_SC(bdp, BD_ENET_TX_STATS);
 662
 663         /*
 664          * Save skb pointer.
 665          */
 666         fep->tx_skbuff[curidx] = skb;
 667
 668         fep->stats.tx_bytes += skb->len;
 669
 670         /*
 671          * Push the data cache so the CPM does not get stale memory data.
 672          */
 673         CBDW_BUFADDR(bdp, dma_map_single(fep->dev,
 674                                 skb->data, skb->len, DMA_TO_DEVICE));
 675         CBDW_DATLEN(bdp, skb->len);
 676
 677         dev->trans_start = jiffies;
 678
 679         /*
 680          * If this was the last BD in the ring, start at the beginning again.
 681          */
 682         if ((CBDR_SC(bdp) & BD_ENET_TX_WRAP) == 0)
 683                 fep->cur_tx++;
 684         else
 685                 fep->cur_tx = fep->tx_bd_base;
 686
 687         if (!--fep->tx_free)
 688                 netif_stop_queue(dev);
 689
 690         /* Trigger transmission start */
 691         sc = BD_ENET_TX_READY | BD_ENET_TX_INTR |
 692              BD_ENET_TX_LAST | BD_ENET_TX_TC;
 693
 694         /* note that while FEC does not have this bit
 695          * it marks it as available for software use
 696          * yay for hw reuse :) */
 697         if (skb->len <= 60)
 698                 sc |= BD_ENET_TX_PAD;
 699         CBDS_SC(bdp, sc);
 700
 701         (*fep->ops->tx_kickstart)(dev);
 702
 703         spin_unlock_irqrestore(&fep->tx_lock, flags);
 704
 705         return NETDEV_TX_OK;
 706 }
 707
 708 static void fs_timeout(struct net_device *dev)
 709 {
 710         struct fs_enet_private *fep = netdev_priv(dev);
 711         unsigned long flags;
 712         int wake = 0;
 713
 714         fep->stats.tx_errors++;
 715
 716         spin_lock_irqsave(&fep->lock, flags);
 717
 718         if (dev->flags & IFF_UP) {
 719                 phy_stop(fep->phydev);
 720                 (*fep->ops->stop)(dev);
 721                 (*fep->ops->restart)(dev);
 722                 phy_start(fep->phydev);
 723         }
 724
 725         phy_start(fep->phydev);
 726         wake = fep->tx_free && !(CBDR_SC(fep->cur_tx) & BD_ENET_TX_READY);
 727         spin_unlock_irqrestore(&fep->lock, flags);
 728
 729         if (wake)
 730                 netif_wake_queue(dev);
 731 }
 732
 733 /*-----------------------------------------------------------------------------
 734  *  generic link-change handler - should be sufficient for most cases
 735  *-----------------------------------------------------------------------------*/
 736 static void generic_adjust_link(struct  net_device *dev)
 737 {
 738         struct fs_enet_private *fep = netdev_priv(dev);
 739         struct phy_device *phydev = fep->phydev;
 740         int new_state = 0;
 741
 742         if (phydev->link) {
 743                 /* adjust to duplex mode */
 744                 if (phydev->duplex != fep->oldduplex) {
 745                         new_state = 1;
 746                         fep->oldduplex = phydev->duplex;
 747                 }
 748
 749                 if (phydev->speed != fep->oldspeed) {
 750                         new_state = 1;
 751                         fep->oldspeed = phydev->speed;
 752                 }
 753
 754                 if (!fep->oldlink) {
 755                         new_state = 1;
 756                         fep->oldlink = 1;
 757                 }
 758
 759                 if (new_state)
 760                         fep->ops->restart(dev);
 761         } else if (fep->oldlink) {
 762                 new_state = 1;
 763                 fep->oldlink = 0;
 764                 fep->oldspeed = 0;
 765                 fep->oldduplex = -1;
 766         }
 767
 768         if (new_state && netif_msg_link(fep))
 769                 phy_print_status(phydev);
 770 }
 771
 772
 773 static void fs_adjust_link(struct net_device *dev)
 774 {
 775         struct fs_enet_private *fep = netdev_priv(dev);
 776         unsigned long flags;
 777
 778         spin_lock_irqsave(&fep->lock, flags);
 779
 780         if(fep->ops->adjust_link)
 781                 fep->ops->adjust_link(dev);
 782         else
 783                 generic_adjust_link(dev);
 784
 785         spin_unlock_irqrestore(&fep->lock, flags);
 786 }
 787
 788 static int fs_init_phy(struct net_device *dev)
 789 {
 790         struct fs_enet_private *fep = netdev_priv(dev);
 791         struct phy_device *phydev;
 792
 793         fep->oldlink = 0;
 794         fep->oldspeed = 0;
 795         fep->oldduplex = -1;
 796
 797         phydev = of_phy_connect(dev, fep->fpi->phy_node, &fs_adjust_link, 0,
 798                                 PHY_INTERFACE_MODE_MII);
 799         if (!phydev) {
 800                 phydev = of_phy_connect_fixed_link(dev, &fs_adjust_link,
 801                                                    PHY_INTERFACE_MODE_MII);
 802         }
 803         if (!phydev) {
 804                 dev_err(&dev->dev, "Could not attach to PHY\n");
 805                 return -ENODEV;
 806         }
 807
 808         fep->phydev = phydev;
 809
 810         return 0;
 811 }
 812
 813 static int fs_enet_open(struct net_device *dev)
 814 {
 815         struct fs_enet_private *fep = netdev_priv(dev);
 816         int r;
 817         int err;
 818
 819         /* to initialize the fep->cur_rx,... */
 820         /* not doing this, will cause a crash in fs_enet_rx_napi */
 821         fs_init_bds(fep->ndev);
 822
 823         if (fep->fpi->use_napi)
 824                 napi_enable(&fep->napi);
 825
 826         /* Install our interrupt handler. */
 827         r = request_irq(fep->interrupt, fs_enet_interrupt, IRQF_SHARED,
 828                         "fs_enet-mac", dev);
 829         if (r != 0) {
 830                 dev_err(fep->dev, "Could not allocate FS_ENET IRQ!");
 831                 if (fep->fpi->use_napi)
 832                         napi_disable(&fep->napi);
 833                 return -EINVAL;
 834         }
 835
 836         err = fs_init_phy(dev);
 837         if (err) {
 838                 free_irq(fep->interrupt, dev);
 839                 if (fep->fpi->use_napi)
 840                         napi_disable(&fep->napi);
 841                 return err;
 842         }
 843         phy_start(fep->phydev);
 844
 845         netif_start_queue(dev);
 846
 847         return 0;
 848 }
 849
 850 static int fs_enet_close(struct net_device *dev)
 851 {
 852         struct fs_enet_private *fep = netdev_priv(dev);
 853         unsigned long flags;
 854
 855         netif_stop_queue(dev);
 856         netif_carrier_off(dev);
 857         if (fep->fpi->use_napi)
 858                 napi_disable(&fep->napi);
 859         phy_stop(fep->phydev);
 860
 861         spin_lock_irqsave(&fep->lock, flags);
 862         spin_lock(&fep->tx_lock);
 863         (*fep->ops->stop)(dev);
 864         spin_unlock(&fep->tx_lock);
 865         spin_unlock_irqrestore(&fep->lock, flags);
 866
 867         /* release any irqs */
 868         phy_disconnect(fep->phydev);
 869         fep->phydev = NULL;
 870         free_irq(fep->interrupt, dev);
 871
 872         return 0;
 873 }
 874
 875 static struct net_device_stats *fs_enet_get_stats(struct net_device *dev)
 876 {
 877         struct fs_enet_private *fep = netdev_priv(dev);
 878         return &fep->stats;
 879 }
 880
 881 /*************************************************************************/
 882
 883 static void fs_get_drvinfo(struct net_device *dev,
 884                             struct ethtool_drvinfo *info)
 885 {
 886         strcpy(info->driver, DRV_MODULE_NAME);
 887         strcpy(info->version, DRV_MODULE_VERSION);
 888 }
 889
 890 static int fs_get_regs_len(struct net_device *dev)
 891 {
 892         struct fs_enet_private *fep = netdev_priv(dev);
 893
 894         return (*fep->ops->get_regs_len)(dev);
 895 }
 896
 897 static void fs_get_regs(struct net_device *dev, struct ethtool_regs *regs,
 898                          void *p)
 899 {
 900         struct fs_enet_private *fep = netdev_priv(dev);
 901         unsigned long flags;
 902         int r, len;
 903
 904         len = regs->len;
 905
 906         spin_lock_irqsave(&fep->lock, flags);
 907         r = (*fep->ops->get_regs)(dev, p, &len);
 908         spin_unlock_irqrestore(&fep->lock, flags);
 909
 910         if (r == 0)
 911                 regs->version = 0;
 912 }
 913
 914 static int fs_get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
 915 {
 916         struct fs_enet_private *fep = netdev_priv(dev);
 917
 918         if (!fep->phydev)
 919                 return -ENODEV;
 920
 921         return phy_ethtool_gset(fep->phydev, cmd);
 922 }
 923
 924 static int fs_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
 925 {
 926         struct fs_enet_private *fep = netdev_priv(dev);
 927
 928         if (!fep->phydev)
 929                 return -ENODEV;
 930
 931         return phy_ethtool_sset(fep->phydev, cmd);
 932 }
 933
 934 static int fs_nway_reset(struct net_device *dev)
 935 {
 936         return 0;
 937 }
 938
 939 static u32 fs_get_msglevel(struct net_device *dev)
 940 {
 941         struct fs_enet_private *fep = netdev_priv(dev);
 942         return fep->msg_enable;
 943 }
 944
 945 static void fs_set_msglevel(struct net_device *dev, u32 value)
 946 {
 947         struct fs_enet_private *fep = netdev_priv(dev);
 948         fep->msg_enable = value;
 949 }
 950
 951 static const struct ethtool_ops fs_ethtool_ops = {
 952         .get_drvinfo = fs_get_drvinfo,
 953         .get_regs_len = fs_get_regs_len,
 954         .get_settings = fs_get_settings,
 955         .set_settings = fs_set_settings,
 956         .nway_reset = fs_nway_reset,
 957         .get_link = ethtool_op_get_link,
 958         .get_msglevel = fs_get_msglevel,
 959         .set_msglevel = fs_set_msglevel,
 960         .set_tx_csum = ethtool_op_set_tx_csum,  /* local! */
 961         .set_sg = ethtool_op_set_sg,
 962         .get_regs = fs_get_regs,
 963 };
 964
 965 static int fs_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
 966 {
 967         struct fs_enet_private *fep = netdev_priv(dev);
 968         struct mii_ioctl_data *mii = (struct mii_ioctl_data *)&rq->ifr_data;
 969
 970         if (!netif_running(dev))
 971                 return -EINVAL;
 972
 973         return phy_mii_ioctl(fep->phydev, mii, cmd);
 974 }
 975
 976 extern int fs_mii_connect(struct net_device *dev);
 977 extern void fs_mii_disconnect(struct net_device *dev);
 978
 979 /**************************************************************************************/
 980
 981 #ifdef CONFIG_FS_ENET_HAS_FEC
 982 #define IS_FEC(match) ((match)->data == &fs_fec_ops)
 983 #else
 984 #define IS_FEC(match) 0
 985 #endif
 986
 987 static const struct net_device_ops fs_enet_netdev_ops = {
 988         .ndo_open               = fs_enet_open,
 989         .ndo_stop               = fs_enet_close,
 990         .ndo_get_stats          = fs_enet_get_stats,
 991         .ndo_start_xmit         = fs_enet_start_xmit,
 992         .ndo_tx_timeout         = fs_timeout,
 993         .ndo_set_multicast_list = fs_set_multicast_list,
 994         .ndo_do_ioctl           = fs_ioctl,
 995         .ndo_validate_addr      = eth_validate_addr,
 996         .ndo_set_mac_address    = eth_mac_addr,
 997         .ndo_change_mtu         = eth_change_mtu,
 998 #ifdef CONFIG_NET_POLL_CONTROLLER
 999         .ndo_poll_controller    = fs_enet_netpoll,
1000 #endif
1001 };
1002
1003 static int __devinit fs_enet_probe(struct of_device *ofdev,
1004                                    const struct of_device_id *match)
1005 {
1006         struct net_device *ndev;
1007         struct fs_enet_private *fep;
1008         struct fs_platform_info *fpi;
1009         const u32 *data;
1010         const u8 *mac_addr;
1011         int privsize, len, ret = -ENODEV;
1012
1013         fpi = kzalloc(sizeof(*fpi), GFP_KERNEL);
1014         if (!fpi)
1015                 return -ENOMEM;
1016
1017         if (!IS_FEC(match)) {
1018                 data = of_get_property(ofdev->node, "fsl,cpm-command", &len);
1019                 if (!data || len != 4)
1020                         goto out_free_fpi;
1021
1022                 fpi->cp_command = *data;
1023         }
1024
1025         fpi->rx_ring = 32;
1026         fpi->tx_ring = 32;
1027         fpi->rx_copybreak = 240;
1028         fpi->use_napi = 1;
1029         fpi->napi_weight = 17;
1030         fpi->phy_node = of_parse_phandle(ofdev->node, "phy-handle", 0);
1031         if ((!fpi->phy_node) && (!of_get_property(ofdev->node, "fixed-link",
1032                                                   NULL)))
1033                 goto out_free_fpi;
1034
1035         privsize = sizeof(*fep) +
1036                    sizeof(struct sk_buff **) *
1037                    (fpi->rx_ring + fpi->tx_ring);
1038
1039         ndev = alloc_etherdev(privsize);
1040         if (!ndev) {
1041                 ret = -ENOMEM;
1042                 goto out_free_fpi;
1043         }
1044
1045         SET_NETDEV_DEV(ndev, &ofdev->dev);
1046         dev_set_drvdata(&ofdev->dev, ndev);
1047
1048         fep = netdev_priv(ndev);
1049         fep->dev = &ofdev->dev;
1050         fep->ndev = ndev;
1051         fep->fpi = fpi;
1052         fep->ops = match->data;
1053
1054         ret = fep->ops->setup_data(ndev);
1055         if (ret)
1056                 goto out_free_dev;
1057
1058         fep->rx_skbuff = (struct sk_buff **)&fep[1];
1059         fep->tx_skbuff = fep->rx_skbuff + fpi->rx_ring;
1060
1061         spin_lock_init(&fep->lock);
1062         spin_lock_init(&fep->tx_lock);
1063
1064         mac_addr = of_get_mac_address(ofdev->node);
1065         if (mac_addr)
1066                 memcpy(ndev->dev_addr, mac_addr, 6);
1067
1068         ret = fep->ops->allocate_bd(ndev);
1069         if (ret)
1070                 goto out_cleanup_data;
1071
1072         fep->rx_bd_base = fep->ring_base;
1073         fep->tx_bd_base = fep->rx_bd_base + fpi->rx_ring;
1074
1075         fep->tx_ring = fpi->tx_ring;
1076         fep->rx_ring = fpi->rx_ring;
1077
1078         ndev->netdev_ops = &fs_enet_netdev_ops;
1079         ndev->watchdog_timeo = 2 * HZ;
1080         if (fpi->use_napi)
1081                 netif_napi_add(ndev, &fep->napi, fs_enet_rx_napi,
1082                                fpi->napi_weight);
1083
1084         ndev->ethtool_ops = &fs_ethtool_ops;
1085
1086         init_timer(&fep->phy_timer_list);
1087
1088         netif_carrier_off(ndev);
1089
1090         ret = register_netdev(ndev);
1091         if (ret)
1092                 goto out_free_bd;
1093
1094         pr_info("%s: fs_enet: %pM\n", ndev->name, ndev->dev_addr);
1095
1096         return 0;
1097
1098 out_free_bd:
1099         fep->ops->free_bd(ndev);
1100 out_cleanup_data:
1101         fep->ops->cleanup_data(ndev);
1102 out_free_dev:
1103         free_netdev(ndev);
1104         dev_set_drvdata(&ofdev->dev, NULL);
1105         of_node_put(fpi->phy_node);
1106 out_free_fpi:
1107         kfree(fpi);
1108         return ret;
1109 }
1110
1111 static int fs_enet_remove(struct of_device *ofdev)
1112 {
1113         struct net_device *ndev = dev_get_drvdata(&ofdev->dev);
1114         struct fs_enet_private *fep = netdev_priv(ndev);
1115
1116         unregister_netdev(ndev);
1117
1118         fep->ops->free_bd(ndev);
1119         fep->ops->cleanup_data(ndev);
1120         dev_set_drvdata(fep->dev, NULL);
1121         of_node_put(fep->fpi->phy_node);
1122         free_netdev(ndev);
1123         return 0;
1124 }
1125
1126 static struct of_device_id fs_enet_match[] = {
1127 #ifdef CONFIG_FS_ENET_HAS_SCC
1128         {
1129                 .compatible = "fsl,cpm1-scc-enet",
1130                 .data = (void *)&fs_scc_ops,
1131         },
1132         {
1133                 .compatible = "fsl,cpm2-scc-enet",
1134                 .data = (void *)&fs_scc_ops,
1135         },
1136 #endif
1137 #ifdef CONFIG_FS_ENET_HAS_FCC
1138         {
1139                 .compatible = "fsl,cpm2-fcc-enet",
1140                 .data = (void *)&fs_fcc_ops,
1141         },
1142 #endif
1143 #ifdef CONFIG_FS_ENET_HAS_FEC
1144 #ifdef CONFIG_FS_ENET_MPC5121_FEC
1145         {
1146                 .compatible = "fsl,mpc5121-fec",
1147                 .data = (void *)&fs_fec_ops,
1148         },
1149 #else
1150         {
1151                 .compatible = "fsl,pq1-fec-enet",
1152                 .data = (void *)&fs_fec_ops,
1153         },
1154 #endif
1155 #endif
1156         {}
1157 };
1158 MODULE_DEVICE_TABLE(of, fs_enet_match);
1159
1160 static struct of_platform_driver fs_enet_driver = {
1161         .name   = "fs_enet",
1162         .match_table = fs_enet_match,
1163         .probe = fs_enet_probe,
1164         .remove = fs_enet_remove,
1165 };
1166
1167 static int __init fs_init(void)
1168 {
1169         return of_register_platform_driver(&fs_enet_driver);
1170 }
1171
1172 static void __exit fs_cleanup(void)
1173 {
1174         of_unregister_platform_driver(&fs_enet_driver);
1175 }
1176
1177 #ifdef CONFIG_NET_POLL_CONTROLLER
1178 static void fs_enet_netpoll(struct net_device *dev)
1179 {
1180        disable_irq(dev->irq);
1181        fs_enet_interrupt(dev->irq, dev);
1182        enable_irq(dev->irq);
1183 }
1184 #endif
1185
1186 /**************************************************************************************/
1187
1188 module_init(fs_init);
1189 module_exit(fs_cleanup);