drivers/soc/fsl/qbman/bman.c

   1 /* Copyright 2008 - 2016 Freescale Semiconductor, Inc.
   2  *
   3  * Redistribution and use in source and binary forms, with or without
   4  * modification, are permitted provided that the following conditions are met:
   5  *     * Redistributions of source code must retain the above copyright
   6  *       notice, this list of conditions and the following disclaimer.
   7  *     * Redistributions in binary form must reproduce the above copyright
   8  *       notice, this list of conditions and the following disclaimer in the
   9  *       documentation and/or other materials provided with the distribution.
  10  *     * Neither the name of Freescale Semiconductor nor the
  11  *       names of its contributors may be used to endorse or promote products
  12  *       derived from this software without specific prior written permission.
  13  *
  14  * ALTERNATIVELY, this software may be distributed under the terms of the
  15  * GNU General Public License ("GPL") as published by the Free Software
  16  * Foundation, either version 2 of that License or (at your option) any
  17  * later version.
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
  20  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  21  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  22  * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
  23  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  24  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  25  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  26  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  28  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29  */
  30
  31 #include "bman_priv.h"
  32
  33 #define IRQNAME         "BMan portal %d"
  34 #define MAX_IRQNAME     16      /* big enough for "BMan portal %d" */
  35
  36 /* Portal register assists */
  37
  38 #if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
  39 /* Cache-inhibited register offsets */
  40 #define BM_REG_RCR_PI_CINH      0x3000
  41 #define BM_REG_RCR_CI_CINH      0x3100
  42 #define BM_REG_RCR_ITR          0x3200
  43 #define BM_REG_CFG              0x3300
  44 #define BM_REG_SCN(n)           (0x3400 + ((n) << 6))
  45 #define BM_REG_ISR              0x3e00
  46 #define BM_REG_IER              0x3e40
  47 #define BM_REG_ISDR             0x3e80
  48 #define BM_REG_IIR              0x3ec0
  49
  50 /* Cache-enabled register offsets */
  51 #define BM_CL_CR                0x0000
  52 #define BM_CL_RR0               0x0100
  53 #define BM_CL_RR1               0x0140
  54 #define BM_CL_RCR               0x1000
  55 #define BM_CL_RCR_PI_CENA       0x3000
  56 #define BM_CL_RCR_CI_CENA       0x3100
  57
  58 #else
  59 /* Cache-inhibited register offsets */
  60 #define BM_REG_RCR_PI_CINH      0x0000
  61 #define BM_REG_RCR_CI_CINH      0x0004
  62 #define BM_REG_RCR_ITR          0x0008
  63 #define BM_REG_CFG              0x0100
  64 #define BM_REG_SCN(n)           (0x0200 + ((n) << 2))
  65 #define BM_REG_ISR              0x0e00
  66 #define BM_REG_IER              0x0e04
  67 #define BM_REG_ISDR             0x0e08
  68 #define BM_REG_IIR              0x0e0c
  69
  70 /* Cache-enabled register offsets */
  71 #define BM_CL_CR                0x0000
  72 #define BM_CL_RR0               0x0100
  73 #define BM_CL_RR1               0x0140
  74 #define BM_CL_RCR               0x1000
  75 #define BM_CL_RCR_PI_CENA       0x3000
  76 #define BM_CL_RCR_CI_CENA       0x3100
  77 #endif
  78
  79 /*
  80  * Portal modes.
  81  *   Enum types;
  82  *     pmode == production mode
  83  *     cmode == consumption mode,
  84  *   Enum values use 3 letter codes. First letter matches the portal mode,
  85  *   remaining two letters indicate;
  86  *     ci == cache-inhibited portal register
  87  *     ce == cache-enabled portal register
  88  *     vb == in-band valid-bit (cache-enabled)
  89  */
  90 enum bm_rcr_pmode {             /* matches BCSP_CFG::RPM */
  91         bm_rcr_pci = 0,         /* PI index, cache-inhibited */
  92         bm_rcr_pce = 1,         /* PI index, cache-enabled */
  93         bm_rcr_pvb = 2          /* valid-bit */
  94 };
  95 enum bm_rcr_cmode {             /* s/w-only */
  96         bm_rcr_cci,             /* CI index, cache-inhibited */
  97         bm_rcr_cce              /* CI index, cache-enabled */
  98 };
  99
 100
 101 /* --- Portal structures --- */
 102
 103 #define BM_RCR_SIZE             8
 104
 105 /* Release Command */
 106 struct bm_rcr_entry {
 107         union {
 108                 struct {
 109                         u8 _ncw_verb; /* writes to this are non-coherent */
 110                         u8 bpid; /* used with BM_RCR_VERB_CMD_BPID_SINGLE */
 111                         u8 __reserved1[62];
 112                 };
 113                 struct bm_buffer bufs[8];
 114         };
 115 };
 116 #define BM_RCR_VERB_VBIT                0x80
 117 #define BM_RCR_VERB_CMD_MASK            0x70    /* one of two values; */
 118 #define BM_RCR_VERB_CMD_BPID_SINGLE     0x20
 119 #define BM_RCR_VERB_CMD_BPID_MULTI      0x30
 120 #define BM_RCR_VERB_BUFCOUNT_MASK       0x0f    /* values 1..8 */
 121
 122 struct bm_rcr {
 123         struct bm_rcr_entry *ring, *cursor;
 124         u8 ci, available, ithresh, vbit;
 125 #ifdef CONFIG_FSL_DPAA_CHECKING
 126         u32 busy;
 127         enum bm_rcr_pmode pmode;
 128         enum bm_rcr_cmode cmode;
 129 #endif
 130 };
 131
 132 /* MC (Management Command) command */
 133 struct bm_mc_command {
 134         u8 _ncw_verb; /* writes to this are non-coherent */
 135         u8 bpid; /* used by acquire command */
 136         u8 __reserved[62];
 137 };
 138 #define BM_MCC_VERB_VBIT                0x80
 139 #define BM_MCC_VERB_CMD_MASK            0x70    /* where the verb contains; */
 140 #define BM_MCC_VERB_CMD_ACQUIRE         0x10
 141 #define BM_MCC_VERB_CMD_QUERY           0x40
 142 #define BM_MCC_VERB_ACQUIRE_BUFCOUNT    0x0f    /* values 1..8 go here */
 143
 144 /* MC result, Acquire and Query Response */
 145 union bm_mc_result {
 146         struct {
 147                 u8 verb;
 148                 u8 bpid;
 149                 u8 __reserved[62];
 150         };
 151         struct bm_buffer bufs[8];
 152 };
 153 #define BM_MCR_VERB_VBIT                0x80
 154 #define BM_MCR_VERB_CMD_MASK            BM_MCC_VERB_CMD_MASK
 155 #define BM_MCR_VERB_CMD_ACQUIRE         BM_MCC_VERB_CMD_ACQUIRE
 156 #define BM_MCR_VERB_CMD_QUERY           BM_MCC_VERB_CMD_QUERY
 157 #define BM_MCR_VERB_CMD_ERR_INVALID     0x60
 158 #define BM_MCR_VERB_CMD_ERR_ECC         0x70
 159 #define BM_MCR_VERB_ACQUIRE_BUFCOUNT    BM_MCC_VERB_ACQUIRE_BUFCOUNT /* 0..8 */
 160 #define BM_MCR_TIMEOUT                  10000 /* us */
 161
 162 struct bm_mc {
 163         struct bm_mc_command *cr;
 164         union bm_mc_result *rr;
 165         u8 rridx, vbit;
 166 #ifdef CONFIG_FSL_DPAA_CHECKING
 167         enum {
 168                 /* Can only be _mc_start()ed */
 169                 mc_idle,
 170                 /* Can only be _mc_commit()ed or _mc_abort()ed */
 171                 mc_user,
 172                 /* Can only be _mc_retry()ed */
 173                 mc_hw
 174         } state;
 175 #endif
 176 };
 177
 178 struct bm_addr {
 179         void *ce;               /* cache-enabled */
 180         __be32 *ce_be;          /* Same as above but for direct access */
 181         void __iomem *ci;       /* cache-inhibited */
 182 };
 183
 184 struct bm_portal {
 185         struct bm_addr addr;
 186         struct bm_rcr rcr;
 187         struct bm_mc mc;
 188 } ____cacheline_aligned;
 189
 190 /* Cache-inhibited register access. */
 191 static inline u32 bm_in(struct bm_portal *p, u32 offset)
 192 {
 193         return ioread32be(p->addr.ci + offset);
 194 }
 195
 196 static inline void bm_out(struct bm_portal *p, u32 offset, u32 val)
 197 {
 198         iowrite32be(val, p->addr.ci + offset);
 199 }
 200
 201 /* Cache Enabled Portal Access */
 202 static inline void bm_cl_invalidate(struct bm_portal *p, u32 offset)
 203 {
 204         dpaa_invalidate(p->addr.ce + offset);
 205 }
 206
 207 static inline void bm_cl_touch_ro(struct bm_portal *p, u32 offset)
 208 {
 209         dpaa_touch_ro(p->addr.ce + offset);
 210 }
 211
 212 static inline u32 bm_ce_in(struct bm_portal *p, u32 offset)
 213 {
 214         return be32_to_cpu(*(p->addr.ce_be + (offset/4)));
 215 }
 216
 217 struct bman_portal {
 218         struct bm_portal p;
 219         /* interrupt sources processed by portal_isr(), configurable */
 220         unsigned long irq_sources;
 221         /* probing time config params for cpu-affine portals */
 222         const struct bm_portal_config *config;
 223         char irqname[MAX_IRQNAME];
 224 };
 225
 226 static cpumask_t affine_mask;
 227 static DEFINE_SPINLOCK(affine_mask_lock);
 228 static DEFINE_PER_CPU(struct bman_portal, bman_affine_portal);
 229
 230 static inline struct bman_portal *get_affine_portal(void)
 231 {
 232         return &get_cpu_var(bman_affine_portal);
 233 }
 234
 235 static inline void put_affine_portal(void)
 236 {
 237         put_cpu_var(bman_affine_portal);
 238 }
 239
 240 /*
 241  * This object type refers to a pool, it isn't *the* pool. There may be
 242  * more than one such object per BMan buffer pool, eg. if different users of the
 243  * pool are operating via different portals.
 244  */
 245 struct bman_pool {
 246         /* index of the buffer pool to encapsulate (0-63) */
 247         u32 bpid;
 248         /* Used for hash-table admin when using depletion notifications. */
 249         struct bman_portal *portal;
 250         struct bman_pool *next;
 251 };
 252
 253 static u32 poll_portal_slow(struct bman_portal *p, u32 is);
 254
 255 static irqreturn_t portal_isr(int irq, void *ptr)
 256 {
 257         struct bman_portal *p = ptr;
 258         struct bm_portal *portal = &p->p;
 259         u32 clear = p->irq_sources;
 260         u32 is = bm_in(portal, BM_REG_ISR) & p->irq_sources;
 261
 262         if (unlikely(!is))
 263                 return IRQ_NONE;
 264
 265         clear |= poll_portal_slow(p, is);
 266         bm_out(portal, BM_REG_ISR, clear);
 267         return IRQ_HANDLED;
 268 }
 269
 270 /* --- RCR API --- */
 271
 272 #define RCR_SHIFT       ilog2(sizeof(struct bm_rcr_entry))
 273 #define RCR_CARRY       (uintptr_t)(BM_RCR_SIZE << RCR_SHIFT)
 274
 275 /* Bit-wise logic to wrap a ring pointer by clearing the "carry bit" */
 276 static struct bm_rcr_entry *rcr_carryclear(struct bm_rcr_entry *p)
 277 {
 278         uintptr_t addr = (uintptr_t)p;
 279
 280         addr &= ~RCR_CARRY;
 281
 282         return (struct bm_rcr_entry *)addr;
 283 }
 284
 285 #ifdef CONFIG_FSL_DPAA_CHECKING
 286 /* Bit-wise logic to convert a ring pointer to a ring index */
 287 static int rcr_ptr2idx(struct bm_rcr_entry *e)
 288 {
 289         return ((uintptr_t)e >> RCR_SHIFT) & (BM_RCR_SIZE - 1);
 290 }
 291 #endif
 292
 293 /* Increment the 'cursor' ring pointer, taking 'vbit' into account */
 294 static inline void rcr_inc(struct bm_rcr *rcr)
 295 {
 296         /* increment to the next RCR pointer and handle overflow and 'vbit' */
 297         struct bm_rcr_entry *partial = rcr->cursor + 1;
 298
 299         rcr->cursor = rcr_carryclear(partial);
 300         if (partial != rcr->cursor)
 301                 rcr->vbit ^= BM_RCR_VERB_VBIT;
 302 }
 303
 304 static int bm_rcr_get_avail(struct bm_portal *portal)
 305 {
 306         struct bm_rcr *rcr = &portal->rcr;
 307
 308         return rcr->available;
 309 }
 310
 311 static int bm_rcr_get_fill(struct bm_portal *portal)
 312 {
 313         struct bm_rcr *rcr = &portal->rcr;
 314
 315         return BM_RCR_SIZE - 1 - rcr->available;
 316 }
 317
 318 static void bm_rcr_set_ithresh(struct bm_portal *portal, u8 ithresh)
 319 {
 320         struct bm_rcr *rcr = &portal->rcr;
 321
 322         rcr->ithresh = ithresh;
 323         bm_out(portal, BM_REG_RCR_ITR, ithresh);
 324 }
 325
 326 static void bm_rcr_cce_prefetch(struct bm_portal *portal)
 327 {
 328         __maybe_unused struct bm_rcr *rcr = &portal->rcr;
 329
 330         DPAA_ASSERT(rcr->cmode == bm_rcr_cce);
 331         bm_cl_touch_ro(portal, BM_CL_RCR_CI_CENA);
 332 }
 333
 334 static u8 bm_rcr_cce_update(struct bm_portal *portal)
 335 {
 336         struct bm_rcr *rcr = &portal->rcr;
 337         u8 diff, old_ci = rcr->ci;
 338
 339         DPAA_ASSERT(rcr->cmode == bm_rcr_cce);
 340         rcr->ci = bm_ce_in(portal, BM_CL_RCR_CI_CENA) & (BM_RCR_SIZE - 1);
 341         bm_cl_invalidate(portal, BM_CL_RCR_CI_CENA);
 342         diff = dpaa_cyc_diff(BM_RCR_SIZE, old_ci, rcr->ci);
 343         rcr->available += diff;
 344         return diff;
 345 }
 346
 347 static inline struct bm_rcr_entry *bm_rcr_start(struct bm_portal *portal)
 348 {
 349         struct bm_rcr *rcr = &portal->rcr;
 350
 351         DPAA_ASSERT(!rcr->busy);
 352         if (!rcr->available)
 353                 return NULL;
 354 #ifdef CONFIG_FSL_DPAA_CHECKING
 355         rcr->busy = 1;
 356 #endif
 357         dpaa_zero(rcr->cursor);
 358         return rcr->cursor;
 359 }
 360
 361 static inline void bm_rcr_pvb_commit(struct bm_portal *portal, u8 myverb)
 362 {
 363         struct bm_rcr *rcr = &portal->rcr;
 364         struct bm_rcr_entry *rcursor;
 365
 366         DPAA_ASSERT(rcr->busy);
 367         DPAA_ASSERT(rcr->pmode == bm_rcr_pvb);
 368         DPAA_ASSERT(rcr->available >= 1);
 369         dma_wmb();
 370         rcursor = rcr->cursor;
 371         rcursor->_ncw_verb = myverb | rcr->vbit;
 372         dpaa_flush(rcursor);
 373         rcr_inc(rcr);
 374         rcr->available--;
 375 #ifdef CONFIG_FSL_DPAA_CHECKING
 376         rcr->busy = 0;
 377 #endif
 378 }
 379
 380 static int bm_rcr_init(struct bm_portal *portal, enum bm_rcr_pmode pmode,
 381                        enum bm_rcr_cmode cmode)
 382 {
 383         struct bm_rcr *rcr = &portal->rcr;
 384         u32 cfg;
 385         u8 pi;
 386
 387         rcr->ring = portal->addr.ce + BM_CL_RCR;
 388         rcr->ci = bm_in(portal, BM_REG_RCR_CI_CINH) & (BM_RCR_SIZE - 1);
 389         pi = bm_in(portal, BM_REG_RCR_PI_CINH) & (BM_RCR_SIZE - 1);
 390         rcr->cursor = rcr->ring + pi;
 391         rcr->vbit = (bm_in(portal, BM_REG_RCR_PI_CINH) & BM_RCR_SIZE) ?
 392                 BM_RCR_VERB_VBIT : 0;
 393         rcr->available = BM_RCR_SIZE - 1
 394                 - dpaa_cyc_diff(BM_RCR_SIZE, rcr->ci, pi);
 395         rcr->ithresh = bm_in(portal, BM_REG_RCR_ITR);
 396 #ifdef CONFIG_FSL_DPAA_CHECKING
 397         rcr->busy = 0;
 398         rcr->pmode = pmode;
 399         rcr->cmode = cmode;
 400 #endif
 401         cfg = (bm_in(portal, BM_REG_CFG) & 0xffffffe0)
 402                 | (pmode & 0x3); /* BCSP_CFG::RPM */
 403         bm_out(portal, BM_REG_CFG, cfg);
 404         return 0;
 405 }
 406
 407 static void bm_rcr_finish(struct bm_portal *portal)
 408 {
 409 #ifdef CONFIG_FSL_DPAA_CHECKING
 410         struct bm_rcr *rcr = &portal->rcr;
 411         int i;
 412
 413         DPAA_ASSERT(!rcr->busy);
 414
 415         i = bm_in(portal, BM_REG_RCR_PI_CINH) & (BM_RCR_SIZE - 1);
 416         if (i != rcr_ptr2idx(rcr->cursor))
 417                 pr_crit("losing uncommitted RCR entries\n");
 418
 419         i = bm_in(portal, BM_REG_RCR_CI_CINH) & (BM_RCR_SIZE - 1);
 420         if (i != rcr->ci)
 421                 pr_crit("missing existing RCR completions\n");
 422         if (rcr->ci != rcr_ptr2idx(rcr->cursor))
 423                 pr_crit("RCR destroyed unquiesced\n");
 424 #endif
 425 }
 426
 427 /* --- Management command API --- */
 428 static int bm_mc_init(struct bm_portal *portal)
 429 {
 430         struct bm_mc *mc = &portal->mc;
 431
 432         mc->cr = portal->addr.ce + BM_CL_CR;
 433         mc->rr = portal->addr.ce + BM_CL_RR0;
 434         mc->rridx = (mc->cr->_ncw_verb & BM_MCC_VERB_VBIT) ?
 435                     0 : 1;
 436         mc->vbit = mc->rridx ? BM_MCC_VERB_VBIT : 0;
 437 #ifdef CONFIG_FSL_DPAA_CHECKING
 438         mc->state = mc_idle;
 439 #endif
 440         return 0;
 441 }
 442
 443 static void bm_mc_finish(struct bm_portal *portal)
 444 {
 445 #ifdef CONFIG_FSL_DPAA_CHECKING
 446         struct bm_mc *mc = &portal->mc;
 447
 448         DPAA_ASSERT(mc->state == mc_idle);
 449         if (mc->state != mc_idle)
 450                 pr_crit("Losing incomplete MC command\n");
 451 #endif
 452 }
 453
 454 static inline struct bm_mc_command *bm_mc_start(struct bm_portal *portal)
 455 {
 456         struct bm_mc *mc = &portal->mc;
 457
 458         DPAA_ASSERT(mc->state == mc_idle);
 459 #ifdef CONFIG_FSL_DPAA_CHECKING
 460         mc->state = mc_user;
 461 #endif
 462         dpaa_zero(mc->cr);
 463         return mc->cr;
 464 }
 465
 466 static inline void bm_mc_commit(struct bm_portal *portal, u8 myverb)
 467 {
 468         struct bm_mc *mc = &portal->mc;
 469         union bm_mc_result *rr = mc->rr + mc->rridx;
 470
 471         DPAA_ASSERT(mc->state == mc_user);
 472         dma_wmb();
 473         mc->cr->_ncw_verb = myverb | mc->vbit;
 474         dpaa_flush(mc->cr);
 475         dpaa_invalidate_touch_ro(rr);
 476 #ifdef CONFIG_FSL_DPAA_CHECKING
 477         mc->state = mc_hw;
 478 #endif
 479 }
 480
 481 static inline union bm_mc_result *bm_mc_result(struct bm_portal *portal)
 482 {
 483         struct bm_mc *mc = &portal->mc;
 484         union bm_mc_result *rr = mc->rr + mc->rridx;
 485
 486         DPAA_ASSERT(mc->state == mc_hw);
 487         /*
 488          * The inactive response register's verb byte always returns zero until
 489          * its command is submitted and completed. This includes the valid-bit,
 490          * in case you were wondering...
 491          */
 492         if (!rr->verb) {
 493                 dpaa_invalidate_touch_ro(rr);
 494                 return NULL;
 495         }
 496         mc->rridx ^= 1;
 497         mc->vbit ^= BM_MCC_VERB_VBIT;
 498 #ifdef CONFIG_FSL_DPAA_CHECKING
 499         mc->state = mc_idle;
 500 #endif
 501         return rr;
 502 }
 503
 504 static inline int bm_mc_result_timeout(struct bm_portal *portal,
 505                                        union bm_mc_result **mcr)
 506 {
 507         int timeout = BM_MCR_TIMEOUT;
 508
 509         do {
 510                 *mcr = bm_mc_result(portal);
 511                 if (*mcr)
 512                         break;
 513                 udelay(1);
 514         } while (--timeout);
 515
 516         return timeout;
 517 }
 518
 519 /* Disable all BSCN interrupts for the portal */
 520 static void bm_isr_bscn_disable(struct bm_portal *portal)
 521 {
 522         bm_out(portal, BM_REG_SCN(0), 0);
 523         bm_out(portal, BM_REG_SCN(1), 0);
 524 }
 525
 526 static int bman_create_portal(struct bman_portal *portal,
 527                               const struct bm_portal_config *c)
 528 {
 529         struct bm_portal *p;
 530         int ret;
 531
 532         p = &portal->p;
 533         /*
 534          * prep the low-level portal struct with the mapped addresses from the
 535          * config, everything that follows depends on it and "config" is more
 536          * for (de)reference...
 537          */
 538         p->addr.ce = c->addr_virt_ce;
 539         p->addr.ce_be = c->addr_virt_ce;
 540         p->addr.ci = c->addr_virt_ci;
 541         if (bm_rcr_init(p, bm_rcr_pvb, bm_rcr_cce)) {
 542                 dev_err(c->dev, "RCR initialisation failed\n");
 543                 goto fail_rcr;
 544         }
 545         if (bm_mc_init(p)) {
 546                 dev_err(c->dev, "MC initialisation failed\n");
 547                 goto fail_mc;
 548         }
 549         /*
 550          * Default to all BPIDs disabled, we enable as required at
 551          * run-time.
 552          */
 553         bm_isr_bscn_disable(p);
 554
 555         /* Write-to-clear any stale interrupt status bits */
 556         bm_out(p, BM_REG_ISDR, 0xffffffff);
 557         portal->irq_sources = 0;
 558         bm_out(p, BM_REG_IER, 0);
 559         bm_out(p, BM_REG_ISR, 0xffffffff);
 560         snprintf(portal->irqname, MAX_IRQNAME, IRQNAME, c->cpu);
 561         if (request_irq(c->irq, portal_isr, 0, portal->irqname, portal)) {
 562                 dev_err(c->dev, "request_irq() failed\n");
 563                 goto fail_irq;
 564         }
 565
 566         if (dpaa_set_portal_irq_affinity(c->dev, c->irq, c->cpu))
 567                 goto fail_affinity;
 568
 569         /* Need RCR to be empty before continuing */
 570         ret = bm_rcr_get_fill(p);
 571         if (ret) {
 572                 dev_err(c->dev, "RCR unclean\n");
 573                 goto fail_rcr_empty;
 574         }
 575         /* Success */
 576         portal->config = c;
 577
 578         bm_out(p, BM_REG_ISDR, 0);
 579         bm_out(p, BM_REG_IIR, 0);
 580
 581         return 0;
 582
 583 fail_rcr_empty:
 584 fail_affinity:
 585         free_irq(c->irq, portal);
 586 fail_irq:
 587         bm_mc_finish(p);
 588 fail_mc:
 589         bm_rcr_finish(p);
 590 fail_rcr:
 591         return -EIO;
 592 }
 593
 594 struct bman_portal *bman_create_affine_portal(const struct bm_portal_config *c)
 595 {
 596         struct bman_portal *portal;
 597         int err;
 598
 599         portal = &per_cpu(bman_affine_portal, c->cpu);
 600         err = bman_create_portal(portal, c);
 601         if (err)
 602                 return NULL;
 603
 604         spin_lock(&affine_mask_lock);
 605         cpumask_set_cpu(c->cpu, &affine_mask);
 606         spin_unlock(&affine_mask_lock);
 607
 608         return portal;
 609 }
 610
 611 static u32 poll_portal_slow(struct bman_portal *p, u32 is)
 612 {
 613         u32 ret = is;
 614
 615         if (is & BM_PIRQ_RCRI) {
 616                 bm_rcr_cce_update(&p->p);
 617                 bm_rcr_set_ithresh(&p->p, 0);
 618                 bm_out(&p->p, BM_REG_ISR, BM_PIRQ_RCRI);
 619                 is &= ~BM_PIRQ_RCRI;
 620         }
 621
 622         /* There should be no status register bits left undefined */
 623         DPAA_ASSERT(!is);
 624         return ret;
 625 }
 626
 627 int bman_p_irqsource_add(struct bman_portal *p, u32 bits)
 628 {
 629         unsigned long irqflags;
 630
 631         local_irq_save(irqflags);
 632         p->irq_sources |= bits & BM_PIRQ_VISIBLE;
 633         bm_out(&p->p, BM_REG_IER, p->irq_sources);
 634         local_irq_restore(irqflags);
 635         return 0;
 636 }
 637
 638 int bm_shutdown_pool(u32 bpid)
 639 {
 640         int err = 0;
 641         struct bm_mc_command *bm_cmd;
 642         union bm_mc_result *bm_res;
 643
 644
 645         struct bman_portal *p = get_affine_portal();
 646         while (1) {
 647                 /* Acquire buffers until empty */
 648                 bm_cmd = bm_mc_start(&p->p);
 649                 bm_cmd->bpid = bpid;
 650                 bm_mc_commit(&p->p, BM_MCC_VERB_CMD_ACQUIRE | 1);
 651                 if (!bm_mc_result_timeout(&p->p, &bm_res)) {
 652                         pr_crit("BMan Acquire Command timedout\n");
 653                         err = -ETIMEDOUT;
 654                         goto done;
 655                 }
 656                 if (!(bm_res->verb & BM_MCR_VERB_ACQUIRE_BUFCOUNT)) {
 657                         /* Pool is empty */
 658                         goto done;
 659                 }
 660         }
 661 done:
 662         put_affine_portal();
 663         return err;
 664 }
 665
 666 struct gen_pool *bm_bpalloc;
 667
 668 static int bm_alloc_bpid_range(u32 *result, u32 count)
 669 {
 670         unsigned long addr;
 671
 672         addr = gen_pool_alloc(bm_bpalloc, count);
 673         if (!addr)
 674                 return -ENOMEM;
 675
 676         *result = addr & ~DPAA_GENALLOC_OFF;
 677
 678         return 0;
 679 }
 680
 681 static int bm_release_bpid(u32 bpid)
 682 {
 683         int ret;
 684
 685         ret = bm_shutdown_pool(bpid);
 686         if (ret) {
 687                 pr_debug("BPID %d leaked\n", bpid);
 688                 return ret;
 689         }
 690
 691         gen_pool_free(bm_bpalloc, bpid | DPAA_GENALLOC_OFF, 1);
 692         return 0;
 693 }
 694
 695 struct bman_pool *bman_new_pool(void)
 696 {
 697         struct bman_pool *pool = NULL;
 698         u32 bpid;
 699
 700         if (bm_alloc_bpid_range(&bpid, 1))
 701                 return NULL;
 702
 703         pool = kmalloc(sizeof(*pool), GFP_KERNEL);
 704         if (!pool)
 705                 goto err;
 706
 707         pool->bpid = bpid;
 708
 709         return pool;
 710 err:
 711         bm_release_bpid(bpid);
 712         kfree(pool);
 713         return NULL;
 714 }
 715 EXPORT_SYMBOL(bman_new_pool);
 716
 717 void bman_free_pool(struct bman_pool *pool)
 718 {
 719         bm_release_bpid(pool->bpid);
 720
 721         kfree(pool);
 722 }
 723 EXPORT_SYMBOL(bman_free_pool);
 724
 725 int bman_get_bpid(const struct bman_pool *pool)
 726 {
 727         return pool->bpid;
 728 }
 729 EXPORT_SYMBOL(bman_get_bpid);
 730
 731 static void update_rcr_ci(struct bman_portal *p, int avail)
 732 {
 733         if (avail)
 734                 bm_rcr_cce_prefetch(&p->p);
 735         else
 736                 bm_rcr_cce_update(&p->p);
 737 }
 738
 739 int bman_release(struct bman_pool *pool, const struct bm_buffer *bufs, u8 num)
 740 {
 741         struct bman_portal *p;
 742         struct bm_rcr_entry *r;
 743         unsigned long irqflags;
 744         int avail, timeout = 1000; /* 1ms */
 745         int i = num - 1;
 746
 747         DPAA_ASSERT(num > 0 && num <= 8);
 748
 749         do {
 750                 p = get_affine_portal();
 751                 local_irq_save(irqflags);
 752                 avail = bm_rcr_get_avail(&p->p);
 753                 if (avail < 2)
 754                         update_rcr_ci(p, avail);
 755                 r = bm_rcr_start(&p->p);
 756                 local_irq_restore(irqflags);
 757                 put_affine_portal();
 758                 if (likely(r))
 759                         break;
 760
 761                 udelay(1);
 762         } while (--timeout);
 763
 764         if (unlikely(!timeout))
 765                 return -ETIMEDOUT;
 766
 767         p = get_affine_portal();
 768         local_irq_save(irqflags);
 769         /*
 770          * we can copy all but the first entry, as this can trigger badness
 771          * with the valid-bit
 772          */
 773         bm_buffer_set64(r->bufs, bm_buffer_get64(bufs));
 774         bm_buffer_set_bpid(r->bufs, pool->bpid);
 775         if (i)
 776                 memcpy(&r->bufs[1], &bufs[1], i * sizeof(bufs[0]));
 777
 778         bm_rcr_pvb_commit(&p->p, BM_RCR_VERB_CMD_BPID_SINGLE |
 779                           (num & BM_RCR_VERB_BUFCOUNT_MASK));
 780
 781         local_irq_restore(irqflags);
 782         put_affine_portal();
 783         return 0;
 784 }
 785 EXPORT_SYMBOL(bman_release);
 786
 787 int bman_acquire(struct bman_pool *pool, struct bm_buffer *bufs, u8 num)
 788 {
 789         struct bman_portal *p = get_affine_portal();
 790         struct bm_mc_command *mcc;
 791         union bm_mc_result *mcr;
 792         int ret;
 793
 794         DPAA_ASSERT(num > 0 && num <= 8);
 795
 796         mcc = bm_mc_start(&p->p);
 797         mcc->bpid = pool->bpid;
 798         bm_mc_commit(&p->p, BM_MCC_VERB_CMD_ACQUIRE |
 799                      (num & BM_MCC_VERB_ACQUIRE_BUFCOUNT));
 800         if (!bm_mc_result_timeout(&p->p, &mcr)) {
 801                 put_affine_portal();
 802                 pr_crit("BMan Acquire Timeout\n");
 803                 return -ETIMEDOUT;
 804         }
 805         ret = mcr->verb & BM_MCR_VERB_ACQUIRE_BUFCOUNT;
 806         if (bufs)
 807                 memcpy(&bufs[0], &mcr->bufs[0], num * sizeof(bufs[0]));
 808
 809         put_affine_portal();
 810         if (ret != num)
 811                 ret = -ENOMEM;
 812         return ret;
 813 }
 814 EXPORT_SYMBOL(bman_acquire);
 815
 816 const struct bm_portal_config *
 817 bman_get_bm_portal_config(const struct bman_portal *portal)
 818 {
 819         return portal->config;
 820 }