?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
mmc: rtsx_pci: Enable MMC_CAP_ERASE to allow erase/discard/trim requests
[linux/fpc-iii.git] / drivers / rapidio / devices / tsi721.c
blobb5b455614f8a6f83dc75ac91ac53c6cd12508d07
1 /*
2 * RapidIO mport driver for Tsi721 PCIExpress-to-SRIO bridge
3 *
4 * Copyright 2011 Integrated Device Technology, Inc.
5 * Alexandre Bounine <alexandre.bounine@idt.com>
6 * Chul Kim <chul.kim@idt.com>
7 *
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms of the GNU General Public License as published by the Free
10 * Software Foundation; either version 2 of the License, or (at your option)
11 * any later version.
12 *
13 * This program is distributed in the hope that it will be useful, but WITHOUT
14 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16 * more details.
17 *
18 * You should have received a copy of the GNU General Public License along with
19 * this program; if not, write to the Free Software Foundation, Inc., 59
20 * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
21 */
22
23 #include <linux/io.h>
24 #include <linux/errno.h>
25 #include <linux/init.h>
26 #include <linux/ioport.h>
27 #include <linux/kernel.h>
28 #include <linux/module.h>
29 #include <linux/pci.h>
30 #include <linux/rio.h>
31 #include <linux/rio_drv.h>
32 #include <linux/dma-mapping.h>
33 #include <linux/interrupt.h>
34 #include <linux/kfifo.h>
35 #include <linux/delay.h>
36
37 #include "tsi721.h"
38
39 #ifdef DEBUG
40 u32 dbg_level = DBG_INIT | DBG_EXIT;
41 module_param(dbg_level, uint, S_IWUSR | S_IRUGO);
42 MODULE_PARM_DESC(dbg_level, "Debugging output level (default 0 = none)");
43 #endif
44
45 static void tsi721_omsg_handler(struct tsi721_device *priv, int ch);
46 static void tsi721_imsg_handler(struct tsi721_device *priv, int ch);
47
48 /**
49 * tsi721_lcread - read from local SREP config space
50 * @mport: RapidIO master port info
51 * @index: ID of RapdiIO interface
52 * @offset: Offset into configuration space
53 * @len: Length (in bytes) of the maintenance transaction
54 * @data: Value to be read into
55 *
56 * Generates a local SREP space read. Returns %0 on
57 * success or %-EINVAL on failure.
58 */
59 static int tsi721_lcread(struct rio_mport *mport, int index, u32 offset,
60 int len, u32 *data)
61 {
62 struct tsi721_device *priv = mport->priv;
63
64 if (len != sizeof(u32))
65 return -EINVAL; /* only 32-bit access is supported */
66
67 *data = ioread32(priv->regs + offset);
68
69 return 0;
70 }
71
72 /**
73 * tsi721_lcwrite - write into local SREP config space
74 * @mport: RapidIO master port info
75 * @index: ID of RapdiIO interface
76 * @offset: Offset into configuration space
77 * @len: Length (in bytes) of the maintenance transaction
78 * @data: Value to be written
79 *
80 * Generates a local write into SREP configuration space. Returns %0 on
81 * success or %-EINVAL on failure.
82 */
83 static int tsi721_lcwrite(struct rio_mport *mport, int index, u32 offset,
84 int len, u32 data)
85 {
86 struct tsi721_device *priv = mport->priv;
87
88 if (len != sizeof(u32))
89 return -EINVAL; /* only 32-bit access is supported */
90
91 iowrite32(data, priv->regs + offset);
92
93 return 0;
94 }
95
96 /**
97 * tsi721_maint_dma - Helper function to generate RapidIO maintenance
98 * transactions using designated Tsi721 DMA channel.
99 * @priv: pointer to tsi721 private data
100 * @sys_size: RapdiIO transport system size
101 * @destid: Destination ID of transaction
102 * @hopcount: Number of hops to target device
103 * @offset: Offset into configuration space
104 * @len: Length (in bytes) of the maintenance transaction
105 * @data: Location to be read from or write into
106 * @do_wr: Operation flag (1 == MAINT_WR)
107 *
108 * Generates a RapidIO maintenance transaction (Read or Write).
109 * Returns %0 on success and %-EINVAL or %-EFAULT on failure.
110 */
111 static int tsi721_maint_dma(struct tsi721_device *priv, u32 sys_size,
112 u16 destid, u8 hopcount, u32 offset, int len,
113 u32 *data, int do_wr)
114 {
115 void __iomem *regs = priv->regs + TSI721_DMAC_BASE(priv->mdma.ch_id);
116 struct tsi721_dma_desc *bd_ptr;
117 u32 rd_count, swr_ptr, ch_stat;
118 int i, err = 0;
119 u32 op = do_wr ? MAINT_WR : MAINT_RD;
120
121 if (offset > (RIO_MAINT_SPACE_SZ - len) || (len != sizeof(u32)))
122 return -EINVAL;
123
124 bd_ptr = priv->mdma.bd_base;
125
126 rd_count = ioread32(regs + TSI721_DMAC_DRDCNT);
127
128 /* Initialize DMA descriptor */
129 bd_ptr[0].type_id = cpu_to_le32((DTYPE2 << 29) | (op << 19) | destid);
130 bd_ptr[0].bcount = cpu_to_le32((sys_size << 26) | 0x04);
131 bd_ptr[0].raddr_lo = cpu_to_le32((hopcount << 24) | offset);
132 bd_ptr[0].raddr_hi = 0;
133 if (do_wr)
134 bd_ptr[0].data[0] = cpu_to_be32p(data);
135 else
136 bd_ptr[0].data[0] = 0xffffffff;
137
138 mb();
139
140 /* Start DMA operation */
141 iowrite32(rd_count + 2, regs + TSI721_DMAC_DWRCNT);
142 ioread32(regs + TSI721_DMAC_DWRCNT);
143 i = 0;
144
145 /* Wait until DMA transfer is finished */
146 while ((ch_stat = ioread32(regs + TSI721_DMAC_STS))
147 & TSI721_DMAC_STS_RUN) {
148 udelay(1);
149 if (++i >= 5000000) {
150 tsi_debug(MAINT, &priv->pdev->dev,
151 "DMA[%d] read timeout ch_status=%x",
152 priv->mdma.ch_id, ch_stat);
153 if (!do_wr)
154 *data = 0xffffffff;
155 err = -EIO;
156 goto err_out;
157 }
158 }
159
160 if (ch_stat & TSI721_DMAC_STS_ABORT) {
161 /* If DMA operation aborted due to error,
162 * reinitialize DMA channel
163 */
164 tsi_debug(MAINT, &priv->pdev->dev, "DMA ABORT ch_stat=%x",
165 ch_stat);
166 tsi_debug(MAINT, &priv->pdev->dev,
167 "OP=%d : destid=%x hc=%x off=%x",
168 do_wr ? MAINT_WR : MAINT_RD,
169 destid, hopcount, offset);
170 iowrite32(TSI721_DMAC_INT_ALL, regs + TSI721_DMAC_INT);
171 iowrite32(TSI721_DMAC_CTL_INIT, regs + TSI721_DMAC_CTL);
172 udelay(10);
173 iowrite32(0, regs + TSI721_DMAC_DWRCNT);
174 udelay(1);
175 if (!do_wr)
176 *data = 0xffffffff;
177 err = -EIO;
178 goto err_out;
179 }
180
181 if (!do_wr)
182 *data = be32_to_cpu(bd_ptr[0].data[0]);
183
184 /*
185 * Update descriptor status FIFO RD pointer.
186 * NOTE: Skipping check and clear FIFO entries because we are waiting
187 * for transfer to be completed.
188 */
189 swr_ptr = ioread32(regs + TSI721_DMAC_DSWP);
190 iowrite32(swr_ptr, regs + TSI721_DMAC_DSRP);
191 err_out:
192
193 return err;
194 }
195
196 /**
197 * tsi721_cread_dma - Generate a RapidIO maintenance read transaction
198 * using Tsi721 BDMA engine.
199 * @mport: RapidIO master port control structure
200 * @index: ID of RapdiIO interface
201 * @destid: Destination ID of transaction
202 * @hopcount: Number of hops to target device
203 * @offset: Offset into configuration space
204 * @len: Length (in bytes) of the maintenance transaction
205 * @val: Location to be read into
206 *
207 * Generates a RapidIO maintenance read transaction.
208 * Returns %0 on success and %-EINVAL or %-EFAULT on failure.
209 */
210 static int tsi721_cread_dma(struct rio_mport *mport, int index, u16 destid,
211 u8 hopcount, u32 offset, int len, u32 *data)
212 {
213 struct tsi721_device *priv = mport->priv;
214
215 return tsi721_maint_dma(priv, mport->sys_size, destid, hopcount,
216 offset, len, data, 0);
217 }
218
219 /**
220 * tsi721_cwrite_dma - Generate a RapidIO maintenance write transaction
221 * using Tsi721 BDMA engine
222 * @mport: RapidIO master port control structure
223 * @index: ID of RapdiIO interface
224 * @destid: Destination ID of transaction
225 * @hopcount: Number of hops to target device
226 * @offset: Offset into configuration space
227 * @len: Length (in bytes) of the maintenance transaction
228 * @val: Value to be written
229 *
230 * Generates a RapidIO maintenance write transaction.
231 * Returns %0 on success and %-EINVAL or %-EFAULT on failure.
232 */
233 static int tsi721_cwrite_dma(struct rio_mport *mport, int index, u16 destid,
234 u8 hopcount, u32 offset, int len, u32 data)
235 {
236 struct tsi721_device *priv = mport->priv;
237 u32 temp = data;
238
239 return tsi721_maint_dma(priv, mport->sys_size, destid, hopcount,
240 offset, len, &temp, 1);
241 }
242
243 /**
244 * tsi721_pw_handler - Tsi721 inbound port-write interrupt handler
245 * @priv: tsi721 device private structure
246 *
247 * Handles inbound port-write interrupts. Copies PW message from an internal
248 * buffer into PW message FIFO and schedules deferred routine to process
249 * queued messages.
250 */
251 static int
252 tsi721_pw_handler(struct tsi721_device *priv)
253 {
254 u32 pw_stat;
255 u32 pw_buf[TSI721_RIO_PW_MSG_SIZE/sizeof(u32)];
256
257
258 pw_stat = ioread32(priv->regs + TSI721_RIO_PW_RX_STAT);
259
260 if (pw_stat & TSI721_RIO_PW_RX_STAT_PW_VAL) {
261 pw_buf[0] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(0));
262 pw_buf[1] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(1));
263 pw_buf[2] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(2));
264 pw_buf[3] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(3));
265
266 /* Queue PW message (if there is room in FIFO),
267 * otherwise discard it.
268 */
269 spin_lock(&priv->pw_fifo_lock);
270 if (kfifo_avail(&priv->pw_fifo) >= TSI721_RIO_PW_MSG_SIZE)
271 kfifo_in(&priv->pw_fifo, pw_buf,
272 TSI721_RIO_PW_MSG_SIZE);
273 else
274 priv->pw_discard_count++;
275 spin_unlock(&priv->pw_fifo_lock);
276 }
277
278 /* Clear pending PW interrupts */
279 iowrite32(TSI721_RIO_PW_RX_STAT_PW_DISC | TSI721_RIO_PW_RX_STAT_PW_VAL,
280 priv->regs + TSI721_RIO_PW_RX_STAT);
281
282 schedule_work(&priv->pw_work);
283
284 return 0;
285 }
286
287 static void tsi721_pw_dpc(struct work_struct *work)
288 {
289 struct tsi721_device *priv = container_of(work, struct tsi721_device,
290 pw_work);
291 union rio_pw_msg pwmsg;
292
293 /*
294 * Process port-write messages
295 */
296 while (kfifo_out_spinlocked(&priv->pw_fifo, (unsigned char *)&pwmsg,
297 TSI721_RIO_PW_MSG_SIZE, &priv->pw_fifo_lock)) {
298 /* Pass the port-write message to RIO core for processing */
299 rio_inb_pwrite_handler(&priv->mport, &pwmsg);
300 }
301 }
302
303 /**
304 * tsi721_pw_enable - enable/disable port-write interface init
305 * @mport: Master port implementing the port write unit
306 * @enable: 1=enable; 0=disable port-write message handling
307 */
308 static int tsi721_pw_enable(struct rio_mport *mport, int enable)
309 {
310 struct tsi721_device *priv = mport->priv;
311 u32 rval;
312
313 rval = ioread32(priv->regs + TSI721_RIO_EM_INT_ENABLE);
314
315 if (enable)
316 rval |= TSI721_RIO_EM_INT_ENABLE_PW_RX;
317 else
318 rval &= ~TSI721_RIO_EM_INT_ENABLE_PW_RX;
319
320 /* Clear pending PW interrupts */
321 iowrite32(TSI721_RIO_PW_RX_STAT_PW_DISC | TSI721_RIO_PW_RX_STAT_PW_VAL,
322 priv->regs + TSI721_RIO_PW_RX_STAT);
323 /* Update enable bits */
324 iowrite32(rval, priv->regs + TSI721_RIO_EM_INT_ENABLE);
325
326 return 0;
327 }
328
329 /**
330 * tsi721_dsend - Send a RapidIO doorbell
331 * @mport: RapidIO master port info
332 * @index: ID of RapidIO interface
333 * @destid: Destination ID of target device
334 * @data: 16-bit info field of RapidIO doorbell
335 *
336 * Sends a RapidIO doorbell message. Always returns %0.
337 */
338 static int tsi721_dsend(struct rio_mport *mport, int index,
339 u16 destid, u16 data)
340 {
341 struct tsi721_device *priv = mport->priv;
342 u32 offset;
343
344 offset = (((mport->sys_size) ? RIO_TT_CODE_16 : RIO_TT_CODE_8) << 18) |
345 (destid << 2);
346
347 tsi_debug(DBELL, &priv->pdev->dev,
348 "Send Doorbell 0x%04x to destID 0x%x", data, destid);
349 iowrite16be(data, priv->odb_base + offset);
350
351 return 0;
352 }
353
354 /**
355 * tsi721_dbell_handler - Tsi721 doorbell interrupt handler
356 * @priv: tsi721 device-specific data structure
357 *
358 * Handles inbound doorbell interrupts. Copies doorbell entry from an internal
359 * buffer into DB message FIFO and schedules deferred routine to process
360 * queued DBs.
361 */
362 static int
363 tsi721_dbell_handler(struct tsi721_device *priv)
364 {
365 u32 regval;
366
367 /* Disable IDB interrupts */
368 regval = ioread32(priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
369 regval &= ~TSI721_SR_CHINT_IDBQRCV;
370 iowrite32(regval,
371 priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
372
373 schedule_work(&priv->idb_work);
374
375 return 0;
376 }
377
378 static void tsi721_db_dpc(struct work_struct *work)
379 {
380 struct tsi721_device *priv = container_of(work, struct tsi721_device,
381 idb_work);
382 struct rio_mport *mport;
383 struct rio_dbell *dbell;
384 int found = 0;
385 u32 wr_ptr, rd_ptr;
386 u64 *idb_entry;
387 u32 regval;
388 union {
389 u64 msg;
390 u8 bytes[8];
391 } idb;
392
393 /*
394 * Process queued inbound doorbells
395 */
396 mport = &priv->mport;
397
398 wr_ptr = ioread32(priv->regs + TSI721_IDQ_WP(IDB_QUEUE)) % IDB_QSIZE;
399 rd_ptr = ioread32(priv->regs + TSI721_IDQ_RP(IDB_QUEUE)) % IDB_QSIZE;
400
401 while (wr_ptr != rd_ptr) {
402 idb_entry = (u64 *)(priv->idb_base +
403 (TSI721_IDB_ENTRY_SIZE * rd_ptr));
404 rd_ptr++;
405 rd_ptr %= IDB_QSIZE;
406 idb.msg = *idb_entry;
407 *idb_entry = 0;
408
409 /* Process one doorbell */
410 list_for_each_entry(dbell, &mport->dbells, node) {
411 if ((dbell->res->start <= DBELL_INF(idb.bytes)) &&
412 (dbell->res->end >= DBELL_INF(idb.bytes))) {
413 found = 1;
414 break;
415 }
416 }
417
418 if (found) {
419 dbell->dinb(mport, dbell->dev_id, DBELL_SID(idb.bytes),
420 DBELL_TID(idb.bytes), DBELL_INF(idb.bytes));
421 } else {
422 tsi_debug(DBELL, &priv->pdev->dev,
423 "spurious IDB sid %2.2x tid %2.2x info %4.4x",
424 DBELL_SID(idb.bytes), DBELL_TID(idb.bytes),
425 DBELL_INF(idb.bytes));
426 }
427
428 wr_ptr = ioread32(priv->regs +
429 TSI721_IDQ_WP(IDB_QUEUE)) % IDB_QSIZE;
430 }
431
432 iowrite32(rd_ptr & (IDB_QSIZE - 1),
433 priv->regs + TSI721_IDQ_RP(IDB_QUEUE));
434
435 /* Re-enable IDB interrupts */
436 regval = ioread32(priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
437 regval |= TSI721_SR_CHINT_IDBQRCV;
438 iowrite32(regval,
439 priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
440
441 wr_ptr = ioread32(priv->regs + TSI721_IDQ_WP(IDB_QUEUE)) % IDB_QSIZE;
442 if (wr_ptr != rd_ptr)
443 schedule_work(&priv->idb_work);
444 }
445
446 /**
447 * tsi721_irqhandler - Tsi721 interrupt handler
448 * @irq: Linux interrupt number
449 * @ptr: Pointer to interrupt-specific data (tsi721_device structure)
450 *
451 * Handles Tsi721 interrupts signaled using MSI and INTA. Checks reported
452 * interrupt events and calls an event-specific handler(s).
453 */
454 static irqreturn_t tsi721_irqhandler(int irq, void *ptr)
455 {
456 struct tsi721_device *priv = (struct tsi721_device *)ptr;
457 u32 dev_int;
458 u32 dev_ch_int;
459 u32 intval;
460 u32 ch_inte;
461
462 /* For MSI mode disable all device-level interrupts */
463 if (priv->flags & TSI721_USING_MSI)
464 iowrite32(0, priv->regs + TSI721_DEV_INTE);
465
466 dev_int = ioread32(priv->regs + TSI721_DEV_INT);
467 if (!dev_int)
468 return IRQ_NONE;
469
470 dev_ch_int = ioread32(priv->regs + TSI721_DEV_CHAN_INT);
471
472 if (dev_int & TSI721_DEV_INT_SR2PC_CH) {
473 /* Service SR2PC Channel interrupts */
474 if (dev_ch_int & TSI721_INT_SR2PC_CHAN(IDB_QUEUE)) {
475 /* Service Inbound Doorbell interrupt */
476 intval = ioread32(priv->regs +
477 TSI721_SR_CHINT(IDB_QUEUE));
478 if (intval & TSI721_SR_CHINT_IDBQRCV)
479 tsi721_dbell_handler(priv);
480 else
481 tsi_info(&priv->pdev->dev,
482 "Unsupported SR_CH_INT %x", intval);
483
484 /* Clear interrupts */
485 iowrite32(intval,
486 priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
487 ioread32(priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
488 }
489 }
490
491 if (dev_int & TSI721_DEV_INT_SMSG_CH) {
492 int ch;
493
494 /*
495 * Service channel interrupts from Messaging Engine
496 */
497
498 if (dev_ch_int & TSI721_INT_IMSG_CHAN_M) { /* Inbound Msg */
499 /* Disable signaled OB MSG Channel interrupts */
500 ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
501 ch_inte &= ~(dev_ch_int & TSI721_INT_IMSG_CHAN_M);
502 iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE);
503
504 /*
505 * Process Inbound Message interrupt for each MBOX
506 */
507 for (ch = 4; ch < RIO_MAX_MBOX + 4; ch++) {
508 if (!(dev_ch_int & TSI721_INT_IMSG_CHAN(ch)))
509 continue;
510 tsi721_imsg_handler(priv, ch);
511 }
512 }
513
514 if (dev_ch_int & TSI721_INT_OMSG_CHAN_M) { /* Outbound Msg */
515 /* Disable signaled OB MSG Channel interrupts */
516 ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
517 ch_inte &= ~(dev_ch_int & TSI721_INT_OMSG_CHAN_M);
518 iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE);
519
520 /*
521 * Process Outbound Message interrupts for each MBOX
522 */
523
524 for (ch = 0; ch < RIO_MAX_MBOX; ch++) {
525 if (!(dev_ch_int & TSI721_INT_OMSG_CHAN(ch)))
526 continue;
527 tsi721_omsg_handler(priv, ch);
528 }
529 }
530 }
531
532 if (dev_int & TSI721_DEV_INT_SRIO) {
533 /* Service SRIO MAC interrupts */
534 intval = ioread32(priv->regs + TSI721_RIO_EM_INT_STAT);
535 if (intval & TSI721_RIO_EM_INT_STAT_PW_RX)
536 tsi721_pw_handler(priv);
537 }
538
539 #ifdef CONFIG_RAPIDIO_DMA_ENGINE
540 if (dev_int & TSI721_DEV_INT_BDMA_CH) {
541 int ch;
542
543 if (dev_ch_int & TSI721_INT_BDMA_CHAN_M) {
544 tsi_debug(DMA, &priv->pdev->dev,
545 "IRQ from DMA channel 0x%08x", dev_ch_int);
546
547 for (ch = 0; ch < TSI721_DMA_MAXCH; ch++) {
548 if (!(dev_ch_int & TSI721_INT_BDMA_CHAN(ch)))
549 continue;
550 tsi721_bdma_handler(&priv->bdma[ch]);
551 }
552 }
553 }
554 #endif
555
556 /* For MSI mode re-enable device-level interrupts */
557 if (priv->flags & TSI721_USING_MSI) {
558 dev_int = TSI721_DEV_INT_SR2PC_CH | TSI721_DEV_INT_SRIO |
559 TSI721_DEV_INT_SMSG_CH | TSI721_DEV_INT_BDMA_CH;
560 iowrite32(dev_int, priv->regs + TSI721_DEV_INTE);
561 }
562
563 return IRQ_HANDLED;
564 }
565
566 static void tsi721_interrupts_init(struct tsi721_device *priv)
567 {
568 u32 intr;
569
570 /* Enable IDB interrupts */
571 iowrite32(TSI721_SR_CHINT_ALL,
572 priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
573 iowrite32(TSI721_SR_CHINT_IDBQRCV,
574 priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
575
576 /* Enable SRIO MAC interrupts */
577 iowrite32(TSI721_RIO_EM_DEV_INT_EN_INT,
578 priv->regs + TSI721_RIO_EM_DEV_INT_EN);
579
580 /* Enable interrupts from channels in use */
581 #ifdef CONFIG_RAPIDIO_DMA_ENGINE
582 intr = TSI721_INT_SR2PC_CHAN(IDB_QUEUE) |
583 (TSI721_INT_BDMA_CHAN_M &
584 ~TSI721_INT_BDMA_CHAN(TSI721_DMACH_MAINT));
585 #else
586 intr = TSI721_INT_SR2PC_CHAN(IDB_QUEUE);
587 #endif
588 iowrite32(intr, priv->regs + TSI721_DEV_CHAN_INTE);
589
590 if (priv->flags & TSI721_USING_MSIX)
591 intr = TSI721_DEV_INT_SRIO;
592 else
593 intr = TSI721_DEV_INT_SR2PC_CH | TSI721_DEV_INT_SRIO |
594 TSI721_DEV_INT_SMSG_CH | TSI721_DEV_INT_BDMA_CH;
595
596 iowrite32(intr, priv->regs + TSI721_DEV_INTE);
597 ioread32(priv->regs + TSI721_DEV_INTE);
598 }
599
600 #ifdef CONFIG_PCI_MSI
601 /**
602 * tsi721_omsg_msix - MSI-X interrupt handler for outbound messaging
603 * @irq: Linux interrupt number
604 * @ptr: Pointer to interrupt-specific data (tsi721_device structure)
605 *
606 * Handles outbound messaging interrupts signaled using MSI-X.
607 */
608 static irqreturn_t tsi721_omsg_msix(int irq, void *ptr)
609 {
610 struct tsi721_device *priv = (struct tsi721_device *)ptr;
611 int mbox;
612
613 mbox = (irq - priv->msix[TSI721_VECT_OMB0_DONE].vector) % RIO_MAX_MBOX;
614 tsi721_omsg_handler(priv, mbox);
615 return IRQ_HANDLED;
616 }
617
618 /**
619 * tsi721_imsg_msix - MSI-X interrupt handler for inbound messaging
620 * @irq: Linux interrupt number
621 * @ptr: Pointer to interrupt-specific data (tsi721_device structure)
622 *
623 * Handles inbound messaging interrupts signaled using MSI-X.
624 */
625 static irqreturn_t tsi721_imsg_msix(int irq, void *ptr)
626 {
627 struct tsi721_device *priv = (struct tsi721_device *)ptr;
628 int mbox;
629
630 mbox = (irq - priv->msix[TSI721_VECT_IMB0_RCV].vector) % RIO_MAX_MBOX;
631 tsi721_imsg_handler(priv, mbox + 4);
632 return IRQ_HANDLED;
633 }
634
635 /**
636 * tsi721_srio_msix - Tsi721 MSI-X SRIO MAC interrupt handler
637 * @irq: Linux interrupt number
638 * @ptr: Pointer to interrupt-specific data (tsi721_device structure)
639 *
640 * Handles Tsi721 interrupts from SRIO MAC.
641 */
642 static irqreturn_t tsi721_srio_msix(int irq, void *ptr)
643 {
644 struct tsi721_device *priv = (struct tsi721_device *)ptr;
645 u32 srio_int;
646
647 /* Service SRIO MAC interrupts */
648 srio_int = ioread32(priv->regs + TSI721_RIO_EM_INT_STAT);
649 if (srio_int & TSI721_RIO_EM_INT_STAT_PW_RX)
650 tsi721_pw_handler(priv);
651
652 return IRQ_HANDLED;
653 }
654
655 /**
656 * tsi721_sr2pc_ch_msix - Tsi721 MSI-X SR2PC Channel interrupt handler
657 * @irq: Linux interrupt number
658 * @ptr: Pointer to interrupt-specific data (tsi721_device structure)
659 *
660 * Handles Tsi721 interrupts from SR2PC Channel.
661 * NOTE: At this moment services only one SR2PC channel associated with inbound
662 * doorbells.
663 */
664 static irqreturn_t tsi721_sr2pc_ch_msix(int irq, void *ptr)
665 {
666 struct tsi721_device *priv = (struct tsi721_device *)ptr;
667 u32 sr_ch_int;
668
669 /* Service Inbound DB interrupt from SR2PC channel */
670 sr_ch_int = ioread32(priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
671 if (sr_ch_int & TSI721_SR_CHINT_IDBQRCV)
672 tsi721_dbell_handler(priv);
673
674 /* Clear interrupts */
675 iowrite32(sr_ch_int, priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
676 /* Read back to ensure that interrupt was cleared */
677 sr_ch_int = ioread32(priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
678
679 return IRQ_HANDLED;
680 }
681
682 /**
683 * tsi721_request_msix - register interrupt service for MSI-X mode.
684 * @priv: tsi721 device-specific data structure
685 *
686 * Registers MSI-X interrupt service routines for interrupts that are active
687 * immediately after mport initialization. Messaging interrupt service routines
688 * should be registered during corresponding open requests.
689 */
690 static int tsi721_request_msix(struct tsi721_device *priv)
691 {
692 int err = 0;
693
694 err = request_irq(priv->msix[TSI721_VECT_IDB].vector,
695 tsi721_sr2pc_ch_msix, 0,
696 priv->msix[TSI721_VECT_IDB].irq_name, (void *)priv);
697 if (err)
698 return err;
699
700 err = request_irq(priv->msix[TSI721_VECT_PWRX].vector,
701 tsi721_srio_msix, 0,
702 priv->msix[TSI721_VECT_PWRX].irq_name, (void *)priv);
703 if (err) {
704 free_irq(priv->msix[TSI721_VECT_IDB].vector, (void *)priv);
705 return err;
706 }
707
708 return 0;
709 }
710
711 /**
712 * tsi721_enable_msix - Attempts to enable MSI-X support for Tsi721.
713 * @priv: pointer to tsi721 private data
714 *
715 * Configures MSI-X support for Tsi721. Supports only an exact number
716 * of requested vectors.
717 */
718 static int tsi721_enable_msix(struct tsi721_device *priv)
719 {
720 struct msix_entry entries[TSI721_VECT_MAX];
721 int err;
722 int i;
723
724 entries[TSI721_VECT_IDB].entry = TSI721_MSIX_SR2PC_IDBQ_RCV(IDB_QUEUE);
725 entries[TSI721_VECT_PWRX].entry = TSI721_MSIX_SRIO_MAC_INT;
726
727 /*
728 * Initialize MSI-X entries for Messaging Engine:
729 * this driver supports four RIO mailboxes (inbound and outbound)
730 * NOTE: Inbound message MBOX 0...4 use IB channels 4...7. Therefore
731 * offset +4 is added to IB MBOX number.
732 */
733 for (i = 0; i < RIO_MAX_MBOX; i++) {
734 entries[TSI721_VECT_IMB0_RCV + i].entry =
735 TSI721_MSIX_IMSG_DQ_RCV(i + 4);
736 entries[TSI721_VECT_IMB0_INT + i].entry =
737 TSI721_MSIX_IMSG_INT(i + 4);
738 entries[TSI721_VECT_OMB0_DONE + i].entry =
739 TSI721_MSIX_OMSG_DONE(i);
740 entries[TSI721_VECT_OMB0_INT + i].entry =
741 TSI721_MSIX_OMSG_INT(i);
742 }
743
744 #ifdef CONFIG_RAPIDIO_DMA_ENGINE
745 /*
746 * Initialize MSI-X entries for Block DMA Engine:
747 * this driver supports XXX DMA channels
748 * (one is reserved for SRIO maintenance transactions)
749 */
750 for (i = 0; i < TSI721_DMA_CHNUM; i++) {
751 entries[TSI721_VECT_DMA0_DONE + i].entry =
752 TSI721_MSIX_DMACH_DONE(i);
753 entries[TSI721_VECT_DMA0_INT + i].entry =
754 TSI721_MSIX_DMACH_INT(i);
755 }
756 #endif /* CONFIG_RAPIDIO_DMA_ENGINE */
757
758 err = pci_enable_msix_exact(priv->pdev, entries, ARRAY_SIZE(entries));
759 if (err) {
760 tsi_err(&priv->pdev->dev,
761 "Failed to enable MSI-X (err=%d)", err);
762 return err;
763 }
764
765 /*
766 * Copy MSI-X vector information into tsi721 private structure
767 */
768 priv->msix[TSI721_VECT_IDB].vector = entries[TSI721_VECT_IDB].vector;
769 snprintf(priv->msix[TSI721_VECT_IDB].irq_name, IRQ_DEVICE_NAME_MAX,
770 DRV_NAME "-idb@pci:%s", pci_name(priv->pdev));
771 priv->msix[TSI721_VECT_PWRX].vector = entries[TSI721_VECT_PWRX].vector;
772 snprintf(priv->msix[TSI721_VECT_PWRX].irq_name, IRQ_DEVICE_NAME_MAX,
773 DRV_NAME "-pwrx@pci:%s", pci_name(priv->pdev));
774
775 for (i = 0; i < RIO_MAX_MBOX; i++) {
776 priv->msix[TSI721_VECT_IMB0_RCV + i].vector =
777 entries[TSI721_VECT_IMB0_RCV + i].vector;
778 snprintf(priv->msix[TSI721_VECT_IMB0_RCV + i].irq_name,
779 IRQ_DEVICE_NAME_MAX, DRV_NAME "-imbr%d@pci:%s",
780 i, pci_name(priv->pdev));
781
782 priv->msix[TSI721_VECT_IMB0_INT + i].vector =
783 entries[TSI721_VECT_IMB0_INT + i].vector;
784 snprintf(priv->msix[TSI721_VECT_IMB0_INT + i].irq_name,
785 IRQ_DEVICE_NAME_MAX, DRV_NAME "-imbi%d@pci:%s",
786 i, pci_name(priv->pdev));
787
788 priv->msix[TSI721_VECT_OMB0_DONE + i].vector =
789 entries[TSI721_VECT_OMB0_DONE + i].vector;
790 snprintf(priv->msix[TSI721_VECT_OMB0_DONE + i].irq_name,
791 IRQ_DEVICE_NAME_MAX, DRV_NAME "-ombd%d@pci:%s",
792 i, pci_name(priv->pdev));
793
794 priv->msix[TSI721_VECT_OMB0_INT + i].vector =
795 entries[TSI721_VECT_OMB0_INT + i].vector;
796 snprintf(priv->msix[TSI721_VECT_OMB0_INT + i].irq_name,
797 IRQ_DEVICE_NAME_MAX, DRV_NAME "-ombi%d@pci:%s",
798 i, pci_name(priv->pdev));
799 }
800
801 #ifdef CONFIG_RAPIDIO_DMA_ENGINE
802 for (i = 0; i < TSI721_DMA_CHNUM; i++) {
803 priv->msix[TSI721_VECT_DMA0_DONE + i].vector =
804 entries[TSI721_VECT_DMA0_DONE + i].vector;
805 snprintf(priv->msix[TSI721_VECT_DMA0_DONE + i].irq_name,
806 IRQ_DEVICE_NAME_MAX, DRV_NAME "-dmad%d@pci:%s",
807 i, pci_name(priv->pdev));
808
809 priv->msix[TSI721_VECT_DMA0_INT + i].vector =
810 entries[TSI721_VECT_DMA0_INT + i].vector;
811 snprintf(priv->msix[TSI721_VECT_DMA0_INT + i].irq_name,
812 IRQ_DEVICE_NAME_MAX, DRV_NAME "-dmai%d@pci:%s",
813 i, pci_name(priv->pdev));
814 }
815 #endif /* CONFIG_RAPIDIO_DMA_ENGINE */
816
817 return 0;
818 }
819 #endif /* CONFIG_PCI_MSI */
820
821 static int tsi721_request_irq(struct tsi721_device *priv)
822 {
823 int err;
824
825 #ifdef CONFIG_PCI_MSI
826 if (priv->flags & TSI721_USING_MSIX)
827 err = tsi721_request_msix(priv);
828 else
829 #endif
830 err = request_irq(priv->pdev->irq, tsi721_irqhandler,
831 (priv->flags & TSI721_USING_MSI) ? 0 : IRQF_SHARED,
832 DRV_NAME, (void *)priv);
833
834 if (err)
835 tsi_err(&priv->pdev->dev,
836 "Unable to allocate interrupt, err=%d", err);
837
838 return err;
839 }
840
841 static void tsi721_free_irq(struct tsi721_device *priv)
842 {
843 #ifdef CONFIG_PCI_MSI
844 if (priv->flags & TSI721_USING_MSIX) {
845 free_irq(priv->msix[TSI721_VECT_IDB].vector, (void *)priv);
846 free_irq(priv->msix[TSI721_VECT_PWRX].vector, (void *)priv);
847 } else
848 #endif
849 free_irq(priv->pdev->irq, (void *)priv);
850 }
851
852 static int
853 tsi721_obw_alloc(struct tsi721_device *priv, struct tsi721_obw_bar *pbar,
854 u32 size, int *win_id)
855 {
856 u64 win_base;
857 u64 bar_base;
858 u64 bar_end;
859 u32 align;
860 struct tsi721_ob_win *win;
861 struct tsi721_ob_win *new_win = NULL;
862 int new_win_idx = -1;
863 int i = 0;
864
865 bar_base = pbar->base;
866 bar_end = bar_base + pbar->size;
867 win_base = bar_base;
868 align = size/TSI721_PC2SR_ZONES;
869
870 while (i < TSI721_IBWIN_NUM) {
871 for (i = 0; i < TSI721_IBWIN_NUM; i++) {
872 if (!priv->ob_win[i].active) {
873 if (new_win == NULL) {
874 new_win = &priv->ob_win[i];
875 new_win_idx = i;
876 }
877 continue;
878 }
879
880 /*
881 * If this window belongs to the current BAR check it
882 * for overlap
883 */
884 win = &priv->ob_win[i];
885
886 if (win->base >= bar_base && win->base < bar_end) {
887 if (win_base < (win->base + win->size) &&
888 (win_base + size) > win->base) {
889 /* Overlap detected */
890 win_base = win->base + win->size;
891 win_base = ALIGN(win_base, align);
892 break;
893 }
894 }
895 }
896 }
897
898 if (win_base + size > bar_end)
899 return -ENOMEM;
900
901 if (!new_win) {
902 tsi_err(&priv->pdev->dev, "OBW count tracking failed");
903 return -EIO;
904 }
905
906 new_win->active = true;
907 new_win->base = win_base;
908 new_win->size = size;
909 new_win->pbar = pbar;
910 priv->obwin_cnt--;
911 pbar->free -= size;
912 *win_id = new_win_idx;
913 return 0;
914 }
915
916 static int tsi721_map_outb_win(struct rio_mport *mport, u16 destid, u64 rstart,
917 u32 size, u32 flags, dma_addr_t *laddr)
918 {
919 struct tsi721_device *priv = mport->priv;
920 int i;
921 struct tsi721_obw_bar *pbar;
922 struct tsi721_ob_win *ob_win;
923 int obw = -1;
924 u32 rval;
925 u64 rio_addr;
926 u32 zsize;
927 int ret = -ENOMEM;
928
929 tsi_debug(OBW, &priv->pdev->dev,
930 "did=%d ra=0x%llx sz=0x%x", destid, rstart, size);
931
932 if (!is_power_of_2(size) || (size < 0x8000) || (rstart & (size - 1)))
933 return -EINVAL;
934
935 if (priv->obwin_cnt == 0)
936 return -EBUSY;
937
938 for (i = 0; i < 2; i++) {
939 if (priv->p2r_bar[i].free >= size) {
940 pbar = &priv->p2r_bar[i];
941 ret = tsi721_obw_alloc(priv, pbar, size, &obw);
942 if (!ret)
943 break;
944 }
945 }
946
947 if (ret)
948 return ret;
949
950 WARN_ON(obw == -1);
951 ob_win = &priv->ob_win[obw];
952 ob_win->destid = destid;
953 ob_win->rstart = rstart;
954 tsi_debug(OBW, &priv->pdev->dev,
955 "allocated OBW%d @%llx", obw, ob_win->base);
956
957 /*
958 * Configure Outbound Window
959 */
960
961 zsize = size/TSI721_PC2SR_ZONES;
962 rio_addr = rstart;
963
964 /*
965 * Program Address Translation Zones:
966 * This implementation uses all 8 zones associated wit window.
967 */
968 for (i = 0; i < TSI721_PC2SR_ZONES; i++) {
969
970 while (ioread32(priv->regs + TSI721_ZONE_SEL) &
971 TSI721_ZONE_SEL_GO) {
972 udelay(1);
973 }
974
975 rval = (u32)(rio_addr & TSI721_LUT_DATA0_ADD) |
976 TSI721_LUT_DATA0_NREAD | TSI721_LUT_DATA0_NWR;
977 iowrite32(rval, priv->regs + TSI721_LUT_DATA0);
978 rval = (u32)(rio_addr >> 32);
979 iowrite32(rval, priv->regs + TSI721_LUT_DATA1);
980 rval = destid;
981 iowrite32(rval, priv->regs + TSI721_LUT_DATA2);
982
983 rval = TSI721_ZONE_SEL_GO | (obw << 3) | i;
984 iowrite32(rval, priv->regs + TSI721_ZONE_SEL);
985
986 rio_addr += zsize;
987 }
988
989 iowrite32(TSI721_OBWIN_SIZE(size) << 8,
990 priv->regs + TSI721_OBWINSZ(obw));
991 iowrite32((u32)(ob_win->base >> 32), priv->regs + TSI721_OBWINUB(obw));
992 iowrite32((u32)(ob_win->base & TSI721_OBWINLB_BA) | TSI721_OBWINLB_WEN,
993 priv->regs + TSI721_OBWINLB(obw));
994
995 *laddr = ob_win->base;
996 return 0;
997 }
998
999 static void tsi721_unmap_outb_win(struct rio_mport *mport,
1000 u16 destid, u64 rstart)
1001 {
1002 struct tsi721_device *priv = mport->priv;
1003 struct tsi721_ob_win *ob_win;
1004 int i;
1005
1006 tsi_debug(OBW, &priv->pdev->dev, "did=%d ra=0x%llx", destid, rstart);
1007
1008 for (i = 0; i < TSI721_OBWIN_NUM; i++) {
1009 ob_win = &priv->ob_win[i];
1010
1011 if (ob_win->active &&
1012 ob_win->destid == destid && ob_win->rstart == rstart) {
1013 tsi_debug(OBW, &priv->pdev->dev,
1014 "free OBW%d @%llx", i, ob_win->base);
1015 ob_win->active = false;
1016 iowrite32(0, priv->regs + TSI721_OBWINLB(i));
1017 ob_win->pbar->free += ob_win->size;
1018 priv->obwin_cnt++;
1019 break;
1020 }
1021 }
1022 }
1023
1024 /**
1025 * tsi721_init_pc2sr_mapping - initializes outbound (PCIe->SRIO)
1026 * translation regions.
1027 * @priv: pointer to tsi721 private data
1028 *
1029 * Disables SREP translation regions.
1030 */
1031 static void tsi721_init_pc2sr_mapping(struct tsi721_device *priv)
1032 {
1033 int i, z;
1034 u32 rval;
1035
1036 /* Disable all PC2SR translation windows */
1037 for (i = 0; i < TSI721_OBWIN_NUM; i++)
1038 iowrite32(0, priv->regs + TSI721_OBWINLB(i));
1039
1040 /* Initialize zone lookup tables to avoid ECC errors on reads */
1041 iowrite32(0, priv->regs + TSI721_LUT_DATA0);
1042 iowrite32(0, priv->regs + TSI721_LUT_DATA1);
1043 iowrite32(0, priv->regs + TSI721_LUT_DATA2);
1044
1045 for (i = 0; i < TSI721_OBWIN_NUM; i++) {
1046 for (z = 0; z < TSI721_PC2SR_ZONES; z++) {
1047 while (ioread32(priv->regs + TSI721_ZONE_SEL) &
1048 TSI721_ZONE_SEL_GO) {
1049 udelay(1);
1050 }
1051 rval = TSI721_ZONE_SEL_GO | (i << 3) | z;
1052 iowrite32(rval, priv->regs + TSI721_ZONE_SEL);
1053 }
1054 }
1055
1056 if (priv->p2r_bar[0].size == 0 && priv->p2r_bar[1].size == 0) {
1057 priv->obwin_cnt = 0;
1058 return;
1059 }
1060
1061 priv->p2r_bar[0].free = priv->p2r_bar[0].size;
1062 priv->p2r_bar[1].free = priv->p2r_bar[1].size;
1063
1064 for (i = 0; i < TSI721_OBWIN_NUM; i++)
1065 priv->ob_win[i].active = false;
1066
1067 priv->obwin_cnt = TSI721_OBWIN_NUM;
1068 }
1069
1070 /**
1071 * tsi721_rio_map_inb_mem -- Mapping inbound memory region.
1072 * @mport: RapidIO master port
1073 * @lstart: Local memory space start address.
1074 * @rstart: RapidIO space start address.
1075 * @size: The mapping region size.
1076 * @flags: Flags for mapping. 0 for using default flags.
1077 *
1078 * Return: 0 -- Success.
1079 *
1080 * This function will create the inbound mapping
1081 * from rstart to lstart.
1082 */
1083 static int tsi721_rio_map_inb_mem(struct rio_mport *mport, dma_addr_t lstart,
1084 u64 rstart, u32 size, u32 flags)
1085 {
1086 struct tsi721_device *priv = mport->priv;
1087 int i, avail = -1;
1088 u32 regval;
1089 struct tsi721_ib_win *ib_win;
1090 bool direct = (lstart == rstart);
1091 u64 ibw_size;
1092 dma_addr_t loc_start;
1093 u64 ibw_start;
1094 struct tsi721_ib_win_mapping *map = NULL;
1095 int ret = -EBUSY;
1096
1097 if (direct) {
1098 /* Calculate minimal acceptable window size and base address */
1099
1100 ibw_size = roundup_pow_of_two(size);
1101 ibw_start = lstart & ~(ibw_size - 1);
1102
1103 tsi_debug(IBW, &priv->pdev->dev,
1104 "Direct (RIO_0x%llx -> PCIe_0x%pad), size=0x%x, ibw_start = 0x%llx",
1105 rstart, &lstart, size, ibw_start);
1106
1107 while ((lstart + size) > (ibw_start + ibw_size)) {
1108 ibw_size *= 2;
1109 ibw_start = lstart & ~(ibw_size - 1);
1110 if (ibw_size > 0x80000000) { /* Limit max size to 2GB */
1111 return -EBUSY;
1112 }
1113 }
1114
1115 loc_start = ibw_start;
1116
1117 map = kzalloc(sizeof(struct tsi721_ib_win_mapping), GFP_ATOMIC);
1118 if (map == NULL)
1119 return -ENOMEM;
1120
1121 } else {
1122 tsi_debug(IBW, &priv->pdev->dev,
1123 "Translated (RIO_0x%llx -> PCIe_0x%pad), size=0x%x",
1124 rstart, &lstart, size);
1125
1126 if (!is_power_of_2(size) || size < 0x1000 ||
1127 ((u64)lstart & (size - 1)) || (rstart & (size - 1)))
1128 return -EINVAL;
1129 if (priv->ibwin_cnt == 0)
1130 return -EBUSY;
1131 ibw_start = rstart;
1132 ibw_size = size;
1133 loc_start = lstart;
1134 }
1135
1136 /*
1137 * Scan for overlapping with active regions and mark the first available
1138 * IB window at the same time.
1139 */
1140 for (i = 0; i < TSI721_IBWIN_NUM; i++) {
1141 ib_win = &priv->ib_win[i];
1142
1143 if (!ib_win->active) {
1144 if (avail == -1) {
1145 avail = i;
1146 ret = 0;
1147 }
1148 } else if (ibw_start < (ib_win->rstart + ib_win->size) &&
1149 (ibw_start + ibw_size) > ib_win->rstart) {
1150 /* Return error if address translation involved */
1151 if (direct && ib_win->xlat) {
1152 ret = -EFAULT;
1153 break;
1154 }
1155
1156 /*
1157 * Direct mappings usually are larger than originally
1158 * requested fragments - check if this new request fits
1159 * into it.
1160 */
1161 if (rstart >= ib_win->rstart &&
1162 (rstart + size) <= (ib_win->rstart +
1163 ib_win->size)) {
1164 /* We are in - no further mapping required */
1165 map->lstart = lstart;
1166 list_add_tail(&map->node, &ib_win->mappings);
1167 return 0;
1168 }
1169
1170 ret = -EFAULT;
1171 break;
1172 }
1173 }
1174
1175 if (ret)
1176 goto out;
1177 i = avail;
1178
1179 /* Sanity check: available IB window must be disabled at this point */
1180 regval = ioread32(priv->regs + TSI721_IBWIN_LB(i));
1181 if (WARN_ON(regval & TSI721_IBWIN_LB_WEN)) {
1182 ret = -EIO;
1183 goto out;
1184 }
1185
1186 ib_win = &priv->ib_win[i];
1187 ib_win->active = true;
1188 ib_win->rstart = ibw_start;
1189 ib_win->lstart = loc_start;
1190 ib_win->size = ibw_size;
1191 ib_win->xlat = (lstart != rstart);
1192 INIT_LIST_HEAD(&ib_win->mappings);
1193
1194 /*
1195 * When using direct IBW mapping and have larger than requested IBW size
1196 * we can have multiple local memory blocks mapped through the same IBW
1197 * To handle this situation we maintain list of "clients" for such IBWs.
1198 */
1199 if (direct) {
1200 map->lstart = lstart;
1201 list_add_tail(&map->node, &ib_win->mappings);
1202 }
1203
1204 iowrite32(TSI721_IBWIN_SIZE(ibw_size) << 8,
1205 priv->regs + TSI721_IBWIN_SZ(i));
1206
1207 iowrite32(((u64)loc_start >> 32), priv->regs + TSI721_IBWIN_TUA(i));
1208 iowrite32(((u64)loc_start & TSI721_IBWIN_TLA_ADD),
1209 priv->regs + TSI721_IBWIN_TLA(i));
1210
1211 iowrite32(ibw_start >> 32, priv->regs + TSI721_IBWIN_UB(i));
1212 iowrite32((ibw_start & TSI721_IBWIN_LB_BA) | TSI721_IBWIN_LB_WEN,
1213 priv->regs + TSI721_IBWIN_LB(i));
1214
1215 priv->ibwin_cnt--;
1216
1217 tsi_debug(IBW, &priv->pdev->dev,
1218 "Configured IBWIN%d (RIO_0x%llx -> PCIe_0x%pad), size=0x%llx",
1219 i, ibw_start, &loc_start, ibw_size);
1220
1221 return 0;
1222 out:
1223 kfree(map);
1224 return ret;
1225 }
1226
1227 /**
1228 * tsi721_rio_unmap_inb_mem -- Unmapping inbound memory region.
1229 * @mport: RapidIO master port
1230 * @lstart: Local memory space start address.
1231 */
1232 static void tsi721_rio_unmap_inb_mem(struct rio_mport *mport,
1233 dma_addr_t lstart)
1234 {
1235 struct tsi721_device *priv = mport->priv;
1236 struct tsi721_ib_win *ib_win;
1237 int i;
1238
1239 tsi_debug(IBW, &priv->pdev->dev,
1240 "Unmap IBW mapped to PCIe_0x%pad", &lstart);
1241
1242 /* Search for matching active inbound translation window */
1243 for (i = 0; i < TSI721_IBWIN_NUM; i++) {
1244 ib_win = &priv->ib_win[i];
1245
1246 /* Address translating IBWs must to be an exact march */
1247 if (!ib_win->active ||
1248 (ib_win->xlat && lstart != ib_win->lstart))
1249 continue;
1250
1251 if (lstart >= ib_win->lstart &&
1252 lstart < (ib_win->lstart + ib_win->size)) {
1253
1254 if (!ib_win->xlat) {
1255 struct tsi721_ib_win_mapping *map;
1256 int found = 0;
1257
1258 list_for_each_entry(map,
1259 &ib_win->mappings, node) {
1260 if (map->lstart == lstart) {
1261 list_del(&map->node);
1262 kfree(map);
1263 found = 1;
1264 break;
1265 }
1266 }
1267
1268 if (!found)
1269 continue;
1270
1271 if (!list_empty(&ib_win->mappings))
1272 break;
1273 }
1274
1275 tsi_debug(IBW, &priv->pdev->dev, "Disable IBWIN_%d", i);
1276 iowrite32(0, priv->regs + TSI721_IBWIN_LB(i));
1277 ib_win->active = false;
1278 priv->ibwin_cnt++;
1279 break;
1280 }
1281 }
1282
1283 if (i == TSI721_IBWIN_NUM)
1284 tsi_debug(IBW, &priv->pdev->dev,
1285 "IB window mapped to %pad not found", &lstart);
1286 }
1287
1288 /**
1289 * tsi721_init_sr2pc_mapping - initializes inbound (SRIO->PCIe)
1290 * translation regions.
1291 * @priv: pointer to tsi721 private data
1292 *
1293 * Disables inbound windows.
1294 */
1295 static void tsi721_init_sr2pc_mapping(struct tsi721_device *priv)
1296 {
1297 int i;
1298
1299 /* Disable all SR2PC inbound windows */
1300 for (i = 0; i < TSI721_IBWIN_NUM; i++)
1301 iowrite32(0, priv->regs + TSI721_IBWIN_LB(i));
1302 priv->ibwin_cnt = TSI721_IBWIN_NUM;
1303 }
1304
1305 /*
1306 * tsi721_close_sr2pc_mapping - closes all active inbound (SRIO->PCIe)
1307 * translation regions.
1308 * @priv: pointer to tsi721 device private data
1309 */
1310 static void tsi721_close_sr2pc_mapping(struct tsi721_device *priv)
1311 {
1312 struct tsi721_ib_win *ib_win;
1313 int i;
1314
1315 /* Disable all active SR2PC inbound windows */
1316 for (i = 0; i < TSI721_IBWIN_NUM; i++) {
1317 ib_win = &priv->ib_win[i];
1318 if (ib_win->active) {
1319 iowrite32(0, priv->regs + TSI721_IBWIN_LB(i));
1320 ib_win->active = false;
1321 }
1322 }
1323 }
1324
1325 /**
1326 * tsi721_port_write_init - Inbound port write interface init
1327 * @priv: pointer to tsi721 private data
1328 *
1329 * Initializes inbound port write handler.
1330 * Returns %0 on success or %-ENOMEM on failure.
1331 */
1332 static int tsi721_port_write_init(struct tsi721_device *priv)
1333 {
1334 priv->pw_discard_count = 0;
1335 INIT_WORK(&priv->pw_work, tsi721_pw_dpc);
1336 spin_lock_init(&priv->pw_fifo_lock);
1337 if (kfifo_alloc(&priv->pw_fifo,
1338 TSI721_RIO_PW_MSG_SIZE * 32, GFP_KERNEL)) {
1339 tsi_err(&priv->pdev->dev, "PW FIFO allocation failed");
1340 return -ENOMEM;
1341 }
1342
1343 /* Use reliable port-write capture mode */
1344 iowrite32(TSI721_RIO_PW_CTL_PWC_REL, priv->regs + TSI721_RIO_PW_CTL);
1345 return 0;
1346 }
1347
1348 static void tsi721_port_write_free(struct tsi721_device *priv)
1349 {
1350 kfifo_free(&priv->pw_fifo);
1351 }
1352
1353 static int tsi721_doorbell_init(struct tsi721_device *priv)
1354 {
1355 /* Outbound Doorbells do not require any setup.
1356 * Tsi721 uses dedicated PCI BAR1 to generate doorbells.
1357 * That BAR1 was mapped during the probe routine.
1358 */
1359
1360 /* Initialize Inbound Doorbell processing DPC and queue */
1361 priv->db_discard_count = 0;
1362 INIT_WORK(&priv->idb_work, tsi721_db_dpc);
1363
1364 /* Allocate buffer for inbound doorbells queue */
1365 priv->idb_base = dma_zalloc_coherent(&priv->pdev->dev,
1366 IDB_QSIZE * TSI721_IDB_ENTRY_SIZE,
1367 &priv->idb_dma, GFP_KERNEL);
1368 if (!priv->idb_base)
1369 return -ENOMEM;
1370
1371 tsi_debug(DBELL, &priv->pdev->dev,
1372 "Allocated IDB buffer @ %p (phys = %pad)",
1373 priv->idb_base, &priv->idb_dma);
1374
1375 iowrite32(TSI721_IDQ_SIZE_VAL(IDB_QSIZE),
1376 priv->regs + TSI721_IDQ_SIZE(IDB_QUEUE));
1377 iowrite32(((u64)priv->idb_dma >> 32),
1378 priv->regs + TSI721_IDQ_BASEU(IDB_QUEUE));
1379 iowrite32(((u64)priv->idb_dma & TSI721_IDQ_BASEL_ADDR),
1380 priv->regs + TSI721_IDQ_BASEL(IDB_QUEUE));
1381 /* Enable accepting all inbound doorbells */
1382 iowrite32(0, priv->regs + TSI721_IDQ_MASK(IDB_QUEUE));
1383
1384 iowrite32(TSI721_IDQ_INIT, priv->regs + TSI721_IDQ_CTL(IDB_QUEUE));
1385
1386 iowrite32(0, priv->regs + TSI721_IDQ_RP(IDB_QUEUE));
1387
1388 return 0;
1389 }
1390
1391 static void tsi721_doorbell_free(struct tsi721_device *priv)
1392 {
1393 if (priv->idb_base == NULL)
1394 return;
1395
1396 /* Free buffer allocated for inbound doorbell queue */
1397 dma_free_coherent(&priv->pdev->dev, IDB_QSIZE * TSI721_IDB_ENTRY_SIZE,
1398 priv->idb_base, priv->idb_dma);
1399 priv->idb_base = NULL;
1400 }
1401
1402 /**
1403 * tsi721_bdma_maint_init - Initialize maintenance request BDMA channel.
1404 * @priv: pointer to tsi721 private data
1405 *
1406 * Initialize BDMA channel allocated for RapidIO maintenance read/write
1407 * request generation
1408 * Returns %0 on success or %-ENOMEM on failure.
1409 */
1410 static int tsi721_bdma_maint_init(struct tsi721_device *priv)
1411 {
1412 struct tsi721_dma_desc *bd_ptr;
1413 u64 *sts_ptr;
1414 dma_addr_t bd_phys, sts_phys;
1415 int sts_size;
1416 int bd_num = 2;
1417 void __iomem *regs;
1418
1419 tsi_debug(MAINT, &priv->pdev->dev,
1420 "Init BDMA_%d Maintenance requests", TSI721_DMACH_MAINT);
1421
1422 /*
1423 * Initialize DMA channel for maintenance requests
1424 */
1425
1426 priv->mdma.ch_id = TSI721_DMACH_MAINT;
1427 regs = priv->regs + TSI721_DMAC_BASE(TSI721_DMACH_MAINT);
1428
1429 /* Allocate space for DMA descriptors */
1430 bd_ptr = dma_zalloc_coherent(&priv->pdev->dev,
1431 bd_num * sizeof(struct tsi721_dma_desc),
1432 &bd_phys, GFP_KERNEL);
1433 if (!bd_ptr)
1434 return -ENOMEM;
1435
1436 priv->mdma.bd_num = bd_num;
1437 priv->mdma.bd_phys = bd_phys;
1438 priv->mdma.bd_base = bd_ptr;
1439
1440 tsi_debug(MAINT, &priv->pdev->dev, "DMA descriptors @ %p (phys = %pad)",
1441 bd_ptr, &bd_phys);
1442
1443 /* Allocate space for descriptor status FIFO */
1444 sts_size = (bd_num >= TSI721_DMA_MINSTSSZ) ?
1445 bd_num : TSI721_DMA_MINSTSSZ;
1446 sts_size = roundup_pow_of_two(sts_size);
1447 sts_ptr = dma_zalloc_coherent(&priv->pdev->dev,
1448 sts_size * sizeof(struct tsi721_dma_sts),
1449 &sts_phys, GFP_KERNEL);
1450 if (!sts_ptr) {
1451 /* Free space allocated for DMA descriptors */
1452 dma_free_coherent(&priv->pdev->dev,
1453 bd_num * sizeof(struct tsi721_dma_desc),
1454 bd_ptr, bd_phys);
1455 priv->mdma.bd_base = NULL;
1456 return -ENOMEM;
1457 }
1458
1459 priv->mdma.sts_phys = sts_phys;
1460 priv->mdma.sts_base = sts_ptr;
1461 priv->mdma.sts_size = sts_size;
1462
1463 tsi_debug(MAINT, &priv->pdev->dev,
1464 "desc status FIFO @ %p (phys = %pad) size=0x%x",
1465 sts_ptr, &sts_phys, sts_size);
1466
1467 /* Initialize DMA descriptors ring */
1468 bd_ptr[bd_num - 1].type_id = cpu_to_le32(DTYPE3 << 29);
1469 bd_ptr[bd_num - 1].next_lo = cpu_to_le32((u64)bd_phys &
1470 TSI721_DMAC_DPTRL_MASK);
1471 bd_ptr[bd_num - 1].next_hi = cpu_to_le32((u64)bd_phys >> 32);
1472
1473 /* Setup DMA descriptor pointers */
1474 iowrite32(((u64)bd_phys >> 32), regs + TSI721_DMAC_DPTRH);
1475 iowrite32(((u64)bd_phys & TSI721_DMAC_DPTRL_MASK),
1476 regs + TSI721_DMAC_DPTRL);
1477
1478 /* Setup descriptor status FIFO */
1479 iowrite32(((u64)sts_phys >> 32), regs + TSI721_DMAC_DSBH);
1480 iowrite32(((u64)sts_phys & TSI721_DMAC_DSBL_MASK),
1481 regs + TSI721_DMAC_DSBL);
1482 iowrite32(TSI721_DMAC_DSSZ_SIZE(sts_size),
1483 regs + TSI721_DMAC_DSSZ);
1484
1485 /* Clear interrupt bits */
1486 iowrite32(TSI721_DMAC_INT_ALL, regs + TSI721_DMAC_INT);
1487
1488 ioread32(regs + TSI721_DMAC_INT);
1489
1490 /* Toggle DMA channel initialization */
1491 iowrite32(TSI721_DMAC_CTL_INIT, regs + TSI721_DMAC_CTL);
1492 ioread32(regs + TSI721_DMAC_CTL);
1493 udelay(10);
1494
1495 return 0;
1496 }
1497
1498 static int tsi721_bdma_maint_free(struct tsi721_device *priv)
1499 {
1500 u32 ch_stat;
1501 struct tsi721_bdma_maint *mdma = &priv->mdma;
1502 void __iomem *regs = priv->regs + TSI721_DMAC_BASE(mdma->ch_id);
1503
1504 if (mdma->bd_base == NULL)
1505 return 0;
1506
1507 /* Check if DMA channel still running */
1508 ch_stat = ioread32(regs + TSI721_DMAC_STS);
1509 if (ch_stat & TSI721_DMAC_STS_RUN)
1510 return -EFAULT;
1511
1512 /* Put DMA channel into init state */
1513 iowrite32(TSI721_DMAC_CTL_INIT, regs + TSI721_DMAC_CTL);
1514
1515 /* Free space allocated for DMA descriptors */
1516 dma_free_coherent(&priv->pdev->dev,
1517 mdma->bd_num * sizeof(struct tsi721_dma_desc),
1518 mdma->bd_base, mdma->bd_phys);
1519 mdma->bd_base = NULL;
1520
1521 /* Free space allocated for status FIFO */
1522 dma_free_coherent(&priv->pdev->dev,
1523 mdma->sts_size * sizeof(struct tsi721_dma_sts),
1524 mdma->sts_base, mdma->sts_phys);
1525 mdma->sts_base = NULL;
1526 return 0;
1527 }
1528
1529 /* Enable Inbound Messaging Interrupts */
1530 static void
1531 tsi721_imsg_interrupt_enable(struct tsi721_device *priv, int ch,
1532 u32 inte_mask)
1533 {
1534 u32 rval;
1535
1536 if (!inte_mask)
1537 return;
1538
1539 /* Clear pending Inbound Messaging interrupts */
1540 iowrite32(inte_mask, priv->regs + TSI721_IBDMAC_INT(ch));
1541
1542 /* Enable Inbound Messaging interrupts */
1543 rval = ioread32(priv->regs + TSI721_IBDMAC_INTE(ch));
1544 iowrite32(rval | inte_mask, priv->regs + TSI721_IBDMAC_INTE(ch));
1545
1546 if (priv->flags & TSI721_USING_MSIX)
1547 return; /* Finished if we are in MSI-X mode */
1548
1549 /*
1550 * For MSI and INTA interrupt signalling we need to enable next levels
1551 */
1552
1553 /* Enable Device Channel Interrupt */
1554 rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1555 iowrite32(rval | TSI721_INT_IMSG_CHAN(ch),
1556 priv->regs + TSI721_DEV_CHAN_INTE);
1557 }
1558
1559 /* Disable Inbound Messaging Interrupts */
1560 static void
1561 tsi721_imsg_interrupt_disable(struct tsi721_device *priv, int ch,
1562 u32 inte_mask)
1563 {
1564 u32 rval;
1565
1566 if (!inte_mask)
1567 return;
1568
1569 /* Clear pending Inbound Messaging interrupts */
1570 iowrite32(inte_mask, priv->regs + TSI721_IBDMAC_INT(ch));
1571
1572 /* Disable Inbound Messaging interrupts */
1573 rval = ioread32(priv->regs + TSI721_IBDMAC_INTE(ch));
1574 rval &= ~inte_mask;
1575 iowrite32(rval, priv->regs + TSI721_IBDMAC_INTE(ch));
1576
1577 if (priv->flags & TSI721_USING_MSIX)
1578 return; /* Finished if we are in MSI-X mode */
1579
1580 /*
1581 * For MSI and INTA interrupt signalling we need to disable next levels
1582 */
1583
1584 /* Disable Device Channel Interrupt */
1585 rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1586 rval &= ~TSI721_INT_IMSG_CHAN(ch);
1587 iowrite32(rval, priv->regs + TSI721_DEV_CHAN_INTE);
1588 }
1589
1590 /* Enable Outbound Messaging interrupts */
1591 static void
1592 tsi721_omsg_interrupt_enable(struct tsi721_device *priv, int ch,
1593 u32 inte_mask)
1594 {
1595 u32 rval;
1596
1597 if (!inte_mask)
1598 return;
1599
1600 /* Clear pending Outbound Messaging interrupts */
1601 iowrite32(inte_mask, priv->regs + TSI721_OBDMAC_INT(ch));
1602
1603 /* Enable Outbound Messaging channel interrupts */
1604 rval = ioread32(priv->regs + TSI721_OBDMAC_INTE(ch));
1605 iowrite32(rval | inte_mask, priv->regs + TSI721_OBDMAC_INTE(ch));
1606
1607 if (priv->flags & TSI721_USING_MSIX)
1608 return; /* Finished if we are in MSI-X mode */
1609
1610 /*
1611 * For MSI and INTA interrupt signalling we need to enable next levels
1612 */
1613
1614 /* Enable Device Channel Interrupt */
1615 rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1616 iowrite32(rval | TSI721_INT_OMSG_CHAN(ch),
1617 priv->regs + TSI721_DEV_CHAN_INTE);
1618 }
1619
1620 /* Disable Outbound Messaging interrupts */
1621 static void
1622 tsi721_omsg_interrupt_disable(struct tsi721_device *priv, int ch,
1623 u32 inte_mask)
1624 {
1625 u32 rval;
1626
1627 if (!inte_mask)
1628 return;
1629
1630 /* Clear pending Outbound Messaging interrupts */
1631 iowrite32(inte_mask, priv->regs + TSI721_OBDMAC_INT(ch));
1632
1633 /* Disable Outbound Messaging interrupts */
1634 rval = ioread32(priv->regs + TSI721_OBDMAC_INTE(ch));
1635 rval &= ~inte_mask;
1636 iowrite32(rval, priv->regs + TSI721_OBDMAC_INTE(ch));
1637
1638 if (priv->flags & TSI721_USING_MSIX)
1639 return; /* Finished if we are in MSI-X mode */
1640
1641 /*
1642 * For MSI and INTA interrupt signalling we need to disable next levels
1643 */
1644
1645 /* Disable Device Channel Interrupt */
1646 rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1647 rval &= ~TSI721_INT_OMSG_CHAN(ch);
1648 iowrite32(rval, priv->regs + TSI721_DEV_CHAN_INTE);
1649 }
1650
1651 /**
1652 * tsi721_add_outb_message - Add message to the Tsi721 outbound message queue
1653 * @mport: Master port with outbound message queue
1654 * @rdev: Target of outbound message
1655 * @mbox: Outbound mailbox
1656 * @buffer: Message to add to outbound queue
1657 * @len: Length of message
1658 */
1659 static int
1660 tsi721_add_outb_message(struct rio_mport *mport, struct rio_dev *rdev, int mbox,
1661 void *buffer, size_t len)
1662 {
1663 struct tsi721_device *priv = mport->priv;
1664 struct tsi721_omsg_desc *desc;
1665 u32 tx_slot;
1666 unsigned long flags;
1667
1668 if (!priv->omsg_init[mbox] ||
1669 len > TSI721_MSG_MAX_SIZE || len < 8)
1670 return -EINVAL;
1671
1672 spin_lock_irqsave(&priv->omsg_ring[mbox].lock, flags);
1673
1674 tx_slot = priv->omsg_ring[mbox].tx_slot;
1675
1676 /* Copy copy message into transfer buffer */
1677 memcpy(priv->omsg_ring[mbox].omq_base[tx_slot], buffer, len);
1678
1679 if (len & 0x7)
1680 len += 8;
1681
1682 /* Build descriptor associated with buffer */
1683 desc = priv->omsg_ring[mbox].omd_base;
1684 desc[tx_slot].type_id = cpu_to_le32((DTYPE4 << 29) | rdev->destid);
1685 #ifdef TSI721_OMSG_DESC_INT
1686 /* Request IOF_DONE interrupt generation for each N-th frame in queue */
1687 if (tx_slot % 4 == 0)
1688 desc[tx_slot].type_id |= cpu_to_le32(TSI721_OMD_IOF);
1689 #endif
1690 desc[tx_slot].msg_info =
1691 cpu_to_le32((mport->sys_size << 26) | (mbox << 22) |
1692 (0xe << 12) | (len & 0xff8));
1693 desc[tx_slot].bufptr_lo =
1694 cpu_to_le32((u64)priv->omsg_ring[mbox].omq_phys[tx_slot] &
1695 0xffffffff);
1696 desc[tx_slot].bufptr_hi =
1697 cpu_to_le32((u64)priv->omsg_ring[mbox].omq_phys[tx_slot] >> 32);
1698
1699 priv->omsg_ring[mbox].wr_count++;
1700
1701 /* Go to next descriptor */
1702 if (++priv->omsg_ring[mbox].tx_slot == priv->omsg_ring[mbox].size) {
1703 priv->omsg_ring[mbox].tx_slot = 0;
1704 /* Move through the ring link descriptor at the end */
1705 priv->omsg_ring[mbox].wr_count++;
1706 }
1707
1708 mb();
1709
1710 /* Set new write count value */
1711 iowrite32(priv->omsg_ring[mbox].wr_count,
1712 priv->regs + TSI721_OBDMAC_DWRCNT(mbox));
1713 ioread32(priv->regs + TSI721_OBDMAC_DWRCNT(mbox));
1714
1715 spin_unlock_irqrestore(&priv->omsg_ring[mbox].lock, flags);
1716
1717 return 0;
1718 }
1719
1720 /**
1721 * tsi721_omsg_handler - Outbound Message Interrupt Handler
1722 * @priv: pointer to tsi721 private data
1723 * @ch: number of OB MSG channel to service
1724 *
1725 * Services channel interrupts from outbound messaging engine.
1726 */
1727 static void tsi721_omsg_handler(struct tsi721_device *priv, int ch)
1728 {
1729 u32 omsg_int;
1730 struct rio_mport *mport = &priv->mport;
1731 void *dev_id = NULL;
1732 u32 tx_slot = 0xffffffff;
1733 int do_callback = 0;
1734
1735 spin_lock(&priv->omsg_ring[ch].lock);
1736
1737 omsg_int = ioread32(priv->regs + TSI721_OBDMAC_INT(ch));
1738
1739 if (omsg_int & TSI721_OBDMAC_INT_ST_FULL)
1740 tsi_info(&priv->pdev->dev,
1741 "OB MBOX%d: Status FIFO is full", ch);
1742
1743 if (omsg_int & (TSI721_OBDMAC_INT_DONE | TSI721_OBDMAC_INT_IOF_DONE)) {
1744 u32 srd_ptr;
1745 u64 *sts_ptr, last_ptr = 0, prev_ptr = 0;
1746 int i, j;
1747
1748 /*
1749 * Find last successfully processed descriptor
1750 */
1751
1752 /* Check and clear descriptor status FIFO entries */
1753 srd_ptr = priv->omsg_ring[ch].sts_rdptr;
1754 sts_ptr = priv->omsg_ring[ch].sts_base;
1755 j = srd_ptr * 8;
1756 while (sts_ptr[j]) {
1757 for (i = 0; i < 8 && sts_ptr[j]; i++, j++) {
1758 prev_ptr = last_ptr;
1759 last_ptr = le64_to_cpu(sts_ptr[j]);
1760 sts_ptr[j] = 0;
1761 }
1762
1763 ++srd_ptr;
1764 srd_ptr %= priv->omsg_ring[ch].sts_size;
1765 j = srd_ptr * 8;
1766 }
1767
1768 if (last_ptr == 0)
1769 goto no_sts_update;
1770
1771 priv->omsg_ring[ch].sts_rdptr = srd_ptr;
1772 iowrite32(srd_ptr, priv->regs + TSI721_OBDMAC_DSRP(ch));
1773
1774 if (!mport->outb_msg[ch].mcback)
1775 goto no_sts_update;
1776
1777 /* Inform upper layer about transfer completion */
1778
1779 tx_slot = (last_ptr - (u64)priv->omsg_ring[ch].omd_phys)/
1780 sizeof(struct tsi721_omsg_desc);
1781
1782 /*
1783 * Check if this is a Link Descriptor (LD).
1784 * If yes, ignore LD and use descriptor processed
1785 * before LD.
1786 */
1787 if (tx_slot == priv->omsg_ring[ch].size) {
1788 if (prev_ptr)
1789 tx_slot = (prev_ptr -
1790 (u64)priv->omsg_ring[ch].omd_phys)/
1791 sizeof(struct tsi721_omsg_desc);
1792 else
1793 goto no_sts_update;
1794 }
1795
1796 if (tx_slot >= priv->omsg_ring[ch].size)
1797 tsi_debug(OMSG, &priv->pdev->dev,
1798 "OB_MSG tx_slot=%x > size=%x",
1799 tx_slot, priv->omsg_ring[ch].size);
1800 WARN_ON(tx_slot >= priv->omsg_ring[ch].size);
1801
1802 /* Move slot index to the next message to be sent */
1803 ++tx_slot;
1804 if (tx_slot == priv->omsg_ring[ch].size)
1805 tx_slot = 0;
1806
1807 dev_id = priv->omsg_ring[ch].dev_id;
1808 do_callback = 1;
1809 }
1810
1811 no_sts_update:
1812
1813 if (omsg_int & TSI721_OBDMAC_INT_ERROR) {
1814 /*
1815 * Outbound message operation aborted due to error,
1816 * reinitialize OB MSG channel
1817 */
1818
1819 tsi_debug(OMSG, &priv->pdev->dev, "OB MSG ABORT ch_stat=%x",
1820 ioread32(priv->regs + TSI721_OBDMAC_STS(ch)));
1821
1822 iowrite32(TSI721_OBDMAC_INT_ERROR,
1823 priv->regs + TSI721_OBDMAC_INT(ch));
1824 iowrite32(TSI721_OBDMAC_CTL_RETRY_THR | TSI721_OBDMAC_CTL_INIT,
1825 priv->regs + TSI721_OBDMAC_CTL(ch));
1826 ioread32(priv->regs + TSI721_OBDMAC_CTL(ch));
1827
1828 /* Inform upper level to clear all pending tx slots */
1829 dev_id = priv->omsg_ring[ch].dev_id;
1830 tx_slot = priv->omsg_ring[ch].tx_slot;
1831 do_callback = 1;
1832
1833 /* Synch tx_slot tracking */
1834 iowrite32(priv->omsg_ring[ch].tx_slot,
1835 priv->regs + TSI721_OBDMAC_DRDCNT(ch));
1836 ioread32(priv->regs + TSI721_OBDMAC_DRDCNT(ch));
1837 priv->omsg_ring[ch].wr_count = priv->omsg_ring[ch].tx_slot;
1838 priv->omsg_ring[ch].sts_rdptr = 0;
1839 }
1840
1841 /* Clear channel interrupts */
1842 iowrite32(omsg_int, priv->regs + TSI721_OBDMAC_INT(ch));
1843
1844 if (!(priv->flags & TSI721_USING_MSIX)) {
1845 u32 ch_inte;
1846
1847 /* Re-enable channel interrupts */
1848 ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1849 ch_inte |= TSI721_INT_OMSG_CHAN(ch);
1850 iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE);
1851 }
1852
1853 spin_unlock(&priv->omsg_ring[ch].lock);
1854
1855 if (mport->outb_msg[ch].mcback && do_callback)
1856 mport->outb_msg[ch].mcback(mport, dev_id, ch, tx_slot);
1857 }
1858
1859 /**
1860 * tsi721_open_outb_mbox - Initialize Tsi721 outbound mailbox
1861 * @mport: Master port implementing Outbound Messaging Engine
1862 * @dev_id: Device specific pointer to pass on event
1863 * @mbox: Mailbox to open
1864 * @entries: Number of entries in the outbound mailbox ring
1865 */
1866 static int tsi721_open_outb_mbox(struct rio_mport *mport, void *dev_id,
1867 int mbox, int entries)
1868 {
1869 struct tsi721_device *priv = mport->priv;
1870 struct tsi721_omsg_desc *bd_ptr;
1871 int i, rc = 0;
1872
1873 if ((entries < TSI721_OMSGD_MIN_RING_SIZE) ||
1874 (entries > (TSI721_OMSGD_RING_SIZE)) ||
1875 (!is_power_of_2(entries)) || mbox >= RIO_MAX_MBOX) {
1876 rc = -EINVAL;
1877 goto out;
1878 }
1879
1880 priv->omsg_ring[mbox].dev_id = dev_id;
1881 priv->omsg_ring[mbox].size = entries;
1882 priv->omsg_ring[mbox].sts_rdptr = 0;
1883 spin_lock_init(&priv->omsg_ring[mbox].lock);
1884
1885 /* Outbound Msg Buffer allocation based on
1886 the number of maximum descriptor entries */
1887 for (i = 0; i < entries; i++) {
1888 priv->omsg_ring[mbox].omq_base[i] =
1889 dma_alloc_coherent(
1890 &priv->pdev->dev, TSI721_MSG_BUFFER_SIZE,
1891 &priv->omsg_ring[mbox].omq_phys[i],
1892 GFP_KERNEL);
1893 if (priv->omsg_ring[mbox].omq_base[i] == NULL) {
1894 tsi_debug(OMSG, &priv->pdev->dev,
1895 "ENOMEM for OB_MSG_%d data buffer", mbox);
1896 rc = -ENOMEM;
1897 goto out_buf;
1898 }
1899 }
1900
1901 /* Outbound message descriptor allocation */
1902 priv->omsg_ring[mbox].omd_base = dma_alloc_coherent(
1903 &priv->pdev->dev,
1904 (entries + 1) * sizeof(struct tsi721_omsg_desc),
1905 &priv->omsg_ring[mbox].omd_phys, GFP_KERNEL);
1906 if (priv->omsg_ring[mbox].omd_base == NULL) {
1907 tsi_debug(OMSG, &priv->pdev->dev,
1908 "ENOMEM for OB_MSG_%d descriptor memory", mbox);
1909 rc = -ENOMEM;
1910 goto out_buf;
1911 }
1912
1913 priv->omsg_ring[mbox].tx_slot = 0;
1914
1915 /* Outbound message descriptor status FIFO allocation */
1916 priv->omsg_ring[mbox].sts_size = roundup_pow_of_two(entries + 1);
1917 priv->omsg_ring[mbox].sts_base = dma_zalloc_coherent(&priv->pdev->dev,
1918 priv->omsg_ring[mbox].sts_size *
1919 sizeof(struct tsi721_dma_sts),
1920 &priv->omsg_ring[mbox].sts_phys, GFP_KERNEL);
1921 if (priv->omsg_ring[mbox].sts_base == NULL) {
1922 tsi_debug(OMSG, &priv->pdev->dev,
1923 "ENOMEM for OB_MSG_%d status FIFO", mbox);
1924 rc = -ENOMEM;
1925 goto out_desc;
1926 }
1927
1928 /*
1929 * Configure Outbound Messaging Engine
1930 */
1931
1932 /* Setup Outbound Message descriptor pointer */
1933 iowrite32(((u64)priv->omsg_ring[mbox].omd_phys >> 32),
1934 priv->regs + TSI721_OBDMAC_DPTRH(mbox));
1935 iowrite32(((u64)priv->omsg_ring[mbox].omd_phys &
1936 TSI721_OBDMAC_DPTRL_MASK),
1937 priv->regs + TSI721_OBDMAC_DPTRL(mbox));
1938
1939 /* Setup Outbound Message descriptor status FIFO */
1940 iowrite32(((u64)priv->omsg_ring[mbox].sts_phys >> 32),
1941 priv->regs + TSI721_OBDMAC_DSBH(mbox));
1942 iowrite32(((u64)priv->omsg_ring[mbox].sts_phys &
1943 TSI721_OBDMAC_DSBL_MASK),
1944 priv->regs + TSI721_OBDMAC_DSBL(mbox));
1945 iowrite32(TSI721_DMAC_DSSZ_SIZE(priv->omsg_ring[mbox].sts_size),
1946 priv->regs + (u32)TSI721_OBDMAC_DSSZ(mbox));
1947
1948 /* Enable interrupts */
1949
1950 #ifdef CONFIG_PCI_MSI
1951 if (priv->flags & TSI721_USING_MSIX) {
1952 int idx = TSI721_VECT_OMB0_DONE + mbox;
1953
1954 /* Request interrupt service if we are in MSI-X mode */
1955 rc = request_irq(priv->msix[idx].vector, tsi721_omsg_msix, 0,
1956 priv->msix[idx].irq_name, (void *)priv);
1957
1958 if (rc) {
1959 tsi_debug(OMSG, &priv->pdev->dev,
1960 "Unable to get MSI-X IRQ for OBOX%d-DONE",
1961 mbox);
1962 goto out_stat;
1963 }
1964
1965 idx = TSI721_VECT_OMB0_INT + mbox;
1966 rc = request_irq(priv->msix[idx].vector, tsi721_omsg_msix, 0,
1967 priv->msix[idx].irq_name, (void *)priv);
1968
1969 if (rc) {
1970 tsi_debug(OMSG, &priv->pdev->dev,
1971 "Unable to get MSI-X IRQ for MBOX%d-INT", mbox);
1972 idx = TSI721_VECT_OMB0_DONE + mbox;
1973 free_irq(priv->msix[idx].vector, (void *)priv);
1974 goto out_stat;
1975 }
1976 }
1977 #endif /* CONFIG_PCI_MSI */
1978
1979 tsi721_omsg_interrupt_enable(priv, mbox, TSI721_OBDMAC_INT_ALL);
1980
1981 /* Initialize Outbound Message descriptors ring */
1982 bd_ptr = priv->omsg_ring[mbox].omd_base;
1983 bd_ptr[entries].type_id = cpu_to_le32(DTYPE5 << 29);
1984 bd_ptr[entries].msg_info = 0;
1985 bd_ptr[entries].next_lo =
1986 cpu_to_le32((u64)priv->omsg_ring[mbox].omd_phys &
1987 TSI721_OBDMAC_DPTRL_MASK);
1988 bd_ptr[entries].next_hi =
1989 cpu_to_le32((u64)priv->omsg_ring[mbox].omd_phys >> 32);
1990 priv->omsg_ring[mbox].wr_count = 0;
1991 mb();
1992
1993 /* Initialize Outbound Message engine */
1994 iowrite32(TSI721_OBDMAC_CTL_RETRY_THR | TSI721_OBDMAC_CTL_INIT,
1995 priv->regs + TSI721_OBDMAC_CTL(mbox));
1996 ioread32(priv->regs + TSI721_OBDMAC_DWRCNT(mbox));
1997 udelay(10);
1998
1999 priv->omsg_init[mbox] = 1;
2000
2001 return 0;
2002
2003 #ifdef CONFIG_PCI_MSI
2004 out_stat:
2005 dma_free_coherent(&priv->pdev->dev,
2006 priv->omsg_ring[mbox].sts_size * sizeof(struct tsi721_dma_sts),
2007 priv->omsg_ring[mbox].sts_base,
2008 priv->omsg_ring[mbox].sts_phys);
2009
2010 priv->omsg_ring[mbox].sts_base = NULL;
2011 #endif /* CONFIG_PCI_MSI */
2012
2013 out_desc:
2014 dma_free_coherent(&priv->pdev->dev,
2015 (entries + 1) * sizeof(struct tsi721_omsg_desc),
2016 priv->omsg_ring[mbox].omd_base,
2017 priv->omsg_ring[mbox].omd_phys);
2018
2019 priv->omsg_ring[mbox].omd_base = NULL;
2020
2021 out_buf:
2022 for (i = 0; i < priv->omsg_ring[mbox].size; i++) {
2023 if (priv->omsg_ring[mbox].omq_base[i]) {
2024 dma_free_coherent(&priv->pdev->dev,
2025 TSI721_MSG_BUFFER_SIZE,
2026 priv->omsg_ring[mbox].omq_base[i],
2027 priv->omsg_ring[mbox].omq_phys[i]);
2028
2029 priv->omsg_ring[mbox].omq_base[i] = NULL;
2030 }
2031 }
2032
2033 out:
2034 return rc;
2035 }
2036
2037 /**
2038 * tsi721_close_outb_mbox - Close Tsi721 outbound mailbox
2039 * @mport: Master port implementing the outbound message unit
2040 * @mbox: Mailbox to close
2041 */
2042 static void tsi721_close_outb_mbox(struct rio_mport *mport, int mbox)
2043 {
2044 struct tsi721_device *priv = mport->priv;
2045 u32 i;
2046
2047 if (!priv->omsg_init[mbox])
2048 return;
2049 priv->omsg_init[mbox] = 0;
2050
2051 /* Disable Interrupts */
2052
2053 tsi721_omsg_interrupt_disable(priv, mbox, TSI721_OBDMAC_INT_ALL);
2054
2055 #ifdef CONFIG_PCI_MSI
2056 if (priv->flags & TSI721_USING_MSIX) {
2057 free_irq(priv->msix[TSI721_VECT_OMB0_DONE + mbox].vector,
2058 (void *)priv);
2059 free_irq(priv->msix[TSI721_VECT_OMB0_INT + mbox].vector,
2060 (void *)priv);
2061 }
2062 #endif /* CONFIG_PCI_MSI */
2063
2064 /* Free OMSG Descriptor Status FIFO */
2065 dma_free_coherent(&priv->pdev->dev,
2066 priv->omsg_ring[mbox].sts_size * sizeof(struct tsi721_dma_sts),
2067 priv->omsg_ring[mbox].sts_base,
2068 priv->omsg_ring[mbox].sts_phys);
2069
2070 priv->omsg_ring[mbox].sts_base = NULL;
2071
2072 /* Free OMSG descriptors */
2073 dma_free_coherent(&priv->pdev->dev,
2074 (priv->omsg_ring[mbox].size + 1) *
2075 sizeof(struct tsi721_omsg_desc),
2076 priv->omsg_ring[mbox].omd_base,
2077 priv->omsg_ring[mbox].omd_phys);
2078
2079 priv->omsg_ring[mbox].omd_base = NULL;
2080
2081 /* Free message buffers */
2082 for (i = 0; i < priv->omsg_ring[mbox].size; i++) {
2083 if (priv->omsg_ring[mbox].omq_base[i]) {
2084 dma_free_coherent(&priv->pdev->dev,
2085 TSI721_MSG_BUFFER_SIZE,
2086 priv->omsg_ring[mbox].omq_base[i],
2087 priv->omsg_ring[mbox].omq_phys[i]);
2088
2089 priv->omsg_ring[mbox].omq_base[i] = NULL;
2090 }
2091 }
2092 }
2093
2094 /**
2095 * tsi721_imsg_handler - Inbound Message Interrupt Handler
2096 * @priv: pointer to tsi721 private data
2097 * @ch: inbound message channel number to service
2098 *
2099 * Services channel interrupts from inbound messaging engine.
2100 */
2101 static void tsi721_imsg_handler(struct tsi721_device *priv, int ch)
2102 {
2103 u32 mbox = ch - 4;
2104 u32 imsg_int;
2105 struct rio_mport *mport = &priv->mport;
2106
2107 spin_lock(&priv->imsg_ring[mbox].lock);
2108
2109 imsg_int = ioread32(priv->regs + TSI721_IBDMAC_INT(ch));
2110
2111 if (imsg_int & TSI721_IBDMAC_INT_SRTO)
2112 tsi_info(&priv->pdev->dev, "IB MBOX%d SRIO timeout", mbox);
2113
2114 if (imsg_int & TSI721_IBDMAC_INT_PC_ERROR)
2115 tsi_info(&priv->pdev->dev, "IB MBOX%d PCIe error", mbox);
2116
2117 if (imsg_int & TSI721_IBDMAC_INT_FQ_LOW)
2118 tsi_info(&priv->pdev->dev, "IB MBOX%d IB free queue low", mbox);
2119
2120 /* Clear IB channel interrupts */
2121 iowrite32(imsg_int, priv->regs + TSI721_IBDMAC_INT(ch));
2122
2123 /* If an IB Msg is received notify the upper layer */
2124 if (imsg_int & TSI721_IBDMAC_INT_DQ_RCV &&
2125 mport->inb_msg[mbox].mcback)
2126 mport->inb_msg[mbox].mcback(mport,
2127 priv->imsg_ring[mbox].dev_id, mbox, -1);
2128
2129 if (!(priv->flags & TSI721_USING_MSIX)) {
2130 u32 ch_inte;
2131
2132 /* Re-enable channel interrupts */
2133 ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
2134 ch_inte |= TSI721_INT_IMSG_CHAN(ch);
2135 iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE);
2136 }
2137
2138 spin_unlock(&priv->imsg_ring[mbox].lock);
2139 }
2140
2141 /**
2142 * tsi721_open_inb_mbox - Initialize Tsi721 inbound mailbox
2143 * @mport: Master port implementing the Inbound Messaging Engine
2144 * @dev_id: Device specific pointer to pass on event
2145 * @mbox: Mailbox to open
2146 * @entries: Number of entries in the inbound mailbox ring
2147 */
2148 static int tsi721_open_inb_mbox(struct rio_mport *mport, void *dev_id,
2149 int mbox, int entries)
2150 {
2151 struct tsi721_device *priv = mport->priv;
2152 int ch = mbox + 4;
2153 int i;
2154 u64 *free_ptr;
2155 int rc = 0;
2156
2157 if ((entries < TSI721_IMSGD_MIN_RING_SIZE) ||
2158 (entries > TSI721_IMSGD_RING_SIZE) ||
2159 (!is_power_of_2(entries)) || mbox >= RIO_MAX_MBOX) {
2160 rc = -EINVAL;
2161 goto out;
2162 }
2163
2164 /* Initialize IB Messaging Ring */
2165 priv->imsg_ring[mbox].dev_id = dev_id;
2166 priv->imsg_ring[mbox].size = entries;
2167 priv->imsg_ring[mbox].rx_slot = 0;
2168 priv->imsg_ring[mbox].desc_rdptr = 0;
2169 priv->imsg_ring[mbox].fq_wrptr = 0;
2170 for (i = 0; i < priv->imsg_ring[mbox].size; i++)
2171 priv->imsg_ring[mbox].imq_base[i] = NULL;
2172 spin_lock_init(&priv->imsg_ring[mbox].lock);
2173
2174 /* Allocate buffers for incoming messages */
2175 priv->imsg_ring[mbox].buf_base =
2176 dma_alloc_coherent(&priv->pdev->dev,
2177 entries * TSI721_MSG_BUFFER_SIZE,
2178 &priv->imsg_ring[mbox].buf_phys,
2179 GFP_KERNEL);
2180
2181 if (priv->imsg_ring[mbox].buf_base == NULL) {
2182 tsi_err(&priv->pdev->dev,
2183 "Failed to allocate buffers for IB MBOX%d", mbox);
2184 rc = -ENOMEM;
2185 goto out;
2186 }
2187
2188 /* Allocate memory for circular free list */
2189 priv->imsg_ring[mbox].imfq_base =
2190 dma_alloc_coherent(&priv->pdev->dev,
2191 entries * 8,
2192 &priv->imsg_ring[mbox].imfq_phys,
2193 GFP_KERNEL);
2194
2195 if (priv->imsg_ring[mbox].imfq_base == NULL) {
2196 tsi_err(&priv->pdev->dev,
2197 "Failed to allocate free queue for IB MBOX%d", mbox);
2198 rc = -ENOMEM;
2199 goto out_buf;
2200 }
2201
2202 /* Allocate memory for Inbound message descriptors */
2203 priv->imsg_ring[mbox].imd_base =
2204 dma_alloc_coherent(&priv->pdev->dev,
2205 entries * sizeof(struct tsi721_imsg_desc),
2206 &priv->imsg_ring[mbox].imd_phys, GFP_KERNEL);
2207
2208 if (priv->imsg_ring[mbox].imd_base == NULL) {
2209 tsi_err(&priv->pdev->dev,
2210 "Failed to allocate descriptor memory for IB MBOX%d",
2211 mbox);
2212 rc = -ENOMEM;
2213 goto out_dma;
2214 }
2215
2216 /* Fill free buffer pointer list */
2217 free_ptr = priv->imsg_ring[mbox].imfq_base;
2218 for (i = 0; i < entries; i++)
2219 free_ptr[i] = cpu_to_le64(
2220 (u64)(priv->imsg_ring[mbox].buf_phys) +
2221 i * 0x1000);
2222
2223 mb();
2224
2225 /*
2226 * For mapping of inbound SRIO Messages into appropriate queues we need
2227 * to set Inbound Device ID register in the messaging engine. We do it
2228 * once when first inbound mailbox is requested.
2229 */
2230 if (!(priv->flags & TSI721_IMSGID_SET)) {
2231 iowrite32((u32)priv->mport.host_deviceid,
2232 priv->regs + TSI721_IB_DEVID);
2233 priv->flags |= TSI721_IMSGID_SET;
2234 }
2235
2236 /*
2237 * Configure Inbound Messaging channel (ch = mbox + 4)
2238 */
2239
2240 /* Setup Inbound Message free queue */
2241 iowrite32(((u64)priv->imsg_ring[mbox].imfq_phys >> 32),
2242 priv->regs + TSI721_IBDMAC_FQBH(ch));
2243 iowrite32(((u64)priv->imsg_ring[mbox].imfq_phys &
2244 TSI721_IBDMAC_FQBL_MASK),
2245 priv->regs+TSI721_IBDMAC_FQBL(ch));
2246 iowrite32(TSI721_DMAC_DSSZ_SIZE(entries),
2247 priv->regs + TSI721_IBDMAC_FQSZ(ch));
2248
2249 /* Setup Inbound Message descriptor queue */
2250 iowrite32(((u64)priv->imsg_ring[mbox].imd_phys >> 32),
2251 priv->regs + TSI721_IBDMAC_DQBH(ch));
2252 iowrite32(((u32)priv->imsg_ring[mbox].imd_phys &
2253 (u32)TSI721_IBDMAC_DQBL_MASK),
2254 priv->regs+TSI721_IBDMAC_DQBL(ch));
2255 iowrite32(TSI721_DMAC_DSSZ_SIZE(entries),
2256 priv->regs + TSI721_IBDMAC_DQSZ(ch));
2257
2258 /* Enable interrupts */
2259
2260 #ifdef CONFIG_PCI_MSI
2261 if (priv->flags & TSI721_USING_MSIX) {
2262 int idx = TSI721_VECT_IMB0_RCV + mbox;
2263
2264 /* Request interrupt service if we are in MSI-X mode */
2265 rc = request_irq(priv->msix[idx].vector, tsi721_imsg_msix, 0,
2266 priv->msix[idx].irq_name, (void *)priv);
2267
2268 if (rc) {
2269 tsi_debug(IMSG, &priv->pdev->dev,
2270 "Unable to get MSI-X IRQ for IBOX%d-DONE",
2271 mbox);
2272 goto out_desc;
2273 }
2274
2275 idx = TSI721_VECT_IMB0_INT + mbox;
2276 rc = request_irq(priv->msix[idx].vector, tsi721_imsg_msix, 0,
2277 priv->msix[idx].irq_name, (void *)priv);
2278
2279 if (rc) {
2280 tsi_debug(IMSG, &priv->pdev->dev,
2281 "Unable to get MSI-X IRQ for IBOX%d-INT", mbox);
2282 free_irq(
2283 priv->msix[TSI721_VECT_IMB0_RCV + mbox].vector,
2284 (void *)priv);
2285 goto out_desc;
2286 }
2287 }
2288 #endif /* CONFIG_PCI_MSI */
2289
2290 tsi721_imsg_interrupt_enable(priv, ch, TSI721_IBDMAC_INT_ALL);
2291
2292 /* Initialize Inbound Message Engine */
2293 iowrite32(TSI721_IBDMAC_CTL_INIT, priv->regs + TSI721_IBDMAC_CTL(ch));
2294 ioread32(priv->regs + TSI721_IBDMAC_CTL(ch));
2295 udelay(10);
2296 priv->imsg_ring[mbox].fq_wrptr = entries - 1;
2297 iowrite32(entries - 1, priv->regs + TSI721_IBDMAC_FQWP(ch));
2298
2299 priv->imsg_init[mbox] = 1;
2300 return 0;
2301
2302 #ifdef CONFIG_PCI_MSI
2303 out_desc:
2304 dma_free_coherent(&priv->pdev->dev,
2305 priv->imsg_ring[mbox].size * sizeof(struct tsi721_imsg_desc),
2306 priv->imsg_ring[mbox].imd_base,
2307 priv->imsg_ring[mbox].imd_phys);
2308
2309 priv->imsg_ring[mbox].imd_base = NULL;
2310 #endif /* CONFIG_PCI_MSI */
2311
2312 out_dma:
2313 dma_free_coherent(&priv->pdev->dev,
2314 priv->imsg_ring[mbox].size * 8,
2315 priv->imsg_ring[mbox].imfq_base,
2316 priv->imsg_ring[mbox].imfq_phys);
2317
2318 priv->imsg_ring[mbox].imfq_base = NULL;
2319
2320 out_buf:
2321 dma_free_coherent(&priv->pdev->dev,
2322 priv->imsg_ring[mbox].size * TSI721_MSG_BUFFER_SIZE,
2323 priv->imsg_ring[mbox].buf_base,
2324 priv->imsg_ring[mbox].buf_phys);
2325
2326 priv->imsg_ring[mbox].buf_base = NULL;
2327
2328 out:
2329 return rc;
2330 }
2331
2332 /**
2333 * tsi721_close_inb_mbox - Shut down Tsi721 inbound mailbox
2334 * @mport: Master port implementing the Inbound Messaging Engine
2335 * @mbox: Mailbox to close
2336 */
2337 static void tsi721_close_inb_mbox(struct rio_mport *mport, int mbox)
2338 {
2339 struct tsi721_device *priv = mport->priv;
2340 u32 rx_slot;
2341 int ch = mbox + 4;
2342
2343 if (!priv->imsg_init[mbox]) /* mbox isn't initialized yet */
2344 return;
2345 priv->imsg_init[mbox] = 0;
2346
2347 /* Disable Inbound Messaging Engine */
2348
2349 /* Disable Interrupts */
2350 tsi721_imsg_interrupt_disable(priv, ch, TSI721_OBDMAC_INT_MASK);
2351
2352 #ifdef CONFIG_PCI_MSI
2353 if (priv->flags & TSI721_USING_MSIX) {
2354 free_irq(priv->msix[TSI721_VECT_IMB0_RCV + mbox].vector,
2355 (void *)priv);
2356 free_irq(priv->msix[TSI721_VECT_IMB0_INT + mbox].vector,
2357 (void *)priv);
2358 }
2359 #endif /* CONFIG_PCI_MSI */
2360
2361 /* Clear Inbound Buffer Queue */
2362 for (rx_slot = 0; rx_slot < priv->imsg_ring[mbox].size; rx_slot++)
2363 priv->imsg_ring[mbox].imq_base[rx_slot] = NULL;
2364
2365 /* Free memory allocated for message buffers */
2366 dma_free_coherent(&priv->pdev->dev,
2367 priv->imsg_ring[mbox].size * TSI721_MSG_BUFFER_SIZE,
2368 priv->imsg_ring[mbox].buf_base,
2369 priv->imsg_ring[mbox].buf_phys);
2370
2371 priv->imsg_ring[mbox].buf_base = NULL;
2372
2373 /* Free memory allocated for free pointr list */
2374 dma_free_coherent(&priv->pdev->dev,
2375 priv->imsg_ring[mbox].size * 8,
2376 priv->imsg_ring[mbox].imfq_base,
2377 priv->imsg_ring[mbox].imfq_phys);
2378
2379 priv->imsg_ring[mbox].imfq_base = NULL;
2380
2381 /* Free memory allocated for RX descriptors */
2382 dma_free_coherent(&priv->pdev->dev,
2383 priv->imsg_ring[mbox].size * sizeof(struct tsi721_imsg_desc),
2384 priv->imsg_ring[mbox].imd_base,
2385 priv->imsg_ring[mbox].imd_phys);
2386
2387 priv->imsg_ring[mbox].imd_base = NULL;
2388 }
2389
2390 /**
2391 * tsi721_add_inb_buffer - Add buffer to the Tsi721 inbound message queue
2392 * @mport: Master port implementing the Inbound Messaging Engine
2393 * @mbox: Inbound mailbox number
2394 * @buf: Buffer to add to inbound queue
2395 */
2396 static int tsi721_add_inb_buffer(struct rio_mport *mport, int mbox, void *buf)
2397 {
2398 struct tsi721_device *priv = mport->priv;
2399 u32 rx_slot;
2400 int rc = 0;
2401
2402 rx_slot = priv->imsg_ring[mbox].rx_slot;
2403 if (priv->imsg_ring[mbox].imq_base[rx_slot]) {
2404 tsi_err(&priv->pdev->dev,
2405 "Error adding inbound buffer %d, buffer exists",
2406 rx_slot);
2407 rc = -EINVAL;
2408 goto out;
2409 }
2410
2411 priv->imsg_ring[mbox].imq_base[rx_slot] = buf;
2412
2413 if (++priv->imsg_ring[mbox].rx_slot == priv->imsg_ring[mbox].size)
2414 priv->imsg_ring[mbox].rx_slot = 0;
2415
2416 out:
2417 return rc;
2418 }
2419
2420 /**
2421 * tsi721_get_inb_message - Fetch inbound message from the Tsi721 MSG Queue
2422 * @mport: Master port implementing the Inbound Messaging Engine
2423 * @mbox: Inbound mailbox number
2424 *
2425 * Returns pointer to the message on success or NULL on failure.
2426 */
2427 static void *tsi721_get_inb_message(struct rio_mport *mport, int mbox)
2428 {
2429 struct tsi721_device *priv = mport->priv;
2430 struct tsi721_imsg_desc *desc;
2431 u32 rx_slot;
2432 void *rx_virt = NULL;
2433 u64 rx_phys;
2434 void *buf = NULL;
2435 u64 *free_ptr;
2436 int ch = mbox + 4;
2437 int msg_size;
2438
2439 if (!priv->imsg_init[mbox])
2440 return NULL;
2441
2442 desc = priv->imsg_ring[mbox].imd_base;
2443 desc += priv->imsg_ring[mbox].desc_rdptr;
2444
2445 if (!(le32_to_cpu(desc->msg_info) & TSI721_IMD_HO))
2446 goto out;
2447
2448 rx_slot = priv->imsg_ring[mbox].rx_slot;
2449 while (priv->imsg_ring[mbox].imq_base[rx_slot] == NULL) {
2450 if (++rx_slot == priv->imsg_ring[mbox].size)
2451 rx_slot = 0;
2452 }
2453
2454 rx_phys = ((u64)le32_to_cpu(desc->bufptr_hi) << 32) |
2455 le32_to_cpu(desc->bufptr_lo);
2456
2457 rx_virt = priv->imsg_ring[mbox].buf_base +
2458 (rx_phys - (u64)priv->imsg_ring[mbox].buf_phys);
2459
2460 buf = priv->imsg_ring[mbox].imq_base[rx_slot];
2461 msg_size = le32_to_cpu(desc->msg_info) & TSI721_IMD_BCOUNT;
2462 if (msg_size == 0)
2463 msg_size = RIO_MAX_MSG_SIZE;
2464
2465 memcpy(buf, rx_virt, msg_size);
2466 priv->imsg_ring[mbox].imq_base[rx_slot] = NULL;
2467
2468 desc->msg_info &= cpu_to_le32(~TSI721_IMD_HO);
2469 if (++priv->imsg_ring[mbox].desc_rdptr == priv->imsg_ring[mbox].size)
2470 priv->imsg_ring[mbox].desc_rdptr = 0;
2471
2472 iowrite32(priv->imsg_ring[mbox].desc_rdptr,
2473 priv->regs + TSI721_IBDMAC_DQRP(ch));
2474
2475 /* Return free buffer into the pointer list */
2476 free_ptr = priv->imsg_ring[mbox].imfq_base;
2477 free_ptr[priv->imsg_ring[mbox].fq_wrptr] = cpu_to_le64(rx_phys);
2478
2479 if (++priv->imsg_ring[mbox].fq_wrptr == priv->imsg_ring[mbox].size)
2480 priv->imsg_ring[mbox].fq_wrptr = 0;
2481
2482 iowrite32(priv->imsg_ring[mbox].fq_wrptr,
2483 priv->regs + TSI721_IBDMAC_FQWP(ch));
2484 out:
2485 return buf;
2486 }
2487
2488 /**
2489 * tsi721_messages_init - Initialization of Messaging Engine
2490 * @priv: pointer to tsi721 private data
2491 *
2492 * Configures Tsi721 messaging engine.
2493 */
2494 static int tsi721_messages_init(struct tsi721_device *priv)
2495 {
2496 int ch;
2497
2498 iowrite32(0, priv->regs + TSI721_SMSG_ECC_LOG);
2499 iowrite32(0, priv->regs + TSI721_RETRY_GEN_CNT);
2500 iowrite32(0, priv->regs + TSI721_RETRY_RX_CNT);
2501
2502 /* Set SRIO Message Request/Response Timeout */
2503 iowrite32(TSI721_RQRPTO_VAL, priv->regs + TSI721_RQRPTO);
2504
2505 /* Initialize Inbound Messaging Engine Registers */
2506 for (ch = 0; ch < TSI721_IMSG_CHNUM; ch++) {
2507 /* Clear interrupt bits */
2508 iowrite32(TSI721_IBDMAC_INT_MASK,
2509 priv->regs + TSI721_IBDMAC_INT(ch));
2510 /* Clear Status */
2511 iowrite32(0, priv->regs + TSI721_IBDMAC_STS(ch));
2512
2513 iowrite32(TSI721_SMSG_ECC_COR_LOG_MASK,
2514 priv->regs + TSI721_SMSG_ECC_COR_LOG(ch));
2515 iowrite32(TSI721_SMSG_ECC_NCOR_MASK,
2516 priv->regs + TSI721_SMSG_ECC_NCOR(ch));
2517 }
2518
2519 return 0;
2520 }
2521
2522 /**
2523 * tsi721_query_mport - Fetch inbound message from the Tsi721 MSG Queue
2524 * @mport: Master port implementing the Inbound Messaging Engine
2525 * @mbox: Inbound mailbox number
2526 *
2527 * Returns pointer to the message on success or NULL on failure.
2528 */
2529 static int tsi721_query_mport(struct rio_mport *mport,
2530 struct rio_mport_attr *attr)
2531 {
2532 struct tsi721_device *priv = mport->priv;
2533 u32 rval;
2534
2535 rval = ioread32(priv->regs + (0x100 + RIO_PORT_N_ERR_STS_CSR(0)));
2536 if (rval & RIO_PORT_N_ERR_STS_PORT_OK) {
2537 rval = ioread32(priv->regs + (0x100 + RIO_PORT_N_CTL2_CSR(0)));
2538 attr->link_speed = (rval & RIO_PORT_N_CTL2_SEL_BAUD) >> 28;
2539 rval = ioread32(priv->regs + (0x100 + RIO_PORT_N_CTL_CSR(0)));
2540 attr->link_width = (rval & RIO_PORT_N_CTL_IPW) >> 27;
2541 } else
2542 attr->link_speed = RIO_LINK_DOWN;
2543
2544 #ifdef CONFIG_RAPIDIO_DMA_ENGINE
2545 attr->flags = RIO_MPORT_DMA | RIO_MPORT_DMA_SG;
2546 attr->dma_max_sge = 0;
2547 attr->dma_max_size = TSI721_BDMA_MAX_BCOUNT;
2548 attr->dma_align = 0;
2549 #else
2550 attr->flags = 0;
2551 #endif
2552 return 0;
2553 }
2554
2555 /**
2556 * tsi721_disable_ints - disables all device interrupts
2557 * @priv: pointer to tsi721 private data
2558 */
2559 static void tsi721_disable_ints(struct tsi721_device *priv)
2560 {
2561 int ch;
2562
2563 /* Disable all device level interrupts */
2564 iowrite32(0, priv->regs + TSI721_DEV_INTE);
2565
2566 /* Disable all Device Channel interrupts */
2567 iowrite32(0, priv->regs + TSI721_DEV_CHAN_INTE);
2568
2569 /* Disable all Inbound Msg Channel interrupts */
2570 for (ch = 0; ch < TSI721_IMSG_CHNUM; ch++)
2571 iowrite32(0, priv->regs + TSI721_IBDMAC_INTE(ch));
2572
2573 /* Disable all Outbound Msg Channel interrupts */
2574 for (ch = 0; ch < TSI721_OMSG_CHNUM; ch++)
2575 iowrite32(0, priv->regs + TSI721_OBDMAC_INTE(ch));
2576
2577 /* Disable all general messaging interrupts */
2578 iowrite32(0, priv->regs + TSI721_SMSG_INTE);
2579
2580 /* Disable all BDMA Channel interrupts */
2581 for (ch = 0; ch < TSI721_DMA_MAXCH; ch++)
2582 iowrite32(0,
2583 priv->regs + TSI721_DMAC_BASE(ch) + TSI721_DMAC_INTE);
2584
2585 /* Disable all general BDMA interrupts */
2586 iowrite32(0, priv->regs + TSI721_BDMA_INTE);
2587
2588 /* Disable all SRIO Channel interrupts */
2589 for (ch = 0; ch < TSI721_SRIO_MAXCH; ch++)
2590 iowrite32(0, priv->regs + TSI721_SR_CHINTE(ch));
2591
2592 /* Disable all general SR2PC interrupts */
2593 iowrite32(0, priv->regs + TSI721_SR2PC_GEN_INTE);
2594
2595 /* Disable all PC2SR interrupts */
2596 iowrite32(0, priv->regs + TSI721_PC2SR_INTE);
2597
2598 /* Disable all I2C interrupts */
2599 iowrite32(0, priv->regs + TSI721_I2C_INT_ENABLE);
2600
2601 /* Disable SRIO MAC interrupts */
2602 iowrite32(0, priv->regs + TSI721_RIO_EM_INT_ENABLE);
2603 iowrite32(0, priv->regs + TSI721_RIO_EM_DEV_INT_EN);
2604 }
2605
2606 static struct rio_ops tsi721_rio_ops = {
2607 .lcread = tsi721_lcread,
2608 .lcwrite = tsi721_lcwrite,
2609 .cread = tsi721_cread_dma,
2610 .cwrite = tsi721_cwrite_dma,
2611 .dsend = tsi721_dsend,
2612 .open_inb_mbox = tsi721_open_inb_mbox,
2613 .close_inb_mbox = tsi721_close_inb_mbox,
2614 .open_outb_mbox = tsi721_open_outb_mbox,
2615 .close_outb_mbox = tsi721_close_outb_mbox,
2616 .add_outb_message = tsi721_add_outb_message,
2617 .add_inb_buffer = tsi721_add_inb_buffer,
2618 .get_inb_message = tsi721_get_inb_message,
2619 .map_inb = tsi721_rio_map_inb_mem,
2620 .unmap_inb = tsi721_rio_unmap_inb_mem,
2621 .pwenable = tsi721_pw_enable,
2622 .query_mport = tsi721_query_mport,
2623 .map_outb = tsi721_map_outb_win,
2624 .unmap_outb = tsi721_unmap_outb_win,
2625 };
2626
2627 static void tsi721_mport_release(struct device *dev)
2628 {
2629 struct rio_mport *mport = to_rio_mport(dev);
2630
2631 tsi_debug(EXIT, dev, "%s id=%d", mport->name, mport->id);
2632 }
2633
2634 /**
2635 * tsi721_setup_mport - Setup Tsi721 as RapidIO subsystem master port
2636 * @priv: pointer to tsi721 private data
2637 *
2638 * Configures Tsi721 as RapidIO master port.
2639 */
2640 static int tsi721_setup_mport(struct tsi721_device *priv)
2641 {
2642 struct pci_dev *pdev = priv->pdev;
2643 int err = 0;
2644 struct rio_mport *mport = &priv->mport;
2645
2646 err = rio_mport_initialize(mport);
2647 if (err)
2648 return err;
2649
2650 mport->ops = &tsi721_rio_ops;
2651 mport->index = 0;
2652 mport->sys_size = 0; /* small system */
2653 mport->phy_type = RIO_PHY_SERIAL;
2654 mport->priv = (void *)priv;
2655 mport->phys_efptr = 0x100;
2656 mport->dev.parent = &pdev->dev;
2657 mport->dev.release = tsi721_mport_release;
2658
2659 INIT_LIST_HEAD(&mport->dbells);
2660
2661 rio_init_dbell_res(&mport->riores[RIO_DOORBELL_RESOURCE], 0, 0xffff);
2662 rio_init_mbox_res(&mport->riores[RIO_INB_MBOX_RESOURCE], 0, 3);
2663 rio_init_mbox_res(&mport->riores[RIO_OUTB_MBOX_RESOURCE], 0, 3);
2664 snprintf(mport->name, RIO_MAX_MPORT_NAME, "%s(%s)",
2665 dev_driver_string(&pdev->dev), dev_name(&pdev->dev));
2666
2667 /* Hook up interrupt handler */
2668
2669 #ifdef CONFIG_PCI_MSI
2670 if (!tsi721_enable_msix(priv))
2671 priv->flags |= TSI721_USING_MSIX;
2672 else if (!pci_enable_msi(pdev))
2673 priv->flags |= TSI721_USING_MSI;
2674 else
2675 tsi_debug(MPORT, &pdev->dev,
2676 "MSI/MSI-X is not available. Using legacy INTx.");
2677 #endif /* CONFIG_PCI_MSI */
2678
2679 err = tsi721_request_irq(priv);
2680
2681 if (err) {
2682 tsi_err(&pdev->dev, "Unable to get PCI IRQ %02X (err=0x%x)",
2683 pdev->irq, err);
2684 return err;
2685 }
2686
2687 #ifdef CONFIG_RAPIDIO_DMA_ENGINE
2688 err = tsi721_register_dma(priv);
2689 if (err)
2690 goto err_exit;
2691 #endif
2692 /* Enable SRIO link */
2693 iowrite32(ioread32(priv->regs + TSI721_DEVCTL) |
2694 TSI721_DEVCTL_SRBOOT_CMPL,
2695 priv->regs + TSI721_DEVCTL);
2696
2697 if (mport->host_deviceid >= 0)
2698 iowrite32(RIO_PORT_GEN_HOST | RIO_PORT_GEN_MASTER |
2699 RIO_PORT_GEN_DISCOVERED,
2700 priv->regs + (0x100 + RIO_PORT_GEN_CTL_CSR));
2701 else
2702 iowrite32(0, priv->regs + (0x100 + RIO_PORT_GEN_CTL_CSR));
2703
2704 err = rio_register_mport(mport);
2705 if (err) {
2706 tsi721_unregister_dma(priv);
2707 goto err_exit;
2708 }
2709
2710 return 0;
2711
2712 err_exit:
2713 tsi721_free_irq(priv);
2714 return err;
2715 }
2716
2717 static int tsi721_probe(struct pci_dev *pdev,
2718 const struct pci_device_id *id)
2719 {
2720 struct tsi721_device *priv;
2721 int err;
2722
2723 priv = kzalloc(sizeof(struct tsi721_device), GFP_KERNEL);
2724 if (!priv) {
2725 err = -ENOMEM;
2726 goto err_exit;
2727 }
2728
2729 err = pci_enable_device(pdev);
2730 if (err) {
2731 tsi_err(&pdev->dev, "Failed to enable PCI device");
2732 goto err_clean;
2733 }
2734
2735 priv->pdev = pdev;
2736
2737 #ifdef DEBUG
2738 {
2739 int i;
2740
2741 for (i = 0; i <= PCI_STD_RESOURCE_END; i++) {
2742 tsi_debug(INIT, &pdev->dev, "res%d %pR",
2743 i, &pdev->resource[i]);
2744 }
2745 }
2746 #endif
2747 /*
2748 * Verify BAR configuration
2749 */
2750
2751 /* BAR_0 (registers) must be 512KB+ in 32-bit address space */
2752 if (!(pci_resource_flags(pdev, BAR_0) & IORESOURCE_MEM) ||
2753 pci_resource_flags(pdev, BAR_0) & IORESOURCE_MEM_64 ||
2754 pci_resource_len(pdev, BAR_0) < TSI721_REG_SPACE_SIZE) {
2755 tsi_err(&pdev->dev, "Missing or misconfigured CSR BAR0");
2756 err = -ENODEV;
2757 goto err_disable_pdev;
2758 }
2759
2760 /* BAR_1 (outbound doorbells) must be 16MB+ in 32-bit address space */
2761 if (!(pci_resource_flags(pdev, BAR_1) & IORESOURCE_MEM) ||
2762 pci_resource_flags(pdev, BAR_1) & IORESOURCE_MEM_64 ||
2763 pci_resource_len(pdev, BAR_1) < TSI721_DB_WIN_SIZE) {
2764 tsi_err(&pdev->dev, "Missing or misconfigured Doorbell BAR1");
2765 err = -ENODEV;
2766 goto err_disable_pdev;
2767 }
2768
2769 /*
2770 * BAR_2 and BAR_4 (outbound translation) must be in 64-bit PCIe address
2771 * space.
2772 * NOTE: BAR_2 and BAR_4 are not used by this version of driver.
2773 * It may be a good idea to keep them disabled using HW configuration
2774 * to save PCI memory space.
2775 */
2776
2777 priv->p2r_bar[0].size = priv->p2r_bar[1].size = 0;
2778
2779 if (pci_resource_flags(pdev, BAR_2) & IORESOURCE_MEM_64) {
2780 if (pci_resource_flags(pdev, BAR_2) & IORESOURCE_PREFETCH)
2781 tsi_debug(INIT, &pdev->dev,
2782 "Prefetchable OBW BAR2 will not be used");
2783 else {
2784 priv->p2r_bar[0].base = pci_resource_start(pdev, BAR_2);
2785 priv->p2r_bar[0].size = pci_resource_len(pdev, BAR_2);
2786 }
2787 }
2788
2789 if (pci_resource_flags(pdev, BAR_4) & IORESOURCE_MEM_64) {
2790 if (pci_resource_flags(pdev, BAR_4) & IORESOURCE_PREFETCH)
2791 tsi_debug(INIT, &pdev->dev,
2792 "Prefetchable OBW BAR4 will not be used");
2793 else {
2794 priv->p2r_bar[1].base = pci_resource_start(pdev, BAR_4);
2795 priv->p2r_bar[1].size = pci_resource_len(pdev, BAR_4);
2796 }
2797 }
2798
2799 err = pci_request_regions(pdev, DRV_NAME);
2800 if (err) {
2801 tsi_err(&pdev->dev, "Unable to obtain PCI resources");
2802 goto err_disable_pdev;
2803 }
2804
2805 pci_set_master(pdev);
2806
2807 priv->regs = pci_ioremap_bar(pdev, BAR_0);
2808 if (!priv->regs) {
2809 tsi_err(&pdev->dev, "Unable to map device registers space");
2810 err = -ENOMEM;
2811 goto err_free_res;
2812 }
2813
2814 priv->odb_base = pci_ioremap_bar(pdev, BAR_1);
2815 if (!priv->odb_base) {
2816 tsi_err(&pdev->dev, "Unable to map outbound doorbells space");
2817 err = -ENOMEM;
2818 goto err_unmap_bars;
2819 }
2820
2821 /* Configure DMA attributes. */
2822 if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
2823 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
2824 if (err) {
2825 tsi_err(&pdev->dev, "Unable to set DMA mask");
2826 goto err_unmap_bars;
2827 }
2828
2829 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)))
2830 tsi_info(&pdev->dev, "Unable to set consistent DMA mask");
2831 } else {
2832 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
2833 if (err)
2834 tsi_info(&pdev->dev, "Unable to set consistent DMA mask");
2835 }
2836
2837 BUG_ON(!pci_is_pcie(pdev));
2838
2839 /* Clear "no snoop" and "relaxed ordering" bits. */
2840 pcie_capability_clear_and_set_word(pdev, PCI_EXP_DEVCTL,
2841 PCI_EXP_DEVCTL_RELAX_EN | PCI_EXP_DEVCTL_NOSNOOP_EN, 0);
2842
2843 /* Adjust PCIe completion timeout. */
2844 pcie_capability_clear_and_set_word(pdev, PCI_EXP_DEVCTL2, 0xf, 0x2);
2845
2846 /*
2847 * FIXUP: correct offsets of MSI-X tables in the MSI-X Capability Block
2848 */
2849 pci_write_config_dword(pdev, TSI721_PCIECFG_EPCTL, 0x01);
2850 pci_write_config_dword(pdev, TSI721_PCIECFG_MSIXTBL,
2851 TSI721_MSIXTBL_OFFSET);
2852 pci_write_config_dword(pdev, TSI721_PCIECFG_MSIXPBA,
2853 TSI721_MSIXPBA_OFFSET);
2854 pci_write_config_dword(pdev, TSI721_PCIECFG_EPCTL, 0);
2855 /* End of FIXUP */
2856
2857 tsi721_disable_ints(priv);
2858
2859 tsi721_init_pc2sr_mapping(priv);
2860 tsi721_init_sr2pc_mapping(priv);
2861
2862 if (tsi721_bdma_maint_init(priv)) {
2863 tsi_err(&pdev->dev, "BDMA initialization failed");
2864 err = -ENOMEM;
2865 goto err_unmap_bars;
2866 }
2867
2868 err = tsi721_doorbell_init(priv);
2869 if (err)
2870 goto err_free_bdma;
2871
2872 tsi721_port_write_init(priv);
2873
2874 err = tsi721_messages_init(priv);
2875 if (err)
2876 goto err_free_consistent;
2877
2878 err = tsi721_setup_mport(priv);
2879 if (err)
2880 goto err_free_consistent;
2881
2882 pci_set_drvdata(pdev, priv);
2883 tsi721_interrupts_init(priv);
2884
2885 return 0;
2886
2887 err_free_consistent:
2888 tsi721_port_write_free(priv);
2889 tsi721_doorbell_free(priv);
2890 err_free_bdma:
2891 tsi721_bdma_maint_free(priv);
2892 err_unmap_bars:
2893 if (priv->regs)
2894 iounmap(priv->regs);
2895 if (priv->odb_base)
2896 iounmap(priv->odb_base);
2897 err_free_res:
2898 pci_release_regions(pdev);
2899 pci_clear_master(pdev);
2900 err_disable_pdev:
2901 pci_disable_device(pdev);
2902 err_clean:
2903 kfree(priv);
2904 err_exit:
2905 return err;
2906 }
2907
2908 static void tsi721_remove(struct pci_dev *pdev)
2909 {
2910 struct tsi721_device *priv = pci_get_drvdata(pdev);
2911
2912 tsi_debug(EXIT, &pdev->dev, "enter");
2913
2914 tsi721_disable_ints(priv);
2915 tsi721_free_irq(priv);
2916 flush_scheduled_work();
2917 rio_unregister_mport(&priv->mport);
2918
2919 tsi721_unregister_dma(priv);
2920 tsi721_bdma_maint_free(priv);
2921 tsi721_doorbell_free(priv);
2922 tsi721_port_write_free(priv);
2923 tsi721_close_sr2pc_mapping(priv);
2924
2925 if (priv->regs)
2926 iounmap(priv->regs);
2927 if (priv->odb_base)
2928 iounmap(priv->odb_base);
2929 #ifdef CONFIG_PCI_MSI
2930 if (priv->flags & TSI721_USING_MSIX)
2931 pci_disable_msix(priv->pdev);
2932 else if (priv->flags & TSI721_USING_MSI)
2933 pci_disable_msi(priv->pdev);
2934 #endif
2935 pci_release_regions(pdev);
2936 pci_clear_master(pdev);
2937 pci_disable_device(pdev);
2938 pci_set_drvdata(pdev, NULL);
2939 kfree(priv);
2940 tsi_debug(EXIT, &pdev->dev, "exit");
2941 }
2942
2943 static void tsi721_shutdown(struct pci_dev *pdev)
2944 {
2945 struct tsi721_device *priv = pci_get_drvdata(pdev);
2946
2947 tsi_debug(EXIT, &pdev->dev, "enter");
2948
2949 tsi721_disable_ints(priv);
2950 tsi721_dma_stop_all(priv);
2951 pci_clear_master(pdev);
2952 pci_disable_device(pdev);
2953 }
2954
2955 static const struct pci_device_id tsi721_pci_tbl[] = {
2956 { PCI_DEVICE(PCI_VENDOR_ID_IDT, PCI_DEVICE_ID_TSI721) },
2957 { 0, } /* terminate list */
2958 };
2959
2960 MODULE_DEVICE_TABLE(pci, tsi721_pci_tbl);
2961
2962 static struct pci_driver tsi721_driver = {
2963 .name = "tsi721",
2964 .id_table = tsi721_pci_tbl,
2965 .probe = tsi721_probe,
2966 .remove = tsi721_remove,
2967 .shutdown = tsi721_shutdown,
2968 };
2969
2970 module_pci_driver(tsi721_driver);
2971
2972 MODULE_DESCRIPTION("IDT Tsi721 PCIExpress-to-SRIO bridge driver");
2973 MODULE_AUTHOR("Integrated Device Technology, Inc.");
2974 MODULE_LICENSE("GPL");
Linux with added FPC-III support