?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
x86/xen: resume timer irqs early
[linux/fpc-iii.git] / drivers / net / wireless / ath / ath10k / ce.c
blobf8b969f518f84ecee55c4e38f5d5f8247d501cf0
1 /*
2 * Copyright (c) 2005-2011 Atheros Communications Inc.
3 * Copyright (c) 2011-2013 Qualcomm Atheros, Inc.
4 *
5 * Permission to use, copy, modify, and/or distribute this software for any
6 * purpose with or without fee is hereby granted, provided that the above
7 * copyright notice and this permission notice appear in all copies.
8 *
9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
16 */
17
18 #include "hif.h"
19 #include "pci.h"
20 #include "ce.h"
21 #include "debug.h"
22
23 /*
24 * Support for Copy Engine hardware, which is mainly used for
25 * communication between Host and Target over a PCIe interconnect.
26 */
27
28 /*
29 * A single CopyEngine (CE) comprises two "rings":
30 * a source ring
31 * a destination ring
32 *
33 * Each ring consists of a number of descriptors which specify
34 * an address, length, and meta-data.
35 *
36 * Typically, one side of the PCIe interconnect (Host or Target)
37 * controls one ring and the other side controls the other ring.
38 * The source side chooses when to initiate a transfer and it
39 * chooses what to send (buffer address, length). The destination
40 * side keeps a supply of "anonymous receive buffers" available and
41 * it handles incoming data as it arrives (when the destination
42 * recieves an interrupt).
43 *
44 * The sender may send a simple buffer (address/length) or it may
45 * send a small list of buffers. When a small list is sent, hardware
46 * "gathers" these and they end up in a single destination buffer
47 * with a single interrupt.
48 *
49 * There are several "contexts" managed by this layer -- more, it
50 * may seem -- than should be needed. These are provided mainly for
51 * maximum flexibility and especially to facilitate a simpler HIF
52 * implementation. There are per-CopyEngine recv, send, and watermark
53 * contexts. These are supplied by the caller when a recv, send,
54 * or watermark handler is established and they are echoed back to
55 * the caller when the respective callbacks are invoked. There is
56 * also a per-transfer context supplied by the caller when a buffer
57 * (or sendlist) is sent and when a buffer is enqueued for recv.
58 * These per-transfer contexts are echoed back to the caller when
59 * the buffer is sent/received.
60 */
61
62 static inline void ath10k_ce_dest_ring_write_index_set(struct ath10k *ar,
63 u32 ce_ctrl_addr,
64 unsigned int n)
65 {
66 ath10k_pci_write32(ar, ce_ctrl_addr + DST_WR_INDEX_ADDRESS, n);
67 }
68
69 static inline u32 ath10k_ce_dest_ring_write_index_get(struct ath10k *ar,
70 u32 ce_ctrl_addr)
71 {
72 return ath10k_pci_read32(ar, ce_ctrl_addr + DST_WR_INDEX_ADDRESS);
73 }
74
75 static inline void ath10k_ce_src_ring_write_index_set(struct ath10k *ar,
76 u32 ce_ctrl_addr,
77 unsigned int n)
78 {
79 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
80 void __iomem *indicator_addr;
81
82 if (!test_bit(ATH10K_PCI_FEATURE_HW_1_0_WORKAROUND, ar_pci->features)) {
83 ath10k_pci_write32(ar, ce_ctrl_addr + SR_WR_INDEX_ADDRESS, n);
84 return;
85 }
86
87 /* workaround for QCA988x_1.0 HW CE */
88 indicator_addr = ar_pci->mem + ce_ctrl_addr + DST_WATERMARK_ADDRESS;
89
90 if (ce_ctrl_addr == ath10k_ce_base_address(CDC_WAR_DATA_CE)) {
91 iowrite32((CDC_WAR_MAGIC_STR | n), indicator_addr);
92 } else {
93 unsigned long irq_flags;
94 local_irq_save(irq_flags);
95 iowrite32(1, indicator_addr);
96
97 /*
98 * PCIE write waits for ACK in IPQ8K, there is no
99 * need to read back value.
100 */
101 (void)ioread32(indicator_addr);
102 (void)ioread32(indicator_addr); /* conservative */
103
104 ath10k_pci_write32(ar, ce_ctrl_addr + SR_WR_INDEX_ADDRESS, n);
105
106 iowrite32(0, indicator_addr);
107 local_irq_restore(irq_flags);
108 }
109 }
110
111 static inline u32 ath10k_ce_src_ring_write_index_get(struct ath10k *ar,
112 u32 ce_ctrl_addr)
113 {
114 return ath10k_pci_read32(ar, ce_ctrl_addr + SR_WR_INDEX_ADDRESS);
115 }
116
117 static inline u32 ath10k_ce_src_ring_read_index_get(struct ath10k *ar,
118 u32 ce_ctrl_addr)
119 {
120 return ath10k_pci_read32(ar, ce_ctrl_addr + CURRENT_SRRI_ADDRESS);
121 }
122
123 static inline void ath10k_ce_src_ring_base_addr_set(struct ath10k *ar,
124 u32 ce_ctrl_addr,
125 unsigned int addr)
126 {
127 ath10k_pci_write32(ar, ce_ctrl_addr + SR_BA_ADDRESS, addr);
128 }
129
130 static inline void ath10k_ce_src_ring_size_set(struct ath10k *ar,
131 u32 ce_ctrl_addr,
132 unsigned int n)
133 {
134 ath10k_pci_write32(ar, ce_ctrl_addr + SR_SIZE_ADDRESS, n);
135 }
136
137 static inline void ath10k_ce_src_ring_dmax_set(struct ath10k *ar,
138 u32 ce_ctrl_addr,
139 unsigned int n)
140 {
141 u32 ctrl1_addr = ath10k_pci_read32((ar),
142 (ce_ctrl_addr) + CE_CTRL1_ADDRESS);
143
144 ath10k_pci_write32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS,
145 (ctrl1_addr & ~CE_CTRL1_DMAX_LENGTH_MASK) |
146 CE_CTRL1_DMAX_LENGTH_SET(n));
147 }
148
149 static inline void ath10k_ce_src_ring_byte_swap_set(struct ath10k *ar,
150 u32 ce_ctrl_addr,
151 unsigned int n)
152 {
153 u32 ctrl1_addr = ath10k_pci_read32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS);
154
155 ath10k_pci_write32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS,
156 (ctrl1_addr & ~CE_CTRL1_SRC_RING_BYTE_SWAP_EN_MASK) |
157 CE_CTRL1_SRC_RING_BYTE_SWAP_EN_SET(n));
158 }
159
160 static inline void ath10k_ce_dest_ring_byte_swap_set(struct ath10k *ar,
161 u32 ce_ctrl_addr,
162 unsigned int n)
163 {
164 u32 ctrl1_addr = ath10k_pci_read32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS);
165
166 ath10k_pci_write32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS,
167 (ctrl1_addr & ~CE_CTRL1_DST_RING_BYTE_SWAP_EN_MASK) |
168 CE_CTRL1_DST_RING_BYTE_SWAP_EN_SET(n));
169 }
170
171 static inline u32 ath10k_ce_dest_ring_read_index_get(struct ath10k *ar,
172 u32 ce_ctrl_addr)
173 {
174 return ath10k_pci_read32(ar, ce_ctrl_addr + CURRENT_DRRI_ADDRESS);
175 }
176
177 static inline void ath10k_ce_dest_ring_base_addr_set(struct ath10k *ar,
178 u32 ce_ctrl_addr,
179 u32 addr)
180 {
181 ath10k_pci_write32(ar, ce_ctrl_addr + DR_BA_ADDRESS, addr);
182 }
183
184 static inline void ath10k_ce_dest_ring_size_set(struct ath10k *ar,
185 u32 ce_ctrl_addr,
186 unsigned int n)
187 {
188 ath10k_pci_write32(ar, ce_ctrl_addr + DR_SIZE_ADDRESS, n);
189 }
190
191 static inline void ath10k_ce_src_ring_highmark_set(struct ath10k *ar,
192 u32 ce_ctrl_addr,
193 unsigned int n)
194 {
195 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS);
196
197 ath10k_pci_write32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS,
198 (addr & ~SRC_WATERMARK_HIGH_MASK) |
199 SRC_WATERMARK_HIGH_SET(n));
200 }
201
202 static inline void ath10k_ce_src_ring_lowmark_set(struct ath10k *ar,
203 u32 ce_ctrl_addr,
204 unsigned int n)
205 {
206 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS);
207
208 ath10k_pci_write32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS,
209 (addr & ~SRC_WATERMARK_LOW_MASK) |
210 SRC_WATERMARK_LOW_SET(n));
211 }
212
213 static inline void ath10k_ce_dest_ring_highmark_set(struct ath10k *ar,
214 u32 ce_ctrl_addr,
215 unsigned int n)
216 {
217 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS);
218
219 ath10k_pci_write32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS,
220 (addr & ~DST_WATERMARK_HIGH_MASK) |
221 DST_WATERMARK_HIGH_SET(n));
222 }
223
224 static inline void ath10k_ce_dest_ring_lowmark_set(struct ath10k *ar,
225 u32 ce_ctrl_addr,
226 unsigned int n)
227 {
228 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS);
229
230 ath10k_pci_write32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS,
231 (addr & ~DST_WATERMARK_LOW_MASK) |
232 DST_WATERMARK_LOW_SET(n));
233 }
234
235 static inline void ath10k_ce_copy_complete_inter_enable(struct ath10k *ar,
236 u32 ce_ctrl_addr)
237 {
238 u32 host_ie_addr = ath10k_pci_read32(ar,
239 ce_ctrl_addr + HOST_IE_ADDRESS);
240
241 ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IE_ADDRESS,
242 host_ie_addr | HOST_IE_COPY_COMPLETE_MASK);
243 }
244
245 static inline void ath10k_ce_copy_complete_intr_disable(struct ath10k *ar,
246 u32 ce_ctrl_addr)
247 {
248 u32 host_ie_addr = ath10k_pci_read32(ar,
249 ce_ctrl_addr + HOST_IE_ADDRESS);
250
251 ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IE_ADDRESS,
252 host_ie_addr & ~HOST_IE_COPY_COMPLETE_MASK);
253 }
254
255 static inline void ath10k_ce_watermark_intr_disable(struct ath10k *ar,
256 u32 ce_ctrl_addr)
257 {
258 u32 host_ie_addr = ath10k_pci_read32(ar,
259 ce_ctrl_addr + HOST_IE_ADDRESS);
260
261 ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IE_ADDRESS,
262 host_ie_addr & ~CE_WATERMARK_MASK);
263 }
264
265 static inline void ath10k_ce_error_intr_enable(struct ath10k *ar,
266 u32 ce_ctrl_addr)
267 {
268 u32 misc_ie_addr = ath10k_pci_read32(ar,
269 ce_ctrl_addr + MISC_IE_ADDRESS);
270
271 ath10k_pci_write32(ar, ce_ctrl_addr + MISC_IE_ADDRESS,
272 misc_ie_addr | CE_ERROR_MASK);
273 }
274
275 static inline void ath10k_ce_engine_int_status_clear(struct ath10k *ar,
276 u32 ce_ctrl_addr,
277 unsigned int mask)
278 {
279 ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IS_ADDRESS, mask);
280 }
281
282
283 /*
284 * Guts of ath10k_ce_send, used by both ath10k_ce_send and
285 * ath10k_ce_sendlist_send.
286 * The caller takes responsibility for any needed locking.
287 */
288 static int ath10k_ce_send_nolock(struct ce_state *ce_state,
289 void *per_transfer_context,
290 u32 buffer,
291 unsigned int nbytes,
292 unsigned int transfer_id,
293 unsigned int flags)
294 {
295 struct ath10k *ar = ce_state->ar;
296 struct ce_ring_state *src_ring = ce_state->src_ring;
297 struct ce_desc *desc, *sdesc;
298 unsigned int nentries_mask = src_ring->nentries_mask;
299 unsigned int sw_index = src_ring->sw_index;
300 unsigned int write_index = src_ring->write_index;
301 u32 ctrl_addr = ce_state->ctrl_addr;
302 u32 desc_flags = 0;
303 int ret = 0;
304
305 if (nbytes > ce_state->src_sz_max)
306 ath10k_warn("%s: send more we can (nbytes: %d, max: %d)\n",
307 __func__, nbytes, ce_state->src_sz_max);
308
309 ath10k_pci_wake(ar);
310
311 if (unlikely(CE_RING_DELTA(nentries_mask,
312 write_index, sw_index - 1) <= 0)) {
313 ret = -EIO;
314 goto exit;
315 }
316
317 desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space,
318 write_index);
319 sdesc = CE_SRC_RING_TO_DESC(src_ring->shadow_base, write_index);
320
321 desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA);
322
323 if (flags & CE_SEND_FLAG_GATHER)
324 desc_flags |= CE_DESC_FLAGS_GATHER;
325 if (flags & CE_SEND_FLAG_BYTE_SWAP)
326 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP;
327
328 sdesc->addr = __cpu_to_le32(buffer);
329 sdesc->nbytes = __cpu_to_le16(nbytes);
330 sdesc->flags = __cpu_to_le16(desc_flags);
331
332 *desc = *sdesc;
333
334 src_ring->per_transfer_context[write_index] = per_transfer_context;
335
336 /* Update Source Ring Write Index */
337 write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
338
339 /* WORKAROUND */
340 if (!(flags & CE_SEND_FLAG_GATHER))
341 ath10k_ce_src_ring_write_index_set(ar, ctrl_addr, write_index);
342
343 src_ring->write_index = write_index;
344 exit:
345 ath10k_pci_sleep(ar);
346 return ret;
347 }
348
349 int ath10k_ce_send(struct ce_state *ce_state,
350 void *per_transfer_context,
351 u32 buffer,
352 unsigned int nbytes,
353 unsigned int transfer_id,
354 unsigned int flags)
355 {
356 struct ath10k *ar = ce_state->ar;
357 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
358 int ret;
359
360 spin_lock_bh(&ar_pci->ce_lock);
361 ret = ath10k_ce_send_nolock(ce_state, per_transfer_context,
362 buffer, nbytes, transfer_id, flags);
363 spin_unlock_bh(&ar_pci->ce_lock);
364
365 return ret;
366 }
367
368 void ath10k_ce_sendlist_buf_add(struct ce_sendlist *sendlist, u32 buffer,
369 unsigned int nbytes, u32 flags)
370 {
371 unsigned int num_items = sendlist->num_items;
372 struct ce_sendlist_item *item;
373
374 item = &sendlist->item[num_items];
375 item->data = buffer;
376 item->u.nbytes = nbytes;
377 item->flags = flags;
378 sendlist->num_items++;
379 }
380
381 int ath10k_ce_sendlist_send(struct ce_state *ce_state,
382 void *per_transfer_context,
383 struct ce_sendlist *sendlist,
384 unsigned int transfer_id)
385 {
386 struct ce_ring_state *src_ring = ce_state->src_ring;
387 struct ce_sendlist_item *item;
388 struct ath10k *ar = ce_state->ar;
389 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
390 unsigned int nentries_mask = src_ring->nentries_mask;
391 unsigned int num_items = sendlist->num_items;
392 unsigned int sw_index;
393 unsigned int write_index;
394 int i, delta, ret = -ENOMEM;
395
396 spin_lock_bh(&ar_pci->ce_lock);
397
398 sw_index = src_ring->sw_index;
399 write_index = src_ring->write_index;
400
401 delta = CE_RING_DELTA(nentries_mask, write_index, sw_index - 1);
402
403 if (delta >= num_items) {
404 /*
405 * Handle all but the last item uniformly.
406 */
407 for (i = 0; i < num_items - 1; i++) {
408 item = &sendlist->item[i];
409 ret = ath10k_ce_send_nolock(ce_state,
410 CE_SENDLIST_ITEM_CTXT,
411 (u32) item->data,
412 item->u.nbytes, transfer_id,
413 item->flags |
414 CE_SEND_FLAG_GATHER);
415 if (ret)
416 ath10k_warn("CE send failed for item: %d\n", i);
417 }
418 /*
419 * Provide valid context pointer for final item.
420 */
421 item = &sendlist->item[i];
422 ret = ath10k_ce_send_nolock(ce_state, per_transfer_context,
423 (u32) item->data, item->u.nbytes,
424 transfer_id, item->flags);
425 if (ret)
426 ath10k_warn("CE send failed for last item: %d\n", i);
427 }
428
429 spin_unlock_bh(&ar_pci->ce_lock);
430
431 return ret;
432 }
433
434 int ath10k_ce_recv_buf_enqueue(struct ce_state *ce_state,
435 void *per_recv_context,
436 u32 buffer)
437 {
438 struct ce_ring_state *dest_ring = ce_state->dest_ring;
439 u32 ctrl_addr = ce_state->ctrl_addr;
440 struct ath10k *ar = ce_state->ar;
441 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
442 unsigned int nentries_mask = dest_ring->nentries_mask;
443 unsigned int write_index;
444 unsigned int sw_index;
445 int ret;
446
447 spin_lock_bh(&ar_pci->ce_lock);
448 write_index = dest_ring->write_index;
449 sw_index = dest_ring->sw_index;
450
451 ath10k_pci_wake(ar);
452
453 if (CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) > 0) {
454 struct ce_desc *base = dest_ring->base_addr_owner_space;
455 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, write_index);
456
457 /* Update destination descriptor */
458 desc->addr = __cpu_to_le32(buffer);
459 desc->nbytes = 0;
460
461 dest_ring->per_transfer_context[write_index] =
462 per_recv_context;
463
464 /* Update Destination Ring Write Index */
465 write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
466 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
467 dest_ring->write_index = write_index;
468 ret = 0;
469 } else {
470 ret = -EIO;
471 }
472 ath10k_pci_sleep(ar);
473 spin_unlock_bh(&ar_pci->ce_lock);
474
475 return ret;
476 }
477
478 /*
479 * Guts of ath10k_ce_completed_recv_next.
480 * The caller takes responsibility for any necessary locking.
481 */
482 static int ath10k_ce_completed_recv_next_nolock(struct ce_state *ce_state,
483 void **per_transfer_contextp,
484 u32 *bufferp,
485 unsigned int *nbytesp,
486 unsigned int *transfer_idp,
487 unsigned int *flagsp)
488 {
489 struct ce_ring_state *dest_ring = ce_state->dest_ring;
490 unsigned int nentries_mask = dest_ring->nentries_mask;
491 unsigned int sw_index = dest_ring->sw_index;
492
493 struct ce_desc *base = dest_ring->base_addr_owner_space;
494 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index);
495 struct ce_desc sdesc;
496 u16 nbytes;
497
498 /* Copy in one go for performance reasons */
499 sdesc = *desc;
500
501 nbytes = __le16_to_cpu(sdesc.nbytes);
502 if (nbytes == 0) {
503 /*
504 * This closes a relatively unusual race where the Host
505 * sees the updated DRRI before the update to the
506 * corresponding descriptor has completed. We treat this
507 * as a descriptor that is not yet done.
508 */
509 return -EIO;
510 }
511
512 desc->nbytes = 0;
513
514 /* Return data from completed destination descriptor */
515 *bufferp = __le32_to_cpu(sdesc.addr);
516 *nbytesp = nbytes;
517 *transfer_idp = MS(__le16_to_cpu(sdesc.flags), CE_DESC_FLAGS_META_DATA);
518
519 if (__le16_to_cpu(sdesc.flags) & CE_DESC_FLAGS_BYTE_SWAP)
520 *flagsp = CE_RECV_FLAG_SWAPPED;
521 else
522 *flagsp = 0;
523
524 if (per_transfer_contextp)
525 *per_transfer_contextp =
526 dest_ring->per_transfer_context[sw_index];
527
528 /* sanity */
529 dest_ring->per_transfer_context[sw_index] = NULL;
530
531 /* Update sw_index */
532 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
533 dest_ring->sw_index = sw_index;
534
535 return 0;
536 }
537
538 int ath10k_ce_completed_recv_next(struct ce_state *ce_state,
539 void **per_transfer_contextp,
540 u32 *bufferp,
541 unsigned int *nbytesp,
542 unsigned int *transfer_idp,
543 unsigned int *flagsp)
544 {
545 struct ath10k *ar = ce_state->ar;
546 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
547 int ret;
548
549 spin_lock_bh(&ar_pci->ce_lock);
550 ret = ath10k_ce_completed_recv_next_nolock(ce_state,
551 per_transfer_contextp,
552 bufferp, nbytesp,
553 transfer_idp, flagsp);
554 spin_unlock_bh(&ar_pci->ce_lock);
555
556 return ret;
557 }
558
559 int ath10k_ce_revoke_recv_next(struct ce_state *ce_state,
560 void **per_transfer_contextp,
561 u32 *bufferp)
562 {
563 struct ce_ring_state *dest_ring;
564 unsigned int nentries_mask;
565 unsigned int sw_index;
566 unsigned int write_index;
567 int ret;
568 struct ath10k *ar;
569 struct ath10k_pci *ar_pci;
570
571 dest_ring = ce_state->dest_ring;
572
573 if (!dest_ring)
574 return -EIO;
575
576 ar = ce_state->ar;
577 ar_pci = ath10k_pci_priv(ar);
578
579 spin_lock_bh(&ar_pci->ce_lock);
580
581 nentries_mask = dest_ring->nentries_mask;
582 sw_index = dest_ring->sw_index;
583 write_index = dest_ring->write_index;
584 if (write_index != sw_index) {
585 struct ce_desc *base = dest_ring->base_addr_owner_space;
586 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index);
587
588 /* Return data from completed destination descriptor */
589 *bufferp = __le32_to_cpu(desc->addr);
590
591 if (per_transfer_contextp)
592 *per_transfer_contextp =
593 dest_ring->per_transfer_context[sw_index];
594
595 /* sanity */
596 dest_ring->per_transfer_context[sw_index] = NULL;
597
598 /* Update sw_index */
599 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
600 dest_ring->sw_index = sw_index;
601 ret = 0;
602 } else {
603 ret = -EIO;
604 }
605
606 spin_unlock_bh(&ar_pci->ce_lock);
607
608 return ret;
609 }
610
611 /*
612 * Guts of ath10k_ce_completed_send_next.
613 * The caller takes responsibility for any necessary locking.
614 */
615 static int ath10k_ce_completed_send_next_nolock(struct ce_state *ce_state,
616 void **per_transfer_contextp,
617 u32 *bufferp,
618 unsigned int *nbytesp,
619 unsigned int *transfer_idp)
620 {
621 struct ce_ring_state *src_ring = ce_state->src_ring;
622 u32 ctrl_addr = ce_state->ctrl_addr;
623 struct ath10k *ar = ce_state->ar;
624 unsigned int nentries_mask = src_ring->nentries_mask;
625 unsigned int sw_index = src_ring->sw_index;
626 unsigned int read_index;
627 int ret = -EIO;
628
629 if (src_ring->hw_index == sw_index) {
630 /*
631 * The SW completion index has caught up with the cached
632 * version of the HW completion index.
633 * Update the cached HW completion index to see whether
634 * the SW has really caught up to the HW, or if the cached
635 * value of the HW index has become stale.
636 */
637 ath10k_pci_wake(ar);
638 src_ring->hw_index =
639 ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
640 src_ring->hw_index &= nentries_mask;
641 ath10k_pci_sleep(ar);
642 }
643 read_index = src_ring->hw_index;
644
645 if ((read_index != sw_index) && (read_index != 0xffffffff)) {
646 struct ce_desc *sbase = src_ring->shadow_base;
647 struct ce_desc *sdesc = CE_SRC_RING_TO_DESC(sbase, sw_index);
648
649 /* Return data from completed source descriptor */
650 *bufferp = __le32_to_cpu(sdesc->addr);
651 *nbytesp = __le16_to_cpu(sdesc->nbytes);
652 *transfer_idp = MS(__le16_to_cpu(sdesc->flags),
653 CE_DESC_FLAGS_META_DATA);
654
655 if (per_transfer_contextp)
656 *per_transfer_contextp =
657 src_ring->per_transfer_context[sw_index];
658
659 /* sanity */
660 src_ring->per_transfer_context[sw_index] = NULL;
661
662 /* Update sw_index */
663 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
664 src_ring->sw_index = sw_index;
665 ret = 0;
666 }
667
668 return ret;
669 }
670
671 /* NB: Modeled after ath10k_ce_completed_send_next */
672 int ath10k_ce_cancel_send_next(struct ce_state *ce_state,
673 void **per_transfer_contextp,
674 u32 *bufferp,
675 unsigned int *nbytesp,
676 unsigned int *transfer_idp)
677 {
678 struct ce_ring_state *src_ring;
679 unsigned int nentries_mask;
680 unsigned int sw_index;
681 unsigned int write_index;
682 int ret;
683 struct ath10k *ar;
684 struct ath10k_pci *ar_pci;
685
686 src_ring = ce_state->src_ring;
687
688 if (!src_ring)
689 return -EIO;
690
691 ar = ce_state->ar;
692 ar_pci = ath10k_pci_priv(ar);
693
694 spin_lock_bh(&ar_pci->ce_lock);
695
696 nentries_mask = src_ring->nentries_mask;
697 sw_index = src_ring->sw_index;
698 write_index = src_ring->write_index;
699
700 if (write_index != sw_index) {
701 struct ce_desc *base = src_ring->base_addr_owner_space;
702 struct ce_desc *desc = CE_SRC_RING_TO_DESC(base, sw_index);
703
704 /* Return data from completed source descriptor */
705 *bufferp = __le32_to_cpu(desc->addr);
706 *nbytesp = __le16_to_cpu(desc->nbytes);
707 *transfer_idp = MS(__le16_to_cpu(desc->flags),
708 CE_DESC_FLAGS_META_DATA);
709
710 if (per_transfer_contextp)
711 *per_transfer_contextp =
712 src_ring->per_transfer_context[sw_index];
713
714 /* sanity */
715 src_ring->per_transfer_context[sw_index] = NULL;
716
717 /* Update sw_index */
718 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
719 src_ring->sw_index = sw_index;
720 ret = 0;
721 } else {
722 ret = -EIO;
723 }
724
725 spin_unlock_bh(&ar_pci->ce_lock);
726
727 return ret;
728 }
729
730 int ath10k_ce_completed_send_next(struct ce_state *ce_state,
731 void **per_transfer_contextp,
732 u32 *bufferp,
733 unsigned int *nbytesp,
734 unsigned int *transfer_idp)
735 {
736 struct ath10k *ar = ce_state->ar;
737 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
738 int ret;
739
740 spin_lock_bh(&ar_pci->ce_lock);
741 ret = ath10k_ce_completed_send_next_nolock(ce_state,
742 per_transfer_contextp,
743 bufferp, nbytesp,
744 transfer_idp);
745 spin_unlock_bh(&ar_pci->ce_lock);
746
747 return ret;
748 }
749
750 /*
751 * Guts of interrupt handler for per-engine interrupts on a particular CE.
752 *
753 * Invokes registered callbacks for recv_complete,
754 * send_complete, and watermarks.
755 */
756 void ath10k_ce_per_engine_service(struct ath10k *ar, unsigned int ce_id)
757 {
758 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
759 struct ce_state *ce_state = ar_pci->ce_id_to_state[ce_id];
760 u32 ctrl_addr = ce_state->ctrl_addr;
761 void *transfer_context;
762 u32 buf;
763 unsigned int nbytes;
764 unsigned int id;
765 unsigned int flags;
766
767 ath10k_pci_wake(ar);
768 spin_lock_bh(&ar_pci->ce_lock);
769
770 /* Clear the copy-complete interrupts that will be handled here. */
771 ath10k_ce_engine_int_status_clear(ar, ctrl_addr,
772 HOST_IS_COPY_COMPLETE_MASK);
773
774 if (ce_state->recv_cb) {
775 /*
776 * Pop completed recv buffers and call the registered
777 * recv callback for each
778 */
779 while (ath10k_ce_completed_recv_next_nolock(ce_state,
780 &transfer_context,
781 &buf, &nbytes,
782 &id, &flags) == 0) {
783 spin_unlock_bh(&ar_pci->ce_lock);
784 ce_state->recv_cb(ce_state, transfer_context, buf,
785 nbytes, id, flags);
786 spin_lock_bh(&ar_pci->ce_lock);
787 }
788 }
789
790 if (ce_state->send_cb) {
791 /*
792 * Pop completed send buffers and call the registered
793 * send callback for each
794 */
795 while (ath10k_ce_completed_send_next_nolock(ce_state,
796 &transfer_context,
797 &buf,
798 &nbytes,
799 &id) == 0) {
800 spin_unlock_bh(&ar_pci->ce_lock);
801 ce_state->send_cb(ce_state, transfer_context,
802 buf, nbytes, id);
803 spin_lock_bh(&ar_pci->ce_lock);
804 }
805 }
806
807 /*
808 * Misc CE interrupts are not being handled, but still need
809 * to be cleared.
810 */
811 ath10k_ce_engine_int_status_clear(ar, ctrl_addr, CE_WATERMARK_MASK);
812
813 spin_unlock_bh(&ar_pci->ce_lock);
814 ath10k_pci_sleep(ar);
815 }
816
817 /*
818 * Handler for per-engine interrupts on ALL active CEs.
819 * This is used in cases where the system is sharing a
820 * single interrput for all CEs
821 */
822
823 void ath10k_ce_per_engine_service_any(struct ath10k *ar)
824 {
825 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
826 int ce_id;
827 u32 intr_summary;
828
829 ath10k_pci_wake(ar);
830 intr_summary = CE_INTERRUPT_SUMMARY(ar);
831
832 for (ce_id = 0; intr_summary && (ce_id < ar_pci->ce_count); ce_id++) {
833 if (intr_summary & (1 << ce_id))
834 intr_summary &= ~(1 << ce_id);
835 else
836 /* no intr pending on this CE */
837 continue;
838
839 ath10k_ce_per_engine_service(ar, ce_id);
840 }
841
842 ath10k_pci_sleep(ar);
843 }
844
845 /*
846 * Adjust interrupts for the copy complete handler.
847 * If it's needed for either send or recv, then unmask
848 * this interrupt; otherwise, mask it.
849 *
850 * Called with ce_lock held.
851 */
852 static void ath10k_ce_per_engine_handler_adjust(struct ce_state *ce_state,
853 int disable_copy_compl_intr)
854 {
855 u32 ctrl_addr = ce_state->ctrl_addr;
856 struct ath10k *ar = ce_state->ar;
857
858 ath10k_pci_wake(ar);
859
860 if ((!disable_copy_compl_intr) &&
861 (ce_state->send_cb || ce_state->recv_cb))
862 ath10k_ce_copy_complete_inter_enable(ar, ctrl_addr);
863 else
864 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr);
865
866 ath10k_ce_watermark_intr_disable(ar, ctrl_addr);
867
868 ath10k_pci_sleep(ar);
869 }
870
871 void ath10k_ce_disable_interrupts(struct ath10k *ar)
872 {
873 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
874 int ce_id;
875
876 ath10k_pci_wake(ar);
877 for (ce_id = 0; ce_id < ar_pci->ce_count; ce_id++) {
878 struct ce_state *ce_state = ar_pci->ce_id_to_state[ce_id];
879 u32 ctrl_addr = ce_state->ctrl_addr;
880
881 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr);
882 }
883 ath10k_pci_sleep(ar);
884 }
885
886 void ath10k_ce_send_cb_register(struct ce_state *ce_state,
887 void (*send_cb) (struct ce_state *ce_state,
888 void *transfer_context,
889 u32 buffer,
890 unsigned int nbytes,
891 unsigned int transfer_id),
892 int disable_interrupts)
893 {
894 struct ath10k *ar = ce_state->ar;
895 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
896
897 spin_lock_bh(&ar_pci->ce_lock);
898 ce_state->send_cb = send_cb;
899 ath10k_ce_per_engine_handler_adjust(ce_state, disable_interrupts);
900 spin_unlock_bh(&ar_pci->ce_lock);
901 }
902
903 void ath10k_ce_recv_cb_register(struct ce_state *ce_state,
904 void (*recv_cb) (struct ce_state *ce_state,
905 void *transfer_context,
906 u32 buffer,
907 unsigned int nbytes,
908 unsigned int transfer_id,
909 unsigned int flags))
910 {
911 struct ath10k *ar = ce_state->ar;
912 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
913
914 spin_lock_bh(&ar_pci->ce_lock);
915 ce_state->recv_cb = recv_cb;
916 ath10k_ce_per_engine_handler_adjust(ce_state, 0);
917 spin_unlock_bh(&ar_pci->ce_lock);
918 }
919
920 static int ath10k_ce_init_src_ring(struct ath10k *ar,
921 unsigned int ce_id,
922 struct ce_state *ce_state,
923 const struct ce_attr *attr)
924 {
925 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
926 struct ce_ring_state *src_ring;
927 unsigned int nentries = attr->src_nentries;
928 unsigned int ce_nbytes;
929 u32 ctrl_addr = ath10k_ce_base_address(ce_id);
930 dma_addr_t base_addr;
931 char *ptr;
932
933 nentries = roundup_pow_of_two(nentries);
934
935 if (ce_state->src_ring) {
936 WARN_ON(ce_state->src_ring->nentries != nentries);
937 return 0;
938 }
939
940 ce_nbytes = sizeof(struct ce_ring_state) + (nentries * sizeof(void *));
941 ptr = kzalloc(ce_nbytes, GFP_KERNEL);
942 if (ptr == NULL)
943 return -ENOMEM;
944
945 ce_state->src_ring = (struct ce_ring_state *)ptr;
946 src_ring = ce_state->src_ring;
947
948 ptr += sizeof(struct ce_ring_state);
949 src_ring->nentries = nentries;
950 src_ring->nentries_mask = nentries - 1;
951
952 ath10k_pci_wake(ar);
953 src_ring->sw_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
954 src_ring->sw_index &= src_ring->nentries_mask;
955 src_ring->hw_index = src_ring->sw_index;
956
957 src_ring->write_index =
958 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr);
959 src_ring->write_index &= src_ring->nentries_mask;
960 ath10k_pci_sleep(ar);
961
962 src_ring->per_transfer_context = (void **)ptr;
963
964 /*
965 * Legacy platforms that do not support cache
966 * coherent DMA are unsupported
967 */
968 src_ring->base_addr_owner_space_unaligned =
969 pci_alloc_consistent(ar_pci->pdev,
970 (nentries * sizeof(struct ce_desc) +
971 CE_DESC_RING_ALIGN),
972 &base_addr);
973 src_ring->base_addr_ce_space_unaligned = base_addr;
974
975 src_ring->base_addr_owner_space = PTR_ALIGN(
976 src_ring->base_addr_owner_space_unaligned,
977 CE_DESC_RING_ALIGN);
978 src_ring->base_addr_ce_space = ALIGN(
979 src_ring->base_addr_ce_space_unaligned,
980 CE_DESC_RING_ALIGN);
981
982 /*
983 * Also allocate a shadow src ring in regular
984 * mem to use for faster access.
985 */
986 src_ring->shadow_base_unaligned =
987 kmalloc((nentries * sizeof(struct ce_desc) +
988 CE_DESC_RING_ALIGN), GFP_KERNEL);
989
990 src_ring->shadow_base = PTR_ALIGN(
991 src_ring->shadow_base_unaligned,
992 CE_DESC_RING_ALIGN);
993
994 ath10k_pci_wake(ar);
995 ath10k_ce_src_ring_base_addr_set(ar, ctrl_addr,
996 src_ring->base_addr_ce_space);
997 ath10k_ce_src_ring_size_set(ar, ctrl_addr, nentries);
998 ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, attr->src_sz_max);
999 ath10k_ce_src_ring_byte_swap_set(ar, ctrl_addr, 0);
1000 ath10k_ce_src_ring_lowmark_set(ar, ctrl_addr, 0);
1001 ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, nentries);
1002 ath10k_pci_sleep(ar);
1003
1004 return 0;
1005 }
1006
1007 static int ath10k_ce_init_dest_ring(struct ath10k *ar,
1008 unsigned int ce_id,
1009 struct ce_state *ce_state,
1010 const struct ce_attr *attr)
1011 {
1012 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
1013 struct ce_ring_state *dest_ring;
1014 unsigned int nentries = attr->dest_nentries;
1015 unsigned int ce_nbytes;
1016 u32 ctrl_addr = ath10k_ce_base_address(ce_id);
1017 dma_addr_t base_addr;
1018 char *ptr;
1019
1020 nentries = roundup_pow_of_two(nentries);
1021
1022 if (ce_state->dest_ring) {
1023 WARN_ON(ce_state->dest_ring->nentries != nentries);
1024 return 0;
1025 }
1026
1027 ce_nbytes = sizeof(struct ce_ring_state) + (nentries * sizeof(void *));
1028 ptr = kzalloc(ce_nbytes, GFP_KERNEL);
1029 if (ptr == NULL)
1030 return -ENOMEM;
1031
1032 ce_state->dest_ring = (struct ce_ring_state *)ptr;
1033 dest_ring = ce_state->dest_ring;
1034
1035 ptr += sizeof(struct ce_ring_state);
1036 dest_ring->nentries = nentries;
1037 dest_ring->nentries_mask = nentries - 1;
1038
1039 ath10k_pci_wake(ar);
1040 dest_ring->sw_index = ath10k_ce_dest_ring_read_index_get(ar, ctrl_addr);
1041 dest_ring->sw_index &= dest_ring->nentries_mask;
1042 dest_ring->write_index =
1043 ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr);
1044 dest_ring->write_index &= dest_ring->nentries_mask;
1045 ath10k_pci_sleep(ar);
1046
1047 dest_ring->per_transfer_context = (void **)ptr;
1048
1049 /*
1050 * Legacy platforms that do not support cache
1051 * coherent DMA are unsupported
1052 */
1053 dest_ring->base_addr_owner_space_unaligned =
1054 pci_alloc_consistent(ar_pci->pdev,
1055 (nentries * sizeof(struct ce_desc) +
1056 CE_DESC_RING_ALIGN),
1057 &base_addr);
1058 dest_ring->base_addr_ce_space_unaligned = base_addr;
1059
1060 /*
1061 * Correctly initialize memory to 0 to prevent garbage
1062 * data crashing system when download firmware
1063 */
1064 memset(dest_ring->base_addr_owner_space_unaligned, 0,
1065 nentries * sizeof(struct ce_desc) + CE_DESC_RING_ALIGN);
1066
1067 dest_ring->base_addr_owner_space = PTR_ALIGN(
1068 dest_ring->base_addr_owner_space_unaligned,
1069 CE_DESC_RING_ALIGN);
1070 dest_ring->base_addr_ce_space = ALIGN(
1071 dest_ring->base_addr_ce_space_unaligned,
1072 CE_DESC_RING_ALIGN);
1073
1074 ath10k_pci_wake(ar);
1075 ath10k_ce_dest_ring_base_addr_set(ar, ctrl_addr,
1076 dest_ring->base_addr_ce_space);
1077 ath10k_ce_dest_ring_size_set(ar, ctrl_addr, nentries);
1078 ath10k_ce_dest_ring_byte_swap_set(ar, ctrl_addr, 0);
1079 ath10k_ce_dest_ring_lowmark_set(ar, ctrl_addr, 0);
1080 ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, nentries);
1081 ath10k_pci_sleep(ar);
1082
1083 return 0;
1084 }
1085
1086 static struct ce_state *ath10k_ce_init_state(struct ath10k *ar,
1087 unsigned int ce_id,
1088 const struct ce_attr *attr)
1089 {
1090 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
1091 struct ce_state *ce_state = NULL;
1092 u32 ctrl_addr = ath10k_ce_base_address(ce_id);
1093
1094 spin_lock_bh(&ar_pci->ce_lock);
1095
1096 if (!ar_pci->ce_id_to_state[ce_id]) {
1097 ce_state = kzalloc(sizeof(*ce_state), GFP_ATOMIC);
1098 if (ce_state == NULL) {
1099 spin_unlock_bh(&ar_pci->ce_lock);
1100 return NULL;
1101 }
1102
1103 ar_pci->ce_id_to_state[ce_id] = ce_state;
1104 ce_state->ar = ar;
1105 ce_state->id = ce_id;
1106 ce_state->ctrl_addr = ctrl_addr;
1107 ce_state->state = CE_RUNNING;
1108 /* Save attribute flags */
1109 ce_state->attr_flags = attr->flags;
1110 ce_state->src_sz_max = attr->src_sz_max;
1111 }
1112
1113 spin_unlock_bh(&ar_pci->ce_lock);
1114
1115 return ce_state;
1116 }
1117
1118 /*
1119 * Initialize a Copy Engine based on caller-supplied attributes.
1120 * This may be called once to initialize both source and destination
1121 * rings or it may be called twice for separate source and destination
1122 * initialization. It may be that only one side or the other is
1123 * initialized by software/firmware.
1124 */
1125 struct ce_state *ath10k_ce_init(struct ath10k *ar,
1126 unsigned int ce_id,
1127 const struct ce_attr *attr)
1128 {
1129 struct ce_state *ce_state;
1130 u32 ctrl_addr = ath10k_ce_base_address(ce_id);
1131
1132 ce_state = ath10k_ce_init_state(ar, ce_id, attr);
1133 if (!ce_state) {
1134 ath10k_err("Failed to initialize CE state for ID: %d\n", ce_id);
1135 return NULL;
1136 }
1137
1138 if (attr->src_nentries) {
1139 if (ath10k_ce_init_src_ring(ar, ce_id, ce_state, attr)) {
1140 ath10k_err("Failed to initialize CE src ring for ID: %d\n",
1141 ce_id);
1142 ath10k_ce_deinit(ce_state);
1143 return NULL;
1144 }
1145 }
1146
1147 if (attr->dest_nentries) {
1148 if (ath10k_ce_init_dest_ring(ar, ce_id, ce_state, attr)) {
1149 ath10k_err("Failed to initialize CE dest ring for ID: %d\n",
1150 ce_id);
1151 ath10k_ce_deinit(ce_state);
1152 return NULL;
1153 }
1154 }
1155
1156 /* Enable CE error interrupts */
1157 ath10k_pci_wake(ar);
1158 ath10k_ce_error_intr_enable(ar, ctrl_addr);
1159 ath10k_pci_sleep(ar);
1160
1161 return ce_state;
1162 }
1163
1164 void ath10k_ce_deinit(struct ce_state *ce_state)
1165 {
1166 unsigned int ce_id = ce_state->id;
1167 struct ath10k *ar = ce_state->ar;
1168 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
1169
1170 ce_state->state = CE_UNUSED;
1171 ar_pci->ce_id_to_state[ce_id] = NULL;
1172
1173 if (ce_state->src_ring) {
1174 kfree(ce_state->src_ring->shadow_base_unaligned);
1175 pci_free_consistent(ar_pci->pdev,
1176 (ce_state->src_ring->nentries *
1177 sizeof(struct ce_desc) +
1178 CE_DESC_RING_ALIGN),
1179 ce_state->src_ring->base_addr_owner_space,
1180 ce_state->src_ring->base_addr_ce_space);
1181 kfree(ce_state->src_ring);
1182 }
1183
1184 if (ce_state->dest_ring) {
1185 pci_free_consistent(ar_pci->pdev,
1186 (ce_state->dest_ring->nentries *
1187 sizeof(struct ce_desc) +
1188 CE_DESC_RING_ALIGN),
1189 ce_state->dest_ring->base_addr_owner_space,
1190 ce_state->dest_ring->base_addr_ce_space);
1191 kfree(ce_state->dest_ring);
1192 }
1193 kfree(ce_state);
1194 }
Linux with added FPC-III support