?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'regmap-fix-v4.9-rc3' of git://git.kernel.org/pub/scm/linux/kernel/git...
[linux/fpc-iii.git] / drivers / net / wireless / ath / ath10k / ce.c
blob0b4d7965988445d0e9660cafeb1a5fd6e3eb64ad
1 /*
2 * Copyright (c) 2005-2011 Atheros Communications Inc.
3 * Copyright (c) 2011-2013 Qualcomm Atheros, Inc.
4 *
5 * Permission to use, copy, modify, and/or distribute this software for any
6 * purpose with or without fee is hereby granted, provided that the above
7 * copyright notice and this permission notice appear in all copies.
8 *
9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
16 */
17
18 #include "hif.h"
19 #include "pci.h"
20 #include "ce.h"
21 #include "debug.h"
22
23 /*
24 * Support for Copy Engine hardware, which is mainly used for
25 * communication between Host and Target over a PCIe interconnect.
26 */
27
28 /*
29 * A single CopyEngine (CE) comprises two "rings":
30 * a source ring
31 * a destination ring
32 *
33 * Each ring consists of a number of descriptors which specify
34 * an address, length, and meta-data.
35 *
36 * Typically, one side of the PCIe interconnect (Host or Target)
37 * controls one ring and the other side controls the other ring.
38 * The source side chooses when to initiate a transfer and it
39 * chooses what to send (buffer address, length). The destination
40 * side keeps a supply of "anonymous receive buffers" available and
41 * it handles incoming data as it arrives (when the destination
42 * receives an interrupt).
43 *
44 * The sender may send a simple buffer (address/length) or it may
45 * send a small list of buffers. When a small list is sent, hardware
46 * "gathers" these and they end up in a single destination buffer
47 * with a single interrupt.
48 *
49 * There are several "contexts" managed by this layer -- more, it
50 * may seem -- than should be needed. These are provided mainly for
51 * maximum flexibility and especially to facilitate a simpler HIF
52 * implementation. There are per-CopyEngine recv, send, and watermark
53 * contexts. These are supplied by the caller when a recv, send,
54 * or watermark handler is established and they are echoed back to
55 * the caller when the respective callbacks are invoked. There is
56 * also a per-transfer context supplied by the caller when a buffer
57 * (or sendlist) is sent and when a buffer is enqueued for recv.
58 * These per-transfer contexts are echoed back to the caller when
59 * the buffer is sent/received.
60 */
61
62 static inline void ath10k_ce_dest_ring_write_index_set(struct ath10k *ar,
63 u32 ce_ctrl_addr,
64 unsigned int n)
65 {
66 ath10k_pci_write32(ar, ce_ctrl_addr + DST_WR_INDEX_ADDRESS, n);
67 }
68
69 static inline u32 ath10k_ce_dest_ring_write_index_get(struct ath10k *ar,
70 u32 ce_ctrl_addr)
71 {
72 return ath10k_pci_read32(ar, ce_ctrl_addr + DST_WR_INDEX_ADDRESS);
73 }
74
75 static inline void ath10k_ce_src_ring_write_index_set(struct ath10k *ar,
76 u32 ce_ctrl_addr,
77 unsigned int n)
78 {
79 ath10k_pci_write32(ar, ce_ctrl_addr + SR_WR_INDEX_ADDRESS, n);
80 }
81
82 static inline u32 ath10k_ce_src_ring_write_index_get(struct ath10k *ar,
83 u32 ce_ctrl_addr)
84 {
85 return ath10k_pci_read32(ar, ce_ctrl_addr + SR_WR_INDEX_ADDRESS);
86 }
87
88 static inline u32 ath10k_ce_src_ring_read_index_get(struct ath10k *ar,
89 u32 ce_ctrl_addr)
90 {
91 return ath10k_pci_read32(ar, ce_ctrl_addr + CURRENT_SRRI_ADDRESS);
92 }
93
94 static inline void ath10k_ce_src_ring_base_addr_set(struct ath10k *ar,
95 u32 ce_ctrl_addr,
96 unsigned int addr)
97 {
98 ath10k_pci_write32(ar, ce_ctrl_addr + SR_BA_ADDRESS, addr);
99 }
100
101 static inline void ath10k_ce_src_ring_size_set(struct ath10k *ar,
102 u32 ce_ctrl_addr,
103 unsigned int n)
104 {
105 ath10k_pci_write32(ar, ce_ctrl_addr + SR_SIZE_ADDRESS, n);
106 }
107
108 static inline void ath10k_ce_src_ring_dmax_set(struct ath10k *ar,
109 u32 ce_ctrl_addr,
110 unsigned int n)
111 {
112 u32 ctrl1_addr = ath10k_pci_read32((ar),
113 (ce_ctrl_addr) + CE_CTRL1_ADDRESS);
114
115 ath10k_pci_write32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS,
116 (ctrl1_addr & ~CE_CTRL1_DMAX_LENGTH_MASK) |
117 CE_CTRL1_DMAX_LENGTH_SET(n));
118 }
119
120 static inline void ath10k_ce_src_ring_byte_swap_set(struct ath10k *ar,
121 u32 ce_ctrl_addr,
122 unsigned int n)
123 {
124 u32 ctrl1_addr = ath10k_pci_read32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS);
125
126 ath10k_pci_write32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS,
127 (ctrl1_addr & ~CE_CTRL1_SRC_RING_BYTE_SWAP_EN_MASK) |
128 CE_CTRL1_SRC_RING_BYTE_SWAP_EN_SET(n));
129 }
130
131 static inline void ath10k_ce_dest_ring_byte_swap_set(struct ath10k *ar,
132 u32 ce_ctrl_addr,
133 unsigned int n)
134 {
135 u32 ctrl1_addr = ath10k_pci_read32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS);
136
137 ath10k_pci_write32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS,
138 (ctrl1_addr & ~CE_CTRL1_DST_RING_BYTE_SWAP_EN_MASK) |
139 CE_CTRL1_DST_RING_BYTE_SWAP_EN_SET(n));
140 }
141
142 static inline u32 ath10k_ce_dest_ring_read_index_get(struct ath10k *ar,
143 u32 ce_ctrl_addr)
144 {
145 return ath10k_pci_read32(ar, ce_ctrl_addr + CURRENT_DRRI_ADDRESS);
146 }
147
148 static inline void ath10k_ce_dest_ring_base_addr_set(struct ath10k *ar,
149 u32 ce_ctrl_addr,
150 u32 addr)
151 {
152 ath10k_pci_write32(ar, ce_ctrl_addr + DR_BA_ADDRESS, addr);
153 }
154
155 static inline void ath10k_ce_dest_ring_size_set(struct ath10k *ar,
156 u32 ce_ctrl_addr,
157 unsigned int n)
158 {
159 ath10k_pci_write32(ar, ce_ctrl_addr + DR_SIZE_ADDRESS, n);
160 }
161
162 static inline void ath10k_ce_src_ring_highmark_set(struct ath10k *ar,
163 u32 ce_ctrl_addr,
164 unsigned int n)
165 {
166 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS);
167
168 ath10k_pci_write32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS,
169 (addr & ~SRC_WATERMARK_HIGH_MASK) |
170 SRC_WATERMARK_HIGH_SET(n));
171 }
172
173 static inline void ath10k_ce_src_ring_lowmark_set(struct ath10k *ar,
174 u32 ce_ctrl_addr,
175 unsigned int n)
176 {
177 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS);
178
179 ath10k_pci_write32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS,
180 (addr & ~SRC_WATERMARK_LOW_MASK) |
181 SRC_WATERMARK_LOW_SET(n));
182 }
183
184 static inline void ath10k_ce_dest_ring_highmark_set(struct ath10k *ar,
185 u32 ce_ctrl_addr,
186 unsigned int n)
187 {
188 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS);
189
190 ath10k_pci_write32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS,
191 (addr & ~DST_WATERMARK_HIGH_MASK) |
192 DST_WATERMARK_HIGH_SET(n));
193 }
194
195 static inline void ath10k_ce_dest_ring_lowmark_set(struct ath10k *ar,
196 u32 ce_ctrl_addr,
197 unsigned int n)
198 {
199 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS);
200
201 ath10k_pci_write32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS,
202 (addr & ~DST_WATERMARK_LOW_MASK) |
203 DST_WATERMARK_LOW_SET(n));
204 }
205
206 static inline void ath10k_ce_copy_complete_inter_enable(struct ath10k *ar,
207 u32 ce_ctrl_addr)
208 {
209 u32 host_ie_addr = ath10k_pci_read32(ar,
210 ce_ctrl_addr + HOST_IE_ADDRESS);
211
212 ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IE_ADDRESS,
213 host_ie_addr | HOST_IE_COPY_COMPLETE_MASK);
214 }
215
216 static inline void ath10k_ce_copy_complete_intr_disable(struct ath10k *ar,
217 u32 ce_ctrl_addr)
218 {
219 u32 host_ie_addr = ath10k_pci_read32(ar,
220 ce_ctrl_addr + HOST_IE_ADDRESS);
221
222 ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IE_ADDRESS,
223 host_ie_addr & ~HOST_IE_COPY_COMPLETE_MASK);
224 }
225
226 static inline void ath10k_ce_watermark_intr_disable(struct ath10k *ar,
227 u32 ce_ctrl_addr)
228 {
229 u32 host_ie_addr = ath10k_pci_read32(ar,
230 ce_ctrl_addr + HOST_IE_ADDRESS);
231
232 ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IE_ADDRESS,
233 host_ie_addr & ~CE_WATERMARK_MASK);
234 }
235
236 static inline void ath10k_ce_error_intr_enable(struct ath10k *ar,
237 u32 ce_ctrl_addr)
238 {
239 u32 misc_ie_addr = ath10k_pci_read32(ar,
240 ce_ctrl_addr + MISC_IE_ADDRESS);
241
242 ath10k_pci_write32(ar, ce_ctrl_addr + MISC_IE_ADDRESS,
243 misc_ie_addr | CE_ERROR_MASK);
244 }
245
246 static inline void ath10k_ce_error_intr_disable(struct ath10k *ar,
247 u32 ce_ctrl_addr)
248 {
249 u32 misc_ie_addr = ath10k_pci_read32(ar,
250 ce_ctrl_addr + MISC_IE_ADDRESS);
251
252 ath10k_pci_write32(ar, ce_ctrl_addr + MISC_IE_ADDRESS,
253 misc_ie_addr & ~CE_ERROR_MASK);
254 }
255
256 static inline void ath10k_ce_engine_int_status_clear(struct ath10k *ar,
257 u32 ce_ctrl_addr,
258 unsigned int mask)
259 {
260 ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IS_ADDRESS, mask);
261 }
262
263 /*
264 * Guts of ath10k_ce_send, used by both ath10k_ce_send and
265 * ath10k_ce_sendlist_send.
266 * The caller takes responsibility for any needed locking.
267 */
268 int ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,
269 void *per_transfer_context,
270 u32 buffer,
271 unsigned int nbytes,
272 unsigned int transfer_id,
273 unsigned int flags)
274 {
275 struct ath10k *ar = ce_state->ar;
276 struct ath10k_ce_ring *src_ring = ce_state->src_ring;
277 struct ce_desc *desc, sdesc;
278 unsigned int nentries_mask = src_ring->nentries_mask;
279 unsigned int sw_index = src_ring->sw_index;
280 unsigned int write_index = src_ring->write_index;
281 u32 ctrl_addr = ce_state->ctrl_addr;
282 u32 desc_flags = 0;
283 int ret = 0;
284
285 if (nbytes > ce_state->src_sz_max)
286 ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n",
287 __func__, nbytes, ce_state->src_sz_max);
288
289 if (unlikely(CE_RING_DELTA(nentries_mask,
290 write_index, sw_index - 1) <= 0)) {
291 ret = -ENOSR;
292 goto exit;
293 }
294
295 desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space,
296 write_index);
297
298 desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA);
299
300 if (flags & CE_SEND_FLAG_GATHER)
301 desc_flags |= CE_DESC_FLAGS_GATHER;
302 if (flags & CE_SEND_FLAG_BYTE_SWAP)
303 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP;
304
305 sdesc.addr = __cpu_to_le32(buffer);
306 sdesc.nbytes = __cpu_to_le16(nbytes);
307 sdesc.flags = __cpu_to_le16(desc_flags);
308
309 *desc = sdesc;
310
311 src_ring->per_transfer_context[write_index] = per_transfer_context;
312
313 /* Update Source Ring Write Index */
314 write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
315
316 /* WORKAROUND */
317 if (!(flags & CE_SEND_FLAG_GATHER))
318 ath10k_ce_src_ring_write_index_set(ar, ctrl_addr, write_index);
319
320 src_ring->write_index = write_index;
321 exit:
322 return ret;
323 }
324
325 void __ath10k_ce_send_revert(struct ath10k_ce_pipe *pipe)
326 {
327 struct ath10k *ar = pipe->ar;
328 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
329 struct ath10k_ce_ring *src_ring = pipe->src_ring;
330 u32 ctrl_addr = pipe->ctrl_addr;
331
332 lockdep_assert_held(&ar_pci->ce_lock);
333
334 /*
335 * This function must be called only if there is an incomplete
336 * scatter-gather transfer (before index register is updated)
337 * that needs to be cleaned up.
338 */
339 if (WARN_ON_ONCE(src_ring->write_index == src_ring->sw_index))
340 return;
341
342 if (WARN_ON_ONCE(src_ring->write_index ==
343 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr)))
344 return;
345
346 src_ring->write_index--;
347 src_ring->write_index &= src_ring->nentries_mask;
348
349 src_ring->per_transfer_context[src_ring->write_index] = NULL;
350 }
351
352 int ath10k_ce_send(struct ath10k_ce_pipe *ce_state,
353 void *per_transfer_context,
354 u32 buffer,
355 unsigned int nbytes,
356 unsigned int transfer_id,
357 unsigned int flags)
358 {
359 struct ath10k *ar = ce_state->ar;
360 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
361 int ret;
362
363 spin_lock_bh(&ar_pci->ce_lock);
364 ret = ath10k_ce_send_nolock(ce_state, per_transfer_context,
365 buffer, nbytes, transfer_id, flags);
366 spin_unlock_bh(&ar_pci->ce_lock);
367
368 return ret;
369 }
370
371 int ath10k_ce_num_free_src_entries(struct ath10k_ce_pipe *pipe)
372 {
373 struct ath10k *ar = pipe->ar;
374 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
375 int delta;
376
377 spin_lock_bh(&ar_pci->ce_lock);
378 delta = CE_RING_DELTA(pipe->src_ring->nentries_mask,
379 pipe->src_ring->write_index,
380 pipe->src_ring->sw_index - 1);
381 spin_unlock_bh(&ar_pci->ce_lock);
382
383 return delta;
384 }
385
386 int __ath10k_ce_rx_num_free_bufs(struct ath10k_ce_pipe *pipe)
387 {
388 struct ath10k *ar = pipe->ar;
389 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
390 struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
391 unsigned int nentries_mask = dest_ring->nentries_mask;
392 unsigned int write_index = dest_ring->write_index;
393 unsigned int sw_index = dest_ring->sw_index;
394
395 lockdep_assert_held(&ar_pci->ce_lock);
396
397 return CE_RING_DELTA(nentries_mask, write_index, sw_index - 1);
398 }
399
400 int __ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx, u32 paddr)
401 {
402 struct ath10k *ar = pipe->ar;
403 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
404 struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
405 unsigned int nentries_mask = dest_ring->nentries_mask;
406 unsigned int write_index = dest_ring->write_index;
407 unsigned int sw_index = dest_ring->sw_index;
408 struct ce_desc *base = dest_ring->base_addr_owner_space;
409 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, write_index);
410 u32 ctrl_addr = pipe->ctrl_addr;
411
412 lockdep_assert_held(&ar_pci->ce_lock);
413
414 if ((pipe->id != 5) &&
415 CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0)
416 return -ENOSPC;
417
418 desc->addr = __cpu_to_le32(paddr);
419 desc->nbytes = 0;
420
421 dest_ring->per_transfer_context[write_index] = ctx;
422 write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
423 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
424 dest_ring->write_index = write_index;
425
426 return 0;
427 }
428
429 void ath10k_ce_rx_update_write_idx(struct ath10k_ce_pipe *pipe, u32 nentries)
430 {
431 struct ath10k *ar = pipe->ar;
432 struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
433 unsigned int nentries_mask = dest_ring->nentries_mask;
434 unsigned int write_index = dest_ring->write_index;
435 u32 ctrl_addr = pipe->ctrl_addr;
436 u32 cur_write_idx = ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr);
437
438 /* Prevent CE ring stuck issue that will occur when ring is full.
439 * Make sure that write index is 1 less than read index.
440 */
441 if ((cur_write_idx + nentries) == dest_ring->sw_index)
442 nentries -= 1;
443
444 write_index = CE_RING_IDX_ADD(nentries_mask, write_index, nentries);
445 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
446 dest_ring->write_index = write_index;
447 }
448
449 int ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx, u32 paddr)
450 {
451 struct ath10k *ar = pipe->ar;
452 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
453 int ret;
454
455 spin_lock_bh(&ar_pci->ce_lock);
456 ret = __ath10k_ce_rx_post_buf(pipe, ctx, paddr);
457 spin_unlock_bh(&ar_pci->ce_lock);
458
459 return ret;
460 }
461
462 /*
463 * Guts of ath10k_ce_completed_recv_next.
464 * The caller takes responsibility for any necessary locking.
465 */
466 int ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,
467 void **per_transfer_contextp,
468 unsigned int *nbytesp)
469 {
470 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
471 unsigned int nentries_mask = dest_ring->nentries_mask;
472 unsigned int sw_index = dest_ring->sw_index;
473
474 struct ce_desc *base = dest_ring->base_addr_owner_space;
475 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index);
476 struct ce_desc sdesc;
477 u16 nbytes;
478
479 /* Copy in one go for performance reasons */
480 sdesc = *desc;
481
482 nbytes = __le16_to_cpu(sdesc.nbytes);
483 if (nbytes == 0) {
484 /*
485 * This closes a relatively unusual race where the Host
486 * sees the updated DRRI before the update to the
487 * corresponding descriptor has completed. We treat this
488 * as a descriptor that is not yet done.
489 */
490 return -EIO;
491 }
492
493 desc->nbytes = 0;
494
495 /* Return data from completed destination descriptor */
496 *nbytesp = nbytes;
497
498 if (per_transfer_contextp)
499 *per_transfer_contextp =
500 dest_ring->per_transfer_context[sw_index];
501
502 /* Copy engine 5 (HTT Rx) will reuse the same transfer context.
503 * So update transfer context all CEs except CE5.
504 */
505 if (ce_state->id != 5)
506 dest_ring->per_transfer_context[sw_index] = NULL;
507
508 /* Update sw_index */
509 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
510 dest_ring->sw_index = sw_index;
511
512 return 0;
513 }
514
515 int ath10k_ce_completed_recv_next(struct ath10k_ce_pipe *ce_state,
516 void **per_transfer_contextp,
517 unsigned int *nbytesp)
518 {
519 struct ath10k *ar = ce_state->ar;
520 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
521 int ret;
522
523 spin_lock_bh(&ar_pci->ce_lock);
524 ret = ath10k_ce_completed_recv_next_nolock(ce_state,
525 per_transfer_contextp,
526 nbytesp);
527 spin_unlock_bh(&ar_pci->ce_lock);
528
529 return ret;
530 }
531
532 int ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,
533 void **per_transfer_contextp,
534 u32 *bufferp)
535 {
536 struct ath10k_ce_ring *dest_ring;
537 unsigned int nentries_mask;
538 unsigned int sw_index;
539 unsigned int write_index;
540 int ret;
541 struct ath10k *ar;
542 struct ath10k_pci *ar_pci;
543
544 dest_ring = ce_state->dest_ring;
545
546 if (!dest_ring)
547 return -EIO;
548
549 ar = ce_state->ar;
550 ar_pci = ath10k_pci_priv(ar);
551
552 spin_lock_bh(&ar_pci->ce_lock);
553
554 nentries_mask = dest_ring->nentries_mask;
555 sw_index = dest_ring->sw_index;
556 write_index = dest_ring->write_index;
557 if (write_index != sw_index) {
558 struct ce_desc *base = dest_ring->base_addr_owner_space;
559 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index);
560
561 /* Return data from completed destination descriptor */
562 *bufferp = __le32_to_cpu(desc->addr);
563
564 if (per_transfer_contextp)
565 *per_transfer_contextp =
566 dest_ring->per_transfer_context[sw_index];
567
568 /* sanity */
569 dest_ring->per_transfer_context[sw_index] = NULL;
570 desc->nbytes = 0;
571
572 /* Update sw_index */
573 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
574 dest_ring->sw_index = sw_index;
575 ret = 0;
576 } else {
577 ret = -EIO;
578 }
579
580 spin_unlock_bh(&ar_pci->ce_lock);
581
582 return ret;
583 }
584
585 /*
586 * Guts of ath10k_ce_completed_send_next.
587 * The caller takes responsibility for any necessary locking.
588 */
589 int ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state,
590 void **per_transfer_contextp)
591 {
592 struct ath10k_ce_ring *src_ring = ce_state->src_ring;
593 u32 ctrl_addr = ce_state->ctrl_addr;
594 struct ath10k *ar = ce_state->ar;
595 unsigned int nentries_mask = src_ring->nentries_mask;
596 unsigned int sw_index = src_ring->sw_index;
597 unsigned int read_index;
598
599 if (src_ring->hw_index == sw_index) {
600 /*
601 * The SW completion index has caught up with the cached
602 * version of the HW completion index.
603 * Update the cached HW completion index to see whether
604 * the SW has really caught up to the HW, or if the cached
605 * value of the HW index has become stale.
606 */
607
608 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
609 if (read_index == 0xffffffff)
610 return -ENODEV;
611
612 read_index &= nentries_mask;
613 src_ring->hw_index = read_index;
614 }
615
616 read_index = src_ring->hw_index;
617
618 if (read_index == sw_index)
619 return -EIO;
620
621 if (per_transfer_contextp)
622 *per_transfer_contextp =
623 src_ring->per_transfer_context[sw_index];
624
625 /* sanity */
626 src_ring->per_transfer_context[sw_index] = NULL;
627
628 /* Update sw_index */
629 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
630 src_ring->sw_index = sw_index;
631
632 return 0;
633 }
634
635 /* NB: Modeled after ath10k_ce_completed_send_next */
636 int ath10k_ce_cancel_send_next(struct ath10k_ce_pipe *ce_state,
637 void **per_transfer_contextp,
638 u32 *bufferp,
639 unsigned int *nbytesp,
640 unsigned int *transfer_idp)
641 {
642 struct ath10k_ce_ring *src_ring;
643 unsigned int nentries_mask;
644 unsigned int sw_index;
645 unsigned int write_index;
646 int ret;
647 struct ath10k *ar;
648 struct ath10k_pci *ar_pci;
649
650 src_ring = ce_state->src_ring;
651
652 if (!src_ring)
653 return -EIO;
654
655 ar = ce_state->ar;
656 ar_pci = ath10k_pci_priv(ar);
657
658 spin_lock_bh(&ar_pci->ce_lock);
659
660 nentries_mask = src_ring->nentries_mask;
661 sw_index = src_ring->sw_index;
662 write_index = src_ring->write_index;
663
664 if (write_index != sw_index) {
665 struct ce_desc *base = src_ring->base_addr_owner_space;
666 struct ce_desc *desc = CE_SRC_RING_TO_DESC(base, sw_index);
667
668 /* Return data from completed source descriptor */
669 *bufferp = __le32_to_cpu(desc->addr);
670 *nbytesp = __le16_to_cpu(desc->nbytes);
671 *transfer_idp = MS(__le16_to_cpu(desc->flags),
672 CE_DESC_FLAGS_META_DATA);
673
674 if (per_transfer_contextp)
675 *per_transfer_contextp =
676 src_ring->per_transfer_context[sw_index];
677
678 /* sanity */
679 src_ring->per_transfer_context[sw_index] = NULL;
680
681 /* Update sw_index */
682 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
683 src_ring->sw_index = sw_index;
684 ret = 0;
685 } else {
686 ret = -EIO;
687 }
688
689 spin_unlock_bh(&ar_pci->ce_lock);
690
691 return ret;
692 }
693
694 int ath10k_ce_completed_send_next(struct ath10k_ce_pipe *ce_state,
695 void **per_transfer_contextp)
696 {
697 struct ath10k *ar = ce_state->ar;
698 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
699 int ret;
700
701 spin_lock_bh(&ar_pci->ce_lock);
702 ret = ath10k_ce_completed_send_next_nolock(ce_state,
703 per_transfer_contextp);
704 spin_unlock_bh(&ar_pci->ce_lock);
705
706 return ret;
707 }
708
709 /*
710 * Guts of interrupt handler for per-engine interrupts on a particular CE.
711 *
712 * Invokes registered callbacks for recv_complete,
713 * send_complete, and watermarks.
714 */
715 void ath10k_ce_per_engine_service(struct ath10k *ar, unsigned int ce_id)
716 {
717 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
718 struct ath10k_ce_pipe *ce_state = &ar_pci->ce_states[ce_id];
719 u32 ctrl_addr = ce_state->ctrl_addr;
720
721 spin_lock_bh(&ar_pci->ce_lock);
722
723 /* Clear the copy-complete interrupts that will be handled here. */
724 ath10k_ce_engine_int_status_clear(ar, ctrl_addr,
725 HOST_IS_COPY_COMPLETE_MASK);
726
727 spin_unlock_bh(&ar_pci->ce_lock);
728
729 if (ce_state->recv_cb)
730 ce_state->recv_cb(ce_state);
731
732 if (ce_state->send_cb)
733 ce_state->send_cb(ce_state);
734
735 spin_lock_bh(&ar_pci->ce_lock);
736
737 /*
738 * Misc CE interrupts are not being handled, but still need
739 * to be cleared.
740 */
741 ath10k_ce_engine_int_status_clear(ar, ctrl_addr, CE_WATERMARK_MASK);
742
743 spin_unlock_bh(&ar_pci->ce_lock);
744 }
745
746 /*
747 * Handler for per-engine interrupts on ALL active CEs.
748 * This is used in cases where the system is sharing a
749 * single interrput for all CEs
750 */
751
752 void ath10k_ce_per_engine_service_any(struct ath10k *ar)
753 {
754 int ce_id;
755 u32 intr_summary;
756
757 intr_summary = CE_INTERRUPT_SUMMARY(ar);
758
759 for (ce_id = 0; intr_summary && (ce_id < CE_COUNT); ce_id++) {
760 if (intr_summary & (1 << ce_id))
761 intr_summary &= ~(1 << ce_id);
762 else
763 /* no intr pending on this CE */
764 continue;
765
766 ath10k_ce_per_engine_service(ar, ce_id);
767 }
768 }
769
770 /*
771 * Adjust interrupts for the copy complete handler.
772 * If it's needed for either send or recv, then unmask
773 * this interrupt; otherwise, mask it.
774 *
775 * Called with ce_lock held.
776 */
777 static void ath10k_ce_per_engine_handler_adjust(struct ath10k_ce_pipe *ce_state)
778 {
779 u32 ctrl_addr = ce_state->ctrl_addr;
780 struct ath10k *ar = ce_state->ar;
781 bool disable_copy_compl_intr = ce_state->attr_flags & CE_ATTR_DIS_INTR;
782
783 if ((!disable_copy_compl_intr) &&
784 (ce_state->send_cb || ce_state->recv_cb))
785 ath10k_ce_copy_complete_inter_enable(ar, ctrl_addr);
786 else
787 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr);
788
789 ath10k_ce_watermark_intr_disable(ar, ctrl_addr);
790 }
791
792 int ath10k_ce_disable_interrupts(struct ath10k *ar)
793 {
794 int ce_id;
795
796 for (ce_id = 0; ce_id < CE_COUNT; ce_id++) {
797 u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
798
799 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr);
800 ath10k_ce_error_intr_disable(ar, ctrl_addr);
801 ath10k_ce_watermark_intr_disable(ar, ctrl_addr);
802 }
803
804 return 0;
805 }
806
807 void ath10k_ce_enable_interrupts(struct ath10k *ar)
808 {
809 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
810 int ce_id;
811
812 /* Skip the last copy engine, CE7 the diagnostic window, as that
813 * uses polling and isn't initialized for interrupts.
814 */
815 for (ce_id = 0; ce_id < CE_COUNT - 1; ce_id++)
816 ath10k_ce_per_engine_handler_adjust(&ar_pci->ce_states[ce_id]);
817 }
818
819 static int ath10k_ce_init_src_ring(struct ath10k *ar,
820 unsigned int ce_id,
821 const struct ce_attr *attr)
822 {
823 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
824 struct ath10k_ce_pipe *ce_state = &ar_pci->ce_states[ce_id];
825 struct ath10k_ce_ring *src_ring = ce_state->src_ring;
826 u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id);
827
828 nentries = roundup_pow_of_two(attr->src_nentries);
829
830 memset(src_ring->base_addr_owner_space, 0,
831 nentries * sizeof(struct ce_desc));
832
833 src_ring->sw_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
834 src_ring->sw_index &= src_ring->nentries_mask;
835 src_ring->hw_index = src_ring->sw_index;
836
837 src_ring->write_index =
838 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr);
839 src_ring->write_index &= src_ring->nentries_mask;
840
841 ath10k_ce_src_ring_base_addr_set(ar, ctrl_addr,
842 src_ring->base_addr_ce_space);
843 ath10k_ce_src_ring_size_set(ar, ctrl_addr, nentries);
844 ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, attr->src_sz_max);
845 ath10k_ce_src_ring_byte_swap_set(ar, ctrl_addr, 0);
846 ath10k_ce_src_ring_lowmark_set(ar, ctrl_addr, 0);
847 ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, nentries);
848
849 ath10k_dbg(ar, ATH10K_DBG_BOOT,
850 "boot init ce src ring id %d entries %d base_addr %pK\n",
851 ce_id, nentries, src_ring->base_addr_owner_space);
852
853 return 0;
854 }
855
856 static int ath10k_ce_init_dest_ring(struct ath10k *ar,
857 unsigned int ce_id,
858 const struct ce_attr *attr)
859 {
860 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
861 struct ath10k_ce_pipe *ce_state = &ar_pci->ce_states[ce_id];
862 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
863 u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id);
864
865 nentries = roundup_pow_of_two(attr->dest_nentries);
866
867 memset(dest_ring->base_addr_owner_space, 0,
868 nentries * sizeof(struct ce_desc));
869
870 dest_ring->sw_index = ath10k_ce_dest_ring_read_index_get(ar, ctrl_addr);
871 dest_ring->sw_index &= dest_ring->nentries_mask;
872 dest_ring->write_index =
873 ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr);
874 dest_ring->write_index &= dest_ring->nentries_mask;
875
876 ath10k_ce_dest_ring_base_addr_set(ar, ctrl_addr,
877 dest_ring->base_addr_ce_space);
878 ath10k_ce_dest_ring_size_set(ar, ctrl_addr, nentries);
879 ath10k_ce_dest_ring_byte_swap_set(ar, ctrl_addr, 0);
880 ath10k_ce_dest_ring_lowmark_set(ar, ctrl_addr, 0);
881 ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, nentries);
882
883 ath10k_dbg(ar, ATH10K_DBG_BOOT,
884 "boot ce dest ring id %d entries %d base_addr %pK\n",
885 ce_id, nentries, dest_ring->base_addr_owner_space);
886
887 return 0;
888 }
889
890 static struct ath10k_ce_ring *
891 ath10k_ce_alloc_src_ring(struct ath10k *ar, unsigned int ce_id,
892 const struct ce_attr *attr)
893 {
894 struct ath10k_ce_ring *src_ring;
895 u32 nentries = attr->src_nentries;
896 dma_addr_t base_addr;
897
898 nentries = roundup_pow_of_two(nentries);
899
900 src_ring = kzalloc(sizeof(*src_ring) +
901 (nentries *
902 sizeof(*src_ring->per_transfer_context)),
903 GFP_KERNEL);
904 if (src_ring == NULL)
905 return ERR_PTR(-ENOMEM);
906
907 src_ring->nentries = nentries;
908 src_ring->nentries_mask = nentries - 1;
909
910 /*
911 * Legacy platforms that do not support cache
912 * coherent DMA are unsupported
913 */
914 src_ring->base_addr_owner_space_unaligned =
915 dma_alloc_coherent(ar->dev,
916 (nentries * sizeof(struct ce_desc) +
917 CE_DESC_RING_ALIGN),
918 &base_addr, GFP_KERNEL);
919 if (!src_ring->base_addr_owner_space_unaligned) {
920 kfree(src_ring);
921 return ERR_PTR(-ENOMEM);
922 }
923
924 src_ring->base_addr_ce_space_unaligned = base_addr;
925
926 src_ring->base_addr_owner_space = PTR_ALIGN(
927 src_ring->base_addr_owner_space_unaligned,
928 CE_DESC_RING_ALIGN);
929 src_ring->base_addr_ce_space = ALIGN(
930 src_ring->base_addr_ce_space_unaligned,
931 CE_DESC_RING_ALIGN);
932
933 return src_ring;
934 }
935
936 static struct ath10k_ce_ring *
937 ath10k_ce_alloc_dest_ring(struct ath10k *ar, unsigned int ce_id,
938 const struct ce_attr *attr)
939 {
940 struct ath10k_ce_ring *dest_ring;
941 u32 nentries;
942 dma_addr_t base_addr;
943
944 nentries = roundup_pow_of_two(attr->dest_nentries);
945
946 dest_ring = kzalloc(sizeof(*dest_ring) +
947 (nentries *
948 sizeof(*dest_ring->per_transfer_context)),
949 GFP_KERNEL);
950 if (dest_ring == NULL)
951 return ERR_PTR(-ENOMEM);
952
953 dest_ring->nentries = nentries;
954 dest_ring->nentries_mask = nentries - 1;
955
956 /*
957 * Legacy platforms that do not support cache
958 * coherent DMA are unsupported
959 */
960 dest_ring->base_addr_owner_space_unaligned =
961 dma_alloc_coherent(ar->dev,
962 (nentries * sizeof(struct ce_desc) +
963 CE_DESC_RING_ALIGN),
964 &base_addr, GFP_KERNEL);
965 if (!dest_ring->base_addr_owner_space_unaligned) {
966 kfree(dest_ring);
967 return ERR_PTR(-ENOMEM);
968 }
969
970 dest_ring->base_addr_ce_space_unaligned = base_addr;
971
972 /*
973 * Correctly initialize memory to 0 to prevent garbage
974 * data crashing system when download firmware
975 */
976 memset(dest_ring->base_addr_owner_space_unaligned, 0,
977 nentries * sizeof(struct ce_desc) + CE_DESC_RING_ALIGN);
978
979 dest_ring->base_addr_owner_space = PTR_ALIGN(
980 dest_ring->base_addr_owner_space_unaligned,
981 CE_DESC_RING_ALIGN);
982 dest_ring->base_addr_ce_space = ALIGN(
983 dest_ring->base_addr_ce_space_unaligned,
984 CE_DESC_RING_ALIGN);
985
986 return dest_ring;
987 }
988
989 /*
990 * Initialize a Copy Engine based on caller-supplied attributes.
991 * This may be called once to initialize both source and destination
992 * rings or it may be called twice for separate source and destination
993 * initialization. It may be that only one side or the other is
994 * initialized by software/firmware.
995 */
996 int ath10k_ce_init_pipe(struct ath10k *ar, unsigned int ce_id,
997 const struct ce_attr *attr)
998 {
999 int ret;
1000
1001 if (attr->src_nentries) {
1002 ret = ath10k_ce_init_src_ring(ar, ce_id, attr);
1003 if (ret) {
1004 ath10k_err(ar, "Failed to initialize CE src ring for ID: %d (%d)\n",
1005 ce_id, ret);
1006 return ret;
1007 }
1008 }
1009
1010 if (attr->dest_nentries) {
1011 ret = ath10k_ce_init_dest_ring(ar, ce_id, attr);
1012 if (ret) {
1013 ath10k_err(ar, "Failed to initialize CE dest ring for ID: %d (%d)\n",
1014 ce_id, ret);
1015 return ret;
1016 }
1017 }
1018
1019 return 0;
1020 }
1021
1022 static void ath10k_ce_deinit_src_ring(struct ath10k *ar, unsigned int ce_id)
1023 {
1024 u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1025
1026 ath10k_ce_src_ring_base_addr_set(ar, ctrl_addr, 0);
1027 ath10k_ce_src_ring_size_set(ar, ctrl_addr, 0);
1028 ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, 0);
1029 ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, 0);
1030 }
1031
1032 static void ath10k_ce_deinit_dest_ring(struct ath10k *ar, unsigned int ce_id)
1033 {
1034 u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1035
1036 ath10k_ce_dest_ring_base_addr_set(ar, ctrl_addr, 0);
1037 ath10k_ce_dest_ring_size_set(ar, ctrl_addr, 0);
1038 ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, 0);
1039 }
1040
1041 void ath10k_ce_deinit_pipe(struct ath10k *ar, unsigned int ce_id)
1042 {
1043 ath10k_ce_deinit_src_ring(ar, ce_id);
1044 ath10k_ce_deinit_dest_ring(ar, ce_id);
1045 }
1046
1047 int ath10k_ce_alloc_pipe(struct ath10k *ar, int ce_id,
1048 const struct ce_attr *attr)
1049 {
1050 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
1051 struct ath10k_ce_pipe *ce_state = &ar_pci->ce_states[ce_id];
1052 int ret;
1053
1054 /*
1055 * Make sure there's enough CE ringbuffer entries for HTT TX to avoid
1056 * additional TX locking checks.
1057 *
1058 * For the lack of a better place do the check here.
1059 */
1060 BUILD_BUG_ON(2 * TARGET_NUM_MSDU_DESC >
1061 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1062 BUILD_BUG_ON(2 * TARGET_10X_NUM_MSDU_DESC >
1063 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1064 BUILD_BUG_ON(2 * TARGET_TLV_NUM_MSDU_DESC >
1065 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1066
1067 ce_state->ar = ar;
1068 ce_state->id = ce_id;
1069 ce_state->ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1070 ce_state->attr_flags = attr->flags;
1071 ce_state->src_sz_max = attr->src_sz_max;
1072
1073 if (attr->src_nentries)
1074 ce_state->send_cb = attr->send_cb;
1075
1076 if (attr->dest_nentries)
1077 ce_state->recv_cb = attr->recv_cb;
1078
1079 if (attr->src_nentries) {
1080 ce_state->src_ring = ath10k_ce_alloc_src_ring(ar, ce_id, attr);
1081 if (IS_ERR(ce_state->src_ring)) {
1082 ret = PTR_ERR(ce_state->src_ring);
1083 ath10k_err(ar, "failed to allocate copy engine source ring %d: %d\n",
1084 ce_id, ret);
1085 ce_state->src_ring = NULL;
1086 return ret;
1087 }
1088 }
1089
1090 if (attr->dest_nentries) {
1091 ce_state->dest_ring = ath10k_ce_alloc_dest_ring(ar, ce_id,
1092 attr);
1093 if (IS_ERR(ce_state->dest_ring)) {
1094 ret = PTR_ERR(ce_state->dest_ring);
1095 ath10k_err(ar, "failed to allocate copy engine destination ring %d: %d\n",
1096 ce_id, ret);
1097 ce_state->dest_ring = NULL;
1098 return ret;
1099 }
1100 }
1101
1102 return 0;
1103 }
1104
1105 void ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)
1106 {
1107 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
1108 struct ath10k_ce_pipe *ce_state = &ar_pci->ce_states[ce_id];
1109
1110 if (ce_state->src_ring) {
1111 dma_free_coherent(ar->dev,
1112 (ce_state->src_ring->nentries *
1113 sizeof(struct ce_desc) +
1114 CE_DESC_RING_ALIGN),
1115 ce_state->src_ring->base_addr_owner_space,
1116 ce_state->src_ring->base_addr_ce_space);
1117 kfree(ce_state->src_ring);
1118 }
1119
1120 if (ce_state->dest_ring) {
1121 dma_free_coherent(ar->dev,
1122 (ce_state->dest_ring->nentries *
1123 sizeof(struct ce_desc) +
1124 CE_DESC_RING_ALIGN),
1125 ce_state->dest_ring->base_addr_owner_space,
1126 ce_state->dest_ring->base_addr_ce_space);
1127 kfree(ce_state->dest_ring);
1128 }
1129
1130 ce_state->src_ring = NULL;
1131 ce_state->dest_ring = NULL;
1132 }
Linux with added FPC-III support