?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
perf tools: Don't clone maps from parent when synthesizing forks
[linux/fpc-iii.git] / drivers / net / wireless / ath / ath10k / ce.c
blobf6d3ecbdd3a32c42a6715247e0aee28ee018d1b4
1 /*
2 * Copyright (c) 2005-2011 Atheros Communications Inc.
3 * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
4 * Copyright (c) 2018 The Linux Foundation. All rights reserved.
5 *
6 * Permission to use, copy, modify, and/or distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18
19 #include "hif.h"
20 #include "ce.h"
21 #include "debug.h"
22
23 /*
24 * Support for Copy Engine hardware, which is mainly used for
25 * communication between Host and Target over a PCIe interconnect.
26 */
27
28 /*
29 * A single CopyEngine (CE) comprises two "rings":
30 * a source ring
31 * a destination ring
32 *
33 * Each ring consists of a number of descriptors which specify
34 * an address, length, and meta-data.
35 *
36 * Typically, one side of the PCIe/AHB/SNOC interconnect (Host or Target)
37 * controls one ring and the other side controls the other ring.
38 * The source side chooses when to initiate a transfer and it
39 * chooses what to send (buffer address, length). The destination
40 * side keeps a supply of "anonymous receive buffers" available and
41 * it handles incoming data as it arrives (when the destination
42 * receives an interrupt).
43 *
44 * The sender may send a simple buffer (address/length) or it may
45 * send a small list of buffers. When a small list is sent, hardware
46 * "gathers" these and they end up in a single destination buffer
47 * with a single interrupt.
48 *
49 * There are several "contexts" managed by this layer -- more, it
50 * may seem -- than should be needed. These are provided mainly for
51 * maximum flexibility and especially to facilitate a simpler HIF
52 * implementation. There are per-CopyEngine recv, send, and watermark
53 * contexts. These are supplied by the caller when a recv, send,
54 * or watermark handler is established and they are echoed back to
55 * the caller when the respective callbacks are invoked. There is
56 * also a per-transfer context supplied by the caller when a buffer
57 * (or sendlist) is sent and when a buffer is enqueued for recv.
58 * These per-transfer contexts are echoed back to the caller when
59 * the buffer is sent/received.
60 */
61
62 static inline u32 shadow_sr_wr_ind_addr(struct ath10k *ar,
63 struct ath10k_ce_pipe *ce_state)
64 {
65 u32 ce_id = ce_state->id;
66 u32 addr = 0;
67
68 switch (ce_id) {
69 case 0:
70 addr = 0x00032000;
71 break;
72 case 3:
73 addr = 0x0003200C;
74 break;
75 case 4:
76 addr = 0x00032010;
77 break;
78 case 5:
79 addr = 0x00032014;
80 break;
81 case 7:
82 addr = 0x0003201C;
83 break;
84 default:
85 ath10k_warn(ar, "invalid CE id: %d", ce_id);
86 break;
87 }
88 return addr;
89 }
90
91 static inline u32 shadow_dst_wr_ind_addr(struct ath10k *ar,
92 struct ath10k_ce_pipe *ce_state)
93 {
94 u32 ce_id = ce_state->id;
95 u32 addr = 0;
96
97 switch (ce_id) {
98 case 1:
99 addr = 0x00032034;
100 break;
101 case 2:
102 addr = 0x00032038;
103 break;
104 case 5:
105 addr = 0x00032044;
106 break;
107 case 7:
108 addr = 0x0003204C;
109 break;
110 case 8:
111 addr = 0x00032050;
112 break;
113 case 9:
114 addr = 0x00032054;
115 break;
116 case 10:
117 addr = 0x00032058;
118 break;
119 case 11:
120 addr = 0x0003205C;
121 break;
122 default:
123 ath10k_warn(ar, "invalid CE id: %d", ce_id);
124 break;
125 }
126
127 return addr;
128 }
129
130 static inline unsigned int
131 ath10k_set_ring_byte(unsigned int offset,
132 struct ath10k_hw_ce_regs_addr_map *addr_map)
133 {
134 return ((offset << addr_map->lsb) & addr_map->mask);
135 }
136
137 static inline unsigned int
138 ath10k_get_ring_byte(unsigned int offset,
139 struct ath10k_hw_ce_regs_addr_map *addr_map)
140 {
141 return ((offset & addr_map->mask) >> (addr_map->lsb));
142 }
143
144 static inline u32 ath10k_ce_read32(struct ath10k *ar, u32 offset)
145 {
146 struct ath10k_ce *ce = ath10k_ce_priv(ar);
147
148 return ce->bus_ops->read32(ar, offset);
149 }
150
151 static inline void ath10k_ce_write32(struct ath10k *ar, u32 offset, u32 value)
152 {
153 struct ath10k_ce *ce = ath10k_ce_priv(ar);
154
155 ce->bus_ops->write32(ar, offset, value);
156 }
157
158 static inline void ath10k_ce_dest_ring_write_index_set(struct ath10k *ar,
159 u32 ce_ctrl_addr,
160 unsigned int n)
161 {
162 ath10k_ce_write32(ar, ce_ctrl_addr +
163 ar->hw_ce_regs->dst_wr_index_addr, n);
164 }
165
166 static inline u32 ath10k_ce_dest_ring_write_index_get(struct ath10k *ar,
167 u32 ce_ctrl_addr)
168 {
169 return ath10k_ce_read32(ar, ce_ctrl_addr +
170 ar->hw_ce_regs->dst_wr_index_addr);
171 }
172
173 static inline void ath10k_ce_src_ring_write_index_set(struct ath10k *ar,
174 u32 ce_ctrl_addr,
175 unsigned int n)
176 {
177 ath10k_ce_write32(ar, ce_ctrl_addr +
178 ar->hw_ce_regs->sr_wr_index_addr, n);
179 }
180
181 static inline u32 ath10k_ce_src_ring_write_index_get(struct ath10k *ar,
182 u32 ce_ctrl_addr)
183 {
184 return ath10k_ce_read32(ar, ce_ctrl_addr +
185 ar->hw_ce_regs->sr_wr_index_addr);
186 }
187
188 static inline u32 ath10k_ce_src_ring_read_index_from_ddr(struct ath10k *ar,
189 u32 ce_id)
190 {
191 struct ath10k_ce *ce = ath10k_ce_priv(ar);
192
193 return ce->vaddr_rri[ce_id] & CE_DDR_RRI_MASK;
194 }
195
196 static inline u32 ath10k_ce_src_ring_read_index_get(struct ath10k *ar,
197 u32 ce_ctrl_addr)
198 {
199 struct ath10k_ce *ce = ath10k_ce_priv(ar);
200 u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr);
201 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
202 u32 index;
203
204 if (ar->hw_params.rri_on_ddr &&
205 (ce_state->attr_flags & CE_ATTR_DIS_INTR))
206 index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_id);
207 else
208 index = ath10k_ce_read32(ar, ce_ctrl_addr +
209 ar->hw_ce_regs->current_srri_addr);
210
211 return index;
212 }
213
214 static inline void
215 ath10k_ce_shadow_src_ring_write_index_set(struct ath10k *ar,
216 struct ath10k_ce_pipe *ce_state,
217 unsigned int value)
218 {
219 ath10k_ce_write32(ar, shadow_sr_wr_ind_addr(ar, ce_state), value);
220 }
221
222 static inline void
223 ath10k_ce_shadow_dest_ring_write_index_set(struct ath10k *ar,
224 struct ath10k_ce_pipe *ce_state,
225 unsigned int value)
226 {
227 ath10k_ce_write32(ar, shadow_dst_wr_ind_addr(ar, ce_state), value);
228 }
229
230 static inline void ath10k_ce_src_ring_base_addr_set(struct ath10k *ar,
231 u32 ce_ctrl_addr,
232 unsigned int addr)
233 {
234 ath10k_ce_write32(ar, ce_ctrl_addr +
235 ar->hw_ce_regs->sr_base_addr, addr);
236 }
237
238 static inline void ath10k_ce_src_ring_size_set(struct ath10k *ar,
239 u32 ce_ctrl_addr,
240 unsigned int n)
241 {
242 ath10k_ce_write32(ar, ce_ctrl_addr +
243 ar->hw_ce_regs->sr_size_addr, n);
244 }
245
246 static inline void ath10k_ce_src_ring_dmax_set(struct ath10k *ar,
247 u32 ce_ctrl_addr,
248 unsigned int n)
249 {
250 struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
251
252 u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
253 ctrl_regs->addr);
254
255 ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
256 (ctrl1_addr & ~(ctrl_regs->dmax->mask)) |
257 ath10k_set_ring_byte(n, ctrl_regs->dmax));
258 }
259
260 static inline void ath10k_ce_src_ring_byte_swap_set(struct ath10k *ar,
261 u32 ce_ctrl_addr,
262 unsigned int n)
263 {
264 struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
265
266 u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
267 ctrl_regs->addr);
268
269 ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
270 (ctrl1_addr & ~(ctrl_regs->src_ring->mask)) |
271 ath10k_set_ring_byte(n, ctrl_regs->src_ring));
272 }
273
274 static inline void ath10k_ce_dest_ring_byte_swap_set(struct ath10k *ar,
275 u32 ce_ctrl_addr,
276 unsigned int n)
277 {
278 struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
279
280 u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
281 ctrl_regs->addr);
282
283 ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
284 (ctrl1_addr & ~(ctrl_regs->dst_ring->mask)) |
285 ath10k_set_ring_byte(n, ctrl_regs->dst_ring));
286 }
287
288 static inline
289 u32 ath10k_ce_dest_ring_read_index_from_ddr(struct ath10k *ar, u32 ce_id)
290 {
291 struct ath10k_ce *ce = ath10k_ce_priv(ar);
292
293 return (ce->vaddr_rri[ce_id] >> CE_DDR_DRRI_SHIFT) &
294 CE_DDR_RRI_MASK;
295 }
296
297 static inline u32 ath10k_ce_dest_ring_read_index_get(struct ath10k *ar,
298 u32 ce_ctrl_addr)
299 {
300 struct ath10k_ce *ce = ath10k_ce_priv(ar);
301 u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr);
302 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
303 u32 index;
304
305 if (ar->hw_params.rri_on_ddr &&
306 (ce_state->attr_flags & CE_ATTR_DIS_INTR))
307 index = ath10k_ce_dest_ring_read_index_from_ddr(ar, ce_id);
308 else
309 index = ath10k_ce_read32(ar, ce_ctrl_addr +
310 ar->hw_ce_regs->current_drri_addr);
311
312 return index;
313 }
314
315 static inline void ath10k_ce_dest_ring_base_addr_set(struct ath10k *ar,
316 u32 ce_ctrl_addr,
317 u32 addr)
318 {
319 ath10k_ce_write32(ar, ce_ctrl_addr +
320 ar->hw_ce_regs->dr_base_addr, addr);
321 }
322
323 static inline void ath10k_ce_dest_ring_size_set(struct ath10k *ar,
324 u32 ce_ctrl_addr,
325 unsigned int n)
326 {
327 ath10k_ce_write32(ar, ce_ctrl_addr +
328 ar->hw_ce_regs->dr_size_addr, n);
329 }
330
331 static inline void ath10k_ce_src_ring_highmark_set(struct ath10k *ar,
332 u32 ce_ctrl_addr,
333 unsigned int n)
334 {
335 struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr;
336 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr);
337
338 ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr,
339 (addr & ~(srcr_wm->wm_high->mask)) |
340 (ath10k_set_ring_byte(n, srcr_wm->wm_high)));
341 }
342
343 static inline void ath10k_ce_src_ring_lowmark_set(struct ath10k *ar,
344 u32 ce_ctrl_addr,
345 unsigned int n)
346 {
347 struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr;
348 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr);
349
350 ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr,
351 (addr & ~(srcr_wm->wm_low->mask)) |
352 (ath10k_set_ring_byte(n, srcr_wm->wm_low)));
353 }
354
355 static inline void ath10k_ce_dest_ring_highmark_set(struct ath10k *ar,
356 u32 ce_ctrl_addr,
357 unsigned int n)
358 {
359 struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr;
360 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr);
361
362 ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr,
363 (addr & ~(dstr_wm->wm_high->mask)) |
364 (ath10k_set_ring_byte(n, dstr_wm->wm_high)));
365 }
366
367 static inline void ath10k_ce_dest_ring_lowmark_set(struct ath10k *ar,
368 u32 ce_ctrl_addr,
369 unsigned int n)
370 {
371 struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr;
372 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr);
373
374 ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr,
375 (addr & ~(dstr_wm->wm_low->mask)) |
376 (ath10k_set_ring_byte(n, dstr_wm->wm_low)));
377 }
378
379 static inline void ath10k_ce_copy_complete_inter_enable(struct ath10k *ar,
380 u32 ce_ctrl_addr)
381 {
382 struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie;
383
384 u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
385 ar->hw_ce_regs->host_ie_addr);
386
387 ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
388 host_ie_addr | host_ie->copy_complete->mask);
389 }
390
391 static inline void ath10k_ce_copy_complete_intr_disable(struct ath10k *ar,
392 u32 ce_ctrl_addr)
393 {
394 struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie;
395
396 u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
397 ar->hw_ce_regs->host_ie_addr);
398
399 ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
400 host_ie_addr & ~(host_ie->copy_complete->mask));
401 }
402
403 static inline void ath10k_ce_watermark_intr_disable(struct ath10k *ar,
404 u32 ce_ctrl_addr)
405 {
406 struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
407
408 u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
409 ar->hw_ce_regs->host_ie_addr);
410
411 ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
412 host_ie_addr & ~(wm_regs->wm_mask));
413 }
414
415 static inline void ath10k_ce_error_intr_enable(struct ath10k *ar,
416 u32 ce_ctrl_addr)
417 {
418 struct ath10k_hw_ce_misc_regs *misc_regs = ar->hw_ce_regs->misc_regs;
419
420 u32 misc_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
421 ar->hw_ce_regs->misc_ie_addr);
422
423 ath10k_ce_write32(ar,
424 ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr,
425 misc_ie_addr | misc_regs->err_mask);
426 }
427
428 static inline void ath10k_ce_error_intr_disable(struct ath10k *ar,
429 u32 ce_ctrl_addr)
430 {
431 struct ath10k_hw_ce_misc_regs *misc_regs = ar->hw_ce_regs->misc_regs;
432
433 u32 misc_ie_addr = ath10k_ce_read32(ar,
434 ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr);
435
436 ath10k_ce_write32(ar,
437 ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr,
438 misc_ie_addr & ~(misc_regs->err_mask));
439 }
440
441 static inline void ath10k_ce_engine_int_status_clear(struct ath10k *ar,
442 u32 ce_ctrl_addr,
443 unsigned int mask)
444 {
445 struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
446
447 ath10k_ce_write32(ar, ce_ctrl_addr + wm_regs->addr, mask);
448 }
449
450 /*
451 * Guts of ath10k_ce_send.
452 * The caller takes responsibility for any needed locking.
453 */
454 static int _ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,
455 void *per_transfer_context,
456 dma_addr_t buffer,
457 unsigned int nbytes,
458 unsigned int transfer_id,
459 unsigned int flags)
460 {
461 struct ath10k *ar = ce_state->ar;
462 struct ath10k_ce_ring *src_ring = ce_state->src_ring;
463 struct ce_desc *desc, sdesc;
464 unsigned int nentries_mask = src_ring->nentries_mask;
465 unsigned int sw_index = src_ring->sw_index;
466 unsigned int write_index = src_ring->write_index;
467 u32 ctrl_addr = ce_state->ctrl_addr;
468 u32 desc_flags = 0;
469 int ret = 0;
470
471 if (nbytes > ce_state->src_sz_max)
472 ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n",
473 __func__, nbytes, ce_state->src_sz_max);
474
475 if (unlikely(CE_RING_DELTA(nentries_mask,
476 write_index, sw_index - 1) <= 0)) {
477 ret = -ENOSR;
478 goto exit;
479 }
480
481 desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space,
482 write_index);
483
484 desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA);
485
486 if (flags & CE_SEND_FLAG_GATHER)
487 desc_flags |= CE_DESC_FLAGS_GATHER;
488 if (flags & CE_SEND_FLAG_BYTE_SWAP)
489 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP;
490
491 sdesc.addr = __cpu_to_le32(buffer);
492 sdesc.nbytes = __cpu_to_le16(nbytes);
493 sdesc.flags = __cpu_to_le16(desc_flags);
494
495 *desc = sdesc;
496
497 src_ring->per_transfer_context[write_index] = per_transfer_context;
498
499 /* Update Source Ring Write Index */
500 write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
501
502 /* WORKAROUND */
503 if (!(flags & CE_SEND_FLAG_GATHER)) {
504 if (ar->hw_params.shadow_reg_support)
505 ath10k_ce_shadow_src_ring_write_index_set(ar, ce_state,
506 write_index);
507 else
508 ath10k_ce_src_ring_write_index_set(ar, ctrl_addr,
509 write_index);
510 }
511
512 src_ring->write_index = write_index;
513 exit:
514 return ret;
515 }
516
517 static int _ath10k_ce_send_nolock_64(struct ath10k_ce_pipe *ce_state,
518 void *per_transfer_context,
519 dma_addr_t buffer,
520 unsigned int nbytes,
521 unsigned int transfer_id,
522 unsigned int flags)
523 {
524 struct ath10k *ar = ce_state->ar;
525 struct ath10k_ce_ring *src_ring = ce_state->src_ring;
526 struct ce_desc_64 *desc, sdesc;
527 unsigned int nentries_mask = src_ring->nentries_mask;
528 unsigned int sw_index;
529 unsigned int write_index = src_ring->write_index;
530 u32 ctrl_addr = ce_state->ctrl_addr;
531 __le32 *addr;
532 u32 desc_flags = 0;
533 int ret = 0;
534
535 if (test_bit(ATH10K_FLAG_CRASH_FLUSH, &ar->dev_flags))
536 return -ESHUTDOWN;
537
538 if (nbytes > ce_state->src_sz_max)
539 ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n",
540 __func__, nbytes, ce_state->src_sz_max);
541
542 if (ar->hw_params.rri_on_ddr)
543 sw_index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_state->id);
544 else
545 sw_index = src_ring->sw_index;
546
547 if (unlikely(CE_RING_DELTA(nentries_mask,
548 write_index, sw_index - 1) <= 0)) {
549 ret = -ENOSR;
550 goto exit;
551 }
552
553 desc = CE_SRC_RING_TO_DESC_64(src_ring->base_addr_owner_space,
554 write_index);
555
556 desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA);
557
558 if (flags & CE_SEND_FLAG_GATHER)
559 desc_flags |= CE_DESC_FLAGS_GATHER;
560
561 if (flags & CE_SEND_FLAG_BYTE_SWAP)
562 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP;
563
564 addr = (__le32 *)&sdesc.addr;
565
566 flags |= upper_32_bits(buffer) & CE_DESC_FLAGS_GET_MASK;
567 addr[0] = __cpu_to_le32(buffer);
568 addr[1] = __cpu_to_le32(flags);
569 if (flags & CE_SEND_FLAG_GATHER)
570 addr[1] |= __cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER);
571 else
572 addr[1] &= ~(__cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER));
573
574 sdesc.nbytes = __cpu_to_le16(nbytes);
575 sdesc.flags = __cpu_to_le16(desc_flags);
576
577 *desc = sdesc;
578
579 src_ring->per_transfer_context[write_index] = per_transfer_context;
580
581 /* Update Source Ring Write Index */
582 write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
583
584 if (!(flags & CE_SEND_FLAG_GATHER))
585 ath10k_ce_src_ring_write_index_set(ar, ctrl_addr, write_index);
586
587 src_ring->write_index = write_index;
588 exit:
589 return ret;
590 }
591
592 int ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,
593 void *per_transfer_context,
594 dma_addr_t buffer,
595 unsigned int nbytes,
596 unsigned int transfer_id,
597 unsigned int flags)
598 {
599 return ce_state->ops->ce_send_nolock(ce_state, per_transfer_context,
600 buffer, nbytes, transfer_id, flags);
601 }
602 EXPORT_SYMBOL(ath10k_ce_send_nolock);
603
604 void __ath10k_ce_send_revert(struct ath10k_ce_pipe *pipe)
605 {
606 struct ath10k *ar = pipe->ar;
607 struct ath10k_ce *ce = ath10k_ce_priv(ar);
608 struct ath10k_ce_ring *src_ring = pipe->src_ring;
609 u32 ctrl_addr = pipe->ctrl_addr;
610
611 lockdep_assert_held(&ce->ce_lock);
612
613 /*
614 * This function must be called only if there is an incomplete
615 * scatter-gather transfer (before index register is updated)
616 * that needs to be cleaned up.
617 */
618 if (WARN_ON_ONCE(src_ring->write_index == src_ring->sw_index))
619 return;
620
621 if (WARN_ON_ONCE(src_ring->write_index ==
622 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr)))
623 return;
624
625 src_ring->write_index--;
626 src_ring->write_index &= src_ring->nentries_mask;
627
628 src_ring->per_transfer_context[src_ring->write_index] = NULL;
629 }
630 EXPORT_SYMBOL(__ath10k_ce_send_revert);
631
632 int ath10k_ce_send(struct ath10k_ce_pipe *ce_state,
633 void *per_transfer_context,
634 dma_addr_t buffer,
635 unsigned int nbytes,
636 unsigned int transfer_id,
637 unsigned int flags)
638 {
639 struct ath10k *ar = ce_state->ar;
640 struct ath10k_ce *ce = ath10k_ce_priv(ar);
641 int ret;
642
643 spin_lock_bh(&ce->ce_lock);
644 ret = ath10k_ce_send_nolock(ce_state, per_transfer_context,
645 buffer, nbytes, transfer_id, flags);
646 spin_unlock_bh(&ce->ce_lock);
647
648 return ret;
649 }
650 EXPORT_SYMBOL(ath10k_ce_send);
651
652 int ath10k_ce_num_free_src_entries(struct ath10k_ce_pipe *pipe)
653 {
654 struct ath10k *ar = pipe->ar;
655 struct ath10k_ce *ce = ath10k_ce_priv(ar);
656 int delta;
657
658 spin_lock_bh(&ce->ce_lock);
659 delta = CE_RING_DELTA(pipe->src_ring->nentries_mask,
660 pipe->src_ring->write_index,
661 pipe->src_ring->sw_index - 1);
662 spin_unlock_bh(&ce->ce_lock);
663
664 return delta;
665 }
666 EXPORT_SYMBOL(ath10k_ce_num_free_src_entries);
667
668 int __ath10k_ce_rx_num_free_bufs(struct ath10k_ce_pipe *pipe)
669 {
670 struct ath10k *ar = pipe->ar;
671 struct ath10k_ce *ce = ath10k_ce_priv(ar);
672 struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
673 unsigned int nentries_mask = dest_ring->nentries_mask;
674 unsigned int write_index = dest_ring->write_index;
675 unsigned int sw_index = dest_ring->sw_index;
676
677 lockdep_assert_held(&ce->ce_lock);
678
679 return CE_RING_DELTA(nentries_mask, write_index, sw_index - 1);
680 }
681 EXPORT_SYMBOL(__ath10k_ce_rx_num_free_bufs);
682
683 static int __ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx,
684 dma_addr_t paddr)
685 {
686 struct ath10k *ar = pipe->ar;
687 struct ath10k_ce *ce = ath10k_ce_priv(ar);
688 struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
689 unsigned int nentries_mask = dest_ring->nentries_mask;
690 unsigned int write_index = dest_ring->write_index;
691 unsigned int sw_index = dest_ring->sw_index;
692 struct ce_desc *base = dest_ring->base_addr_owner_space;
693 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, write_index);
694 u32 ctrl_addr = pipe->ctrl_addr;
695
696 lockdep_assert_held(&ce->ce_lock);
697
698 if ((pipe->id != 5) &&
699 CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0)
700 return -ENOSPC;
701
702 desc->addr = __cpu_to_le32(paddr);
703 desc->nbytes = 0;
704
705 dest_ring->per_transfer_context[write_index] = ctx;
706 write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
707 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
708 dest_ring->write_index = write_index;
709
710 return 0;
711 }
712
713 static int __ath10k_ce_rx_post_buf_64(struct ath10k_ce_pipe *pipe,
714 void *ctx,
715 dma_addr_t paddr)
716 {
717 struct ath10k *ar = pipe->ar;
718 struct ath10k_ce *ce = ath10k_ce_priv(ar);
719 struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
720 unsigned int nentries_mask = dest_ring->nentries_mask;
721 unsigned int write_index = dest_ring->write_index;
722 unsigned int sw_index = dest_ring->sw_index;
723 struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
724 struct ce_desc_64 *desc =
725 CE_DEST_RING_TO_DESC_64(base, write_index);
726 u32 ctrl_addr = pipe->ctrl_addr;
727
728 lockdep_assert_held(&ce->ce_lock);
729
730 if (CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0)
731 return -ENOSPC;
732
733 desc->addr = __cpu_to_le64(paddr);
734 desc->addr &= __cpu_to_le64(CE_DESC_37BIT_ADDR_MASK);
735
736 desc->nbytes = 0;
737
738 dest_ring->per_transfer_context[write_index] = ctx;
739 write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
740 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
741 dest_ring->write_index = write_index;
742
743 return 0;
744 }
745
746 void ath10k_ce_rx_update_write_idx(struct ath10k_ce_pipe *pipe, u32 nentries)
747 {
748 struct ath10k *ar = pipe->ar;
749 struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
750 unsigned int nentries_mask = dest_ring->nentries_mask;
751 unsigned int write_index = dest_ring->write_index;
752 u32 ctrl_addr = pipe->ctrl_addr;
753 u32 cur_write_idx = ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr);
754
755 /* Prevent CE ring stuck issue that will occur when ring is full.
756 * Make sure that write index is 1 less than read index.
757 */
758 if (((cur_write_idx + nentries) & nentries_mask) == dest_ring->sw_index)
759 nentries -= 1;
760
761 write_index = CE_RING_IDX_ADD(nentries_mask, write_index, nentries);
762 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
763 dest_ring->write_index = write_index;
764 }
765 EXPORT_SYMBOL(ath10k_ce_rx_update_write_idx);
766
767 int ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx,
768 dma_addr_t paddr)
769 {
770 struct ath10k *ar = pipe->ar;
771 struct ath10k_ce *ce = ath10k_ce_priv(ar);
772 int ret;
773
774 spin_lock_bh(&ce->ce_lock);
775 ret = pipe->ops->ce_rx_post_buf(pipe, ctx, paddr);
776 spin_unlock_bh(&ce->ce_lock);
777
778 return ret;
779 }
780 EXPORT_SYMBOL(ath10k_ce_rx_post_buf);
781
782 /*
783 * Guts of ath10k_ce_completed_recv_next.
784 * The caller takes responsibility for any necessary locking.
785 */
786 static int
787 _ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,
788 void **per_transfer_contextp,
789 unsigned int *nbytesp)
790 {
791 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
792 unsigned int nentries_mask = dest_ring->nentries_mask;
793 unsigned int sw_index = dest_ring->sw_index;
794
795 struct ce_desc *base = dest_ring->base_addr_owner_space;
796 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index);
797 struct ce_desc sdesc;
798 u16 nbytes;
799
800 /* Copy in one go for performance reasons */
801 sdesc = *desc;
802
803 nbytes = __le16_to_cpu(sdesc.nbytes);
804 if (nbytes == 0) {
805 /*
806 * This closes a relatively unusual race where the Host
807 * sees the updated DRRI before the update to the
808 * corresponding descriptor has completed. We treat this
809 * as a descriptor that is not yet done.
810 */
811 return -EIO;
812 }
813
814 desc->nbytes = 0;
815
816 /* Return data from completed destination descriptor */
817 *nbytesp = nbytes;
818
819 if (per_transfer_contextp)
820 *per_transfer_contextp =
821 dest_ring->per_transfer_context[sw_index];
822
823 /* Copy engine 5 (HTT Rx) will reuse the same transfer context.
824 * So update transfer context all CEs except CE5.
825 */
826 if (ce_state->id != 5)
827 dest_ring->per_transfer_context[sw_index] = NULL;
828
829 /* Update sw_index */
830 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
831 dest_ring->sw_index = sw_index;
832
833 return 0;
834 }
835
836 static int
837 _ath10k_ce_completed_recv_next_nolock_64(struct ath10k_ce_pipe *ce_state,
838 void **per_transfer_contextp,
839 unsigned int *nbytesp)
840 {
841 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
842 unsigned int nentries_mask = dest_ring->nentries_mask;
843 unsigned int sw_index = dest_ring->sw_index;
844 struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
845 struct ce_desc_64 *desc =
846 CE_DEST_RING_TO_DESC_64(base, sw_index);
847 struct ce_desc_64 sdesc;
848 u16 nbytes;
849
850 /* Copy in one go for performance reasons */
851 sdesc = *desc;
852
853 nbytes = __le16_to_cpu(sdesc.nbytes);
854 if (nbytes == 0) {
855 /* This closes a relatively unusual race where the Host
856 * sees the updated DRRI before the update to the
857 * corresponding descriptor has completed. We treat this
858 * as a descriptor that is not yet done.
859 */
860 return -EIO;
861 }
862
863 desc->nbytes = 0;
864
865 /* Return data from completed destination descriptor */
866 *nbytesp = nbytes;
867
868 if (per_transfer_contextp)
869 *per_transfer_contextp =
870 dest_ring->per_transfer_context[sw_index];
871
872 /* Copy engine 5 (HTT Rx) will reuse the same transfer context.
873 * So update transfer context all CEs except CE5.
874 */
875 if (ce_state->id != 5)
876 dest_ring->per_transfer_context[sw_index] = NULL;
877
878 /* Update sw_index */
879 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
880 dest_ring->sw_index = sw_index;
881
882 return 0;
883 }
884
885 int ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,
886 void **per_transfer_ctx,
887 unsigned int *nbytesp)
888 {
889 return ce_state->ops->ce_completed_recv_next_nolock(ce_state,
890 per_transfer_ctx,
891 nbytesp);
892 }
893 EXPORT_SYMBOL(ath10k_ce_completed_recv_next_nolock);
894
895 int ath10k_ce_completed_recv_next(struct ath10k_ce_pipe *ce_state,
896 void **per_transfer_contextp,
897 unsigned int *nbytesp)
898 {
899 struct ath10k *ar = ce_state->ar;
900 struct ath10k_ce *ce = ath10k_ce_priv(ar);
901 int ret;
902
903 spin_lock_bh(&ce->ce_lock);
904 ret = ce_state->ops->ce_completed_recv_next_nolock(ce_state,
905 per_transfer_contextp,
906 nbytesp);
907
908 spin_unlock_bh(&ce->ce_lock);
909
910 return ret;
911 }
912 EXPORT_SYMBOL(ath10k_ce_completed_recv_next);
913
914 static int _ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,
915 void **per_transfer_contextp,
916 dma_addr_t *bufferp)
917 {
918 struct ath10k_ce_ring *dest_ring;
919 unsigned int nentries_mask;
920 unsigned int sw_index;
921 unsigned int write_index;
922 int ret;
923 struct ath10k *ar;
924 struct ath10k_ce *ce;
925
926 dest_ring = ce_state->dest_ring;
927
928 if (!dest_ring)
929 return -EIO;
930
931 ar = ce_state->ar;
932 ce = ath10k_ce_priv(ar);
933
934 spin_lock_bh(&ce->ce_lock);
935
936 nentries_mask = dest_ring->nentries_mask;
937 sw_index = dest_ring->sw_index;
938 write_index = dest_ring->write_index;
939 if (write_index != sw_index) {
940 struct ce_desc *base = dest_ring->base_addr_owner_space;
941 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index);
942
943 /* Return data from completed destination descriptor */
944 *bufferp = __le32_to_cpu(desc->addr);
945
946 if (per_transfer_contextp)
947 *per_transfer_contextp =
948 dest_ring->per_transfer_context[sw_index];
949
950 /* sanity */
951 dest_ring->per_transfer_context[sw_index] = NULL;
952 desc->nbytes = 0;
953
954 /* Update sw_index */
955 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
956 dest_ring->sw_index = sw_index;
957 ret = 0;
958 } else {
959 ret = -EIO;
960 }
961
962 spin_unlock_bh(&ce->ce_lock);
963
964 return ret;
965 }
966
967 static int _ath10k_ce_revoke_recv_next_64(struct ath10k_ce_pipe *ce_state,
968 void **per_transfer_contextp,
969 dma_addr_t *bufferp)
970 {
971 struct ath10k_ce_ring *dest_ring;
972 unsigned int nentries_mask;
973 unsigned int sw_index;
974 unsigned int write_index;
975 int ret;
976 struct ath10k *ar;
977 struct ath10k_ce *ce;
978
979 dest_ring = ce_state->dest_ring;
980
981 if (!dest_ring)
982 return -EIO;
983
984 ar = ce_state->ar;
985 ce = ath10k_ce_priv(ar);
986
987 spin_lock_bh(&ce->ce_lock);
988
989 nentries_mask = dest_ring->nentries_mask;
990 sw_index = dest_ring->sw_index;
991 write_index = dest_ring->write_index;
992 if (write_index != sw_index) {
993 struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
994 struct ce_desc_64 *desc =
995 CE_DEST_RING_TO_DESC_64(base, sw_index);
996
997 /* Return data from completed destination descriptor */
998 *bufferp = __le64_to_cpu(desc->addr);
999
1000 if (per_transfer_contextp)
1001 *per_transfer_contextp =
1002 dest_ring->per_transfer_context[sw_index];
1003
1004 /* sanity */
1005 dest_ring->per_transfer_context[sw_index] = NULL;
1006 desc->nbytes = 0;
1007
1008 /* Update sw_index */
1009 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1010 dest_ring->sw_index = sw_index;
1011 ret = 0;
1012 } else {
1013 ret = -EIO;
1014 }
1015
1016 spin_unlock_bh(&ce->ce_lock);
1017
1018 return ret;
1019 }
1020
1021 int ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,
1022 void **per_transfer_contextp,
1023 dma_addr_t *bufferp)
1024 {
1025 return ce_state->ops->ce_revoke_recv_next(ce_state,
1026 per_transfer_contextp,
1027 bufferp);
1028 }
1029 EXPORT_SYMBOL(ath10k_ce_revoke_recv_next);
1030
1031 /*
1032 * Guts of ath10k_ce_completed_send_next.
1033 * The caller takes responsibility for any necessary locking.
1034 */
1035 int ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state,
1036 void **per_transfer_contextp)
1037 {
1038 struct ath10k_ce_ring *src_ring = ce_state->src_ring;
1039 u32 ctrl_addr = ce_state->ctrl_addr;
1040 struct ath10k *ar = ce_state->ar;
1041 unsigned int nentries_mask = src_ring->nentries_mask;
1042 unsigned int sw_index = src_ring->sw_index;
1043 unsigned int read_index;
1044 struct ce_desc *desc;
1045
1046 if (src_ring->hw_index == sw_index) {
1047 /*
1048 * The SW completion index has caught up with the cached
1049 * version of the HW completion index.
1050 * Update the cached HW completion index to see whether
1051 * the SW has really caught up to the HW, or if the cached
1052 * value of the HW index has become stale.
1053 */
1054
1055 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1056 if (read_index == 0xffffffff)
1057 return -ENODEV;
1058
1059 read_index &= nentries_mask;
1060 src_ring->hw_index = read_index;
1061 }
1062
1063 if (ar->hw_params.rri_on_ddr)
1064 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1065 else
1066 read_index = src_ring->hw_index;
1067
1068 if (read_index == sw_index)
1069 return -EIO;
1070
1071 if (per_transfer_contextp)
1072 *per_transfer_contextp =
1073 src_ring->per_transfer_context[sw_index];
1074
1075 /* sanity */
1076 src_ring->per_transfer_context[sw_index] = NULL;
1077 desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space,
1078 sw_index);
1079 desc->nbytes = 0;
1080
1081 /* Update sw_index */
1082 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1083 src_ring->sw_index = sw_index;
1084
1085 return 0;
1086 }
1087 EXPORT_SYMBOL(ath10k_ce_completed_send_next_nolock);
1088
1089 static void ath10k_ce_extract_desc_data(struct ath10k *ar,
1090 struct ath10k_ce_ring *src_ring,
1091 u32 sw_index,
1092 dma_addr_t *bufferp,
1093 u32 *nbytesp,
1094 u32 *transfer_idp)
1095 {
1096 struct ce_desc *base = src_ring->base_addr_owner_space;
1097 struct ce_desc *desc = CE_SRC_RING_TO_DESC(base, sw_index);
1098
1099 /* Return data from completed source descriptor */
1100 *bufferp = __le32_to_cpu(desc->addr);
1101 *nbytesp = __le16_to_cpu(desc->nbytes);
1102 *transfer_idp = MS(__le16_to_cpu(desc->flags),
1103 CE_DESC_FLAGS_META_DATA);
1104 }
1105
1106 static void ath10k_ce_extract_desc_data_64(struct ath10k *ar,
1107 struct ath10k_ce_ring *src_ring,
1108 u32 sw_index,
1109 dma_addr_t *bufferp,
1110 u32 *nbytesp,
1111 u32 *transfer_idp)
1112 {
1113 struct ce_desc_64 *base = src_ring->base_addr_owner_space;
1114 struct ce_desc_64 *desc =
1115 CE_SRC_RING_TO_DESC_64(base, sw_index);
1116
1117 /* Return data from completed source descriptor */
1118 *bufferp = __le64_to_cpu(desc->addr);
1119 *nbytesp = __le16_to_cpu(desc->nbytes);
1120 *transfer_idp = MS(__le16_to_cpu(desc->flags),
1121 CE_DESC_FLAGS_META_DATA);
1122 }
1123
1124 /* NB: Modeled after ath10k_ce_completed_send_next */
1125 int ath10k_ce_cancel_send_next(struct ath10k_ce_pipe *ce_state,
1126 void **per_transfer_contextp,
1127 dma_addr_t *bufferp,
1128 unsigned int *nbytesp,
1129 unsigned int *transfer_idp)
1130 {
1131 struct ath10k_ce_ring *src_ring;
1132 unsigned int nentries_mask;
1133 unsigned int sw_index;
1134 unsigned int write_index;
1135 int ret;
1136 struct ath10k *ar;
1137 struct ath10k_ce *ce;
1138
1139 src_ring = ce_state->src_ring;
1140
1141 if (!src_ring)
1142 return -EIO;
1143
1144 ar = ce_state->ar;
1145 ce = ath10k_ce_priv(ar);
1146
1147 spin_lock_bh(&ce->ce_lock);
1148
1149 nentries_mask = src_ring->nentries_mask;
1150 sw_index = src_ring->sw_index;
1151 write_index = src_ring->write_index;
1152
1153 if (write_index != sw_index) {
1154 ce_state->ops->ce_extract_desc_data(ar, src_ring, sw_index,
1155 bufferp, nbytesp,
1156 transfer_idp);
1157
1158 if (per_transfer_contextp)
1159 *per_transfer_contextp =
1160 src_ring->per_transfer_context[sw_index];
1161
1162 /* sanity */
1163 src_ring->per_transfer_context[sw_index] = NULL;
1164
1165 /* Update sw_index */
1166 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1167 src_ring->sw_index = sw_index;
1168 ret = 0;
1169 } else {
1170 ret = -EIO;
1171 }
1172
1173 spin_unlock_bh(&ce->ce_lock);
1174
1175 return ret;
1176 }
1177 EXPORT_SYMBOL(ath10k_ce_cancel_send_next);
1178
1179 int ath10k_ce_completed_send_next(struct ath10k_ce_pipe *ce_state,
1180 void **per_transfer_contextp)
1181 {
1182 struct ath10k *ar = ce_state->ar;
1183 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1184 int ret;
1185
1186 spin_lock_bh(&ce->ce_lock);
1187 ret = ath10k_ce_completed_send_next_nolock(ce_state,
1188 per_transfer_contextp);
1189 spin_unlock_bh(&ce->ce_lock);
1190
1191 return ret;
1192 }
1193 EXPORT_SYMBOL(ath10k_ce_completed_send_next);
1194
1195 /*
1196 * Guts of interrupt handler for per-engine interrupts on a particular CE.
1197 *
1198 * Invokes registered callbacks for recv_complete,
1199 * send_complete, and watermarks.
1200 */
1201 void ath10k_ce_per_engine_service(struct ath10k *ar, unsigned int ce_id)
1202 {
1203 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1204 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1205 struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
1206 u32 ctrl_addr = ce_state->ctrl_addr;
1207
1208 spin_lock_bh(&ce->ce_lock);
1209
1210 /* Clear the copy-complete interrupts that will be handled here. */
1211 ath10k_ce_engine_int_status_clear(ar, ctrl_addr,
1212 wm_regs->cc_mask);
1213
1214 spin_unlock_bh(&ce->ce_lock);
1215
1216 if (ce_state->recv_cb)
1217 ce_state->recv_cb(ce_state);
1218
1219 if (ce_state->send_cb)
1220 ce_state->send_cb(ce_state);
1221
1222 spin_lock_bh(&ce->ce_lock);
1223
1224 /*
1225 * Misc CE interrupts are not being handled, but still need
1226 * to be cleared.
1227 */
1228 ath10k_ce_engine_int_status_clear(ar, ctrl_addr, wm_regs->wm_mask);
1229
1230 spin_unlock_bh(&ce->ce_lock);
1231 }
1232 EXPORT_SYMBOL(ath10k_ce_per_engine_service);
1233
1234 /*
1235 * Handler for per-engine interrupts on ALL active CEs.
1236 * This is used in cases where the system is sharing a
1237 * single interrput for all CEs
1238 */
1239
1240 void ath10k_ce_per_engine_service_any(struct ath10k *ar)
1241 {
1242 int ce_id;
1243 u32 intr_summary;
1244
1245 intr_summary = ath10k_ce_interrupt_summary(ar);
1246
1247 for (ce_id = 0; intr_summary && (ce_id < CE_COUNT); ce_id++) {
1248 if (intr_summary & (1 << ce_id))
1249 intr_summary &= ~(1 << ce_id);
1250 else
1251 /* no intr pending on this CE */
1252 continue;
1253
1254 ath10k_ce_per_engine_service(ar, ce_id);
1255 }
1256 }
1257 EXPORT_SYMBOL(ath10k_ce_per_engine_service_any);
1258
1259 /*
1260 * Adjust interrupts for the copy complete handler.
1261 * If it's needed for either send or recv, then unmask
1262 * this interrupt; otherwise, mask it.
1263 *
1264 * Called with ce_lock held.
1265 */
1266 static void ath10k_ce_per_engine_handler_adjust(struct ath10k_ce_pipe *ce_state)
1267 {
1268 u32 ctrl_addr = ce_state->ctrl_addr;
1269 struct ath10k *ar = ce_state->ar;
1270 bool disable_copy_compl_intr = ce_state->attr_flags & CE_ATTR_DIS_INTR;
1271
1272 if ((!disable_copy_compl_intr) &&
1273 (ce_state->send_cb || ce_state->recv_cb))
1274 ath10k_ce_copy_complete_inter_enable(ar, ctrl_addr);
1275 else
1276 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr);
1277
1278 ath10k_ce_watermark_intr_disable(ar, ctrl_addr);
1279 }
1280
1281 int ath10k_ce_disable_interrupts(struct ath10k *ar)
1282 {
1283 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1284 struct ath10k_ce_pipe *ce_state;
1285 u32 ctrl_addr;
1286 int ce_id;
1287
1288 for (ce_id = 0; ce_id < CE_COUNT; ce_id++) {
1289 ce_state = &ce->ce_states[ce_id];
1290 if (ce_state->attr_flags & CE_ATTR_POLL)
1291 continue;
1292
1293 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1294
1295 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr);
1296 ath10k_ce_error_intr_disable(ar, ctrl_addr);
1297 ath10k_ce_watermark_intr_disable(ar, ctrl_addr);
1298 }
1299
1300 return 0;
1301 }
1302 EXPORT_SYMBOL(ath10k_ce_disable_interrupts);
1303
1304 void ath10k_ce_enable_interrupts(struct ath10k *ar)
1305 {
1306 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1307 int ce_id;
1308 struct ath10k_ce_pipe *ce_state;
1309
1310 /* Enable interrupts for copy engine that
1311 * are not using polling mode.
1312 */
1313 for (ce_id = 0; ce_id < CE_COUNT; ce_id++) {
1314 ce_state = &ce->ce_states[ce_id];
1315 if (ce_state->attr_flags & CE_ATTR_POLL)
1316 continue;
1317
1318 ath10k_ce_per_engine_handler_adjust(ce_state);
1319 }
1320 }
1321 EXPORT_SYMBOL(ath10k_ce_enable_interrupts);
1322
1323 static int ath10k_ce_init_src_ring(struct ath10k *ar,
1324 unsigned int ce_id,
1325 const struct ce_attr *attr)
1326 {
1327 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1328 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1329 struct ath10k_ce_ring *src_ring = ce_state->src_ring;
1330 u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1331
1332 nentries = roundup_pow_of_two(attr->src_nentries);
1333
1334 if (ar->hw_params.target_64bit)
1335 memset(src_ring->base_addr_owner_space, 0,
1336 nentries * sizeof(struct ce_desc_64));
1337 else
1338 memset(src_ring->base_addr_owner_space, 0,
1339 nentries * sizeof(struct ce_desc));
1340
1341 src_ring->sw_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1342 src_ring->sw_index &= src_ring->nentries_mask;
1343 src_ring->hw_index = src_ring->sw_index;
1344
1345 src_ring->write_index =
1346 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr);
1347 src_ring->write_index &= src_ring->nentries_mask;
1348
1349 ath10k_ce_src_ring_base_addr_set(ar, ctrl_addr,
1350 src_ring->base_addr_ce_space);
1351 ath10k_ce_src_ring_size_set(ar, ctrl_addr, nentries);
1352 ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, attr->src_sz_max);
1353 ath10k_ce_src_ring_byte_swap_set(ar, ctrl_addr, 0);
1354 ath10k_ce_src_ring_lowmark_set(ar, ctrl_addr, 0);
1355 ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, nentries);
1356
1357 ath10k_dbg(ar, ATH10K_DBG_BOOT,
1358 "boot init ce src ring id %d entries %d base_addr %pK\n",
1359 ce_id, nentries, src_ring->base_addr_owner_space);
1360
1361 return 0;
1362 }
1363
1364 static int ath10k_ce_init_dest_ring(struct ath10k *ar,
1365 unsigned int ce_id,
1366 const struct ce_attr *attr)
1367 {
1368 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1369 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1370 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
1371 u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1372
1373 nentries = roundup_pow_of_two(attr->dest_nentries);
1374
1375 if (ar->hw_params.target_64bit)
1376 memset(dest_ring->base_addr_owner_space, 0,
1377 nentries * sizeof(struct ce_desc_64));
1378 else
1379 memset(dest_ring->base_addr_owner_space, 0,
1380 nentries * sizeof(struct ce_desc));
1381
1382 dest_ring->sw_index = ath10k_ce_dest_ring_read_index_get(ar, ctrl_addr);
1383 dest_ring->sw_index &= dest_ring->nentries_mask;
1384 dest_ring->write_index =
1385 ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr);
1386 dest_ring->write_index &= dest_ring->nentries_mask;
1387
1388 ath10k_ce_dest_ring_base_addr_set(ar, ctrl_addr,
1389 dest_ring->base_addr_ce_space);
1390 ath10k_ce_dest_ring_size_set(ar, ctrl_addr, nentries);
1391 ath10k_ce_dest_ring_byte_swap_set(ar, ctrl_addr, 0);
1392 ath10k_ce_dest_ring_lowmark_set(ar, ctrl_addr, 0);
1393 ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, nentries);
1394
1395 ath10k_dbg(ar, ATH10K_DBG_BOOT,
1396 "boot ce dest ring id %d entries %d base_addr %pK\n",
1397 ce_id, nentries, dest_ring->base_addr_owner_space);
1398
1399 return 0;
1400 }
1401
1402 static int ath10k_ce_alloc_shadow_base(struct ath10k *ar,
1403 struct ath10k_ce_ring *src_ring,
1404 u32 nentries)
1405 {
1406 src_ring->shadow_base_unaligned = kcalloc(nentries,
1407 sizeof(struct ce_desc),
1408 GFP_KERNEL);
1409 if (!src_ring->shadow_base_unaligned)
1410 return -ENOMEM;
1411
1412 src_ring->shadow_base = (struct ce_desc *)
1413 PTR_ALIGN(src_ring->shadow_base_unaligned,
1414 CE_DESC_RING_ALIGN);
1415 return 0;
1416 }
1417
1418 static struct ath10k_ce_ring *
1419 ath10k_ce_alloc_src_ring(struct ath10k *ar, unsigned int ce_id,
1420 const struct ce_attr *attr)
1421 {
1422 struct ath10k_ce_ring *src_ring;
1423 u32 nentries = attr->src_nentries;
1424 dma_addr_t base_addr;
1425 int ret;
1426
1427 nentries = roundup_pow_of_two(nentries);
1428
1429 src_ring = kzalloc(struct_size(src_ring, per_transfer_context,
1430 nentries), GFP_KERNEL);
1431 if (src_ring == NULL)
1432 return ERR_PTR(-ENOMEM);
1433
1434 src_ring->nentries = nentries;
1435 src_ring->nentries_mask = nentries - 1;
1436
1437 /*
1438 * Legacy platforms that do not support cache
1439 * coherent DMA are unsupported
1440 */
1441 src_ring->base_addr_owner_space_unaligned =
1442 dma_alloc_coherent(ar->dev,
1443 (nentries * sizeof(struct ce_desc) +
1444 CE_DESC_RING_ALIGN),
1445 &base_addr, GFP_KERNEL);
1446 if (!src_ring->base_addr_owner_space_unaligned) {
1447 kfree(src_ring);
1448 return ERR_PTR(-ENOMEM);
1449 }
1450
1451 src_ring->base_addr_ce_space_unaligned = base_addr;
1452
1453 src_ring->base_addr_owner_space =
1454 PTR_ALIGN(src_ring->base_addr_owner_space_unaligned,
1455 CE_DESC_RING_ALIGN);
1456 src_ring->base_addr_ce_space =
1457 ALIGN(src_ring->base_addr_ce_space_unaligned,
1458 CE_DESC_RING_ALIGN);
1459
1460 if (ar->hw_params.shadow_reg_support) {
1461 ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries);
1462 if (ret) {
1463 dma_free_coherent(ar->dev,
1464 (nentries * sizeof(struct ce_desc) +
1465 CE_DESC_RING_ALIGN),
1466 src_ring->base_addr_owner_space_unaligned,
1467 base_addr);
1468 kfree(src_ring);
1469 return ERR_PTR(ret);
1470 }
1471 }
1472
1473 return src_ring;
1474 }
1475
1476 static struct ath10k_ce_ring *
1477 ath10k_ce_alloc_src_ring_64(struct ath10k *ar, unsigned int ce_id,
1478 const struct ce_attr *attr)
1479 {
1480 struct ath10k_ce_ring *src_ring;
1481 u32 nentries = attr->src_nentries;
1482 dma_addr_t base_addr;
1483 int ret;
1484
1485 nentries = roundup_pow_of_two(nentries);
1486
1487 src_ring = kzalloc(struct_size(src_ring, per_transfer_context,
1488 nentries), GFP_KERNEL);
1489 if (!src_ring)
1490 return ERR_PTR(-ENOMEM);
1491
1492 src_ring->nentries = nentries;
1493 src_ring->nentries_mask = nentries - 1;
1494
1495 /* Legacy platforms that do not support cache
1496 * coherent DMA are unsupported
1497 */
1498 src_ring->base_addr_owner_space_unaligned =
1499 dma_alloc_coherent(ar->dev,
1500 (nentries * sizeof(struct ce_desc_64) +
1501 CE_DESC_RING_ALIGN),
1502 &base_addr, GFP_KERNEL);
1503 if (!src_ring->base_addr_owner_space_unaligned) {
1504 kfree(src_ring);
1505 return ERR_PTR(-ENOMEM);
1506 }
1507
1508 src_ring->base_addr_ce_space_unaligned = base_addr;
1509
1510 src_ring->base_addr_owner_space =
1511 PTR_ALIGN(src_ring->base_addr_owner_space_unaligned,
1512 CE_DESC_RING_ALIGN);
1513 src_ring->base_addr_ce_space =
1514 ALIGN(src_ring->base_addr_ce_space_unaligned,
1515 CE_DESC_RING_ALIGN);
1516
1517 if (ar->hw_params.shadow_reg_support) {
1518 ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries);
1519 if (ret) {
1520 dma_free_coherent(ar->dev,
1521 (nentries * sizeof(struct ce_desc_64) +
1522 CE_DESC_RING_ALIGN),
1523 src_ring->base_addr_owner_space_unaligned,
1524 base_addr);
1525 kfree(src_ring);
1526 return ERR_PTR(ret);
1527 }
1528 }
1529
1530 return src_ring;
1531 }
1532
1533 static struct ath10k_ce_ring *
1534 ath10k_ce_alloc_dest_ring(struct ath10k *ar, unsigned int ce_id,
1535 const struct ce_attr *attr)
1536 {
1537 struct ath10k_ce_ring *dest_ring;
1538 u32 nentries;
1539 dma_addr_t base_addr;
1540
1541 nentries = roundup_pow_of_two(attr->dest_nentries);
1542
1543 dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context,
1544 nentries), GFP_KERNEL);
1545 if (dest_ring == NULL)
1546 return ERR_PTR(-ENOMEM);
1547
1548 dest_ring->nentries = nentries;
1549 dest_ring->nentries_mask = nentries - 1;
1550
1551 /*
1552 * Legacy platforms that do not support cache
1553 * coherent DMA are unsupported
1554 */
1555 dest_ring->base_addr_owner_space_unaligned =
1556 dma_zalloc_coherent(ar->dev,
1557 (nentries * sizeof(struct ce_desc) +
1558 CE_DESC_RING_ALIGN),
1559 &base_addr, GFP_KERNEL);
1560 if (!dest_ring->base_addr_owner_space_unaligned) {
1561 kfree(dest_ring);
1562 return ERR_PTR(-ENOMEM);
1563 }
1564
1565 dest_ring->base_addr_ce_space_unaligned = base_addr;
1566
1567 dest_ring->base_addr_owner_space =
1568 PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned,
1569 CE_DESC_RING_ALIGN);
1570 dest_ring->base_addr_ce_space =
1571 ALIGN(dest_ring->base_addr_ce_space_unaligned,
1572 CE_DESC_RING_ALIGN);
1573
1574 return dest_ring;
1575 }
1576
1577 static struct ath10k_ce_ring *
1578 ath10k_ce_alloc_dest_ring_64(struct ath10k *ar, unsigned int ce_id,
1579 const struct ce_attr *attr)
1580 {
1581 struct ath10k_ce_ring *dest_ring;
1582 u32 nentries;
1583 dma_addr_t base_addr;
1584
1585 nentries = roundup_pow_of_two(attr->dest_nentries);
1586
1587 dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context,
1588 nentries), GFP_KERNEL);
1589 if (!dest_ring)
1590 return ERR_PTR(-ENOMEM);
1591
1592 dest_ring->nentries = nentries;
1593 dest_ring->nentries_mask = nentries - 1;
1594
1595 /* Legacy platforms that do not support cache
1596 * coherent DMA are unsupported
1597 */
1598 dest_ring->base_addr_owner_space_unaligned =
1599 dma_alloc_coherent(ar->dev,
1600 (nentries * sizeof(struct ce_desc_64) +
1601 CE_DESC_RING_ALIGN),
1602 &base_addr, GFP_KERNEL);
1603 if (!dest_ring->base_addr_owner_space_unaligned) {
1604 kfree(dest_ring);
1605 return ERR_PTR(-ENOMEM);
1606 }
1607
1608 dest_ring->base_addr_ce_space_unaligned = base_addr;
1609
1610 /* Correctly initialize memory to 0 to prevent garbage
1611 * data crashing system when download firmware
1612 */
1613 memset(dest_ring->base_addr_owner_space_unaligned, 0,
1614 nentries * sizeof(struct ce_desc_64) + CE_DESC_RING_ALIGN);
1615
1616 dest_ring->base_addr_owner_space =
1617 PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned,
1618 CE_DESC_RING_ALIGN);
1619 dest_ring->base_addr_ce_space =
1620 ALIGN(dest_ring->base_addr_ce_space_unaligned,
1621 CE_DESC_RING_ALIGN);
1622
1623 return dest_ring;
1624 }
1625
1626 /*
1627 * Initialize a Copy Engine based on caller-supplied attributes.
1628 * This may be called once to initialize both source and destination
1629 * rings or it may be called twice for separate source and destination
1630 * initialization. It may be that only one side or the other is
1631 * initialized by software/firmware.
1632 */
1633 int ath10k_ce_init_pipe(struct ath10k *ar, unsigned int ce_id,
1634 const struct ce_attr *attr)
1635 {
1636 int ret;
1637
1638 if (attr->src_nentries) {
1639 ret = ath10k_ce_init_src_ring(ar, ce_id, attr);
1640 if (ret) {
1641 ath10k_err(ar, "Failed to initialize CE src ring for ID: %d (%d)\n",
1642 ce_id, ret);
1643 return ret;
1644 }
1645 }
1646
1647 if (attr->dest_nentries) {
1648 ret = ath10k_ce_init_dest_ring(ar, ce_id, attr);
1649 if (ret) {
1650 ath10k_err(ar, "Failed to initialize CE dest ring for ID: %d (%d)\n",
1651 ce_id, ret);
1652 return ret;
1653 }
1654 }
1655
1656 return 0;
1657 }
1658 EXPORT_SYMBOL(ath10k_ce_init_pipe);
1659
1660 static void ath10k_ce_deinit_src_ring(struct ath10k *ar, unsigned int ce_id)
1661 {
1662 u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1663
1664 ath10k_ce_src_ring_base_addr_set(ar, ctrl_addr, 0);
1665 ath10k_ce_src_ring_size_set(ar, ctrl_addr, 0);
1666 ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, 0);
1667 ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, 0);
1668 }
1669
1670 static void ath10k_ce_deinit_dest_ring(struct ath10k *ar, unsigned int ce_id)
1671 {
1672 u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1673
1674 ath10k_ce_dest_ring_base_addr_set(ar, ctrl_addr, 0);
1675 ath10k_ce_dest_ring_size_set(ar, ctrl_addr, 0);
1676 ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, 0);
1677 }
1678
1679 void ath10k_ce_deinit_pipe(struct ath10k *ar, unsigned int ce_id)
1680 {
1681 ath10k_ce_deinit_src_ring(ar, ce_id);
1682 ath10k_ce_deinit_dest_ring(ar, ce_id);
1683 }
1684 EXPORT_SYMBOL(ath10k_ce_deinit_pipe);
1685
1686 static void _ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)
1687 {
1688 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1689 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1690
1691 if (ce_state->src_ring) {
1692 if (ar->hw_params.shadow_reg_support)
1693 kfree(ce_state->src_ring->shadow_base_unaligned);
1694 dma_free_coherent(ar->dev,
1695 (ce_state->src_ring->nentries *
1696 sizeof(struct ce_desc) +
1697 CE_DESC_RING_ALIGN),
1698 ce_state->src_ring->base_addr_owner_space,
1699 ce_state->src_ring->base_addr_ce_space);
1700 kfree(ce_state->src_ring);
1701 }
1702
1703 if (ce_state->dest_ring) {
1704 dma_free_coherent(ar->dev,
1705 (ce_state->dest_ring->nentries *
1706 sizeof(struct ce_desc) +
1707 CE_DESC_RING_ALIGN),
1708 ce_state->dest_ring->base_addr_owner_space,
1709 ce_state->dest_ring->base_addr_ce_space);
1710 kfree(ce_state->dest_ring);
1711 }
1712
1713 ce_state->src_ring = NULL;
1714 ce_state->dest_ring = NULL;
1715 }
1716
1717 static void _ath10k_ce_free_pipe_64(struct ath10k *ar, int ce_id)
1718 {
1719 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1720 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1721
1722 if (ce_state->src_ring) {
1723 if (ar->hw_params.shadow_reg_support)
1724 kfree(ce_state->src_ring->shadow_base_unaligned);
1725 dma_free_coherent(ar->dev,
1726 (ce_state->src_ring->nentries *
1727 sizeof(struct ce_desc_64) +
1728 CE_DESC_RING_ALIGN),
1729 ce_state->src_ring->base_addr_owner_space,
1730 ce_state->src_ring->base_addr_ce_space);
1731 kfree(ce_state->src_ring);
1732 }
1733
1734 if (ce_state->dest_ring) {
1735 dma_free_coherent(ar->dev,
1736 (ce_state->dest_ring->nentries *
1737 sizeof(struct ce_desc_64) +
1738 CE_DESC_RING_ALIGN),
1739 ce_state->dest_ring->base_addr_owner_space,
1740 ce_state->dest_ring->base_addr_ce_space);
1741 kfree(ce_state->dest_ring);
1742 }
1743
1744 ce_state->src_ring = NULL;
1745 ce_state->dest_ring = NULL;
1746 }
1747
1748 void ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)
1749 {
1750 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1751 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1752
1753 ce_state->ops->ce_free_pipe(ar, ce_id);
1754 }
1755 EXPORT_SYMBOL(ath10k_ce_free_pipe);
1756
1757 void ath10k_ce_dump_registers(struct ath10k *ar,
1758 struct ath10k_fw_crash_data *crash_data)
1759 {
1760 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1761 struct ath10k_ce_crash_data ce_data;
1762 u32 addr, id;
1763
1764 lockdep_assert_held(&ar->data_lock);
1765
1766 ath10k_err(ar, "Copy Engine register dump:\n");
1767
1768 spin_lock_bh(&ce->ce_lock);
1769 for (id = 0; id < CE_COUNT; id++) {
1770 addr = ath10k_ce_base_address(ar, id);
1771 ce_data.base_addr = cpu_to_le32(addr);
1772
1773 ce_data.src_wr_idx =
1774 cpu_to_le32(ath10k_ce_src_ring_write_index_get(ar, addr));
1775 ce_data.src_r_idx =
1776 cpu_to_le32(ath10k_ce_src_ring_read_index_get(ar, addr));
1777 ce_data.dst_wr_idx =
1778 cpu_to_le32(ath10k_ce_dest_ring_write_index_get(ar, addr));
1779 ce_data.dst_r_idx =
1780 cpu_to_le32(ath10k_ce_dest_ring_read_index_get(ar, addr));
1781
1782 if (crash_data)
1783 crash_data->ce_crash_data[id] = ce_data;
1784
1785 ath10k_err(ar, "[%02d]: 0x%08x %3u %3u %3u %3u", id,
1786 le32_to_cpu(ce_data.base_addr),
1787 le32_to_cpu(ce_data.src_wr_idx),
1788 le32_to_cpu(ce_data.src_r_idx),
1789 le32_to_cpu(ce_data.dst_wr_idx),
1790 le32_to_cpu(ce_data.dst_r_idx));
1791 }
1792
1793 spin_unlock_bh(&ce->ce_lock);
1794 }
1795 EXPORT_SYMBOL(ath10k_ce_dump_registers);
1796
1797 static const struct ath10k_ce_ops ce_ops = {
1798 .ce_alloc_src_ring = ath10k_ce_alloc_src_ring,
1799 .ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring,
1800 .ce_rx_post_buf = __ath10k_ce_rx_post_buf,
1801 .ce_completed_recv_next_nolock = _ath10k_ce_completed_recv_next_nolock,
1802 .ce_revoke_recv_next = _ath10k_ce_revoke_recv_next,
1803 .ce_extract_desc_data = ath10k_ce_extract_desc_data,
1804 .ce_free_pipe = _ath10k_ce_free_pipe,
1805 .ce_send_nolock = _ath10k_ce_send_nolock,
1806 };
1807
1808 static const struct ath10k_ce_ops ce_64_ops = {
1809 .ce_alloc_src_ring = ath10k_ce_alloc_src_ring_64,
1810 .ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring_64,
1811 .ce_rx_post_buf = __ath10k_ce_rx_post_buf_64,
1812 .ce_completed_recv_next_nolock =
1813 _ath10k_ce_completed_recv_next_nolock_64,
1814 .ce_revoke_recv_next = _ath10k_ce_revoke_recv_next_64,
1815 .ce_extract_desc_data = ath10k_ce_extract_desc_data_64,
1816 .ce_free_pipe = _ath10k_ce_free_pipe_64,
1817 .ce_send_nolock = _ath10k_ce_send_nolock_64,
1818 };
1819
1820 static void ath10k_ce_set_ops(struct ath10k *ar,
1821 struct ath10k_ce_pipe *ce_state)
1822 {
1823 switch (ar->hw_rev) {
1824 case ATH10K_HW_WCN3990:
1825 ce_state->ops = &ce_64_ops;
1826 break;
1827 default:
1828 ce_state->ops = &ce_ops;
1829 break;
1830 }
1831 }
1832
1833 int ath10k_ce_alloc_pipe(struct ath10k *ar, int ce_id,
1834 const struct ce_attr *attr)
1835 {
1836 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1837 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1838 int ret;
1839
1840 ath10k_ce_set_ops(ar, ce_state);
1841 /* Make sure there's enough CE ringbuffer entries for HTT TX to avoid
1842 * additional TX locking checks.
1843 *
1844 * For the lack of a better place do the check here.
1845 */
1846 BUILD_BUG_ON(2 * TARGET_NUM_MSDU_DESC >
1847 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1848 BUILD_BUG_ON(2 * TARGET_10_4_NUM_MSDU_DESC_PFC >
1849 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1850 BUILD_BUG_ON(2 * TARGET_TLV_NUM_MSDU_DESC >
1851 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1852
1853 ce_state->ar = ar;
1854 ce_state->id = ce_id;
1855 ce_state->ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1856 ce_state->attr_flags = attr->flags;
1857 ce_state->src_sz_max = attr->src_sz_max;
1858
1859 if (attr->src_nentries)
1860 ce_state->send_cb = attr->send_cb;
1861
1862 if (attr->dest_nentries)
1863 ce_state->recv_cb = attr->recv_cb;
1864
1865 if (attr->src_nentries) {
1866 ce_state->src_ring =
1867 ce_state->ops->ce_alloc_src_ring(ar, ce_id, attr);
1868 if (IS_ERR(ce_state->src_ring)) {
1869 ret = PTR_ERR(ce_state->src_ring);
1870 ath10k_err(ar, "failed to alloc CE src ring %d: %d\n",
1871 ce_id, ret);
1872 ce_state->src_ring = NULL;
1873 return ret;
1874 }
1875 }
1876
1877 if (attr->dest_nentries) {
1878 ce_state->dest_ring = ce_state->ops->ce_alloc_dst_ring(ar,
1879 ce_id,
1880 attr);
1881 if (IS_ERR(ce_state->dest_ring)) {
1882 ret = PTR_ERR(ce_state->dest_ring);
1883 ath10k_err(ar, "failed to alloc CE dest ring %d: %d\n",
1884 ce_id, ret);
1885 ce_state->dest_ring = NULL;
1886 return ret;
1887 }
1888 }
1889
1890 return 0;
1891 }
1892 EXPORT_SYMBOL(ath10k_ce_alloc_pipe);
1893
1894 void ath10k_ce_alloc_rri(struct ath10k *ar)
1895 {
1896 int i;
1897 u32 value;
1898 u32 ctrl1_regs;
1899 u32 ce_base_addr;
1900 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1901
1902 ce->vaddr_rri = dma_alloc_coherent(ar->dev,
1903 (CE_COUNT * sizeof(u32)),
1904 &ce->paddr_rri, GFP_KERNEL);
1905
1906 if (!ce->vaddr_rri)
1907 return;
1908
1909 ath10k_ce_write32(ar, ar->hw_ce_regs->ce_rri_low,
1910 lower_32_bits(ce->paddr_rri));
1911 ath10k_ce_write32(ar, ar->hw_ce_regs->ce_rri_high,
1912 (upper_32_bits(ce->paddr_rri) &
1913 CE_DESC_FLAGS_GET_MASK));
1914
1915 for (i = 0; i < CE_COUNT; i++) {
1916 ctrl1_regs = ar->hw_ce_regs->ctrl1_regs->addr;
1917 ce_base_addr = ath10k_ce_base_address(ar, i);
1918 value = ath10k_ce_read32(ar, ce_base_addr + ctrl1_regs);
1919 value |= ar->hw_ce_regs->upd->mask;
1920 ath10k_ce_write32(ar, ce_base_addr + ctrl1_regs, value);
1921 }
1922
1923 memset(ce->vaddr_rri, 0, CE_COUNT * sizeof(u32));
1924 }
1925 EXPORT_SYMBOL(ath10k_ce_alloc_rri);
1926
1927 void ath10k_ce_free_rri(struct ath10k *ar)
1928 {
1929 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1930
1931 dma_free_coherent(ar->dev, (CE_COUNT * sizeof(u32)),
1932 ce->vaddr_rri,
1933 ce->paddr_rri);
1934 }
1935 EXPORT_SYMBOL(ath10k_ce_free_rri);
Linux with added FPC-III support