?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
treewide: remove redundant IS_ERR() before error code check
[linux/fpc-iii.git] / drivers / soc / ti / k3-ringacc.c
blob5fb2ee2ac978aa71f4918d03221152bca693c655
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * TI K3 NAVSS Ring Accelerator subsystem driver
4 *
5 * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com
6 */
7
8 #include <linux/dma-mapping.h>
9 #include <linux/io.h>
10 #include <linux/init.h>
11 #include <linux/of.h>
12 #include <linux/platform_device.h>
13 #include <linux/soc/ti/k3-ringacc.h>
14 #include <linux/soc/ti/ti_sci_protocol.h>
15 #include <linux/soc/ti/ti_sci_inta_msi.h>
16 #include <linux/of_irq.h>
17 #include <linux/irqdomain.h>
18
19 static LIST_HEAD(k3_ringacc_list);
20 static DEFINE_MUTEX(k3_ringacc_list_lock);
21
22 #define K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK GENMASK(19, 0)
23
24 /**
25 * struct k3_ring_rt_regs - The RA realtime Control/Status Registers region
26 *
27 * @resv_16: Reserved
28 * @db: Ring Doorbell Register
29 * @resv_4: Reserved
30 * @occ: Ring Occupancy Register
31 * @indx: Ring Current Index Register
32 * @hwocc: Ring Hardware Occupancy Register
33 * @hwindx: Ring Hardware Current Index Register
34 */
35 struct k3_ring_rt_regs {
36 u32 resv_16[4];
37 u32 db;
38 u32 resv_4[1];
39 u32 occ;
40 u32 indx;
41 u32 hwocc;
42 u32 hwindx;
43 };
44
45 #define K3_RINGACC_RT_REGS_STEP 0x1000
46
47 /**
48 * struct k3_ring_fifo_regs - The Ring Accelerator Queues Registers region
49 *
50 * @head_data: Ring Head Entry Data Registers
51 * @tail_data: Ring Tail Entry Data Registers
52 * @peek_head_data: Ring Peek Head Entry Data Regs
53 * @peek_tail_data: Ring Peek Tail Entry Data Regs
54 */
55 struct k3_ring_fifo_regs {
56 u32 head_data[128];
57 u32 tail_data[128];
58 u32 peek_head_data[128];
59 u32 peek_tail_data[128];
60 };
61
62 /**
63 * struct k3_ringacc_proxy_gcfg_regs - RA Proxy Global Config MMIO Region
64 *
65 * @revision: Revision Register
66 * @config: Config Register
67 */
68 struct k3_ringacc_proxy_gcfg_regs {
69 u32 revision;
70 u32 config;
71 };
72
73 #define K3_RINGACC_PROXY_CFG_THREADS_MASK GENMASK(15, 0)
74
75 /**
76 * struct k3_ringacc_proxy_target_regs - Proxy Datapath MMIO Region
77 *
78 * @control: Proxy Control Register
79 * @status: Proxy Status Register
80 * @resv_512: Reserved
81 * @data: Proxy Data Register
82 */
83 struct k3_ringacc_proxy_target_regs {
84 u32 control;
85 u32 status;
86 u8 resv_512[504];
87 u32 data[128];
88 };
89
90 #define K3_RINGACC_PROXY_TARGET_STEP 0x1000
91 #define K3_RINGACC_PROXY_NOT_USED (-1)
92
93 enum k3_ringacc_proxy_access_mode {
94 PROXY_ACCESS_MODE_HEAD = 0,
95 PROXY_ACCESS_MODE_TAIL = 1,
96 PROXY_ACCESS_MODE_PEEK_HEAD = 2,
97 PROXY_ACCESS_MODE_PEEK_TAIL = 3,
98 };
99
100 #define K3_RINGACC_FIFO_WINDOW_SIZE_BYTES (512U)
101 #define K3_RINGACC_FIFO_REGS_STEP 0x1000
102 #define K3_RINGACC_MAX_DB_RING_CNT (127U)
103
104 struct k3_ring_ops {
105 int (*push_tail)(struct k3_ring *ring, void *elm);
106 int (*push_head)(struct k3_ring *ring, void *elm);
107 int (*pop_tail)(struct k3_ring *ring, void *elm);
108 int (*pop_head)(struct k3_ring *ring, void *elm);
109 };
110
111 /**
112 * struct k3_ring - RA Ring descriptor
113 *
114 * @rt: Ring control/status registers
115 * @fifos: Ring queues registers
116 * @proxy: Ring Proxy Datapath registers
117 * @ring_mem_dma: Ring buffer dma address
118 * @ring_mem_virt: Ring buffer virt address
119 * @ops: Ring operations
120 * @size: Ring size in elements
121 * @elm_size: Size of the ring element
122 * @mode: Ring mode
123 * @flags: flags
124 * @free: Number of free elements
125 * @occ: Ring occupancy
126 * @windex: Write index (only for @K3_RINGACC_RING_MODE_RING)
127 * @rindex: Read index (only for @K3_RINGACC_RING_MODE_RING)
128 * @ring_id: Ring Id
129 * @parent: Pointer on struct @k3_ringacc
130 * @use_count: Use count for shared rings
131 * @proxy_id: RA Ring Proxy Id (only if @K3_RINGACC_RING_USE_PROXY)
132 */
133 struct k3_ring {
134 struct k3_ring_rt_regs __iomem *rt;
135 struct k3_ring_fifo_regs __iomem *fifos;
136 struct k3_ringacc_proxy_target_regs __iomem *proxy;
137 dma_addr_t ring_mem_dma;
138 void *ring_mem_virt;
139 struct k3_ring_ops *ops;
140 u32 size;
141 enum k3_ring_size elm_size;
142 enum k3_ring_mode mode;
143 u32 flags;
144 #define K3_RING_FLAG_BUSY BIT(1)
145 #define K3_RING_FLAG_SHARED BIT(2)
146 u32 free;
147 u32 occ;
148 u32 windex;
149 u32 rindex;
150 u32 ring_id;
151 struct k3_ringacc *parent;
152 u32 use_count;
153 int proxy_id;
154 };
155
156 /**
157 * struct k3_ringacc - Rings accelerator descriptor
158 *
159 * @dev: pointer on RA device
160 * @proxy_gcfg: RA proxy global config registers
161 * @proxy_target_base: RA proxy datapath region
162 * @num_rings: number of ring in RA
163 * @rings_inuse: bitfield for ring usage tracking
164 * @rm_gp_range: general purpose rings range from tisci
165 * @dma_ring_reset_quirk: DMA reset w/a enable
166 * @num_proxies: number of RA proxies
167 * @proxy_inuse: bitfield for proxy usage tracking
168 * @rings: array of rings descriptors (struct @k3_ring)
169 * @list: list of RAs in the system
170 * @req_lock: protect rings allocation
171 * @tisci: pointer ti-sci handle
172 * @tisci_ring_ops: ti-sci rings ops
173 * @tisci_dev_id: ti-sci device id
174 */
175 struct k3_ringacc {
176 struct device *dev;
177 struct k3_ringacc_proxy_gcfg_regs __iomem *proxy_gcfg;
178 void __iomem *proxy_target_base;
179 u32 num_rings; /* number of rings in Ringacc module */
180 unsigned long *rings_inuse;
181 struct ti_sci_resource *rm_gp_range;
182
183 bool dma_ring_reset_quirk;
184 u32 num_proxies;
185 unsigned long *proxy_inuse;
186
187 struct k3_ring *rings;
188 struct list_head list;
189 struct mutex req_lock; /* protect rings allocation */
190
191 const struct ti_sci_handle *tisci;
192 const struct ti_sci_rm_ringacc_ops *tisci_ring_ops;
193 u32 tisci_dev_id;
194 };
195
196 static long k3_ringacc_ring_get_fifo_pos(struct k3_ring *ring)
197 {
198 return K3_RINGACC_FIFO_WINDOW_SIZE_BYTES -
199 (4 << ring->elm_size);
200 }
201
202 static void *k3_ringacc_get_elm_addr(struct k3_ring *ring, u32 idx)
203 {
204 return (ring->ring_mem_virt + idx * (4 << ring->elm_size));
205 }
206
207 static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem);
208 static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem);
209
210 static struct k3_ring_ops k3_ring_mode_ring_ops = {
211 .push_tail = k3_ringacc_ring_push_mem,
212 .pop_head = k3_ringacc_ring_pop_mem,
213 };
214
215 static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem);
216 static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem);
217 static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem);
218 static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem);
219
220 static struct k3_ring_ops k3_ring_mode_msg_ops = {
221 .push_tail = k3_ringacc_ring_push_io,
222 .push_head = k3_ringacc_ring_push_head_io,
223 .pop_tail = k3_ringacc_ring_pop_tail_io,
224 .pop_head = k3_ringacc_ring_pop_io,
225 };
226
227 static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem);
228 static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem);
229 static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem);
230 static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem);
231
232 static struct k3_ring_ops k3_ring_mode_proxy_ops = {
233 .push_tail = k3_ringacc_ring_push_tail_proxy,
234 .push_head = k3_ringacc_ring_push_head_proxy,
235 .pop_tail = k3_ringacc_ring_pop_tail_proxy,
236 .pop_head = k3_ringacc_ring_pop_head_proxy,
237 };
238
239 static void k3_ringacc_ring_dump(struct k3_ring *ring)
240 {
241 struct device *dev = ring->parent->dev;
242
243 dev_dbg(dev, "dump ring: %d\n", ring->ring_id);
244 dev_dbg(dev, "dump mem virt %p, dma %pad\n", ring->ring_mem_virt,
245 &ring->ring_mem_dma);
246 dev_dbg(dev, "dump elmsize %d, size %d, mode %d, proxy_id %d\n",
247 ring->elm_size, ring->size, ring->mode, ring->proxy_id);
248
249 dev_dbg(dev, "dump ring_rt_regs: db%08x\n", readl(&ring->rt->db));
250 dev_dbg(dev, "dump occ%08x\n", readl(&ring->rt->occ));
251 dev_dbg(dev, "dump indx%08x\n", readl(&ring->rt->indx));
252 dev_dbg(dev, "dump hwocc%08x\n", readl(&ring->rt->hwocc));
253 dev_dbg(dev, "dump hwindx%08x\n", readl(&ring->rt->hwindx));
254
255 if (ring->ring_mem_virt)
256 print_hex_dump_debug("dump ring_mem_virt ", DUMP_PREFIX_NONE,
257 16, 1, ring->ring_mem_virt, 16 * 8, false);
258 }
259
260 struct k3_ring *k3_ringacc_request_ring(struct k3_ringacc *ringacc,
261 int id, u32 flags)
262 {
263 int proxy_id = K3_RINGACC_PROXY_NOT_USED;
264
265 mutex_lock(&ringacc->req_lock);
266
267 if (id == K3_RINGACC_RING_ID_ANY) {
268 /* Request for any general purpose ring */
269 struct ti_sci_resource_desc *gp_rings =
270 &ringacc->rm_gp_range->desc[0];
271 unsigned long size;
272
273 size = gp_rings->start + gp_rings->num;
274 id = find_next_zero_bit(ringacc->rings_inuse, size,
275 gp_rings->start);
276 if (id == size)
277 goto error;
278 } else if (id < 0) {
279 goto error;
280 }
281
282 if (test_bit(id, ringacc->rings_inuse) &&
283 !(ringacc->rings[id].flags & K3_RING_FLAG_SHARED))
284 goto error;
285 else if (ringacc->rings[id].flags & K3_RING_FLAG_SHARED)
286 goto out;
287
288 if (flags & K3_RINGACC_RING_USE_PROXY) {
289 proxy_id = find_next_zero_bit(ringacc->proxy_inuse,
290 ringacc->num_proxies, 0);
291 if (proxy_id == ringacc->num_proxies)
292 goto error;
293 }
294
295 if (proxy_id != K3_RINGACC_PROXY_NOT_USED) {
296 set_bit(proxy_id, ringacc->proxy_inuse);
297 ringacc->rings[id].proxy_id = proxy_id;
298 dev_dbg(ringacc->dev, "Giving ring#%d proxy#%d\n", id,
299 proxy_id);
300 } else {
301 dev_dbg(ringacc->dev, "Giving ring#%d\n", id);
302 }
303
304 set_bit(id, ringacc->rings_inuse);
305 out:
306 ringacc->rings[id].use_count++;
307 mutex_unlock(&ringacc->req_lock);
308 return &ringacc->rings[id];
309
310 error:
311 mutex_unlock(&ringacc->req_lock);
312 return NULL;
313 }
314 EXPORT_SYMBOL_GPL(k3_ringacc_request_ring);
315
316 static void k3_ringacc_ring_reset_sci(struct k3_ring *ring)
317 {
318 struct k3_ringacc *ringacc = ring->parent;
319 int ret;
320
321 ret = ringacc->tisci_ring_ops->config(
322 ringacc->tisci,
323 TI_SCI_MSG_VALUE_RM_RING_COUNT_VALID,
324 ringacc->tisci_dev_id,
325 ring->ring_id,
326 0,
327 0,
328 ring->size,
329 0,
330 0,
331 0);
332 if (ret)
333 dev_err(ringacc->dev, "TISCI reset ring fail (%d) ring_idx %d\n",
334 ret, ring->ring_id);
335 }
336
337 void k3_ringacc_ring_reset(struct k3_ring *ring)
338 {
339 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
340 return;
341
342 ring->occ = 0;
343 ring->free = 0;
344 ring->rindex = 0;
345 ring->windex = 0;
346
347 k3_ringacc_ring_reset_sci(ring);
348 }
349 EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset);
350
351 static void k3_ringacc_ring_reconfig_qmode_sci(struct k3_ring *ring,
352 enum k3_ring_mode mode)
353 {
354 struct k3_ringacc *ringacc = ring->parent;
355 int ret;
356
357 ret = ringacc->tisci_ring_ops->config(
358 ringacc->tisci,
359 TI_SCI_MSG_VALUE_RM_RING_MODE_VALID,
360 ringacc->tisci_dev_id,
361 ring->ring_id,
362 0,
363 0,
364 0,
365 mode,
366 0,
367 0);
368 if (ret)
369 dev_err(ringacc->dev, "TISCI reconf qmode fail (%d) ring_idx %d\n",
370 ret, ring->ring_id);
371 }
372
373 void k3_ringacc_ring_reset_dma(struct k3_ring *ring, u32 occ)
374 {
375 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
376 return;
377
378 if (!ring->parent->dma_ring_reset_quirk)
379 goto reset;
380
381 if (!occ)
382 occ = readl(&ring->rt->occ);
383
384 if (occ) {
385 u32 db_ring_cnt, db_ring_cnt_cur;
386
387 dev_dbg(ring->parent->dev, "%s %u occ: %u\n", __func__,
388 ring->ring_id, occ);
389 /* TI-SCI ring reset */
390 k3_ringacc_ring_reset_sci(ring);
391
392 /*
393 * Setup the ring in ring/doorbell mode (if not already in this
394 * mode)
395 */
396 if (ring->mode != K3_RINGACC_RING_MODE_RING)
397 k3_ringacc_ring_reconfig_qmode_sci(
398 ring, K3_RINGACC_RING_MODE_RING);
399 /*
400 * Ring the doorbell 2**22 – ringOcc times.
401 * This will wrap the internal UDMAP ring state occupancy
402 * counter (which is 21-bits wide) to 0.
403 */
404 db_ring_cnt = (1U << 22) - occ;
405
406 while (db_ring_cnt != 0) {
407 /*
408 * Ring the doorbell with the maximum count each
409 * iteration if possible to minimize the total
410 * of writes
411 */
412 if (db_ring_cnt > K3_RINGACC_MAX_DB_RING_CNT)
413 db_ring_cnt_cur = K3_RINGACC_MAX_DB_RING_CNT;
414 else
415 db_ring_cnt_cur = db_ring_cnt;
416
417 writel(db_ring_cnt_cur, &ring->rt->db);
418 db_ring_cnt -= db_ring_cnt_cur;
419 }
420
421 /* Restore the original ring mode (if not ring mode) */
422 if (ring->mode != K3_RINGACC_RING_MODE_RING)
423 k3_ringacc_ring_reconfig_qmode_sci(ring, ring->mode);
424 }
425
426 reset:
427 /* Reset the ring */
428 k3_ringacc_ring_reset(ring);
429 }
430 EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset_dma);
431
432 static void k3_ringacc_ring_free_sci(struct k3_ring *ring)
433 {
434 struct k3_ringacc *ringacc = ring->parent;
435 int ret;
436
437 ret = ringacc->tisci_ring_ops->config(
438 ringacc->tisci,
439 TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER,
440 ringacc->tisci_dev_id,
441 ring->ring_id,
442 0,
443 0,
444 0,
445 0,
446 0,
447 0);
448 if (ret)
449 dev_err(ringacc->dev, "TISCI ring free fail (%d) ring_idx %d\n",
450 ret, ring->ring_id);
451 }
452
453 int k3_ringacc_ring_free(struct k3_ring *ring)
454 {
455 struct k3_ringacc *ringacc;
456
457 if (!ring)
458 return -EINVAL;
459
460 ringacc = ring->parent;
461
462 dev_dbg(ring->parent->dev, "flags: 0x%08x\n", ring->flags);
463
464 if (!test_bit(ring->ring_id, ringacc->rings_inuse))
465 return -EINVAL;
466
467 mutex_lock(&ringacc->req_lock);
468
469 if (--ring->use_count)
470 goto out;
471
472 if (!(ring->flags & K3_RING_FLAG_BUSY))
473 goto no_init;
474
475 k3_ringacc_ring_free_sci(ring);
476
477 dma_free_coherent(ringacc->dev,
478 ring->size * (4 << ring->elm_size),
479 ring->ring_mem_virt, ring->ring_mem_dma);
480 ring->flags = 0;
481 ring->ops = NULL;
482 if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) {
483 clear_bit(ring->proxy_id, ringacc->proxy_inuse);
484 ring->proxy = NULL;
485 ring->proxy_id = K3_RINGACC_PROXY_NOT_USED;
486 }
487
488 no_init:
489 clear_bit(ring->ring_id, ringacc->rings_inuse);
490
491 out:
492 mutex_unlock(&ringacc->req_lock);
493 return 0;
494 }
495 EXPORT_SYMBOL_GPL(k3_ringacc_ring_free);
496
497 u32 k3_ringacc_get_ring_id(struct k3_ring *ring)
498 {
499 if (!ring)
500 return -EINVAL;
501
502 return ring->ring_id;
503 }
504 EXPORT_SYMBOL_GPL(k3_ringacc_get_ring_id);
505
506 u32 k3_ringacc_get_tisci_dev_id(struct k3_ring *ring)
507 {
508 if (!ring)
509 return -EINVAL;
510
511 return ring->parent->tisci_dev_id;
512 }
513 EXPORT_SYMBOL_GPL(k3_ringacc_get_tisci_dev_id);
514
515 int k3_ringacc_get_ring_irq_num(struct k3_ring *ring)
516 {
517 int irq_num;
518
519 if (!ring)
520 return -EINVAL;
521
522 irq_num = ti_sci_inta_msi_get_virq(ring->parent->dev, ring->ring_id);
523 if (irq_num <= 0)
524 irq_num = -EINVAL;
525 return irq_num;
526 }
527 EXPORT_SYMBOL_GPL(k3_ringacc_get_ring_irq_num);
528
529 static int k3_ringacc_ring_cfg_sci(struct k3_ring *ring)
530 {
531 struct k3_ringacc *ringacc = ring->parent;
532 u32 ring_idx;
533 int ret;
534
535 if (!ringacc->tisci)
536 return -EINVAL;
537
538 ring_idx = ring->ring_id;
539 ret = ringacc->tisci_ring_ops->config(
540 ringacc->tisci,
541 TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER,
542 ringacc->tisci_dev_id,
543 ring_idx,
544 lower_32_bits(ring->ring_mem_dma),
545 upper_32_bits(ring->ring_mem_dma),
546 ring->size,
547 ring->mode,
548 ring->elm_size,
549 0);
550 if (ret)
551 dev_err(ringacc->dev, "TISCI config ring fail (%d) ring_idx %d\n",
552 ret, ring_idx);
553
554 return ret;
555 }
556
557 int k3_ringacc_ring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg)
558 {
559 struct k3_ringacc *ringacc = ring->parent;
560 int ret = 0;
561
562 if (!ring || !cfg)
563 return -EINVAL;
564 if (cfg->elm_size > K3_RINGACC_RING_ELSIZE_256 ||
565 cfg->mode >= K3_RINGACC_RING_MODE_INVALID ||
566 cfg->size & ~K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK ||
567 !test_bit(ring->ring_id, ringacc->rings_inuse))
568 return -EINVAL;
569
570 if (cfg->mode == K3_RINGACC_RING_MODE_MESSAGE &&
571 ring->proxy_id == K3_RINGACC_PROXY_NOT_USED &&
572 cfg->elm_size > K3_RINGACC_RING_ELSIZE_8) {
573 dev_err(ringacc->dev,
574 "Message mode must use proxy for %u element size\n",
575 4 << ring->elm_size);
576 return -EINVAL;
577 }
578
579 /*
580 * In case of shared ring only the first user (master user) can
581 * configure the ring. The sequence should be by the client:
582 * ring = k3_ringacc_request_ring(ringacc, ring_id, 0); # master user
583 * k3_ringacc_ring_cfg(ring, cfg); # master configuration
584 * k3_ringacc_request_ring(ringacc, ring_id, K3_RING_FLAG_SHARED);
585 * k3_ringacc_request_ring(ringacc, ring_id, K3_RING_FLAG_SHARED);
586 */
587 if (ring->use_count != 1)
588 return 0;
589
590 ring->size = cfg->size;
591 ring->elm_size = cfg->elm_size;
592 ring->mode = cfg->mode;
593 ring->occ = 0;
594 ring->free = 0;
595 ring->rindex = 0;
596 ring->windex = 0;
597
598 if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED)
599 ring->proxy = ringacc->proxy_target_base +
600 ring->proxy_id * K3_RINGACC_PROXY_TARGET_STEP;
601
602 switch (ring->mode) {
603 case K3_RINGACC_RING_MODE_RING:
604 ring->ops = &k3_ring_mode_ring_ops;
605 break;
606 case K3_RINGACC_RING_MODE_MESSAGE:
607 if (ring->proxy)
608 ring->ops = &k3_ring_mode_proxy_ops;
609 else
610 ring->ops = &k3_ring_mode_msg_ops;
611 break;
612 default:
613 ring->ops = NULL;
614 ret = -EINVAL;
615 goto err_free_proxy;
616 };
617
618 ring->ring_mem_virt = dma_alloc_coherent(ringacc->dev,
619 ring->size * (4 << ring->elm_size),
620 &ring->ring_mem_dma, GFP_KERNEL);
621 if (!ring->ring_mem_virt) {
622 dev_err(ringacc->dev, "Failed to alloc ring mem\n");
623 ret = -ENOMEM;
624 goto err_free_ops;
625 }
626
627 ret = k3_ringacc_ring_cfg_sci(ring);
628
629 if (ret)
630 goto err_free_mem;
631
632 ring->flags |= K3_RING_FLAG_BUSY;
633 ring->flags |= (cfg->flags & K3_RINGACC_RING_SHARED) ?
634 K3_RING_FLAG_SHARED : 0;
635
636 k3_ringacc_ring_dump(ring);
637
638 return 0;
639
640 err_free_mem:
641 dma_free_coherent(ringacc->dev,
642 ring->size * (4 << ring->elm_size),
643 ring->ring_mem_virt,
644 ring->ring_mem_dma);
645 err_free_ops:
646 ring->ops = NULL;
647 err_free_proxy:
648 ring->proxy = NULL;
649 return ret;
650 }
651 EXPORT_SYMBOL_GPL(k3_ringacc_ring_cfg);
652
653 u32 k3_ringacc_ring_get_size(struct k3_ring *ring)
654 {
655 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
656 return -EINVAL;
657
658 return ring->size;
659 }
660 EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_size);
661
662 u32 k3_ringacc_ring_get_free(struct k3_ring *ring)
663 {
664 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
665 return -EINVAL;
666
667 if (!ring->free)
668 ring->free = ring->size - readl(&ring->rt->occ);
669
670 return ring->free;
671 }
672 EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_free);
673
674 u32 k3_ringacc_ring_get_occ(struct k3_ring *ring)
675 {
676 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
677 return -EINVAL;
678
679 return readl(&ring->rt->occ);
680 }
681 EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_occ);
682
683 u32 k3_ringacc_ring_is_full(struct k3_ring *ring)
684 {
685 return !k3_ringacc_ring_get_free(ring);
686 }
687 EXPORT_SYMBOL_GPL(k3_ringacc_ring_is_full);
688
689 enum k3_ringacc_access_mode {
690 K3_RINGACC_ACCESS_MODE_PUSH_HEAD,
691 K3_RINGACC_ACCESS_MODE_POP_HEAD,
692 K3_RINGACC_ACCESS_MODE_PUSH_TAIL,
693 K3_RINGACC_ACCESS_MODE_POP_TAIL,
694 K3_RINGACC_ACCESS_MODE_PEEK_HEAD,
695 K3_RINGACC_ACCESS_MODE_PEEK_TAIL,
696 };
697
698 #define K3_RINGACC_PROXY_MODE(x) (((x) & 0x3) << 16)
699 #define K3_RINGACC_PROXY_ELSIZE(x) (((x) & 0x7) << 24)
700 static int k3_ringacc_ring_cfg_proxy(struct k3_ring *ring,
701 enum k3_ringacc_proxy_access_mode mode)
702 {
703 u32 val;
704
705 val = ring->ring_id;
706 val |= K3_RINGACC_PROXY_MODE(mode);
707 val |= K3_RINGACC_PROXY_ELSIZE(ring->elm_size);
708 writel(val, &ring->proxy->control);
709 return 0;
710 }
711
712 static int k3_ringacc_ring_access_proxy(struct k3_ring *ring, void *elem,
713 enum k3_ringacc_access_mode access_mode)
714 {
715 void __iomem *ptr;
716
717 ptr = (void __iomem *)&ring->proxy->data;
718
719 switch (access_mode) {
720 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
721 case K3_RINGACC_ACCESS_MODE_POP_HEAD:
722 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_HEAD);
723 break;
724 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
725 case K3_RINGACC_ACCESS_MODE_POP_TAIL:
726 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_TAIL);
727 break;
728 default:
729 return -EINVAL;
730 }
731
732 ptr += k3_ringacc_ring_get_fifo_pos(ring);
733
734 switch (access_mode) {
735 case K3_RINGACC_ACCESS_MODE_POP_HEAD:
736 case K3_RINGACC_ACCESS_MODE_POP_TAIL:
737 dev_dbg(ring->parent->dev,
738 "proxy:memcpy_fromio(x): --> ptr(%p), mode:%d\n", ptr,
739 access_mode);
740 memcpy_fromio(elem, ptr, (4 << ring->elm_size));
741 ring->occ--;
742 break;
743 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
744 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
745 dev_dbg(ring->parent->dev,
746 "proxy:memcpy_toio(x): --> ptr(%p), mode:%d\n", ptr,
747 access_mode);
748 memcpy_toio(ptr, elem, (4 << ring->elm_size));
749 ring->free--;
750 break;
751 default:
752 return -EINVAL;
753 }
754
755 dev_dbg(ring->parent->dev, "proxy: free%d occ%d\n", ring->free,
756 ring->occ);
757 return 0;
758 }
759
760 static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem)
761 {
762 return k3_ringacc_ring_access_proxy(ring, elem,
763 K3_RINGACC_ACCESS_MODE_PUSH_HEAD);
764 }
765
766 static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem)
767 {
768 return k3_ringacc_ring_access_proxy(ring, elem,
769 K3_RINGACC_ACCESS_MODE_PUSH_TAIL);
770 }
771
772 static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem)
773 {
774 return k3_ringacc_ring_access_proxy(ring, elem,
775 K3_RINGACC_ACCESS_MODE_POP_HEAD);
776 }
777
778 static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem)
779 {
780 return k3_ringacc_ring_access_proxy(ring, elem,
781 K3_RINGACC_ACCESS_MODE_POP_HEAD);
782 }
783
784 static int k3_ringacc_ring_access_io(struct k3_ring *ring, void *elem,
785 enum k3_ringacc_access_mode access_mode)
786 {
787 void __iomem *ptr;
788
789 switch (access_mode) {
790 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
791 case K3_RINGACC_ACCESS_MODE_POP_HEAD:
792 ptr = (void __iomem *)&ring->fifos->head_data;
793 break;
794 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
795 case K3_RINGACC_ACCESS_MODE_POP_TAIL:
796 ptr = (void __iomem *)&ring->fifos->tail_data;
797 break;
798 default:
799 return -EINVAL;
800 }
801
802 ptr += k3_ringacc_ring_get_fifo_pos(ring);
803
804 switch (access_mode) {
805 case K3_RINGACC_ACCESS_MODE_POP_HEAD:
806 case K3_RINGACC_ACCESS_MODE_POP_TAIL:
807 dev_dbg(ring->parent->dev,
808 "memcpy_fromio(x): --> ptr(%p), mode:%d\n", ptr,
809 access_mode);
810 memcpy_fromio(elem, ptr, (4 << ring->elm_size));
811 ring->occ--;
812 break;
813 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
814 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
815 dev_dbg(ring->parent->dev,
816 "memcpy_toio(x): --> ptr(%p), mode:%d\n", ptr,
817 access_mode);
818 memcpy_toio(ptr, elem, (4 << ring->elm_size));
819 ring->free--;
820 break;
821 default:
822 return -EINVAL;
823 }
824
825 dev_dbg(ring->parent->dev, "free%d index%d occ%d index%d\n", ring->free,
826 ring->windex, ring->occ, ring->rindex);
827 return 0;
828 }
829
830 static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem)
831 {
832 return k3_ringacc_ring_access_io(ring, elem,
833 K3_RINGACC_ACCESS_MODE_PUSH_HEAD);
834 }
835
836 static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem)
837 {
838 return k3_ringacc_ring_access_io(ring, elem,
839 K3_RINGACC_ACCESS_MODE_PUSH_TAIL);
840 }
841
842 static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem)
843 {
844 return k3_ringacc_ring_access_io(ring, elem,
845 K3_RINGACC_ACCESS_MODE_POP_HEAD);
846 }
847
848 static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem)
849 {
850 return k3_ringacc_ring_access_io(ring, elem,
851 K3_RINGACC_ACCESS_MODE_POP_HEAD);
852 }
853
854 static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem)
855 {
856 void *elem_ptr;
857
858 elem_ptr = k3_ringacc_get_elm_addr(ring, ring->windex);
859
860 memcpy(elem_ptr, elem, (4 << ring->elm_size));
861
862 ring->windex = (ring->windex + 1) % ring->size;
863 ring->free--;
864 writel(1, &ring->rt->db);
865
866 dev_dbg(ring->parent->dev, "ring_push_mem: free%d index%d\n",
867 ring->free, ring->windex);
868
869 return 0;
870 }
871
872 static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem)
873 {
874 void *elem_ptr;
875
876 elem_ptr = k3_ringacc_get_elm_addr(ring, ring->rindex);
877
878 memcpy(elem, elem_ptr, (4 << ring->elm_size));
879
880 ring->rindex = (ring->rindex + 1) % ring->size;
881 ring->occ--;
882 writel(-1, &ring->rt->db);
883
884 dev_dbg(ring->parent->dev, "ring_pop_mem: occ%d index%d pos_ptr%p\n",
885 ring->occ, ring->rindex, elem_ptr);
886 return 0;
887 }
888
889 int k3_ringacc_ring_push(struct k3_ring *ring, void *elem)
890 {
891 int ret = -EOPNOTSUPP;
892
893 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
894 return -EINVAL;
895
896 dev_dbg(ring->parent->dev, "ring_push: free%d index%d\n", ring->free,
897 ring->windex);
898
899 if (k3_ringacc_ring_is_full(ring))
900 return -ENOMEM;
901
902 if (ring->ops && ring->ops->push_tail)
903 ret = ring->ops->push_tail(ring, elem);
904
905 return ret;
906 }
907 EXPORT_SYMBOL_GPL(k3_ringacc_ring_push);
908
909 int k3_ringacc_ring_push_head(struct k3_ring *ring, void *elem)
910 {
911 int ret = -EOPNOTSUPP;
912
913 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
914 return -EINVAL;
915
916 dev_dbg(ring->parent->dev, "ring_push_head: free%d index%d\n",
917 ring->free, ring->windex);
918
919 if (k3_ringacc_ring_is_full(ring))
920 return -ENOMEM;
921
922 if (ring->ops && ring->ops->push_head)
923 ret = ring->ops->push_head(ring, elem);
924
925 return ret;
926 }
927 EXPORT_SYMBOL_GPL(k3_ringacc_ring_push_head);
928
929 int k3_ringacc_ring_pop(struct k3_ring *ring, void *elem)
930 {
931 int ret = -EOPNOTSUPP;
932
933 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
934 return -EINVAL;
935
936 if (!ring->occ)
937 ring->occ = k3_ringacc_ring_get_occ(ring);
938
939 dev_dbg(ring->parent->dev, "ring_pop: occ%d index%d\n", ring->occ,
940 ring->rindex);
941
942 if (!ring->occ)
943 return -ENODATA;
944
945 if (ring->ops && ring->ops->pop_head)
946 ret = ring->ops->pop_head(ring, elem);
947
948 return ret;
949 }
950 EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop);
951
952 int k3_ringacc_ring_pop_tail(struct k3_ring *ring, void *elem)
953 {
954 int ret = -EOPNOTSUPP;
955
956 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
957 return -EINVAL;
958
959 if (!ring->occ)
960 ring->occ = k3_ringacc_ring_get_occ(ring);
961
962 dev_dbg(ring->parent->dev, "ring_pop_tail: occ%d index%d\n", ring->occ,
963 ring->rindex);
964
965 if (!ring->occ)
966 return -ENODATA;
967
968 if (ring->ops && ring->ops->pop_tail)
969 ret = ring->ops->pop_tail(ring, elem);
970
971 return ret;
972 }
973 EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop_tail);
974
975 struct k3_ringacc *of_k3_ringacc_get_by_phandle(struct device_node *np,
976 const char *property)
977 {
978 struct device_node *ringacc_np;
979 struct k3_ringacc *ringacc = ERR_PTR(-EPROBE_DEFER);
980 struct k3_ringacc *entry;
981
982 ringacc_np = of_parse_phandle(np, property, 0);
983 if (!ringacc_np)
984 return ERR_PTR(-ENODEV);
985
986 mutex_lock(&k3_ringacc_list_lock);
987 list_for_each_entry(entry, &k3_ringacc_list, list)
988 if (entry->dev->of_node == ringacc_np) {
989 ringacc = entry;
990 break;
991 }
992 mutex_unlock(&k3_ringacc_list_lock);
993 of_node_put(ringacc_np);
994
995 return ringacc;
996 }
997 EXPORT_SYMBOL_GPL(of_k3_ringacc_get_by_phandle);
998
999 static int k3_ringacc_probe_dt(struct k3_ringacc *ringacc)
1000 {
1001 struct device_node *node = ringacc->dev->of_node;
1002 struct device *dev = ringacc->dev;
1003 struct platform_device *pdev = to_platform_device(dev);
1004 int ret;
1005
1006 if (!node) {
1007 dev_err(dev, "device tree info unavailable\n");
1008 return -ENODEV;
1009 }
1010
1011 ret = of_property_read_u32(node, "ti,num-rings", &ringacc->num_rings);
1012 if (ret) {
1013 dev_err(dev, "ti,num-rings read failure %d\n", ret);
1014 return ret;
1015 }
1016
1017 ringacc->dma_ring_reset_quirk =
1018 of_property_read_bool(node, "ti,dma-ring-reset-quirk");
1019
1020 ringacc->tisci = ti_sci_get_by_phandle(node, "ti,sci");
1021 if (IS_ERR(ringacc->tisci)) {
1022 ret = PTR_ERR(ringacc->tisci);
1023 if (ret != -EPROBE_DEFER)
1024 dev_err(dev, "ti,sci read fail %d\n", ret);
1025 ringacc->tisci = NULL;
1026 return ret;
1027 }
1028
1029 ret = of_property_read_u32(node, "ti,sci-dev-id",
1030 &ringacc->tisci_dev_id);
1031 if (ret) {
1032 dev_err(dev, "ti,sci-dev-id read fail %d\n", ret);
1033 return ret;
1034 }
1035
1036 pdev->id = ringacc->tisci_dev_id;
1037
1038 ringacc->rm_gp_range = devm_ti_sci_get_of_resource(ringacc->tisci, dev,
1039 ringacc->tisci_dev_id,
1040 "ti,sci-rm-range-gp-rings");
1041 if (IS_ERR(ringacc->rm_gp_range)) {
1042 dev_err(dev, "Failed to allocate MSI interrupts\n");
1043 return PTR_ERR(ringacc->rm_gp_range);
1044 }
1045
1046 return ti_sci_inta_msi_domain_alloc_irqs(ringacc->dev,
1047 ringacc->rm_gp_range);
1048 }
1049
1050 static int k3_ringacc_probe(struct platform_device *pdev)
1051 {
1052 struct k3_ringacc *ringacc;
1053 void __iomem *base_fifo, *base_rt;
1054 struct device *dev = &pdev->dev;
1055 struct resource *res;
1056 int ret, i;
1057
1058 ringacc = devm_kzalloc(dev, sizeof(*ringacc), GFP_KERNEL);
1059 if (!ringacc)
1060 return -ENOMEM;
1061
1062 ringacc->dev = dev;
1063 mutex_init(&ringacc->req_lock);
1064
1065 dev->msi_domain = of_msi_get_domain(dev, dev->of_node,
1066 DOMAIN_BUS_TI_SCI_INTA_MSI);
1067 if (!dev->msi_domain) {
1068 dev_err(dev, "Failed to get MSI domain\n");
1069 return -EPROBE_DEFER;
1070 }
1071
1072 ret = k3_ringacc_probe_dt(ringacc);
1073 if (ret)
1074 return ret;
1075
1076 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "rt");
1077 base_rt = devm_ioremap_resource(dev, res);
1078 if (IS_ERR(base_rt))
1079 return PTR_ERR(base_rt);
1080
1081 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "fifos");
1082 base_fifo = devm_ioremap_resource(dev, res);
1083 if (IS_ERR(base_fifo))
1084 return PTR_ERR(base_fifo);
1085
1086 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "proxy_gcfg");
1087 ringacc->proxy_gcfg = devm_ioremap_resource(dev, res);
1088 if (IS_ERR(ringacc->proxy_gcfg))
1089 return PTR_ERR(ringacc->proxy_gcfg);
1090
1091 res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
1092 "proxy_target");
1093 ringacc->proxy_target_base = devm_ioremap_resource(dev, res);
1094 if (IS_ERR(ringacc->proxy_target_base))
1095 return PTR_ERR(ringacc->proxy_target_base);
1096
1097 ringacc->num_proxies = readl(&ringacc->proxy_gcfg->config) &
1098 K3_RINGACC_PROXY_CFG_THREADS_MASK;
1099
1100 ringacc->rings = devm_kzalloc(dev,
1101 sizeof(*ringacc->rings) *
1102 ringacc->num_rings,
1103 GFP_KERNEL);
1104 ringacc->rings_inuse = devm_kcalloc(dev,
1105 BITS_TO_LONGS(ringacc->num_rings),
1106 sizeof(unsigned long), GFP_KERNEL);
1107 ringacc->proxy_inuse = devm_kcalloc(dev,
1108 BITS_TO_LONGS(ringacc->num_proxies),
1109 sizeof(unsigned long), GFP_KERNEL);
1110
1111 if (!ringacc->rings || !ringacc->rings_inuse || !ringacc->proxy_inuse)
1112 return -ENOMEM;
1113
1114 for (i = 0; i < ringacc->num_rings; i++) {
1115 ringacc->rings[i].rt = base_rt +
1116 K3_RINGACC_RT_REGS_STEP * i;
1117 ringacc->rings[i].fifos = base_fifo +
1118 K3_RINGACC_FIFO_REGS_STEP * i;
1119 ringacc->rings[i].parent = ringacc;
1120 ringacc->rings[i].ring_id = i;
1121 ringacc->rings[i].proxy_id = K3_RINGACC_PROXY_NOT_USED;
1122 }
1123 dev_set_drvdata(dev, ringacc);
1124
1125 ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops;
1126
1127 mutex_lock(&k3_ringacc_list_lock);
1128 list_add_tail(&ringacc->list, &k3_ringacc_list);
1129 mutex_unlock(&k3_ringacc_list_lock);
1130
1131 dev_info(dev, "Ring Accelerator probed rings:%u, gp-rings[%u,%u] sci-dev-id:%u\n",
1132 ringacc->num_rings,
1133 ringacc->rm_gp_range->desc[0].start,
1134 ringacc->rm_gp_range->desc[0].num,
1135 ringacc->tisci_dev_id);
1136 dev_info(dev, "dma-ring-reset-quirk: %s\n",
1137 ringacc->dma_ring_reset_quirk ? "enabled" : "disabled");
1138 dev_info(dev, "RA Proxy rev. %08x, num_proxies:%u\n",
1139 readl(&ringacc->proxy_gcfg->revision), ringacc->num_proxies);
1140 return 0;
1141 }
1142
1143 /* Match table for of_platform binding */
1144 static const struct of_device_id k3_ringacc_of_match[] = {
1145 { .compatible = "ti,am654-navss-ringacc", },
1146 {},
1147 };
1148
1149 static struct platform_driver k3_ringacc_driver = {
1150 .probe = k3_ringacc_probe,
1151 .driver = {
1152 .name = "k3-ringacc",
1153 .of_match_table = k3_ringacc_of_match,
1154 .suppress_bind_attrs = true,
1155 },
1156 };
1157 builtin_platform_driver(k3_ringacc_driver);
Linux with added FPC-III support