drivers/dma/ti/k3-udma-glue.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * K3 NAVSS DMA glue interface
   4  *
   5  * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com
   6  *
   7  */
   8
   9 #include <linux/atomic.h>
  10 #include <linux/delay.h>
  11 #include <linux/dma-mapping.h>
  12 #include <linux/io.h>
  13 #include <linux/init.h>
  14 #include <linux/of.h>
  15 #include <linux/platform_device.h>
  16 #include <linux/soc/ti/k3-ringacc.h>
  17 #include <linux/dma/ti-cppi5.h>
  18 #include <linux/dma/k3-udma-glue.h>
  19
  20 #include "k3-udma.h"
  21 #include "k3-psil-priv.h"
  22
  23 struct k3_udma_glue_common {
  24         struct device *dev;
  25         struct udma_dev *udmax;
  26         const struct udma_tisci_rm *tisci_rm;
  27         struct k3_ringacc *ringacc;
  28         u32 src_thread;
  29         u32 dst_thread;
  30
  31         u32  hdesc_size;
  32         bool epib;
  33         u32  psdata_size;
  34         u32  swdata_size;
  35         u32  atype;
  36 };
  37
  38 struct k3_udma_glue_tx_channel {
  39         struct k3_udma_glue_common common;
  40
  41         struct udma_tchan *udma_tchanx;
  42         int udma_tchan_id;
  43
  44         struct k3_ring *ringtx;
  45         struct k3_ring *ringtxcq;
  46
  47         bool psil_paired;
  48
  49         int virq;
  50
  51         atomic_t free_pkts;
  52         bool tx_pause_on_err;
  53         bool tx_filt_einfo;
  54         bool tx_filt_pswords;
  55         bool tx_supr_tdpkt;
  56 };
  57
  58 struct k3_udma_glue_rx_flow {
  59         struct udma_rflow *udma_rflow;
  60         int udma_rflow_id;
  61         struct k3_ring *ringrx;
  62         struct k3_ring *ringrxfdq;
  63
  64         int virq;
  65 };
  66
  67 struct k3_udma_glue_rx_channel {
  68         struct k3_udma_glue_common common;
  69
  70         struct udma_rchan *udma_rchanx;
  71         int udma_rchan_id;
  72         bool remote;
  73
  74         bool psil_paired;
  75
  76         u32  swdata_size;
  77         int  flow_id_base;
  78
  79         struct k3_udma_glue_rx_flow *flows;
  80         u32 flow_num;
  81         u32 flows_ready;
  82 };
  83
  84 #define K3_UDMAX_TDOWN_TIMEOUT_US 1000
  85
  86 static int of_k3_udma_glue_parse(struct device_node *udmax_np,
  87                                  struct k3_udma_glue_common *common)
  88 {
  89         common->ringacc = of_k3_ringacc_get_by_phandle(udmax_np,
  90                                                        "ti,ringacc");
  91         if (IS_ERR(common->ringacc))
  92                 return PTR_ERR(common->ringacc);
  93
  94         common->udmax = of_xudma_dev_get(udmax_np, NULL);
  95         if (IS_ERR(common->udmax))
  96                 return PTR_ERR(common->udmax);
  97
  98         common->tisci_rm = xudma_dev_get_tisci_rm(common->udmax);
  99
 100         return 0;
 101 }
 102
 103 static int of_k3_udma_glue_parse_chn(struct device_node *chn_np,
 104                 const char *name, struct k3_udma_glue_common *common,
 105                 bool tx_chn)
 106 {
 107         struct psil_endpoint_config *ep_config;
 108         struct of_phandle_args dma_spec;
 109         u32 thread_id;
 110         int ret = 0;
 111         int index;
 112
 113         if (unlikely(!name))
 114                 return -EINVAL;
 115
 116         index = of_property_match_string(chn_np, "dma-names", name);
 117         if (index < 0)
 118                 return index;
 119
 120         if (of_parse_phandle_with_args(chn_np, "dmas", "#dma-cells", index,
 121                                        &dma_spec))
 122                 return -ENOENT;
 123
 124         thread_id = dma_spec.args[0];
 125         if (dma_spec.args_count == 2) {
 126                 if (dma_spec.args[1] > 2) {
 127                         dev_err(common->dev, "Invalid channel atype: %u\n",
 128                                 dma_spec.args[1]);
 129                         ret = -EINVAL;
 130                         goto out_put_spec;
 131                 }
 132                 common->atype = dma_spec.args[1];
 133         }
 134
 135         if (tx_chn && !(thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) {
 136                 ret = -EINVAL;
 137                 goto out_put_spec;
 138         }
 139
 140         if (!tx_chn && (thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) {
 141                 ret = -EINVAL;
 142                 goto out_put_spec;
 143         }
 144
 145         /* get psil endpoint config */
 146         ep_config = psil_get_ep_config(thread_id);
 147         if (IS_ERR(ep_config)) {
 148                 dev_err(common->dev,
 149                         "No configuration for psi-l thread 0x%04x\n",
 150                         thread_id);
 151                 ret = PTR_ERR(ep_config);
 152                 goto out_put_spec;
 153         }
 154
 155         common->epib = ep_config->needs_epib;
 156         common->psdata_size = ep_config->psd_size;
 157
 158         if (tx_chn)
 159                 common->dst_thread = thread_id;
 160         else
 161                 common->src_thread = thread_id;
 162
 163         ret = of_k3_udma_glue_parse(dma_spec.np, common);
 164
 165 out_put_spec:
 166         of_node_put(dma_spec.np);
 167         return ret;
 168 };
 169
 170 static void k3_udma_glue_dump_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
 171 {
 172         struct device *dev = tx_chn->common.dev;
 173
 174         dev_dbg(dev, "dump_tx_chn:\n"
 175                 "udma_tchan_id: %d\n"
 176                 "src_thread: %08x\n"
 177                 "dst_thread: %08x\n",
 178                 tx_chn->udma_tchan_id,
 179                 tx_chn->common.src_thread,
 180                 tx_chn->common.dst_thread);
 181 }
 182
 183 static void k3_udma_glue_dump_tx_rt_chn(struct k3_udma_glue_tx_channel *chn,
 184                                         char *mark)
 185 {
 186         struct device *dev = chn->common.dev;
 187
 188         dev_dbg(dev, "=== dump ===> %s\n", mark);
 189         dev_dbg(dev, "0x%08X: %08X\n", UDMA_TCHAN_RT_CTL_REG,
 190                 xudma_tchanrt_read(chn->udma_tchanx, UDMA_TCHAN_RT_CTL_REG));
 191         dev_dbg(dev, "0x%08X: %08X\n", UDMA_TCHAN_RT_PEER_RT_EN_REG,
 192                 xudma_tchanrt_read(chn->udma_tchanx,
 193                                    UDMA_TCHAN_RT_PEER_RT_EN_REG));
 194         dev_dbg(dev, "0x%08X: %08X\n", UDMA_TCHAN_RT_PCNT_REG,
 195                 xudma_tchanrt_read(chn->udma_tchanx, UDMA_TCHAN_RT_PCNT_REG));
 196         dev_dbg(dev, "0x%08X: %08X\n", UDMA_TCHAN_RT_BCNT_REG,
 197                 xudma_tchanrt_read(chn->udma_tchanx, UDMA_TCHAN_RT_BCNT_REG));
 198         dev_dbg(dev, "0x%08X: %08X\n", UDMA_TCHAN_RT_SBCNT_REG,
 199                 xudma_tchanrt_read(chn->udma_tchanx, UDMA_TCHAN_RT_SBCNT_REG));
 200 }
 201
 202 static int k3_udma_glue_cfg_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
 203 {
 204         const struct udma_tisci_rm *tisci_rm = tx_chn->common.tisci_rm;
 205         struct ti_sci_msg_rm_udmap_tx_ch_cfg req;
 206
 207         memset(&req, 0, sizeof(req));
 208
 209         req.valid_params = TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID |
 210                         TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_EINFO_VALID |
 211                         TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_PSWORDS_VALID |
 212                         TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID |
 213                         TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID |
 214                         TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID |
 215                         TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID |
 216                         TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID;
 217         req.nav_id = tisci_rm->tisci_dev_id;
 218         req.index = tx_chn->udma_tchan_id;
 219         if (tx_chn->tx_pause_on_err)
 220                 req.tx_pause_on_err = 1;
 221         if (tx_chn->tx_filt_einfo)
 222                 req.tx_filt_einfo = 1;
 223         if (tx_chn->tx_filt_pswords)
 224                 req.tx_filt_pswords = 1;
 225         req.tx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR;
 226         if (tx_chn->tx_supr_tdpkt)
 227                 req.tx_supr_tdpkt = 1;
 228         req.tx_fetch_size = tx_chn->common.hdesc_size >> 2;
 229         req.txcq_qnum = k3_ringacc_get_ring_id(tx_chn->ringtxcq);
 230         req.tx_atype = tx_chn->common.atype;
 231
 232         return tisci_rm->tisci_udmap_ops->tx_ch_cfg(tisci_rm->tisci, &req);
 233 }
 234
 235 struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev,
 236                 const char *name, struct k3_udma_glue_tx_channel_cfg *cfg)
 237 {
 238         struct k3_udma_glue_tx_channel *tx_chn;
 239         int ret;
 240
 241         tx_chn = devm_kzalloc(dev, sizeof(*tx_chn), GFP_KERNEL);
 242         if (!tx_chn)
 243                 return ERR_PTR(-ENOMEM);
 244
 245         tx_chn->common.dev = dev;
 246         tx_chn->common.swdata_size = cfg->swdata_size;
 247         tx_chn->tx_pause_on_err = cfg->tx_pause_on_err;
 248         tx_chn->tx_filt_einfo = cfg->tx_filt_einfo;
 249         tx_chn->tx_filt_pswords = cfg->tx_filt_pswords;
 250         tx_chn->tx_supr_tdpkt = cfg->tx_supr_tdpkt;
 251
 252         /* parse of udmap channel */
 253         ret = of_k3_udma_glue_parse_chn(dev->of_node, name,
 254                                         &tx_chn->common, true);
 255         if (ret)
 256                 goto err;
 257
 258         tx_chn->common.hdesc_size = cppi5_hdesc_calc_size(tx_chn->common.epib,
 259                                                 tx_chn->common.psdata_size,
 260                                                 tx_chn->common.swdata_size);
 261
 262         /* request and cfg UDMAP TX channel */
 263         tx_chn->udma_tchanx = xudma_tchan_get(tx_chn->common.udmax, -1);
 264         if (IS_ERR(tx_chn->udma_tchanx)) {
 265                 ret = PTR_ERR(tx_chn->udma_tchanx);
 266                 dev_err(dev, "UDMAX tchanx get err %d\n", ret);
 267                 goto err;
 268         }
 269         tx_chn->udma_tchan_id = xudma_tchan_get_id(tx_chn->udma_tchanx);
 270
 271         atomic_set(&tx_chn->free_pkts, cfg->txcq_cfg.size);
 272
 273         /* request and cfg rings */
 274         tx_chn->ringtx = k3_ringacc_request_ring(tx_chn->common.ringacc,
 275                                                  tx_chn->udma_tchan_id, 0);
 276         if (!tx_chn->ringtx) {
 277                 ret = -ENODEV;
 278                 dev_err(dev, "Failed to get TX ring %u\n",
 279                         tx_chn->udma_tchan_id);
 280                 goto err;
 281         }
 282
 283         tx_chn->ringtxcq = k3_ringacc_request_ring(tx_chn->common.ringacc,
 284                                                    -1, 0);
 285         if (!tx_chn->ringtxcq) {
 286                 ret = -ENODEV;
 287                 dev_err(dev, "Failed to get TXCQ ring\n");
 288                 goto err;
 289         }
 290
 291         ret = k3_ringacc_ring_cfg(tx_chn->ringtx, &cfg->tx_cfg);
 292         if (ret) {
 293                 dev_err(dev, "Failed to cfg ringtx %d\n", ret);
 294                 goto err;
 295         }
 296
 297         ret = k3_ringacc_ring_cfg(tx_chn->ringtxcq, &cfg->txcq_cfg);
 298         if (ret) {
 299                 dev_err(dev, "Failed to cfg ringtx %d\n", ret);
 300                 goto err;
 301         }
 302
 303         /* request and cfg psi-l */
 304         tx_chn->common.src_thread =
 305                         xudma_dev_get_psil_base(tx_chn->common.udmax) +
 306                         tx_chn->udma_tchan_id;
 307
 308         ret = k3_udma_glue_cfg_tx_chn(tx_chn);
 309         if (ret) {
 310                 dev_err(dev, "Failed to cfg tchan %d\n", ret);
 311                 goto err;
 312         }
 313
 314         ret = xudma_navss_psil_pair(tx_chn->common.udmax,
 315                                     tx_chn->common.src_thread,
 316                                     tx_chn->common.dst_thread);
 317         if (ret) {
 318                 dev_err(dev, "PSI-L request err %d\n", ret);
 319                 goto err;
 320         }
 321
 322         tx_chn->psil_paired = true;
 323
 324         /* reset TX RT registers */
 325         k3_udma_glue_disable_tx_chn(tx_chn);
 326
 327         k3_udma_glue_dump_tx_chn(tx_chn);
 328
 329         return tx_chn;
 330
 331 err:
 332         k3_udma_glue_release_tx_chn(tx_chn);
 333         return ERR_PTR(ret);
 334 }
 335 EXPORT_SYMBOL_GPL(k3_udma_glue_request_tx_chn);
 336
 337 void k3_udma_glue_release_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
 338 {
 339         if (tx_chn->psil_paired) {
 340                 xudma_navss_psil_unpair(tx_chn->common.udmax,
 341                                         tx_chn->common.src_thread,
 342                                         tx_chn->common.dst_thread);
 343                 tx_chn->psil_paired = false;
 344         }
 345
 346         if (!IS_ERR_OR_NULL(tx_chn->udma_tchanx))
 347                 xudma_tchan_put(tx_chn->common.udmax,
 348                                 tx_chn->udma_tchanx);
 349
 350         if (tx_chn->ringtxcq)
 351                 k3_ringacc_ring_free(tx_chn->ringtxcq);
 352
 353         if (tx_chn->ringtx)
 354                 k3_ringacc_ring_free(tx_chn->ringtx);
 355 }
 356 EXPORT_SYMBOL_GPL(k3_udma_glue_release_tx_chn);
 357
 358 int k3_udma_glue_push_tx_chn(struct k3_udma_glue_tx_channel *tx_chn,
 359                              struct cppi5_host_desc_t *desc_tx,
 360                              dma_addr_t desc_dma)
 361 {
 362         u32 ringtxcq_id;
 363
 364         if (!atomic_add_unless(&tx_chn->free_pkts, -1, 0))
 365                 return -ENOMEM;
 366
 367         ringtxcq_id = k3_ringacc_get_ring_id(tx_chn->ringtxcq);
 368         cppi5_desc_set_retpolicy(&desc_tx->hdr, 0, ringtxcq_id);
 369
 370         return k3_ringacc_ring_push(tx_chn->ringtx, &desc_dma);
 371 }
 372 EXPORT_SYMBOL_GPL(k3_udma_glue_push_tx_chn);
 373
 374 int k3_udma_glue_pop_tx_chn(struct k3_udma_glue_tx_channel *tx_chn,
 375                             dma_addr_t *desc_dma)
 376 {
 377         int ret;
 378
 379         ret = k3_ringacc_ring_pop(tx_chn->ringtxcq, desc_dma);
 380         if (!ret)
 381                 atomic_inc(&tx_chn->free_pkts);
 382
 383         return ret;
 384 }
 385 EXPORT_SYMBOL_GPL(k3_udma_glue_pop_tx_chn);
 386
 387 int k3_udma_glue_enable_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
 388 {
 389         u32 txrt_ctl;
 390
 391         txrt_ctl = UDMA_PEER_RT_EN_ENABLE;
 392         xudma_tchanrt_write(tx_chn->udma_tchanx,
 393                             UDMA_TCHAN_RT_PEER_RT_EN_REG,
 394                             txrt_ctl);
 395
 396         txrt_ctl = xudma_tchanrt_read(tx_chn->udma_tchanx,
 397                                       UDMA_TCHAN_RT_CTL_REG);
 398         txrt_ctl |= UDMA_CHAN_RT_CTL_EN;
 399         xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_TCHAN_RT_CTL_REG,
 400                             txrt_ctl);
 401
 402         k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn en");
 403         return 0;
 404 }
 405 EXPORT_SYMBOL_GPL(k3_udma_glue_enable_tx_chn);
 406
 407 void k3_udma_glue_disable_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
 408 {
 409         k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn dis1");
 410
 411         xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_TCHAN_RT_CTL_REG, 0);
 412
 413         xudma_tchanrt_write(tx_chn->udma_tchanx,
 414                             UDMA_TCHAN_RT_PEER_RT_EN_REG, 0);
 415         k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn dis2");
 416 }
 417 EXPORT_SYMBOL_GPL(k3_udma_glue_disable_tx_chn);
 418
 419 void k3_udma_glue_tdown_tx_chn(struct k3_udma_glue_tx_channel *tx_chn,
 420                                bool sync)
 421 {
 422         int i = 0;
 423         u32 val;
 424
 425         k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn tdown1");
 426
 427         xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_TCHAN_RT_CTL_REG,
 428                             UDMA_CHAN_RT_CTL_EN | UDMA_CHAN_RT_CTL_TDOWN);
 429
 430         val = xudma_tchanrt_read(tx_chn->udma_tchanx, UDMA_TCHAN_RT_CTL_REG);
 431
 432         while (sync && (val & UDMA_CHAN_RT_CTL_EN)) {
 433                 val = xudma_tchanrt_read(tx_chn->udma_tchanx,
 434                                          UDMA_TCHAN_RT_CTL_REG);
 435                 udelay(1);
 436                 if (i > K3_UDMAX_TDOWN_TIMEOUT_US) {
 437                         dev_err(tx_chn->common.dev, "TX tdown timeout\n");
 438                         break;
 439                 }
 440                 i++;
 441         }
 442
 443         val = xudma_tchanrt_read(tx_chn->udma_tchanx,
 444                                  UDMA_TCHAN_RT_PEER_RT_EN_REG);
 445         if (sync && (val & UDMA_PEER_RT_EN_ENABLE))
 446                 dev_err(tx_chn->common.dev, "TX tdown peer not stopped\n");
 447         k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn tdown2");
 448 }
 449 EXPORT_SYMBOL_GPL(k3_udma_glue_tdown_tx_chn);
 450
 451 void k3_udma_glue_reset_tx_chn(struct k3_udma_glue_tx_channel *tx_chn,
 452                                void *data,
 453                                void (*cleanup)(void *data, dma_addr_t desc_dma))
 454 {
 455         dma_addr_t desc_dma;
 456         int occ_tx, i, ret;
 457
 458         /* reset TXCQ as it is not input for udma - expected to be empty */
 459         if (tx_chn->ringtxcq)
 460                 k3_ringacc_ring_reset(tx_chn->ringtxcq);
 461
 462         /*
 463          * TXQ reset need to be special way as it is input for udma and its
 464          * state cached by udma, so:
 465          * 1) save TXQ occ
 466          * 2) clean up TXQ and call callback .cleanup() for each desc
 467          * 3) reset TXQ in a special way
 468          */
 469         occ_tx = k3_ringacc_ring_get_occ(tx_chn->ringtx);
 470         dev_dbg(tx_chn->common.dev, "TX reset occ_tx %u\n", occ_tx);
 471
 472         for (i = 0; i < occ_tx; i++) {
 473                 ret = k3_ringacc_ring_pop(tx_chn->ringtx, &desc_dma);
 474                 if (ret) {
 475                         dev_err(tx_chn->common.dev, "TX reset pop %d\n", ret);
 476                         break;
 477                 }
 478                 cleanup(data, desc_dma);
 479         }
 480
 481         k3_ringacc_ring_reset_dma(tx_chn->ringtx, occ_tx);
 482 }
 483 EXPORT_SYMBOL_GPL(k3_udma_glue_reset_tx_chn);
 484
 485 u32 k3_udma_glue_tx_get_hdesc_size(struct k3_udma_glue_tx_channel *tx_chn)
 486 {
 487         return tx_chn->common.hdesc_size;
 488 }
 489 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_hdesc_size);
 490
 491 u32 k3_udma_glue_tx_get_txcq_id(struct k3_udma_glue_tx_channel *tx_chn)
 492 {
 493         return k3_ringacc_get_ring_id(tx_chn->ringtxcq);
 494 }
 495 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_txcq_id);
 496
 497 int k3_udma_glue_tx_get_irq(struct k3_udma_glue_tx_channel *tx_chn)
 498 {
 499         tx_chn->virq = k3_ringacc_get_ring_irq_num(tx_chn->ringtxcq);
 500
 501         return tx_chn->virq;
 502 }
 503 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_irq);
 504
 505 static int k3_udma_glue_cfg_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
 506 {
 507         const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm;
 508         struct ti_sci_msg_rm_udmap_rx_ch_cfg req;
 509         int ret;
 510
 511         memset(&req, 0, sizeof(req));
 512
 513         req.valid_params = TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID |
 514                            TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID |
 515                            TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID |
 516                            TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID |
 517                            TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID |
 518                            TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID;
 519
 520         req.nav_id = tisci_rm->tisci_dev_id;
 521         req.index = rx_chn->udma_rchan_id;
 522         req.rx_fetch_size = rx_chn->common.hdesc_size >> 2;
 523         /*
 524          * TODO: we can't support rxcq_qnum/RCHAN[a]_RCQ cfg with current sysfw
 525          * and udmax impl, so just configure it to invalid value.
 526          * req.rxcq_qnum = k3_ringacc_get_ring_id(rx_chn->flows[0].ringrx);
 527          */
 528         req.rxcq_qnum = 0xFFFF;
 529         if (rx_chn->flow_num && rx_chn->flow_id_base != rx_chn->udma_rchan_id) {
 530                 /* Default flow + extra ones */
 531                 req.flowid_start = rx_chn->flow_id_base;
 532                 req.flowid_cnt = rx_chn->flow_num;
 533         }
 534         req.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR;
 535         req.rx_atype = rx_chn->common.atype;
 536
 537         ret = tisci_rm->tisci_udmap_ops->rx_ch_cfg(tisci_rm->tisci, &req);
 538         if (ret)
 539                 dev_err(rx_chn->common.dev, "rchan%d cfg failed %d\n",
 540                         rx_chn->udma_rchan_id, ret);
 541
 542         return ret;
 543 }
 544
 545 static void k3_udma_glue_release_rx_flow(struct k3_udma_glue_rx_channel *rx_chn,
 546                                          u32 flow_num)
 547 {
 548         struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num];
 549
 550         if (IS_ERR_OR_NULL(flow->udma_rflow))
 551                 return;
 552
 553         if (flow->ringrxfdq)
 554                 k3_ringacc_ring_free(flow->ringrxfdq);
 555
 556         if (flow->ringrx)
 557                 k3_ringacc_ring_free(flow->ringrx);
 558
 559         xudma_rflow_put(rx_chn->common.udmax, flow->udma_rflow);
 560         flow->udma_rflow = NULL;
 561         rx_chn->flows_ready--;
 562 }
 563
 564 static int k3_udma_glue_cfg_rx_flow(struct k3_udma_glue_rx_channel *rx_chn,
 565                                     u32 flow_idx,
 566                                     struct k3_udma_glue_rx_flow_cfg *flow_cfg)
 567 {
 568         struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_idx];
 569         const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm;
 570         struct device *dev = rx_chn->common.dev;
 571         struct ti_sci_msg_rm_udmap_flow_cfg req;
 572         int rx_ring_id;
 573         int rx_ringfdq_id;
 574         int ret = 0;
 575
 576         flow->udma_rflow = xudma_rflow_get(rx_chn->common.udmax,
 577                                            flow->udma_rflow_id);
 578         if (IS_ERR(flow->udma_rflow)) {
 579                 ret = PTR_ERR(flow->udma_rflow);
 580                 dev_err(dev, "UDMAX rflow get err %d\n", ret);
 581                 return ret;
 582         }
 583
 584         if (flow->udma_rflow_id != xudma_rflow_get_id(flow->udma_rflow)) {
 585                 ret = -ENODEV;
 586                 goto err_rflow_put;
 587         }
 588
 589         /* request and cfg rings */
 590         flow->ringrx = k3_ringacc_request_ring(rx_chn->common.ringacc,
 591                                                flow_cfg->ring_rxq_id, 0);
 592         if (!flow->ringrx) {
 593                 ret = -ENODEV;
 594                 dev_err(dev, "Failed to get RX ring\n");
 595                 goto err_rflow_put;
 596         }
 597
 598         flow->ringrxfdq = k3_ringacc_request_ring(rx_chn->common.ringacc,
 599                                                   flow_cfg->ring_rxfdq0_id, 0);
 600         if (!flow->ringrxfdq) {
 601                 ret = -ENODEV;
 602                 dev_err(dev, "Failed to get RXFDQ ring\n");
 603                 goto err_ringrx_free;
 604         }
 605
 606         ret = k3_ringacc_ring_cfg(flow->ringrx, &flow_cfg->rx_cfg);
 607         if (ret) {
 608                 dev_err(dev, "Failed to cfg ringrx %d\n", ret);
 609                 goto err_ringrxfdq_free;
 610         }
 611
 612         ret = k3_ringacc_ring_cfg(flow->ringrxfdq, &flow_cfg->rxfdq_cfg);
 613         if (ret) {
 614                 dev_err(dev, "Failed to cfg ringrxfdq %d\n", ret);
 615                 goto err_ringrxfdq_free;
 616         }
 617
 618         if (rx_chn->remote) {
 619                 rx_ring_id = TI_SCI_RESOURCE_NULL;
 620                 rx_ringfdq_id = TI_SCI_RESOURCE_NULL;
 621         } else {
 622                 rx_ring_id = k3_ringacc_get_ring_id(flow->ringrx);
 623                 rx_ringfdq_id = k3_ringacc_get_ring_id(flow->ringrxfdq);
 624         }
 625
 626         memset(&req, 0, sizeof(req));
 627
 628         req.valid_params =
 629                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID |
 630                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID |
 631                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID |
 632                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DESC_TYPE_VALID |
 633                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID |
 634                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_HI_SEL_VALID |
 635                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_LO_SEL_VALID |
 636                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_HI_SEL_VALID |
 637                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_LO_SEL_VALID |
 638                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID |
 639                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID |
 640                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID |
 641                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID;
 642         req.nav_id = tisci_rm->tisci_dev_id;
 643         req.flow_index = flow->udma_rflow_id;
 644         if (rx_chn->common.epib)
 645                 req.rx_einfo_present = 1;
 646         if (rx_chn->common.psdata_size)
 647                 req.rx_psinfo_present = 1;
 648         if (flow_cfg->rx_error_handling)
 649                 req.rx_error_handling = 1;
 650         req.rx_desc_type = 0;
 651         req.rx_dest_qnum = rx_ring_id;
 652         req.rx_src_tag_hi_sel = 0;
 653         req.rx_src_tag_lo_sel = flow_cfg->src_tag_lo_sel;
 654         req.rx_dest_tag_hi_sel = 0;
 655         req.rx_dest_tag_lo_sel = 0;
 656         req.rx_fdq0_sz0_qnum = rx_ringfdq_id;
 657         req.rx_fdq1_qnum = rx_ringfdq_id;
 658         req.rx_fdq2_qnum = rx_ringfdq_id;
 659         req.rx_fdq3_qnum = rx_ringfdq_id;
 660
 661         ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req);
 662         if (ret) {
 663                 dev_err(dev, "flow%d config failed: %d\n", flow->udma_rflow_id,
 664                         ret);
 665                 goto err_ringrxfdq_free;
 666         }
 667
 668         rx_chn->flows_ready++;
 669         dev_dbg(dev, "flow%d config done. ready:%d\n",
 670                 flow->udma_rflow_id, rx_chn->flows_ready);
 671
 672         return 0;
 673
 674 err_ringrxfdq_free:
 675         k3_ringacc_ring_free(flow->ringrxfdq);
 676
 677 err_ringrx_free:
 678         k3_ringacc_ring_free(flow->ringrx);
 679
 680 err_rflow_put:
 681         xudma_rflow_put(rx_chn->common.udmax, flow->udma_rflow);
 682         flow->udma_rflow = NULL;
 683
 684         return ret;
 685 }
 686
 687 static void k3_udma_glue_dump_rx_chn(struct k3_udma_glue_rx_channel *chn)
 688 {
 689         struct device *dev = chn->common.dev;
 690
 691         dev_dbg(dev, "dump_rx_chn:\n"
 692                 "udma_rchan_id: %d\n"
 693                 "src_thread: %08x\n"
 694                 "dst_thread: %08x\n"
 695                 "epib: %d\n"
 696                 "hdesc_size: %u\n"
 697                 "psdata_size: %u\n"
 698                 "swdata_size: %u\n"
 699                 "flow_id_base: %d\n"
 700                 "flow_num: %d\n",
 701                 chn->udma_rchan_id,
 702                 chn->common.src_thread,
 703                 chn->common.dst_thread,
 704                 chn->common.epib,
 705                 chn->common.hdesc_size,
 706                 chn->common.psdata_size,
 707                 chn->common.swdata_size,
 708                 chn->flow_id_base,
 709                 chn->flow_num);
 710 }
 711
 712 static void k3_udma_glue_dump_rx_rt_chn(struct k3_udma_glue_rx_channel *chn,
 713                                         char *mark)
 714 {
 715         struct device *dev = chn->common.dev;
 716
 717         dev_dbg(dev, "=== dump ===> %s\n", mark);
 718
 719         dev_dbg(dev, "0x%08X: %08X\n", UDMA_RCHAN_RT_CTL_REG,
 720                 xudma_rchanrt_read(chn->udma_rchanx, UDMA_RCHAN_RT_CTL_REG));
 721         dev_dbg(dev, "0x%08X: %08X\n", UDMA_RCHAN_RT_PEER_RT_EN_REG,
 722                 xudma_rchanrt_read(chn->udma_rchanx,
 723                                    UDMA_RCHAN_RT_PEER_RT_EN_REG));
 724         dev_dbg(dev, "0x%08X: %08X\n", UDMA_RCHAN_RT_PCNT_REG,
 725                 xudma_rchanrt_read(chn->udma_rchanx, UDMA_RCHAN_RT_PCNT_REG));
 726         dev_dbg(dev, "0x%08X: %08X\n", UDMA_RCHAN_RT_BCNT_REG,
 727                 xudma_rchanrt_read(chn->udma_rchanx, UDMA_RCHAN_RT_BCNT_REG));
 728         dev_dbg(dev, "0x%08X: %08X\n", UDMA_RCHAN_RT_SBCNT_REG,
 729                 xudma_rchanrt_read(chn->udma_rchanx, UDMA_RCHAN_RT_SBCNT_REG));
 730 }
 731
 732 static int
 733 k3_udma_glue_allocate_rx_flows(struct k3_udma_glue_rx_channel *rx_chn,
 734                                struct k3_udma_glue_rx_channel_cfg *cfg)
 735 {
 736         int ret;
 737
 738         /* default rflow */
 739         if (cfg->flow_id_use_rxchan_id)
 740                 return 0;
 741
 742         /* not a GP rflows */
 743         if (rx_chn->flow_id_base != -1 &&
 744             !xudma_rflow_is_gp(rx_chn->common.udmax, rx_chn->flow_id_base))
 745                 return 0;
 746
 747         /* Allocate range of GP rflows */
 748         ret = xudma_alloc_gp_rflow_range(rx_chn->common.udmax,
 749                                          rx_chn->flow_id_base,
 750                                          rx_chn->flow_num);
 751         if (ret < 0) {
 752                 dev_err(rx_chn->common.dev, "UDMAX reserve_rflow %d cnt:%d err: %d\n",
 753                         rx_chn->flow_id_base, rx_chn->flow_num, ret);
 754                 return ret;
 755         }
 756         rx_chn->flow_id_base = ret;
 757
 758         return 0;
 759 }
 760
 761 static struct k3_udma_glue_rx_channel *
 762 k3_udma_glue_request_rx_chn_priv(struct device *dev, const char *name,
 763                                  struct k3_udma_glue_rx_channel_cfg *cfg)
 764 {
 765         struct k3_udma_glue_rx_channel *rx_chn;
 766         int ret, i;
 767
 768         if (cfg->flow_id_num <= 0)
 769                 return ERR_PTR(-EINVAL);
 770
 771         if (cfg->flow_id_num != 1 &&
 772             (cfg->def_flow_cfg || cfg->flow_id_use_rxchan_id))
 773                 return ERR_PTR(-EINVAL);
 774
 775         rx_chn = devm_kzalloc(dev, sizeof(*rx_chn), GFP_KERNEL);
 776         if (!rx_chn)
 777                 return ERR_PTR(-ENOMEM);
 778
 779         rx_chn->common.dev = dev;
 780         rx_chn->common.swdata_size = cfg->swdata_size;
 781         rx_chn->remote = false;
 782
 783         /* parse of udmap channel */
 784         ret = of_k3_udma_glue_parse_chn(dev->of_node, name,
 785                                         &rx_chn->common, false);
 786         if (ret)
 787                 goto err;
 788
 789         rx_chn->common.hdesc_size = cppi5_hdesc_calc_size(rx_chn->common.epib,
 790                                                 rx_chn->common.psdata_size,
 791                                                 rx_chn->common.swdata_size);
 792
 793         /* request and cfg UDMAP RX channel */
 794         rx_chn->udma_rchanx = xudma_rchan_get(rx_chn->common.udmax, -1);
 795         if (IS_ERR(rx_chn->udma_rchanx)) {
 796                 ret = PTR_ERR(rx_chn->udma_rchanx);
 797                 dev_err(dev, "UDMAX rchanx get err %d\n", ret);
 798                 goto err;
 799         }
 800         rx_chn->udma_rchan_id = xudma_rchan_get_id(rx_chn->udma_rchanx);
 801
 802         rx_chn->flow_num = cfg->flow_id_num;
 803         rx_chn->flow_id_base = cfg->flow_id_base;
 804
 805         /* Use RX channel id as flow id: target dev can't generate flow_id */
 806         if (cfg->flow_id_use_rxchan_id)
 807                 rx_chn->flow_id_base = rx_chn->udma_rchan_id;
 808
 809         rx_chn->flows = devm_kcalloc(dev, rx_chn->flow_num,
 810                                      sizeof(*rx_chn->flows), GFP_KERNEL);
 811         if (!rx_chn->flows) {
 812                 ret = -ENOMEM;
 813                 goto err;
 814         }
 815
 816         ret = k3_udma_glue_allocate_rx_flows(rx_chn, cfg);
 817         if (ret)
 818                 goto err;
 819
 820         for (i = 0; i < rx_chn->flow_num; i++)
 821                 rx_chn->flows[i].udma_rflow_id = rx_chn->flow_id_base + i;
 822
 823         /* request and cfg psi-l */
 824         rx_chn->common.dst_thread =
 825                         xudma_dev_get_psil_base(rx_chn->common.udmax) +
 826                         rx_chn->udma_rchan_id;
 827
 828         ret = k3_udma_glue_cfg_rx_chn(rx_chn);
 829         if (ret) {
 830                 dev_err(dev, "Failed to cfg rchan %d\n", ret);
 831                 goto err;
 832         }
 833
 834         /* init default RX flow only if flow_num = 1 */
 835         if (cfg->def_flow_cfg) {
 836                 ret = k3_udma_glue_cfg_rx_flow(rx_chn, 0, cfg->def_flow_cfg);
 837                 if (ret)
 838                         goto err;
 839         }
 840
 841         ret = xudma_navss_psil_pair(rx_chn->common.udmax,
 842                                     rx_chn->common.src_thread,
 843                                     rx_chn->common.dst_thread);
 844         if (ret) {
 845                 dev_err(dev, "PSI-L request err %d\n", ret);
 846                 goto err;
 847         }
 848
 849         rx_chn->psil_paired = true;
 850
 851         /* reset RX RT registers */
 852         k3_udma_glue_disable_rx_chn(rx_chn);
 853
 854         k3_udma_glue_dump_rx_chn(rx_chn);
 855
 856         return rx_chn;
 857
 858 err:
 859         k3_udma_glue_release_rx_chn(rx_chn);
 860         return ERR_PTR(ret);
 861 }
 862
 863 static struct k3_udma_glue_rx_channel *
 864 k3_udma_glue_request_remote_rx_chn(struct device *dev, const char *name,
 865                                    struct k3_udma_glue_rx_channel_cfg *cfg)
 866 {
 867         struct k3_udma_glue_rx_channel *rx_chn;
 868         int ret, i;
 869
 870         if (cfg->flow_id_num <= 0 ||
 871             cfg->flow_id_use_rxchan_id ||
 872             cfg->def_flow_cfg ||
 873             cfg->flow_id_base < 0)
 874                 return ERR_PTR(-EINVAL);
 875
 876         /*
 877          * Remote RX channel is under control of Remote CPU core, so
 878          * Linux can only request and manipulate by dedicated RX flows
 879          */
 880
 881         rx_chn = devm_kzalloc(dev, sizeof(*rx_chn), GFP_KERNEL);
 882         if (!rx_chn)
 883                 return ERR_PTR(-ENOMEM);
 884
 885         rx_chn->common.dev = dev;
 886         rx_chn->common.swdata_size = cfg->swdata_size;
 887         rx_chn->remote = true;
 888         rx_chn->udma_rchan_id = -1;
 889         rx_chn->flow_num = cfg->flow_id_num;
 890         rx_chn->flow_id_base = cfg->flow_id_base;
 891         rx_chn->psil_paired = false;
 892
 893         /* parse of udmap channel */
 894         ret = of_k3_udma_glue_parse_chn(dev->of_node, name,
 895                                         &rx_chn->common, false);
 896         if (ret)
 897                 goto err;
 898
 899         rx_chn->common.hdesc_size = cppi5_hdesc_calc_size(rx_chn->common.epib,
 900                                                 rx_chn->common.psdata_size,
 901                                                 rx_chn->common.swdata_size);
 902
 903         rx_chn->flows = devm_kcalloc(dev, rx_chn->flow_num,
 904                                      sizeof(*rx_chn->flows), GFP_KERNEL);
 905         if (!rx_chn->flows) {
 906                 ret = -ENOMEM;
 907                 goto err;
 908         }
 909
 910         ret = k3_udma_glue_allocate_rx_flows(rx_chn, cfg);
 911         if (ret)
 912                 goto err;
 913
 914         for (i = 0; i < rx_chn->flow_num; i++)
 915                 rx_chn->flows[i].udma_rflow_id = rx_chn->flow_id_base + i;
 916
 917         k3_udma_glue_dump_rx_chn(rx_chn);
 918
 919         return rx_chn;
 920
 921 err:
 922         k3_udma_glue_release_rx_chn(rx_chn);
 923         return ERR_PTR(ret);
 924 }
 925
 926 struct k3_udma_glue_rx_channel *
 927 k3_udma_glue_request_rx_chn(struct device *dev, const char *name,
 928                             struct k3_udma_glue_rx_channel_cfg *cfg)
 929 {
 930         if (cfg->remote)
 931                 return k3_udma_glue_request_remote_rx_chn(dev, name, cfg);
 932         else
 933                 return k3_udma_glue_request_rx_chn_priv(dev, name, cfg);
 934 }
 935 EXPORT_SYMBOL_GPL(k3_udma_glue_request_rx_chn);
 936
 937 void k3_udma_glue_release_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
 938 {
 939         int i;
 940
 941         if (IS_ERR_OR_NULL(rx_chn->common.udmax))
 942                 return;
 943
 944         if (rx_chn->psil_paired) {
 945                 xudma_navss_psil_unpair(rx_chn->common.udmax,
 946                                         rx_chn->common.src_thread,
 947                                         rx_chn->common.dst_thread);
 948                 rx_chn->psil_paired = false;
 949         }
 950
 951         for (i = 0; i < rx_chn->flow_num; i++)
 952                 k3_udma_glue_release_rx_flow(rx_chn, i);
 953
 954         if (xudma_rflow_is_gp(rx_chn->common.udmax, rx_chn->flow_id_base))
 955                 xudma_free_gp_rflow_range(rx_chn->common.udmax,
 956                                           rx_chn->flow_id_base,
 957                                           rx_chn->flow_num);
 958
 959         if (!IS_ERR_OR_NULL(rx_chn->udma_rchanx))
 960                 xudma_rchan_put(rx_chn->common.udmax,
 961                                 rx_chn->udma_rchanx);
 962 }
 963 EXPORT_SYMBOL_GPL(k3_udma_glue_release_rx_chn);
 964
 965 int k3_udma_glue_rx_flow_init(struct k3_udma_glue_rx_channel *rx_chn,
 966                               u32 flow_idx,
 967                               struct k3_udma_glue_rx_flow_cfg *flow_cfg)
 968 {
 969         if (flow_idx >= rx_chn->flow_num)
 970                 return -EINVAL;
 971
 972         return k3_udma_glue_cfg_rx_flow(rx_chn, flow_idx, flow_cfg);
 973 }
 974 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_init);
 975
 976 u32 k3_udma_glue_rx_flow_get_fdq_id(struct k3_udma_glue_rx_channel *rx_chn,
 977                                     u32 flow_idx)
 978 {
 979         struct k3_udma_glue_rx_flow *flow;
 980
 981         if (flow_idx >= rx_chn->flow_num)
 982                 return -EINVAL;
 983
 984         flow = &rx_chn->flows[flow_idx];
 985
 986         return k3_ringacc_get_ring_id(flow->ringrxfdq);
 987 }
 988 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_get_fdq_id);
 989
 990 u32 k3_udma_glue_rx_get_flow_id_base(struct k3_udma_glue_rx_channel *rx_chn)
 991 {
 992         return rx_chn->flow_id_base;
 993 }
 994 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_flow_id_base);
 995
 996 int k3_udma_glue_rx_flow_enable(struct k3_udma_glue_rx_channel *rx_chn,
 997                                 u32 flow_idx)
 998 {
 999         struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_idx];
1000         const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm;
1001         struct device *dev = rx_chn->common.dev;
1002         struct ti_sci_msg_rm_udmap_flow_cfg req;
1003         int rx_ring_id;
1004         int rx_ringfdq_id;
1005         int ret = 0;
1006
1007         if (!rx_chn->remote)
1008                 return -EINVAL;
1009
1010         rx_ring_id = k3_ringacc_get_ring_id(flow->ringrx);
1011         rx_ringfdq_id = k3_ringacc_get_ring_id(flow->ringrxfdq);
1012
1013         memset(&req, 0, sizeof(req));
1014
1015         req.valid_params =
1016                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID |
1017                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID |
1018                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID |
1019                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID |
1020                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID;
1021         req.nav_id = tisci_rm->tisci_dev_id;
1022         req.flow_index = flow->udma_rflow_id;
1023         req.rx_dest_qnum = rx_ring_id;
1024         req.rx_fdq0_sz0_qnum = rx_ringfdq_id;
1025         req.rx_fdq1_qnum = rx_ringfdq_id;
1026         req.rx_fdq2_qnum = rx_ringfdq_id;
1027         req.rx_fdq3_qnum = rx_ringfdq_id;
1028
1029         ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req);
1030         if (ret) {
1031                 dev_err(dev, "flow%d enable failed: %d\n", flow->udma_rflow_id,
1032                         ret);
1033         }
1034
1035         return ret;
1036 }
1037 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_enable);
1038
1039 int k3_udma_glue_rx_flow_disable(struct k3_udma_glue_rx_channel *rx_chn,
1040                                  u32 flow_idx)
1041 {
1042         struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_idx];
1043         const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm;
1044         struct device *dev = rx_chn->common.dev;
1045         struct ti_sci_msg_rm_udmap_flow_cfg req;
1046         int ret = 0;
1047
1048         if (!rx_chn->remote)
1049                 return -EINVAL;
1050
1051         memset(&req, 0, sizeof(req));
1052         req.valid_params =
1053                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID |
1054                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID |
1055                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID |
1056                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID |
1057                         TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID;
1058         req.nav_id = tisci_rm->tisci_dev_id;
1059         req.flow_index = flow->udma_rflow_id;
1060         req.rx_dest_qnum = TI_SCI_RESOURCE_NULL;
1061         req.rx_fdq0_sz0_qnum = TI_SCI_RESOURCE_NULL;
1062         req.rx_fdq1_qnum = TI_SCI_RESOURCE_NULL;
1063         req.rx_fdq2_qnum = TI_SCI_RESOURCE_NULL;
1064         req.rx_fdq3_qnum = TI_SCI_RESOURCE_NULL;
1065
1066         ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req);
1067         if (ret) {
1068                 dev_err(dev, "flow%d disable failed: %d\n", flow->udma_rflow_id,
1069                         ret);
1070         }
1071
1072         return ret;
1073 }
1074 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_disable);
1075
1076 int k3_udma_glue_enable_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
1077 {
1078         u32 rxrt_ctl;
1079
1080         if (rx_chn->remote)
1081                 return -EINVAL;
1082
1083         if (rx_chn->flows_ready < rx_chn->flow_num)
1084                 return -EINVAL;
1085
1086         rxrt_ctl = xudma_rchanrt_read(rx_chn->udma_rchanx,
1087                                       UDMA_RCHAN_RT_CTL_REG);
1088         rxrt_ctl |= UDMA_CHAN_RT_CTL_EN;
1089         xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_RCHAN_RT_CTL_REG,
1090                             rxrt_ctl);
1091
1092         xudma_rchanrt_write(rx_chn->udma_rchanx,
1093                             UDMA_RCHAN_RT_PEER_RT_EN_REG,
1094                             UDMA_PEER_RT_EN_ENABLE);
1095
1096         k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt en");
1097         return 0;
1098 }
1099 EXPORT_SYMBOL_GPL(k3_udma_glue_enable_rx_chn);
1100
1101 void k3_udma_glue_disable_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
1102 {
1103         k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt dis1");
1104
1105         xudma_rchanrt_write(rx_chn->udma_rchanx,
1106                             UDMA_RCHAN_RT_PEER_RT_EN_REG,
1107                             0);
1108         xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_RCHAN_RT_CTL_REG, 0);
1109
1110         k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt dis2");
1111 }
1112 EXPORT_SYMBOL_GPL(k3_udma_glue_disable_rx_chn);
1113
1114 void k3_udma_glue_tdown_rx_chn(struct k3_udma_glue_rx_channel *rx_chn,
1115                                bool sync)
1116 {
1117         int i = 0;
1118         u32 val;
1119
1120         if (rx_chn->remote)
1121                 return;
1122
1123         k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt tdown1");
1124
1125         xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_RCHAN_RT_PEER_RT_EN_REG,
1126                             UDMA_PEER_RT_EN_ENABLE | UDMA_PEER_RT_EN_TEARDOWN);
1127
1128         val = xudma_rchanrt_read(rx_chn->udma_rchanx, UDMA_RCHAN_RT_CTL_REG);
1129
1130         while (sync && (val & UDMA_CHAN_RT_CTL_EN)) {
1131                 val = xudma_rchanrt_read(rx_chn->udma_rchanx,
1132                                          UDMA_RCHAN_RT_CTL_REG);
1133                 udelay(1);
1134                 if (i > K3_UDMAX_TDOWN_TIMEOUT_US) {
1135                         dev_err(rx_chn->common.dev, "RX tdown timeout\n");
1136                         break;
1137                 }
1138                 i++;
1139         }
1140
1141         val = xudma_rchanrt_read(rx_chn->udma_rchanx,
1142                                  UDMA_RCHAN_RT_PEER_RT_EN_REG);
1143         if (sync && (val & UDMA_PEER_RT_EN_ENABLE))
1144                 dev_err(rx_chn->common.dev, "TX tdown peer not stopped\n");
1145         k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt tdown2");
1146 }
1147 EXPORT_SYMBOL_GPL(k3_udma_glue_tdown_rx_chn);
1148
1149 void k3_udma_glue_reset_rx_chn(struct k3_udma_glue_rx_channel *rx_chn,
1150                 u32 flow_num, void *data,
1151                 void (*cleanup)(void *data, dma_addr_t desc_dma), bool skip_fdq)
1152 {
1153         struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num];
1154         struct device *dev = rx_chn->common.dev;
1155         dma_addr_t desc_dma;
1156         int occ_rx, i, ret;
1157
1158         /* reset RXCQ as it is not input for udma - expected to be empty */
1159         occ_rx = k3_ringacc_ring_get_occ(flow->ringrx);
1160         dev_dbg(dev, "RX reset flow %u occ_rx %u\n", flow_num, occ_rx);
1161         if (flow->ringrx)
1162                 k3_ringacc_ring_reset(flow->ringrx);
1163
1164         /* Skip RX FDQ in case one FDQ is used for the set of flows */
1165         if (skip_fdq)
1166                 return;
1167
1168         /*
1169          * RX FDQ reset need to be special way as it is input for udma and its
1170          * state cached by udma, so:
1171          * 1) save RX FDQ occ
1172          * 2) clean up RX FDQ and call callback .cleanup() for each desc
1173          * 3) reset RX FDQ in a special way
1174          */
1175         occ_rx = k3_ringacc_ring_get_occ(flow->ringrxfdq);
1176         dev_dbg(dev, "RX reset flow %u occ_rx_fdq %u\n", flow_num, occ_rx);
1177
1178         for (i = 0; i < occ_rx; i++) {
1179                 ret = k3_ringacc_ring_pop(flow->ringrxfdq, &desc_dma);
1180                 if (ret) {
1181                         dev_err(dev, "RX reset pop %d\n", ret);
1182                         break;
1183                 }
1184                 cleanup(data, desc_dma);
1185         }
1186
1187         k3_ringacc_ring_reset_dma(flow->ringrxfdq, occ_rx);
1188 }
1189 EXPORT_SYMBOL_GPL(k3_udma_glue_reset_rx_chn);
1190
1191 int k3_udma_glue_push_rx_chn(struct k3_udma_glue_rx_channel *rx_chn,
1192                              u32 flow_num, struct cppi5_host_desc_t *desc_rx,
1193                              dma_addr_t desc_dma)
1194 {
1195         struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num];
1196
1197         return k3_ringacc_ring_push(flow->ringrxfdq, &desc_dma);
1198 }
1199 EXPORT_SYMBOL_GPL(k3_udma_glue_push_rx_chn);
1200
1201 int k3_udma_glue_pop_rx_chn(struct k3_udma_glue_rx_channel *rx_chn,
1202                             u32 flow_num, dma_addr_t *desc_dma)
1203 {
1204         struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num];
1205
1206         return k3_ringacc_ring_pop(flow->ringrx, desc_dma);
1207 }
1208 EXPORT_SYMBOL_GPL(k3_udma_glue_pop_rx_chn);
1209
1210 int k3_udma_glue_rx_get_irq(struct k3_udma_glue_rx_channel *rx_chn,
1211                             u32 flow_num)
1212 {
1213         struct k3_udma_glue_rx_flow *flow;
1214
1215         flow = &rx_chn->flows[flow_num];
1216
1217         flow->virq = k3_ringacc_get_ring_irq_num(flow->ringrx);
1218
1219         return flow->virq;
1220 }
1221 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_irq);