arch/tile/gxio/dma_queue.c

   1 /*
   2  * Copyright 2012 Tilera Corporation. All Rights Reserved.
   3  *
   4  *   This program is free software; you can redistribute it and/or
   5  *   modify it under the terms of the GNU General Public License
   6  *   as published by the Free Software Foundation, version 2.
   7  *
   8  *   This program is distributed in the hope that it will be useful, but
   9  *   WITHOUT ANY WARRANTY; without even the implied warranty of
  10  *   MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
  11  *   NON INFRINGEMENT.  See the GNU General Public License for
  12  *   more details.
  13  */
  14
  15 #include <linux/io.h>
  16 #include <linux/atomic.h>
  17 #include <linux/module.h>
  18 #include <gxio/dma_queue.h>
  19
  20 /* Wait for a memory read to complete. */
  21 #define wait_for_value(val)                             \
  22   __asm__ __volatile__("move %0, %0" :: "r"(val))
  23
  24 /* The index is in the low 16. */
  25 #define DMA_QUEUE_INDEX_MASK ((1 << 16) - 1)
  26
  27 /*
  28  * The hardware descriptor-ring type.
  29  * This matches the types used by mpipe (MPIPE_EDMA_POST_REGION_VAL_t)
  30  * and trio (TRIO_PUSH_DMA_REGION_VAL_t or TRIO_PULL_DMA_REGION_VAL_t).
  31  * See those types for more documentation on the individual fields.
  32  */
  33 typedef union {
  34         struct {
  35 #ifndef __BIG_ENDIAN__
  36                 uint64_t ring_idx:16;
  37                 uint64_t count:16;
  38                 uint64_t gen:1;
  39                 uint64_t __reserved:31;
  40 #else
  41                 uint64_t __reserved:31;
  42                 uint64_t gen:1;
  43                 uint64_t count:16;
  44                 uint64_t ring_idx:16;
  45 #endif
  46         };
  47         uint64_t word;
  48 } __gxio_ring_t;
  49
  50 void __gxio_dma_queue_init(__gxio_dma_queue_t *dma_queue,
  51                            void *post_region_addr, unsigned int num_entries)
  52 {
  53         /*
  54          * Limit 65536 entry rings to 65535 credits because we only have a
  55          * 16 bit completion counter.
  56          */
  57         int64_t credits = (num_entries < 65536) ? num_entries : 65535;
  58
  59         memset(dma_queue, 0, sizeof(*dma_queue));
  60
  61         dma_queue->post_region_addr = post_region_addr;
  62         dma_queue->hw_complete_count = 0;
  63         dma_queue->credits_and_next_index = credits << DMA_QUEUE_CREDIT_SHIFT;
  64 }
  65
  66 EXPORT_SYMBOL_GPL(__gxio_dma_queue_init);
  67
  68 void __gxio_dma_queue_update_credits(__gxio_dma_queue_t *dma_queue)
  69 {
  70         __gxio_ring_t val;
  71         uint64_t count;
  72         uint64_t delta;
  73         uint64_t new_count;
  74
  75         /*
  76          * Read the 64-bit completion count without touching the cache, so
  77          * we later avoid having to evict any sharers of this cache line
  78          * when we update it below.
  79          */
  80         uint64_t orig_hw_complete_count =
  81                 cmpxchg(&dma_queue->hw_complete_count,
  82                         -1, -1);
  83
  84         /* Make sure the load completes before we access the hardware. */
  85         wait_for_value(orig_hw_complete_count);
  86
  87         /* Read the 16-bit count of how many packets it has completed. */
  88         val.word = __gxio_mmio_read(dma_queue->post_region_addr);
  89         count = val.count;
  90
  91         /*
  92          * Calculate the number of completions since we last updated the
  93          * 64-bit counter.  It's safe to ignore the high bits because the
  94          * maximum credit value is 65535.
  95          */
  96         delta = (count - orig_hw_complete_count) & 0xffff;
  97         if (delta == 0)
  98                 return;
  99
 100         /*
 101          * Try to write back the count, advanced by delta.  If we race with
 102          * another thread, this might fail, in which case we return
 103          * immediately on the assumption that some credits are (or at least
 104          * were) available.
 105          */
 106         new_count = orig_hw_complete_count + delta;
 107         if (cmpxchg(&dma_queue->hw_complete_count,
 108                     orig_hw_complete_count,
 109                     new_count) != orig_hw_complete_count)
 110                 return;
 111
 112         /*
 113          * We succeeded in advancing the completion count; add back the
 114          * corresponding number of egress credits.
 115          */
 116         __insn_fetchadd(&dma_queue->credits_and_next_index,
 117                         (delta << DMA_QUEUE_CREDIT_SHIFT));
 118 }
 119
 120 EXPORT_SYMBOL_GPL(__gxio_dma_queue_update_credits);
 121
 122 /*
 123  * A separate 'blocked' method for put() so that backtraces and
 124  * profiles will clearly indicate that we're wasting time spinning on
 125  * egress availability rather than actually posting commands.
 126  */
 127 int64_t __gxio_dma_queue_wait_for_credits(__gxio_dma_queue_t *dma_queue,
 128                                           int64_t modifier)
 129 {
 130         int backoff = 16;
 131         int64_t old;
 132
 133         do {
 134                 int i;
 135                 /* Back off to avoid spamming memory networks. */
 136                 for (i = backoff; i > 0; i--)
 137                         __insn_mfspr(SPR_PASS);
 138
 139                 /* Check credits again. */
 140                 __gxio_dma_queue_update_credits(dma_queue);
 141                 old = __insn_fetchaddgez(&dma_queue->credits_and_next_index,
 142                                          modifier);
 143
 144                 /* Calculate bounded exponential backoff for next iteration. */
 145                 if (backoff < 256)
 146                         backoff *= 2;
 147         } while (old + modifier < 0);
 148
 149         return old;
 150 }
 151
 152 EXPORT_SYMBOL_GPL(__gxio_dma_queue_wait_for_credits);
 153
 154 int64_t __gxio_dma_queue_reserve_aux(__gxio_dma_queue_t *dma_queue,
 155                                      unsigned int num, int wait)
 156 {
 157         return __gxio_dma_queue_reserve(dma_queue, num, wait != 0, true);
 158 }
 159
 160 EXPORT_SYMBOL_GPL(__gxio_dma_queue_reserve_aux);
 161
 162 int __gxio_dma_queue_is_complete(__gxio_dma_queue_t *dma_queue,
 163                                  int64_t completion_slot, int update)
 164 {
 165         if (update) {
 166                 if (ACCESS_ONCE(dma_queue->hw_complete_count) >
 167                     completion_slot)
 168                         return 1;
 169
 170                 __gxio_dma_queue_update_credits(dma_queue);
 171         }
 172
 173         return ACCESS_ONCE(dma_queue->hw_complete_count) > completion_slot;
 174 }
 175
 176 EXPORT_SYMBOL_GPL(__gxio_dma_queue_is_complete);