drivers/gpu/host1x/intr.c

   1 /*
   2  * Tegra host1x Interrupt Management
   3  *
   4  * Copyright (c) 2010-2013, NVIDIA Corporation.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms and conditions of the GNU General Public License,
   8  * version 2, as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17  */
  18
  19 #include <linux/clk.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/slab.h>
  22 #include <linux/irq.h>
  23
  24 #include <trace/events/host1x.h>
  25 #include "channel.h"
  26 #include "dev.h"
  27 #include "intr.h"
  28
  29 /* Wait list management */
  30
  31 enum waitlist_state {
  32         WLS_PENDING,
  33         WLS_REMOVED,
  34         WLS_CANCELLED,
  35         WLS_HANDLED
  36 };
  37
  38 static void waiter_release(struct kref *kref)
  39 {
  40         kfree(container_of(kref, struct host1x_waitlist, refcount));
  41 }
  42
  43 /*
  44  * add a waiter to a waiter queue, sorted by threshold
  45  * returns true if it was added at the head of the queue
  46  */
  47 static bool add_waiter_to_queue(struct host1x_waitlist *waiter,
  48                                 struct list_head *queue)
  49 {
  50         struct host1x_waitlist *pos;
  51         u32 thresh = waiter->thresh;
  52
  53         list_for_each_entry_reverse(pos, queue, list)
  54                 if ((s32)(pos->thresh - thresh) <= 0) {
  55                         list_add(&waiter->list, &pos->list);
  56                         return false;
  57                 }
  58
  59         list_add(&waiter->list, queue);
  60         return true;
  61 }
  62
  63 /*
  64  * run through a waiter queue for a single sync point ID
  65  * and gather all completed waiters into lists by actions
  66  */
  67 static void remove_completed_waiters(struct list_head *head, u32 sync,
  68                         struct list_head completed[HOST1X_INTR_ACTION_COUNT])
  69 {
  70         struct list_head *dest;
  71         struct host1x_waitlist *waiter, *next, *prev;
  72
  73         list_for_each_entry_safe(waiter, next, head, list) {
  74                 if ((s32)(waiter->thresh - sync) > 0)
  75                         break;
  76
  77                 dest = completed + waiter->action;
  78
  79                 /* consolidate submit cleanups */
  80                 if (waiter->action == HOST1X_INTR_ACTION_SUBMIT_COMPLETE &&
  81                     !list_empty(dest)) {
  82                         prev = list_entry(dest->prev,
  83                                           struct host1x_waitlist, list);
  84                         if (prev->data == waiter->data) {
  85                                 prev->count++;
  86                                 dest = NULL;
  87                         }
  88                 }
  89
  90                 /* PENDING->REMOVED or CANCELLED->HANDLED */
  91                 if (atomic_inc_return(&waiter->state) == WLS_HANDLED || !dest) {
  92                         list_del(&waiter->list);
  93                         kref_put(&waiter->refcount, waiter_release);
  94                 } else
  95                         list_move_tail(&waiter->list, dest);
  96         }
  97 }
  98
  99 static void reset_threshold_interrupt(struct host1x *host,
 100                                       struct list_head *head,
 101                                       unsigned int id)
 102 {
 103         u32 thresh =
 104                 list_first_entry(head, struct host1x_waitlist, list)->thresh;
 105
 106         host1x_hw_intr_set_syncpt_threshold(host, id, thresh);
 107         host1x_hw_intr_enable_syncpt_intr(host, id);
 108 }
 109
 110 static void action_submit_complete(struct host1x_waitlist *waiter)
 111 {
 112         struct host1x_channel *channel = waiter->data;
 113
 114         host1x_cdma_update(&channel->cdma);
 115
 116         /*  Add nr_completed to trace */
 117         trace_host1x_channel_submit_complete(dev_name(channel->dev),
 118                                              waiter->count, waiter->thresh);
 119
 120 }
 121
 122 static void action_wakeup(struct host1x_waitlist *waiter)
 123 {
 124         wait_queue_head_t *wq = waiter->data;
 125         wake_up(wq);
 126 }
 127
 128 static void action_wakeup_interruptible(struct host1x_waitlist *waiter)
 129 {
 130         wait_queue_head_t *wq = waiter->data;
 131         wake_up_interruptible(wq);
 132 }
 133
 134 typedef void (*action_handler)(struct host1x_waitlist *waiter);
 135
 136 static action_handler action_handlers[HOST1X_INTR_ACTION_COUNT] = {
 137         action_submit_complete,
 138         action_wakeup,
 139         action_wakeup_interruptible,
 140 };
 141
 142 static void run_handlers(struct list_head completed[HOST1X_INTR_ACTION_COUNT])
 143 {
 144         struct list_head *head = completed;
 145         int i;
 146
 147         for (i = 0; i < HOST1X_INTR_ACTION_COUNT; ++i, ++head) {
 148                 action_handler handler = action_handlers[i];
 149                 struct host1x_waitlist *waiter, *next;
 150
 151                 list_for_each_entry_safe(waiter, next, head, list) {
 152                         list_del(&waiter->list);
 153                         handler(waiter);
 154                         WARN_ON(atomic_xchg(&waiter->state, WLS_HANDLED) !=
 155                                 WLS_REMOVED);
 156                         kref_put(&waiter->refcount, waiter_release);
 157                 }
 158         }
 159 }
 160
 161 /*
 162  * Remove & handle all waiters that have completed for the given syncpt
 163  */
 164 static int process_wait_list(struct host1x *host,
 165                              struct host1x_syncpt *syncpt,
 166                              u32 threshold)
 167 {
 168         struct list_head completed[HOST1X_INTR_ACTION_COUNT];
 169         unsigned int i;
 170         int empty;
 171
 172         for (i = 0; i < HOST1X_INTR_ACTION_COUNT; ++i)
 173                 INIT_LIST_HEAD(completed + i);
 174
 175         spin_lock(&syncpt->intr.lock);
 176
 177         remove_completed_waiters(&syncpt->intr.wait_head, threshold,
 178                                  completed);
 179
 180         empty = list_empty(&syncpt->intr.wait_head);
 181         if (empty)
 182                 host1x_hw_intr_disable_syncpt_intr(host, syncpt->id);
 183         else
 184                 reset_threshold_interrupt(host, &syncpt->intr.wait_head,
 185                                           syncpt->id);
 186
 187         spin_unlock(&syncpt->intr.lock);
 188
 189         run_handlers(completed);
 190
 191         return empty;
 192 }
 193
 194 /*
 195  * Sync point threshold interrupt service thread function
 196  * Handles sync point threshold triggers, in thread context
 197  */
 198
 199 static void syncpt_thresh_work(struct work_struct *work)
 200 {
 201         struct host1x_syncpt_intr *syncpt_intr =
 202                 container_of(work, struct host1x_syncpt_intr, work);
 203         struct host1x_syncpt *syncpt =
 204                 container_of(syncpt_intr, struct host1x_syncpt, intr);
 205         unsigned int id = syncpt->id;
 206         struct host1x *host = syncpt->host;
 207
 208         (void)process_wait_list(host, syncpt,
 209                                 host1x_syncpt_load(host->syncpt + id));
 210 }
 211
 212 int host1x_intr_add_action(struct host1x *host, u32 id, u32 thresh,
 213                            enum host1x_intr_action action, void *data,
 214                            struct host1x_waitlist *waiter, void **ref)
 215 {
 216         struct host1x_syncpt *syncpt;
 217         int queue_was_empty;
 218
 219         if (waiter == NULL) {
 220                 pr_warn("%s: NULL waiter\n", __func__);
 221                 return -EINVAL;
 222         }
 223
 224         /* initialize a new waiter */
 225         INIT_LIST_HEAD(&waiter->list);
 226         kref_init(&waiter->refcount);
 227         if (ref)
 228                 kref_get(&waiter->refcount);
 229         waiter->thresh = thresh;
 230         waiter->action = action;
 231         atomic_set(&waiter->state, WLS_PENDING);
 232         waiter->data = data;
 233         waiter->count = 1;
 234
 235         syncpt = host->syncpt + id;
 236
 237         spin_lock(&syncpt->intr.lock);
 238
 239         queue_was_empty = list_empty(&syncpt->intr.wait_head);
 240
 241         if (add_waiter_to_queue(waiter, &syncpt->intr.wait_head)) {
 242                 /* added at head of list - new threshold value */
 243                 host1x_hw_intr_set_syncpt_threshold(host, id, thresh);
 244
 245                 /* added as first waiter - enable interrupt */
 246                 if (queue_was_empty)
 247                         host1x_hw_intr_enable_syncpt_intr(host, id);
 248         }
 249
 250         spin_unlock(&syncpt->intr.lock);
 251
 252         if (ref)
 253                 *ref = waiter;
 254         return 0;
 255 }
 256
 257 void host1x_intr_put_ref(struct host1x *host, u32 id, void *ref)
 258 {
 259         struct host1x_waitlist *waiter = ref;
 260         struct host1x_syncpt *syncpt;
 261
 262         while (atomic_cmpxchg(&waiter->state, WLS_PENDING, WLS_CANCELLED) ==
 263                WLS_REMOVED)
 264                 schedule();
 265
 266         syncpt = host->syncpt + id;
 267         (void)process_wait_list(host, syncpt,
 268                                 host1x_syncpt_load(host->syncpt + id));
 269
 270         kref_put(&waiter->refcount, waiter_release);
 271 }
 272
 273 int host1x_intr_init(struct host1x *host, unsigned int irq_sync)
 274 {
 275         unsigned int id;
 276         u32 nb_pts = host1x_syncpt_nb_pts(host);
 277
 278         mutex_init(&host->intr_mutex);
 279         host->intr_syncpt_irq = irq_sync;
 280         host->intr_wq = create_workqueue("host_syncpt");
 281         if (!host->intr_wq)
 282                 return -ENOMEM;
 283
 284         for (id = 0; id < nb_pts; ++id) {
 285                 struct host1x_syncpt *syncpt = host->syncpt + id;
 286
 287                 spin_lock_init(&syncpt->intr.lock);
 288                 INIT_LIST_HEAD(&syncpt->intr.wait_head);
 289                 snprintf(syncpt->intr.thresh_irq_name,
 290                          sizeof(syncpt->intr.thresh_irq_name),
 291                          "host1x_sp_%02d", id);
 292         }
 293
 294         host1x_intr_start(host);
 295
 296         return 0;
 297 }
 298
 299 void host1x_intr_deinit(struct host1x *host)
 300 {
 301         host1x_intr_stop(host);
 302         destroy_workqueue(host->intr_wq);
 303 }
 304
 305 void host1x_intr_start(struct host1x *host)
 306 {
 307         u32 hz = clk_get_rate(host->clk);
 308         int err;
 309
 310         mutex_lock(&host->intr_mutex);
 311         err = host1x_hw_intr_init_host_sync(host, DIV_ROUND_UP(hz, 1000000),
 312                                             syncpt_thresh_work);
 313         if (err) {
 314                 mutex_unlock(&host->intr_mutex);
 315                 return;
 316         }
 317         mutex_unlock(&host->intr_mutex);
 318 }
 319
 320 void host1x_intr_stop(struct host1x *host)
 321 {
 322         unsigned int id;
 323         struct host1x_syncpt *syncpt = host->syncpt;
 324         u32 nb_pts = host1x_syncpt_nb_pts(host);
 325
 326         mutex_lock(&host->intr_mutex);
 327
 328         host1x_hw_intr_disable_all_syncpt_intrs(host);
 329
 330         for (id = 0; id < nb_pts; ++id) {
 331                 struct host1x_waitlist *waiter, *next;
 332
 333                 list_for_each_entry_safe(waiter, next,
 334                         &syncpt[id].intr.wait_head, list) {
 335                         if (atomic_cmpxchg(&waiter->state,
 336                             WLS_CANCELLED, WLS_HANDLED) == WLS_CANCELLED) {
 337                                 list_del(&waiter->list);
 338                                 kref_put(&waiter->refcount, waiter_release);
 339                         }
 340                 }
 341
 342                 if (!list_empty(&syncpt[id].intr.wait_head)) {
 343                         /* output diagnostics */
 344                         mutex_unlock(&host->intr_mutex);
 345                         pr_warn("%s cannot stop syncpt intr id=%d\n",
 346                                 __func__, id);
 347                         return;
 348                 }
 349         }
 350
 351         host1x_hw_intr_free_syncpt_irq(host);
 352
 353         mutex_unlock(&host->intr_mutex);
 354 }