drivers/gpu/drm/nouveau/nv04_fifo.c

   1 /*
   2  * Copyright (C) 2007 Ben Skeggs.
   3  * All Rights Reserved.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining
   6  * a copy of this software and associated documentation files (the
   7  * "Software"), to deal in the Software without restriction, including
   8  * without limitation the rights to use, copy, modify, merge, publish,
   9  * distribute, sublicense, and/or sell copies of the Software, and to
  10  * permit persons to whom the Software is furnished to do so, subject to
  11  * the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the
  14  * next paragraph) shall be included in all copies or substantial
  15  * portions of the Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  18  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  19  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  20  * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  21  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  22  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  23  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  24  *
  25  */
  26
  27 #include "drmP.h"
  28 #include "drm.h"
  29 #include "nouveau_drv.h"
  30 #include "nouveau_ramht.h"
  31 #include "nouveau_util.h"
  32
  33 #define NV04_RAMFC(c) (dev_priv->ramfc->pinst + ((c) * NV04_RAMFC__SIZE))
  34 #define NV04_RAMFC__SIZE 32
  35 #define NV04_RAMFC_DMA_PUT                                       0x00
  36 #define NV04_RAMFC_DMA_GET                                       0x04
  37 #define NV04_RAMFC_DMA_INSTANCE                                  0x08
  38 #define NV04_RAMFC_DMA_STATE                                     0x0C
  39 #define NV04_RAMFC_DMA_FETCH                                     0x10
  40 #define NV04_RAMFC_ENGINE                                        0x14
  41 #define NV04_RAMFC_PULL1_ENGINE                                  0x18
  42
  43 #define RAMFC_WR(offset, val) nv_wo32(chan->ramfc, NV04_RAMFC_##offset, (val))
  44 #define RAMFC_RD(offset)      nv_ro32(chan->ramfc, NV04_RAMFC_##offset)
  45
  46 void
  47 nv04_fifo_disable(struct drm_device *dev)
  48 {
  49         uint32_t tmp;
  50
  51         tmp = nv_rd32(dev, NV04_PFIFO_CACHE1_DMA_PUSH);
  52         nv_wr32(dev, NV04_PFIFO_CACHE1_DMA_PUSH, tmp & ~1);
  53         nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH0, 0);
  54         tmp = nv_rd32(dev, NV03_PFIFO_CACHE1_PULL1);
  55         nv_wr32(dev, NV04_PFIFO_CACHE1_PULL0, tmp & ~1);
  56 }
  57
  58 void
  59 nv04_fifo_enable(struct drm_device *dev)
  60 {
  61         nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH0, 1);
  62         nv_wr32(dev, NV04_PFIFO_CACHE1_PULL0, 1);
  63 }
  64
  65 bool
  66 nv04_fifo_reassign(struct drm_device *dev, bool enable)
  67 {
  68         uint32_t reassign = nv_rd32(dev, NV03_PFIFO_CACHES);
  69
  70         nv_wr32(dev, NV03_PFIFO_CACHES, enable ? 1 : 0);
  71         return (reassign == 1);
  72 }
  73
  74 bool
  75 nv04_fifo_cache_pull(struct drm_device *dev, bool enable)
  76 {
  77         int pull = nv_mask(dev, NV04_PFIFO_CACHE1_PULL0, 1, enable);
  78
  79         if (!enable) {
  80                 /* In some cases the PFIFO puller may be left in an
  81                  * inconsistent state if you try to stop it when it's
  82                  * busy translating handles. Sometimes you get a
  83                  * PFIFO_CACHE_ERROR, sometimes it just fails silently
  84                  * sending incorrect instance offsets to PGRAPH after
  85                  * it's started up again. To avoid the latter we
  86                  * invalidate the most recently calculated instance.
  87                  */
  88                 if (!nv_wait(dev, NV04_PFIFO_CACHE1_PULL0,
  89                              NV04_PFIFO_CACHE1_PULL0_HASH_BUSY, 0))
  90                         NV_ERROR(dev, "Timeout idling the PFIFO puller.\n");
  91
  92                 if (nv_rd32(dev, NV04_PFIFO_CACHE1_PULL0) &
  93                     NV04_PFIFO_CACHE1_PULL0_HASH_FAILED)
  94                         nv_wr32(dev, NV03_PFIFO_INTR_0,
  95                                 NV_PFIFO_INTR_CACHE_ERROR);
  96
  97                 nv_wr32(dev, NV04_PFIFO_CACHE1_HASH, 0);
  98         }
  99
 100         return pull & 1;
 101 }
 102
 103 int
 104 nv04_fifo_channel_id(struct drm_device *dev)
 105 {
 106         return nv_rd32(dev, NV03_PFIFO_CACHE1_PUSH1) &
 107                         NV03_PFIFO_CACHE1_PUSH1_CHID_MASK;
 108 }
 109
 110 #ifdef __BIG_ENDIAN
 111 #define DMA_FETCH_ENDIANNESS NV_PFIFO_CACHE1_BIG_ENDIAN
 112 #else
 113 #define DMA_FETCH_ENDIANNESS 0
 114 #endif
 115
 116 int
 117 nv04_fifo_create_context(struct nouveau_channel *chan)
 118 {
 119         struct drm_device *dev = chan->dev;
 120         struct drm_nouveau_private *dev_priv = dev->dev_private;
 121         unsigned long flags;
 122         int ret;
 123
 124         ret = nouveau_gpuobj_new_fake(dev, NV04_RAMFC(chan->id), ~0,
 125                                                 NV04_RAMFC__SIZE,
 126                                                 NVOBJ_FLAG_ZERO_ALLOC |
 127                                                 NVOBJ_FLAG_ZERO_FREE,
 128                                                 &chan->ramfc);
 129         if (ret)
 130                 return ret;
 131
 132         chan->user = ioremap(pci_resource_start(dev->pdev, 0) +
 133                              NV03_USER(chan->id), PAGE_SIZE);
 134         if (!chan->user)
 135                 return -ENOMEM;
 136
 137         spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
 138
 139         /* Setup initial state */
 140         RAMFC_WR(DMA_PUT, chan->pushbuf_base);
 141         RAMFC_WR(DMA_GET, chan->pushbuf_base);
 142         RAMFC_WR(DMA_INSTANCE, chan->pushbuf->pinst >> 4);
 143         RAMFC_WR(DMA_FETCH, (NV_PFIFO_CACHE1_DMA_FETCH_TRIG_128_BYTES |
 144                              NV_PFIFO_CACHE1_DMA_FETCH_SIZE_128_BYTES |
 145                              NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_8 |
 146                              DMA_FETCH_ENDIANNESS));
 147
 148         /* enable the fifo dma operation */
 149         nv_wr32(dev, NV04_PFIFO_MODE,
 150                 nv_rd32(dev, NV04_PFIFO_MODE) | (1 << chan->id));
 151
 152         spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
 153         return 0;
 154 }
 155
 156 void
 157 nv04_fifo_destroy_context(struct nouveau_channel *chan)
 158 {
 159         struct drm_device *dev = chan->dev;
 160         struct drm_nouveau_private *dev_priv = dev->dev_private;
 161         struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
 162         unsigned long flags;
 163
 164         spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
 165         pfifo->reassign(dev, false);
 166
 167         /* Unload the context if it's the currently active one */
 168         if (pfifo->channel_id(dev) == chan->id) {
 169                 pfifo->disable(dev);
 170                 pfifo->unload_context(dev);
 171                 pfifo->enable(dev);
 172         }
 173
 174         /* Keep it from being rescheduled */
 175         nv_mask(dev, NV04_PFIFO_MODE, 1 << chan->id, 0);
 176
 177         pfifo->reassign(dev, true);
 178         spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
 179
 180         /* Free the channel resources */
 181         if (chan->user) {
 182                 iounmap(chan->user);
 183                 chan->user = NULL;
 184         }
 185         nouveau_gpuobj_ref(NULL, &chan->ramfc);
 186 }
 187
 188 static void
 189 nv04_fifo_do_load_context(struct drm_device *dev, int chid)
 190 {
 191         struct drm_nouveau_private *dev_priv = dev->dev_private;
 192         uint32_t fc = NV04_RAMFC(chid), tmp;
 193
 194         nv_wr32(dev, NV04_PFIFO_CACHE1_DMA_PUT, nv_ri32(dev, fc + 0));
 195         nv_wr32(dev, NV04_PFIFO_CACHE1_DMA_GET, nv_ri32(dev, fc + 4));
 196         tmp = nv_ri32(dev, fc + 8);
 197         nv_wr32(dev, NV04_PFIFO_CACHE1_DMA_INSTANCE, tmp & 0xFFFF);
 198         nv_wr32(dev, NV04_PFIFO_CACHE1_DMA_DCOUNT, tmp >> 16);
 199         nv_wr32(dev, NV04_PFIFO_CACHE1_DMA_STATE, nv_ri32(dev, fc + 12));
 200         nv_wr32(dev, NV04_PFIFO_CACHE1_DMA_FETCH, nv_ri32(dev, fc + 16));
 201         nv_wr32(dev, NV04_PFIFO_CACHE1_ENGINE, nv_ri32(dev, fc + 20));
 202         nv_wr32(dev, NV04_PFIFO_CACHE1_PULL1, nv_ri32(dev, fc + 24));
 203
 204         nv_wr32(dev, NV03_PFIFO_CACHE1_GET, 0);
 205         nv_wr32(dev, NV03_PFIFO_CACHE1_PUT, 0);
 206 }
 207
 208 int
 209 nv04_fifo_load_context(struct nouveau_channel *chan)
 210 {
 211         uint32_t tmp;
 212
 213         nv_wr32(chan->dev, NV03_PFIFO_CACHE1_PUSH1,
 214                            NV03_PFIFO_CACHE1_PUSH1_DMA | chan->id);
 215         nv04_fifo_do_load_context(chan->dev, chan->id);
 216         nv_wr32(chan->dev, NV04_PFIFO_CACHE1_DMA_PUSH, 1);
 217
 218         /* Reset NV04_PFIFO_CACHE1_DMA_CTL_AT_INFO to INVALID */
 219         tmp = nv_rd32(chan->dev, NV04_PFIFO_CACHE1_DMA_CTL) & ~(1 << 31);
 220         nv_wr32(chan->dev, NV04_PFIFO_CACHE1_DMA_CTL, tmp);
 221
 222         return 0;
 223 }
 224
 225 int
 226 nv04_fifo_unload_context(struct drm_device *dev)
 227 {
 228         struct drm_nouveau_private *dev_priv = dev->dev_private;
 229         struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
 230         struct nouveau_channel *chan = NULL;
 231         uint32_t tmp;
 232         int chid;
 233
 234         chid = pfifo->channel_id(dev);
 235         if (chid < 0 || chid >= dev_priv->engine.fifo.channels)
 236                 return 0;
 237
 238         chan = dev_priv->channels.ptr[chid];
 239         if (!chan) {
 240                 NV_ERROR(dev, "Inactive channel on PFIFO: %d\n", chid);
 241                 return -EINVAL;
 242         }
 243
 244         RAMFC_WR(DMA_PUT, nv_rd32(dev, NV04_PFIFO_CACHE1_DMA_PUT));
 245         RAMFC_WR(DMA_GET, nv_rd32(dev, NV04_PFIFO_CACHE1_DMA_GET));
 246         tmp  = nv_rd32(dev, NV04_PFIFO_CACHE1_DMA_DCOUNT) << 16;
 247         tmp |= nv_rd32(dev, NV04_PFIFO_CACHE1_DMA_INSTANCE);
 248         RAMFC_WR(DMA_INSTANCE, tmp);
 249         RAMFC_WR(DMA_STATE, nv_rd32(dev, NV04_PFIFO_CACHE1_DMA_STATE));
 250         RAMFC_WR(DMA_FETCH, nv_rd32(dev, NV04_PFIFO_CACHE1_DMA_FETCH));
 251         RAMFC_WR(ENGINE, nv_rd32(dev, NV04_PFIFO_CACHE1_ENGINE));
 252         RAMFC_WR(PULL1_ENGINE, nv_rd32(dev, NV04_PFIFO_CACHE1_PULL1));
 253
 254         nv04_fifo_do_load_context(dev, pfifo->channels - 1);
 255         nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH1, pfifo->channels - 1);
 256         return 0;
 257 }
 258
 259 static void
 260 nv04_fifo_init_reset(struct drm_device *dev)
 261 {
 262         nv_wr32(dev, NV03_PMC_ENABLE,
 263                 nv_rd32(dev, NV03_PMC_ENABLE) & ~NV_PMC_ENABLE_PFIFO);
 264         nv_wr32(dev, NV03_PMC_ENABLE,
 265                 nv_rd32(dev, NV03_PMC_ENABLE) |  NV_PMC_ENABLE_PFIFO);
 266
 267         nv_wr32(dev, 0x003224, 0x000f0078);
 268         nv_wr32(dev, 0x002044, 0x0101ffff);
 269         nv_wr32(dev, 0x002040, 0x000000ff);
 270         nv_wr32(dev, 0x002500, 0x00000000);
 271         nv_wr32(dev, 0x003000, 0x00000000);
 272         nv_wr32(dev, 0x003050, 0x00000000);
 273         nv_wr32(dev, 0x003200, 0x00000000);
 274         nv_wr32(dev, 0x003250, 0x00000000);
 275         nv_wr32(dev, 0x003220, 0x00000000);
 276
 277         nv_wr32(dev, 0x003250, 0x00000000);
 278         nv_wr32(dev, 0x003270, 0x00000000);
 279         nv_wr32(dev, 0x003210, 0x00000000);
 280 }
 281
 282 static void
 283 nv04_fifo_init_ramxx(struct drm_device *dev)
 284 {
 285         struct drm_nouveau_private *dev_priv = dev->dev_private;
 286
 287         nv_wr32(dev, NV03_PFIFO_RAMHT, (0x03 << 24) /* search 128 */ |
 288                                        ((dev_priv->ramht->bits - 9) << 16) |
 289                                        (dev_priv->ramht->gpuobj->pinst >> 8));
 290         nv_wr32(dev, NV03_PFIFO_RAMRO, dev_priv->ramro->pinst >> 8);
 291         nv_wr32(dev, NV03_PFIFO_RAMFC, dev_priv->ramfc->pinst >> 8);
 292 }
 293
 294 static void
 295 nv04_fifo_init_intr(struct drm_device *dev)
 296 {
 297         nouveau_irq_register(dev, 8, nv04_fifo_isr);
 298         nv_wr32(dev, 0x002100, 0xffffffff);
 299         nv_wr32(dev, 0x002140, 0xffffffff);
 300 }
 301
 302 int
 303 nv04_fifo_init(struct drm_device *dev)
 304 {
 305         struct drm_nouveau_private *dev_priv = dev->dev_private;
 306         struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
 307         int i;
 308
 309         nv04_fifo_init_reset(dev);
 310         nv04_fifo_init_ramxx(dev);
 311
 312         nv04_fifo_do_load_context(dev, pfifo->channels - 1);
 313         nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH1, pfifo->channels - 1);
 314
 315         nv04_fifo_init_intr(dev);
 316         pfifo->enable(dev);
 317         pfifo->reassign(dev, true);
 318
 319         for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
 320                 if (dev_priv->channels.ptr[i]) {
 321                         uint32_t mode = nv_rd32(dev, NV04_PFIFO_MODE);
 322                         nv_wr32(dev, NV04_PFIFO_MODE, mode | (1 << i));
 323                 }
 324         }
 325
 326         return 0;
 327 }
 328
 329 void
 330 nv04_fifo_fini(struct drm_device *dev)
 331 {
 332         nv_wr32(dev, 0x2140, 0x00000000);
 333         nouveau_irq_unregister(dev, 8);
 334 }
 335
 336 static bool
 337 nouveau_fifo_swmthd(struct drm_device *dev, u32 chid, u32 addr, u32 data)
 338 {
 339         struct drm_nouveau_private *dev_priv = dev->dev_private;
 340         struct nouveau_channel *chan = NULL;
 341         struct nouveau_gpuobj *obj;
 342         unsigned long flags;
 343         const int subc = (addr >> 13) & 0x7;
 344         const int mthd = addr & 0x1ffc;
 345         bool handled = false;
 346         u32 engine;
 347
 348         spin_lock_irqsave(&dev_priv->channels.lock, flags);
 349         if (likely(chid >= 0 && chid < dev_priv->engine.fifo.channels))
 350                 chan = dev_priv->channels.ptr[chid];
 351         if (unlikely(!chan))
 352                 goto out;
 353
 354         switch (mthd) {
 355         case 0x0000: /* bind object to subchannel */
 356                 obj = nouveau_ramht_find(chan, data);
 357                 if (unlikely(!obj || obj->engine != NVOBJ_ENGINE_SW))
 358                         break;
 359
 360                 chan->sw_subchannel[subc] = obj->class;
 361                 engine = 0x0000000f << (subc * 4);
 362
 363                 nv_mask(dev, NV04_PFIFO_CACHE1_ENGINE, engine, 0x00000000);
 364                 handled = true;
 365                 break;
 366         default:
 367                 engine = nv_rd32(dev, NV04_PFIFO_CACHE1_ENGINE);
 368                 if (unlikely(((engine >> (subc * 4)) & 0xf) != 0))
 369                         break;
 370
 371                 if (!nouveau_gpuobj_mthd_call(chan, chan->sw_subchannel[subc],
 372                                               mthd, data))
 373                         handled = true;
 374                 break;
 375         }
 376
 377 out:
 378         spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
 379         return handled;
 380 }
 381
 382 static const char *nv_dma_state_err(u32 state)
 383 {
 384         static const char * const desc[] = {
 385                 "NONE", "CALL_SUBR_ACTIVE", "INVALID_MTHD", "RET_SUBR_INACTIVE",
 386                 "INVALID_CMD", "IB_EMPTY"/* NV50+ */, "MEM_FAULT", "UNK"
 387         };
 388         return desc[(state >> 29) & 0x7];
 389 }
 390
 391 void
 392 nv04_fifo_isr(struct drm_device *dev)
 393 {
 394         struct drm_nouveau_private *dev_priv = dev->dev_private;
 395         struct nouveau_engine *engine = &dev_priv->engine;
 396         uint32_t status, reassign;
 397         int cnt = 0;
 398
 399         reassign = nv_rd32(dev, NV03_PFIFO_CACHES) & 1;
 400         while ((status = nv_rd32(dev, NV03_PFIFO_INTR_0)) && (cnt++ < 100)) {
 401                 uint32_t chid, get;
 402
 403                 nv_wr32(dev, NV03_PFIFO_CACHES, 0);
 404
 405                 chid = engine->fifo.channel_id(dev);
 406                 get  = nv_rd32(dev, NV03_PFIFO_CACHE1_GET);
 407
 408                 if (status & NV_PFIFO_INTR_CACHE_ERROR) {
 409                         uint32_t mthd, data;
 410                         int ptr;
 411
 412                         /* NV_PFIFO_CACHE1_GET actually goes to 0xffc before
 413                          * wrapping on my G80 chips, but CACHE1 isn't big
 414                          * enough for this much data.. Tests show that it
 415                          * wraps around to the start at GET=0x800.. No clue
 416                          * as to why..
 417                          */
 418                         ptr = (get & 0x7ff) >> 2;
 419
 420                         if (dev_priv->card_type < NV_40) {
 421                                 mthd = nv_rd32(dev,
 422                                         NV04_PFIFO_CACHE1_METHOD(ptr));
 423                                 data = nv_rd32(dev,
 424                                         NV04_PFIFO_CACHE1_DATA(ptr));
 425                         } else {
 426                                 mthd = nv_rd32(dev,
 427                                         NV40_PFIFO_CACHE1_METHOD(ptr));
 428                                 data = nv_rd32(dev,
 429                                         NV40_PFIFO_CACHE1_DATA(ptr));
 430                         }
 431
 432                         if (!nouveau_fifo_swmthd(dev, chid, mthd, data)) {
 433                                 NV_INFO(dev, "PFIFO_CACHE_ERROR - Ch %d/%d "
 434                                              "Mthd 0x%04x Data 0x%08x\n",
 435                                         chid, (mthd >> 13) & 7, mthd & 0x1ffc,
 436                                         data);
 437                         }
 438
 439                         nv_wr32(dev, NV04_PFIFO_CACHE1_DMA_PUSH, 0);
 440                         nv_wr32(dev, NV03_PFIFO_INTR_0,
 441                                                 NV_PFIFO_INTR_CACHE_ERROR);
 442
 443                         nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH0,
 444                                 nv_rd32(dev, NV03_PFIFO_CACHE1_PUSH0) & ~1);
 445                         nv_wr32(dev, NV03_PFIFO_CACHE1_GET, get + 4);
 446                         nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH0,
 447                                 nv_rd32(dev, NV03_PFIFO_CACHE1_PUSH0) | 1);
 448                         nv_wr32(dev, NV04_PFIFO_CACHE1_HASH, 0);
 449
 450                         nv_wr32(dev, NV04_PFIFO_CACHE1_DMA_PUSH,
 451                                 nv_rd32(dev, NV04_PFIFO_CACHE1_DMA_PUSH) | 1);
 452                         nv_wr32(dev, NV04_PFIFO_CACHE1_PULL0, 1);
 453
 454                         status &= ~NV_PFIFO_INTR_CACHE_ERROR;
 455                 }
 456
 457                 if (status & NV_PFIFO_INTR_DMA_PUSHER) {
 458                         u32 dma_get = nv_rd32(dev, 0x003244);
 459                         u32 dma_put = nv_rd32(dev, 0x003240);
 460                         u32 push = nv_rd32(dev, 0x003220);
 461                         u32 state = nv_rd32(dev, 0x003228);
 462
 463                         if (dev_priv->card_type == NV_50) {
 464                                 u32 ho_get = nv_rd32(dev, 0x003328);
 465                                 u32 ho_put = nv_rd32(dev, 0x003320);
 466                                 u32 ib_get = nv_rd32(dev, 0x003334);
 467                                 u32 ib_put = nv_rd32(dev, 0x003330);
 468
 469                                 if (nouveau_ratelimit())
 470                                         NV_INFO(dev, "PFIFO_DMA_PUSHER - Ch %d Get 0x%02x%08x "
 471                                              "Put 0x%02x%08x IbGet 0x%08x IbPut 0x%08x "
 472                                              "State 0x%08x (err: %s) Push 0x%08x\n",
 473                                                 chid, ho_get, dma_get, ho_put,
 474                                                 dma_put, ib_get, ib_put, state,
 475                                                 nv_dma_state_err(state),
 476                                                 push);
 477
 478                                 /* METHOD_COUNT, in DMA_STATE on earlier chipsets */
 479                                 nv_wr32(dev, 0x003364, 0x00000000);
 480                                 if (dma_get != dma_put || ho_get != ho_put) {
 481                                         nv_wr32(dev, 0x003244, dma_put);
 482                                         nv_wr32(dev, 0x003328, ho_put);
 483                                 } else
 484                                 if (ib_get != ib_put) {
 485                                         nv_wr32(dev, 0x003334, ib_put);
 486                                 }
 487                         } else {
 488                                 NV_INFO(dev, "PFIFO_DMA_PUSHER - Ch %d Get 0x%08x "
 489                                              "Put 0x%08x State 0x%08x (err: %s) Push 0x%08x\n",
 490                                         chid, dma_get, dma_put, state,
 491                                         nv_dma_state_err(state), push);
 492
 493                                 if (dma_get != dma_put)
 494                                         nv_wr32(dev, 0x003244, dma_put);
 495                         }
 496
 497                         nv_wr32(dev, 0x003228, 0x00000000);
 498                         nv_wr32(dev, 0x003220, 0x00000001);
 499                         nv_wr32(dev, 0x002100, NV_PFIFO_INTR_DMA_PUSHER);
 500                         status &= ~NV_PFIFO_INTR_DMA_PUSHER;
 501                 }
 502
 503                 if (status & NV_PFIFO_INTR_SEMAPHORE) {
 504                         uint32_t sem;
 505
 506                         status &= ~NV_PFIFO_INTR_SEMAPHORE;
 507                         nv_wr32(dev, NV03_PFIFO_INTR_0,
 508                                 NV_PFIFO_INTR_SEMAPHORE);
 509
 510                         sem = nv_rd32(dev, NV10_PFIFO_CACHE1_SEMAPHORE);
 511                         nv_wr32(dev, NV10_PFIFO_CACHE1_SEMAPHORE, sem | 0x1);
 512
 513                         nv_wr32(dev, NV03_PFIFO_CACHE1_GET, get + 4);
 514                         nv_wr32(dev, NV04_PFIFO_CACHE1_PULL0, 1);
 515                 }
 516
 517                 if (dev_priv->card_type == NV_50) {
 518                         if (status & 0x00000010) {
 519                                 nv50_fb_vm_trap(dev, nouveau_ratelimit());
 520                                 status &= ~0x00000010;
 521                                 nv_wr32(dev, 0x002100, 0x00000010);
 522                         }
 523                 }
 524
 525                 if (status) {
 526                         if (nouveau_ratelimit())
 527                                 NV_INFO(dev, "PFIFO_INTR 0x%08x - Ch %d\n",
 528                                         status, chid);
 529                         nv_wr32(dev, NV03_PFIFO_INTR_0, status);
 530                         status = 0;
 531                 }
 532
 533                 nv_wr32(dev, NV03_PFIFO_CACHES, reassign);
 534         }
 535
 536         if (status) {
 537                 NV_INFO(dev, "PFIFO still angry after %d spins, halt\n", cnt);
 538                 nv_wr32(dev, 0x2140, 0);
 539                 nv_wr32(dev, 0x140, 0);
 540         }
 541
 542         nv_wr32(dev, NV03_PMC_INTR_0, NV_PMC_INTR_0_PFIFO_PENDING);
 543 }