drivers/gpu/drm/radeon/radeon_cs.c

   1 /*
   2  * Copyright 2008 Jerome Glisse.
   3  * All Rights Reserved.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  22  * DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors:
  25  *    Jerome Glisse <glisse@freedesktop.org>
  26  */
  27 #include <drm/drmP.h>
  28 #include <drm/radeon_drm.h>
  29 #include "radeon_reg.h"
  30 #include "radeon.h"
  31 #include "radeon_trace.h"
  32
  33 static int radeon_cs_parser_relocs(struct radeon_cs_parser *p)
  34 {
  35         struct drm_device *ddev = p->rdev->ddev;
  36         struct radeon_cs_chunk *chunk;
  37         unsigned i, j;
  38         bool duplicate;
  39
  40         if (p->chunk_relocs_idx == -1) {
  41                 return 0;
  42         }
  43         chunk = &p->chunks[p->chunk_relocs_idx];
  44         p->dma_reloc_idx = 0;
  45         /* FIXME: we assume that each relocs use 4 dwords */
  46         p->nrelocs = chunk->length_dw / 4;
  47         p->relocs_ptr = kcalloc(p->nrelocs, sizeof(void *), GFP_KERNEL);
  48         if (p->relocs_ptr == NULL) {
  49                 return -ENOMEM;
  50         }
  51         p->relocs = kcalloc(p->nrelocs, sizeof(struct radeon_cs_reloc), GFP_KERNEL);
  52         if (p->relocs == NULL) {
  53                 return -ENOMEM;
  54         }
  55         for (i = 0; i < p->nrelocs; i++) {
  56                 struct drm_radeon_cs_reloc *r;
  57
  58                 duplicate = false;
  59                 r = (struct drm_radeon_cs_reloc *)&chunk->kdata[i*4];
  60                 for (j = 0; j < i; j++) {
  61                         if (r->handle == p->relocs[j].handle) {
  62                                 p->relocs_ptr[i] = &p->relocs[j];
  63                                 duplicate = true;
  64                                 break;
  65                         }
  66                 }
  67                 if (duplicate) {
  68                         p->relocs[i].handle = 0;
  69                         continue;
  70                 }
  71
  72                 p->relocs[i].gobj = drm_gem_object_lookup(ddev, p->filp,
  73                                                           r->handle);
  74                 if (p->relocs[i].gobj == NULL) {
  75                         DRM_ERROR("gem object lookup failed 0x%x\n",
  76                                   r->handle);
  77                         return -ENOENT;
  78                 }
  79                 p->relocs_ptr[i] = &p->relocs[i];
  80                 p->relocs[i].robj = gem_to_radeon_bo(p->relocs[i].gobj);
  81                 p->relocs[i].lobj.bo = p->relocs[i].robj;
  82                 p->relocs[i].lobj.written = !!r->write_domain;
  83
  84                 /* the first reloc of an UVD job is the msg and that must be in
  85                    VRAM, also but everything into VRAM on AGP cards to avoid
  86                    image corruptions */
  87                 if (p->ring == R600_RING_TYPE_UVD_INDEX &&
  88                     (i == 0 || drm_pci_device_is_agp(p->rdev->ddev))) {
  89                         /* TODO: is this still needed for NI+ ? */
  90                         p->relocs[i].lobj.domain =
  91                                 RADEON_GEM_DOMAIN_VRAM;
  92
  93                         p->relocs[i].lobj.alt_domain =
  94                                 RADEON_GEM_DOMAIN_VRAM;
  95
  96                 } else {
  97                         uint32_t domain = r->write_domain ?
  98                                 r->write_domain : r->read_domains;
  99
 100                         if (domain & RADEON_GEM_DOMAIN_CPU) {
 101                                 DRM_ERROR("RADEON_GEM_DOMAIN_CPU is not valid "
 102                                           "for command submission\n");
 103                                 return -EINVAL;
 104                         }
 105
 106                         p->relocs[i].lobj.domain = domain;
 107                         if (domain == RADEON_GEM_DOMAIN_VRAM)
 108                                 domain |= RADEON_GEM_DOMAIN_GTT;
 109                         p->relocs[i].lobj.alt_domain = domain;
 110                 }
 111
 112                 p->relocs[i].lobj.tv.bo = &p->relocs[i].robj->tbo;
 113                 p->relocs[i].handle = r->handle;
 114
 115                 radeon_bo_list_add_object(&p->relocs[i].lobj,
 116                                           &p->validated);
 117         }
 118         return radeon_bo_list_validate(&p->ticket, &p->validated, p->ring);
 119 }
 120
 121 static int radeon_cs_get_ring(struct radeon_cs_parser *p, u32 ring, s32 priority)
 122 {
 123         p->priority = priority;
 124
 125         switch (ring) {
 126         default:
 127                 DRM_ERROR("unknown ring id: %d\n", ring);
 128                 return -EINVAL;
 129         case RADEON_CS_RING_GFX:
 130                 p->ring = RADEON_RING_TYPE_GFX_INDEX;
 131                 break;
 132         case RADEON_CS_RING_COMPUTE:
 133                 if (p->rdev->family >= CHIP_TAHITI) {
 134                         if (p->priority > 0)
 135                                 p->ring = CAYMAN_RING_TYPE_CP1_INDEX;
 136                         else
 137                                 p->ring = CAYMAN_RING_TYPE_CP2_INDEX;
 138                 } else
 139                         p->ring = RADEON_RING_TYPE_GFX_INDEX;
 140                 break;
 141         case RADEON_CS_RING_DMA:
 142                 if (p->rdev->family >= CHIP_CAYMAN) {
 143                         if (p->priority > 0)
 144                                 p->ring = R600_RING_TYPE_DMA_INDEX;
 145                         else
 146                                 p->ring = CAYMAN_RING_TYPE_DMA1_INDEX;
 147                 } else if (p->rdev->family >= CHIP_R600) {
 148                         p->ring = R600_RING_TYPE_DMA_INDEX;
 149                 } else {
 150                         return -EINVAL;
 151                 }
 152                 break;
 153         case RADEON_CS_RING_UVD:
 154                 p->ring = R600_RING_TYPE_UVD_INDEX;
 155                 break;
 156         }
 157         return 0;
 158 }
 159
 160 static void radeon_cs_sync_rings(struct radeon_cs_parser *p)
 161 {
 162         int i;
 163
 164         for (i = 0; i < p->nrelocs; i++) {
 165                 if (!p->relocs[i].robj)
 166                         continue;
 167
 168                 radeon_ib_sync_to(&p->ib, p->relocs[i].robj->tbo.sync_obj);
 169         }
 170 }
 171
 172 /* XXX: note that this is called from the legacy UMS CS ioctl as well */
 173 int radeon_cs_parser_init(struct radeon_cs_parser *p, void *data)
 174 {
 175         struct drm_radeon_cs *cs = data;
 176         uint64_t *chunk_array_ptr;
 177         unsigned size, i;
 178         u32 ring = RADEON_CS_RING_GFX;
 179         s32 priority = 0;
 180
 181         if (!cs->num_chunks) {
 182                 return 0;
 183         }
 184         /* get chunks */
 185         INIT_LIST_HEAD(&p->validated);
 186         p->idx = 0;
 187         p->ib.sa_bo = NULL;
 188         p->ib.semaphore = NULL;
 189         p->const_ib.sa_bo = NULL;
 190         p->const_ib.semaphore = NULL;
 191         p->chunk_ib_idx = -1;
 192         p->chunk_relocs_idx = -1;
 193         p->chunk_flags_idx = -1;
 194         p->chunk_const_ib_idx = -1;
 195         p->chunks_array = kcalloc(cs->num_chunks, sizeof(uint64_t), GFP_KERNEL);
 196         if (p->chunks_array == NULL) {
 197                 return -ENOMEM;
 198         }
 199         chunk_array_ptr = (uint64_t *)(unsigned long)(cs->chunks);
 200         if (DRM_COPY_FROM_USER(p->chunks_array, chunk_array_ptr,
 201                                sizeof(uint64_t)*cs->num_chunks)) {
 202                 return -EFAULT;
 203         }
 204         p->cs_flags = 0;
 205         p->nchunks = cs->num_chunks;
 206         p->chunks = kcalloc(p->nchunks, sizeof(struct radeon_cs_chunk), GFP_KERNEL);
 207         if (p->chunks == NULL) {
 208                 return -ENOMEM;
 209         }
 210         for (i = 0; i < p->nchunks; i++) {
 211                 struct drm_radeon_cs_chunk __user **chunk_ptr = NULL;
 212                 struct drm_radeon_cs_chunk user_chunk;
 213                 uint32_t __user *cdata;
 214
 215                 chunk_ptr = (void __user*)(unsigned long)p->chunks_array[i];
 216                 if (DRM_COPY_FROM_USER(&user_chunk, chunk_ptr,
 217                                        sizeof(struct drm_radeon_cs_chunk))) {
 218                         return -EFAULT;
 219                 }
 220                 p->chunks[i].length_dw = user_chunk.length_dw;
 221                 p->chunks[i].kdata = NULL;
 222                 p->chunks[i].chunk_id = user_chunk.chunk_id;
 223                 p->chunks[i].user_ptr = (void __user *)(unsigned long)user_chunk.chunk_data;
 224                 if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_RELOCS) {
 225                         p->chunk_relocs_idx = i;
 226                 }
 227                 if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_IB) {
 228                         p->chunk_ib_idx = i;
 229                         /* zero length IB isn't useful */
 230                         if (p->chunks[i].length_dw == 0)
 231                                 return -EINVAL;
 232                 }
 233                 if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_CONST_IB) {
 234                         p->chunk_const_ib_idx = i;
 235                         /* zero length CONST IB isn't useful */
 236                         if (p->chunks[i].length_dw == 0)
 237                                 return -EINVAL;
 238                 }
 239                 if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_FLAGS) {
 240                         p->chunk_flags_idx = i;
 241                         /* zero length flags aren't useful */
 242                         if (p->chunks[i].length_dw == 0)
 243                                 return -EINVAL;
 244                 }
 245
 246                 cdata = (uint32_t *)(unsigned long)user_chunk.chunk_data;
 247                 if ((p->chunks[i].chunk_id == RADEON_CHUNK_ID_RELOCS) ||
 248                     (p->chunks[i].chunk_id == RADEON_CHUNK_ID_FLAGS)) {
 249                         size = p->chunks[i].length_dw * sizeof(uint32_t);
 250                         p->chunks[i].kdata = kmalloc(size, GFP_KERNEL);
 251                         if (p->chunks[i].kdata == NULL) {
 252                                 return -ENOMEM;
 253                         }
 254                         if (DRM_COPY_FROM_USER(p->chunks[i].kdata,
 255                                                p->chunks[i].user_ptr, size)) {
 256                                 return -EFAULT;
 257                         }
 258                         if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_FLAGS) {
 259                                 p->cs_flags = p->chunks[i].kdata[0];
 260                                 if (p->chunks[i].length_dw > 1)
 261                                         ring = p->chunks[i].kdata[1];
 262                                 if (p->chunks[i].length_dw > 2)
 263                                         priority = (s32)p->chunks[i].kdata[2];
 264                         }
 265                 }
 266         }
 267
 268         /* these are KMS only */
 269         if (p->rdev) {
 270                 if ((p->cs_flags & RADEON_CS_USE_VM) &&
 271                     !p->rdev->vm_manager.enabled) {
 272                         DRM_ERROR("VM not active on asic!\n");
 273                         return -EINVAL;
 274                 }
 275
 276                 if (radeon_cs_get_ring(p, ring, priority))
 277                         return -EINVAL;
 278
 279                 /* we only support VM on some SI+ rings */
 280                 if ((p->cs_flags & RADEON_CS_USE_VM) == 0) {
 281                         if (p->rdev->asic->ring[p->ring]->cs_parse == NULL) {
 282                                 DRM_ERROR("Ring %d requires VM!\n", p->ring);
 283                                 return -EINVAL;
 284                         }
 285                 } else {
 286                         if (p->rdev->asic->ring[p->ring]->ib_parse == NULL) {
 287                                 DRM_ERROR("VM not supported on ring %d!\n",
 288                                           p->ring);
 289                                 return -EINVAL;
 290                         }
 291                 }
 292         }
 293
 294         /* deal with non-vm */
 295         if ((p->chunk_ib_idx != -1) &&
 296             ((p->cs_flags & RADEON_CS_USE_VM) == 0) &&
 297             (p->chunks[p->chunk_ib_idx].chunk_id == RADEON_CHUNK_ID_IB)) {
 298                 if (p->chunks[p->chunk_ib_idx].length_dw > (16 * 1024)) {
 299                         DRM_ERROR("cs IB too big: %d\n",
 300                                   p->chunks[p->chunk_ib_idx].length_dw);
 301                         return -EINVAL;
 302                 }
 303                 if (p->rdev && (p->rdev->flags & RADEON_IS_AGP)) {
 304                         p->chunks[p->chunk_ib_idx].kpage[0] = kmalloc(PAGE_SIZE, GFP_KERNEL);
 305                         p->chunks[p->chunk_ib_idx].kpage[1] = kmalloc(PAGE_SIZE, GFP_KERNEL);
 306                         if (p->chunks[p->chunk_ib_idx].kpage[0] == NULL ||
 307                             p->chunks[p->chunk_ib_idx].kpage[1] == NULL) {
 308                                 kfree(p->chunks[p->chunk_ib_idx].kpage[0]);
 309                                 kfree(p->chunks[p->chunk_ib_idx].kpage[1]);
 310                                 p->chunks[p->chunk_ib_idx].kpage[0] = NULL;
 311                                 p->chunks[p->chunk_ib_idx].kpage[1] = NULL;
 312                                 return -ENOMEM;
 313                         }
 314                 }
 315                 p->chunks[p->chunk_ib_idx].kpage_idx[0] = -1;
 316                 p->chunks[p->chunk_ib_idx].kpage_idx[1] = -1;
 317                 p->chunks[p->chunk_ib_idx].last_copied_page = -1;
 318                 p->chunks[p->chunk_ib_idx].last_page_index =
 319                         ((p->chunks[p->chunk_ib_idx].length_dw * 4) - 1) / PAGE_SIZE;
 320         }
 321
 322         return 0;
 323 }
 324
 325 /**
 326  * cs_parser_fini() - clean parser states
 327  * @parser:     parser structure holding parsing context.
 328  * @error:      error number
 329  *
 330  * If error is set than unvalidate buffer, otherwise just free memory
 331  * used by parsing context.
 332  **/
 333 static void radeon_cs_parser_fini(struct radeon_cs_parser *parser, int error, bool backoff)
 334 {
 335         unsigned i;
 336
 337         if (!error) {
 338                 ttm_eu_fence_buffer_objects(&parser->ticket,
 339                                             &parser->validated,
 340                                             parser->ib.fence);
 341         } else if (backoff) {
 342                 ttm_eu_backoff_reservation(&parser->ticket,
 343                                            &parser->validated);
 344         }
 345
 346         if (parser->relocs != NULL) {
 347                 for (i = 0; i < parser->nrelocs; i++) {
 348                         if (parser->relocs[i].gobj)
 349                                 drm_gem_object_unreference_unlocked(parser->relocs[i].gobj);
 350                 }
 351         }
 352         kfree(parser->track);
 353         kfree(parser->relocs);
 354         kfree(parser->relocs_ptr);
 355         for (i = 0; i < parser->nchunks; i++) {
 356                 kfree(parser->chunks[i].kdata);
 357                 if ((parser->rdev->flags & RADEON_IS_AGP)) {
 358                         kfree(parser->chunks[i].kpage[0]);
 359                         kfree(parser->chunks[i].kpage[1]);
 360                 }
 361         }
 362         kfree(parser->chunks);
 363         kfree(parser->chunks_array);
 364         radeon_ib_free(parser->rdev, &parser->ib);
 365         radeon_ib_free(parser->rdev, &parser->const_ib);
 366 }
 367
 368 static int radeon_cs_ib_chunk(struct radeon_device *rdev,
 369                               struct radeon_cs_parser *parser)
 370 {
 371         struct radeon_cs_chunk *ib_chunk;
 372         int r;
 373
 374         if (parser->chunk_ib_idx == -1)
 375                 return 0;
 376
 377         if (parser->cs_flags & RADEON_CS_USE_VM)
 378                 return 0;
 379
 380         ib_chunk = &parser->chunks[parser->chunk_ib_idx];
 381         /* Copy the packet into the IB, the parser will read from the
 382          * input memory (cached) and write to the IB (which can be
 383          * uncached).
 384          */
 385         r =  radeon_ib_get(rdev, parser->ring, &parser->ib,
 386                            NULL, ib_chunk->length_dw * 4);
 387         if (r) {
 388                 DRM_ERROR("Failed to get ib !\n");
 389                 return r;
 390         }
 391         parser->ib.length_dw = ib_chunk->length_dw;
 392         r = radeon_cs_parse(rdev, parser->ring, parser);
 393         if (r || parser->parser_error) {
 394                 DRM_ERROR("Invalid command stream !\n");
 395                 return r;
 396         }
 397         r = radeon_cs_finish_pages(parser);
 398         if (r) {
 399                 DRM_ERROR("Invalid command stream !\n");
 400                 return r;
 401         }
 402
 403         if (parser->ring == R600_RING_TYPE_UVD_INDEX)
 404                 radeon_uvd_note_usage(rdev);
 405
 406         radeon_cs_sync_rings(parser);
 407         r = radeon_ib_schedule(rdev, &parser->ib, NULL);
 408         if (r) {
 409                 DRM_ERROR("Failed to schedule IB !\n");
 410         }
 411         return r;
 412 }
 413
 414 static int radeon_bo_vm_update_pte(struct radeon_cs_parser *parser,
 415                                    struct radeon_vm *vm)
 416 {
 417         struct radeon_device *rdev = parser->rdev;
 418         struct radeon_bo_list *lobj;
 419         struct radeon_bo *bo;
 420         int r;
 421
 422         r = radeon_vm_bo_update_pte(rdev, vm, rdev->ring_tmp_bo.bo, &rdev->ring_tmp_bo.bo->tbo.mem);
 423         if (r) {
 424                 return r;
 425         }
 426         list_for_each_entry(lobj, &parser->validated, tv.head) {
 427                 bo = lobj->bo;
 428                 r = radeon_vm_bo_update_pte(parser->rdev, vm, bo, &bo->tbo.mem);
 429                 if (r) {
 430                         return r;
 431                 }
 432         }
 433         return 0;
 434 }
 435
 436 static int radeon_cs_ib_vm_chunk(struct radeon_device *rdev,
 437                                  struct radeon_cs_parser *parser)
 438 {
 439         struct radeon_cs_chunk *ib_chunk;
 440         struct radeon_fpriv *fpriv = parser->filp->driver_priv;
 441         struct radeon_vm *vm = &fpriv->vm;
 442         int r;
 443
 444         if (parser->chunk_ib_idx == -1)
 445                 return 0;
 446         if ((parser->cs_flags & RADEON_CS_USE_VM) == 0)
 447                 return 0;
 448
 449         if ((rdev->family >= CHIP_TAHITI) &&
 450             (parser->chunk_const_ib_idx != -1)) {
 451                 ib_chunk = &parser->chunks[parser->chunk_const_ib_idx];
 452                 if (ib_chunk->length_dw > RADEON_IB_VM_MAX_SIZE) {
 453                         DRM_ERROR("cs IB CONST too big: %d\n", ib_chunk->length_dw);
 454                         return -EINVAL;
 455                 }
 456                 r =  radeon_ib_get(rdev, parser->ring, &parser->const_ib,
 457                                    vm, ib_chunk->length_dw * 4);
 458                 if (r) {
 459                         DRM_ERROR("Failed to get const ib !\n");
 460                         return r;
 461                 }
 462                 parser->const_ib.is_const_ib = true;
 463                 parser->const_ib.length_dw = ib_chunk->length_dw;
 464                 /* Copy the packet into the IB */
 465                 if (DRM_COPY_FROM_USER(parser->const_ib.ptr, ib_chunk->user_ptr,
 466                                        ib_chunk->length_dw * 4)) {
 467                         return -EFAULT;
 468                 }
 469                 r = radeon_ring_ib_parse(rdev, parser->ring, &parser->const_ib);
 470                 if (r) {
 471                         return r;
 472                 }
 473         }
 474
 475         ib_chunk = &parser->chunks[parser->chunk_ib_idx];
 476         if (ib_chunk->length_dw > RADEON_IB_VM_MAX_SIZE) {
 477                 DRM_ERROR("cs IB too big: %d\n", ib_chunk->length_dw);
 478                 return -EINVAL;
 479         }
 480         r =  radeon_ib_get(rdev, parser->ring, &parser->ib,
 481                            vm, ib_chunk->length_dw * 4);
 482         if (r) {
 483                 DRM_ERROR("Failed to get ib !\n");
 484                 return r;
 485         }
 486         parser->ib.length_dw = ib_chunk->length_dw;
 487         /* Copy the packet into the IB */
 488         if (DRM_COPY_FROM_USER(parser->ib.ptr, ib_chunk->user_ptr,
 489                                ib_chunk->length_dw * 4)) {
 490                 return -EFAULT;
 491         }
 492         r = radeon_ring_ib_parse(rdev, parser->ring, &parser->ib);
 493         if (r) {
 494                 return r;
 495         }
 496
 497         if (parser->ring == R600_RING_TYPE_UVD_INDEX)
 498                 radeon_uvd_note_usage(rdev);
 499
 500         mutex_lock(&rdev->vm_manager.lock);
 501         mutex_lock(&vm->mutex);
 502         r = radeon_vm_alloc_pt(rdev, vm);
 503         if (r) {
 504                 goto out;
 505         }
 506         r = radeon_bo_vm_update_pte(parser, vm);
 507         if (r) {
 508                 goto out;
 509         }
 510         radeon_cs_sync_rings(parser);
 511         radeon_ib_sync_to(&parser->ib, vm->fence);
 512         radeon_ib_sync_to(&parser->ib, radeon_vm_grab_id(
 513                 rdev, vm, parser->ring));
 514
 515         if ((rdev->family >= CHIP_TAHITI) &&
 516             (parser->chunk_const_ib_idx != -1)) {
 517                 r = radeon_ib_schedule(rdev, &parser->ib, &parser->const_ib);
 518         } else {
 519                 r = radeon_ib_schedule(rdev, &parser->ib, NULL);
 520         }
 521
 522         if (!r) {
 523                 radeon_vm_fence(rdev, vm, parser->ib.fence);
 524         }
 525
 526 out:
 527         radeon_vm_add_to_lru(rdev, vm);
 528         mutex_unlock(&vm->mutex);
 529         mutex_unlock(&rdev->vm_manager.lock);
 530         return r;
 531 }
 532
 533 static int radeon_cs_handle_lockup(struct radeon_device *rdev, int r)
 534 {
 535         if (r == -EDEADLK) {
 536                 r = radeon_gpu_reset(rdev);
 537                 if (!r)
 538                         r = -EAGAIN;
 539         }
 540         return r;
 541 }
 542
 543 int radeon_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
 544 {
 545         struct radeon_device *rdev = dev->dev_private;
 546         struct radeon_cs_parser parser;
 547         int r;
 548
 549         down_read(&rdev->exclusive_lock);
 550         if (!rdev->accel_working) {
 551                 up_read(&rdev->exclusive_lock);
 552                 return -EBUSY;
 553         }
 554         /* initialize parser */
 555         memset(&parser, 0, sizeof(struct radeon_cs_parser));
 556         parser.filp = filp;
 557         parser.rdev = rdev;
 558         parser.dev = rdev->dev;
 559         parser.family = rdev->family;
 560         r = radeon_cs_parser_init(&parser, data);
 561         if (r) {
 562                 DRM_ERROR("Failed to initialize parser !\n");
 563                 radeon_cs_parser_fini(&parser, r, false);
 564                 up_read(&rdev->exclusive_lock);
 565                 r = radeon_cs_handle_lockup(rdev, r);
 566                 return r;
 567         }
 568         r = radeon_cs_parser_relocs(&parser);
 569         if (r) {
 570                 if (r != -ERESTARTSYS)
 571                         DRM_ERROR("Failed to parse relocation %d!\n", r);
 572                 radeon_cs_parser_fini(&parser, r, false);
 573                 up_read(&rdev->exclusive_lock);
 574                 r = radeon_cs_handle_lockup(rdev, r);
 575                 return r;
 576         }
 577
 578         trace_radeon_cs(&parser);
 579
 580         r = radeon_cs_ib_chunk(rdev, &parser);
 581         if (r) {
 582                 goto out;
 583         }
 584         r = radeon_cs_ib_vm_chunk(rdev, &parser);
 585         if (r) {
 586                 goto out;
 587         }
 588 out:
 589         radeon_cs_parser_fini(&parser, r, true);
 590         up_read(&rdev->exclusive_lock);
 591         r = radeon_cs_handle_lockup(rdev, r);
 592         return r;
 593 }
 594
 595 int radeon_cs_finish_pages(struct radeon_cs_parser *p)
 596 {
 597         struct radeon_cs_chunk *ibc = &p->chunks[p->chunk_ib_idx];
 598         int i;
 599         int size = PAGE_SIZE;
 600
 601         for (i = ibc->last_copied_page + 1; i <= ibc->last_page_index; i++) {
 602                 if (i == ibc->last_page_index) {
 603                         size = (ibc->length_dw * 4) % PAGE_SIZE;
 604                         if (size == 0)
 605                                 size = PAGE_SIZE;
 606                 }
 607
 608                 if (DRM_COPY_FROM_USER(p->ib.ptr + (i * (PAGE_SIZE/4)),
 609                                        ibc->user_ptr + (i * PAGE_SIZE),
 610                                        size))
 611                         return -EFAULT;
 612         }
 613         return 0;
 614 }
 615
 616 static int radeon_cs_update_pages(struct radeon_cs_parser *p, int pg_idx)
 617 {
 618         int new_page;
 619         struct radeon_cs_chunk *ibc = &p->chunks[p->chunk_ib_idx];
 620         int i;
 621         int size = PAGE_SIZE;
 622         bool copy1 = (p->rdev && (p->rdev->flags & RADEON_IS_AGP)) ?
 623                 false : true;
 624
 625         for (i = ibc->last_copied_page + 1; i < pg_idx; i++) {
 626                 if (DRM_COPY_FROM_USER(p->ib.ptr + (i * (PAGE_SIZE/4)),
 627                                        ibc->user_ptr + (i * PAGE_SIZE),
 628                                        PAGE_SIZE)) {
 629                         p->parser_error = -EFAULT;
 630                         return 0;
 631                 }
 632         }
 633
 634         if (pg_idx == ibc->last_page_index) {
 635                 size = (ibc->length_dw * 4) % PAGE_SIZE;
 636                 if (size == 0)
 637                         size = PAGE_SIZE;
 638         }
 639
 640         new_page = ibc->kpage_idx[0] < ibc->kpage_idx[1] ? 0 : 1;
 641         if (copy1)
 642                 ibc->kpage[new_page] = p->ib.ptr + (pg_idx * (PAGE_SIZE / 4));
 643
 644         if (DRM_COPY_FROM_USER(ibc->kpage[new_page],
 645                                ibc->user_ptr + (pg_idx * PAGE_SIZE),
 646                                size)) {
 647                 p->parser_error = -EFAULT;
 648                 return 0;
 649         }
 650
 651         /* copy to IB for non single case */
 652         if (!copy1)
 653                 memcpy((void *)(p->ib.ptr+(pg_idx*(PAGE_SIZE/4))), ibc->kpage[new_page], size);
 654
 655         ibc->last_copied_page = pg_idx;
 656         ibc->kpage_idx[new_page] = pg_idx;
 657
 658         return new_page;
 659 }
 660
 661 u32 radeon_get_ib_value(struct radeon_cs_parser *p, int idx)
 662 {
 663         struct radeon_cs_chunk *ibc = &p->chunks[p->chunk_ib_idx];
 664         u32 pg_idx, pg_offset;
 665         u32 idx_value = 0;
 666         int new_page;
 667
 668         pg_idx = (idx * 4) / PAGE_SIZE;
 669         pg_offset = (idx * 4) % PAGE_SIZE;
 670
 671         if (ibc->kpage_idx[0] == pg_idx)
 672                 return ibc->kpage[0][pg_offset/4];
 673         if (ibc->kpage_idx[1] == pg_idx)
 674                 return ibc->kpage[1][pg_offset/4];
 675
 676         new_page = radeon_cs_update_pages(p, pg_idx);
 677         if (new_page < 0) {
 678                 p->parser_error = new_page;
 679                 return 0;
 680         }
 681
 682         idx_value = ibc->kpage[new_page][pg_offset/4];
 683         return idx_value;
 684 }
 685
 686 /**
 687  * radeon_cs_packet_parse() - parse cp packet and point ib index to next packet
 688  * @parser:     parser structure holding parsing context.
 689  * @pkt:        where to store packet information
 690  *
 691  * Assume that chunk_ib_index is properly set. Will return -EINVAL
 692  * if packet is bigger than remaining ib size. or if packets is unknown.
 693  **/
 694 int radeon_cs_packet_parse(struct radeon_cs_parser *p,
 695                            struct radeon_cs_packet *pkt,
 696                            unsigned idx)
 697 {
 698         struct radeon_cs_chunk *ib_chunk = &p->chunks[p->chunk_ib_idx];
 699         struct radeon_device *rdev = p->rdev;
 700         uint32_t header;
 701
 702         if (idx >= ib_chunk->length_dw) {
 703                 DRM_ERROR("Can not parse packet at %d after CS end %d !\n",
 704                           idx, ib_chunk->length_dw);
 705                 return -EINVAL;
 706         }
 707         header = radeon_get_ib_value(p, idx);
 708         pkt->idx = idx;
 709         pkt->type = RADEON_CP_PACKET_GET_TYPE(header);
 710         pkt->count = RADEON_CP_PACKET_GET_COUNT(header);
 711         pkt->one_reg_wr = 0;
 712         switch (pkt->type) {
 713         case RADEON_PACKET_TYPE0:
 714                 if (rdev->family < CHIP_R600) {
 715                         pkt->reg = R100_CP_PACKET0_GET_REG(header);
 716                         pkt->one_reg_wr =
 717                                 RADEON_CP_PACKET0_GET_ONE_REG_WR(header);
 718                 } else
 719                         pkt->reg = R600_CP_PACKET0_GET_REG(header);
 720                 break;
 721         case RADEON_PACKET_TYPE3:
 722                 pkt->opcode = RADEON_CP_PACKET3_GET_OPCODE(header);
 723                 break;
 724         case RADEON_PACKET_TYPE2:
 725                 pkt->count = -1;
 726                 break;
 727         default:
 728                 DRM_ERROR("Unknown packet type %d at %d !\n", pkt->type, idx);
 729                 return -EINVAL;
 730         }
 731         if ((pkt->count + 1 + pkt->idx) >= ib_chunk->length_dw) {
 732                 DRM_ERROR("Packet (%d:%d:%d) end after CS buffer (%d) !\n",
 733                           pkt->idx, pkt->type, pkt->count, ib_chunk->length_dw);
 734                 return -EINVAL;
 735         }
 736         return 0;
 737 }
 738
 739 /**
 740  * radeon_cs_packet_next_is_pkt3_nop() - test if the next packet is P3 NOP
 741  * @p:          structure holding the parser context.
 742  *
 743  * Check if the next packet is NOP relocation packet3.
 744  **/
 745 bool radeon_cs_packet_next_is_pkt3_nop(struct radeon_cs_parser *p)
 746 {
 747         struct radeon_cs_packet p3reloc;
 748         int r;
 749
 750         r = radeon_cs_packet_parse(p, &p3reloc, p->idx);
 751         if (r)
 752                 return false;
 753         if (p3reloc.type != RADEON_PACKET_TYPE3)
 754                 return false;
 755         if (p3reloc.opcode != RADEON_PACKET3_NOP)
 756                 return false;
 757         return true;
 758 }
 759
 760 /**
 761  * radeon_cs_dump_packet() - dump raw packet context
 762  * @p:          structure holding the parser context.
 763  * @pkt:        structure holding the packet.
 764  *
 765  * Used mostly for debugging and error reporting.
 766  **/
 767 void radeon_cs_dump_packet(struct radeon_cs_parser *p,
 768                            struct radeon_cs_packet *pkt)
 769 {
 770         volatile uint32_t *ib;
 771         unsigned i;
 772         unsigned idx;
 773
 774         ib = p->ib.ptr;
 775         idx = pkt->idx;
 776         for (i = 0; i <= (pkt->count + 1); i++, idx++)
 777                 DRM_INFO("ib[%d]=0x%08X\n", idx, ib[idx]);
 778 }
 779
 780 /**
 781  * radeon_cs_packet_next_reloc() - parse next (should be reloc) packet
 782  * @parser:             parser structure holding parsing context.
 783  * @data:               pointer to relocation data
 784  * @offset_start:       starting offset
 785  * @offset_mask:        offset mask (to align start offset on)
 786  * @reloc:              reloc informations
 787  *
 788  * Check if next packet is relocation packet3, do bo validation and compute
 789  * GPU offset using the provided start.
 790  **/
 791 int radeon_cs_packet_next_reloc(struct radeon_cs_parser *p,
 792                                 struct radeon_cs_reloc **cs_reloc,
 793                                 int nomm)
 794 {
 795         struct radeon_cs_chunk *relocs_chunk;
 796         struct radeon_cs_packet p3reloc;
 797         unsigned idx;
 798         int r;
 799
 800         if (p->chunk_relocs_idx == -1) {
 801                 DRM_ERROR("No relocation chunk !\n");
 802                 return -EINVAL;
 803         }
 804         *cs_reloc = NULL;
 805         relocs_chunk = &p->chunks[p->chunk_relocs_idx];
 806         r = radeon_cs_packet_parse(p, &p3reloc, p->idx);
 807         if (r)
 808                 return r;
 809         p->idx += p3reloc.count + 2;
 810         if (p3reloc.type != RADEON_PACKET_TYPE3 ||
 811             p3reloc.opcode != RADEON_PACKET3_NOP) {
 812                 DRM_ERROR("No packet3 for relocation for packet at %d.\n",
 813                           p3reloc.idx);
 814                 radeon_cs_dump_packet(p, &p3reloc);
 815                 return -EINVAL;
 816         }
 817         idx = radeon_get_ib_value(p, p3reloc.idx + 1);
 818         if (idx >= relocs_chunk->length_dw) {
 819                 DRM_ERROR("Relocs at %d after relocations chunk end %d !\n",
 820                           idx, relocs_chunk->length_dw);
 821                 radeon_cs_dump_packet(p, &p3reloc);
 822                 return -EINVAL;
 823         }
 824         /* FIXME: we assume reloc size is 4 dwords */
 825         if (nomm) {
 826                 *cs_reloc = p->relocs;
 827                 (*cs_reloc)->lobj.gpu_offset =
 828                         (u64)relocs_chunk->kdata[idx + 3] << 32;
 829                 (*cs_reloc)->lobj.gpu_offset |= relocs_chunk->kdata[idx + 0];
 830         } else
 831                 *cs_reloc = p->relocs_ptr[(idx / 4)];
 832         return 0;
 833 }