drivers/gpu/drm/radeon/radeon_ring.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <linux/seq_file.h>
  29 #include "drmP.h"
  30 #include "radeon_drm.h"
  31 #include "radeon_reg.h"
  32 #include "radeon.h"
  33 #include "atom.h"
  34
  35 int radeon_debugfs_ib_init(struct radeon_device *rdev);
  36
  37 /*
  38  * IB.
  39  */
  40 int radeon_ib_get(struct radeon_device *rdev, struct radeon_ib **ib)
  41 {
  42         struct radeon_fence *fence;
  43         struct radeon_ib *nib;
  44         unsigned long i;
  45         int r = 0;
  46
  47         *ib = NULL;
  48         r = radeon_fence_create(rdev, &fence);
  49         if (r) {
  50                 DRM_ERROR("failed to create fence for new IB\n");
  51                 return r;
  52         }
  53         mutex_lock(&rdev->ib_pool.mutex);
  54         i = find_first_zero_bit(rdev->ib_pool.alloc_bm, RADEON_IB_POOL_SIZE);
  55         if (i < RADEON_IB_POOL_SIZE) {
  56                 set_bit(i, rdev->ib_pool.alloc_bm);
  57                 rdev->ib_pool.ibs[i].length_dw = 0;
  58                 *ib = &rdev->ib_pool.ibs[i];
  59                 mutex_unlock(&rdev->ib_pool.mutex);
  60                 goto out;
  61         }
  62         if (list_empty(&rdev->ib_pool.scheduled_ibs)) {
  63                 /* we go do nothings here */
  64                 mutex_unlock(&rdev->ib_pool.mutex);
  65                 DRM_ERROR("all IB allocated none scheduled.\n");
  66                 r = -EINVAL;
  67                 goto out;
  68         }
  69         /* get the first ib on the scheduled list */
  70         nib = list_entry(rdev->ib_pool.scheduled_ibs.next,
  71                          struct radeon_ib, list);
  72         if (nib->fence == NULL) {
  73                 /* we go do nothings here */
  74                 mutex_unlock(&rdev->ib_pool.mutex);
  75                 DRM_ERROR("IB %lu scheduled without a fence.\n", nib->idx);
  76                 r = -EINVAL;
  77                 goto out;
  78         }
  79         mutex_unlock(&rdev->ib_pool.mutex);
  80
  81         r = radeon_fence_wait(nib->fence, false);
  82         if (r) {
  83                 DRM_ERROR("radeon: IB(%lu:0x%016lX:%u)\n", nib->idx,
  84                           (unsigned long)nib->gpu_addr, nib->length_dw);
  85                 DRM_ERROR("radeon: GPU lockup detected, fail to get a IB\n");
  86                 goto out;
  87         }
  88         radeon_fence_unref(&nib->fence);
  89
  90         nib->length_dw = 0;
  91
  92         /* scheduled list is accessed here */
  93         mutex_lock(&rdev->ib_pool.mutex);
  94         list_del(&nib->list);
  95         INIT_LIST_HEAD(&nib->list);
  96         mutex_unlock(&rdev->ib_pool.mutex);
  97
  98         *ib = nib;
  99 out:
 100         if (r) {
 101                 radeon_fence_unref(&fence);
 102         } else {
 103                 (*ib)->fence = fence;
 104         }
 105         return r;
 106 }
 107
 108 void radeon_ib_free(struct radeon_device *rdev, struct radeon_ib **ib)
 109 {
 110         struct radeon_ib *tmp = *ib;
 111
 112         *ib = NULL;
 113         if (tmp == NULL) {
 114                 return;
 115         }
 116         mutex_lock(&rdev->ib_pool.mutex);
 117         if (!list_empty(&tmp->list) && !radeon_fence_signaled(tmp->fence)) {
 118                 /* IB is scheduled & not signaled don't do anythings */
 119                 mutex_unlock(&rdev->ib_pool.mutex);
 120                 return;
 121         }
 122         list_del(&tmp->list);
 123         INIT_LIST_HEAD(&tmp->list);
 124         if (tmp->fence)
 125                 radeon_fence_unref(&tmp->fence);
 126
 127         tmp->length_dw = 0;
 128         clear_bit(tmp->idx, rdev->ib_pool.alloc_bm);
 129         mutex_unlock(&rdev->ib_pool.mutex);
 130 }
 131
 132 int radeon_ib_schedule(struct radeon_device *rdev, struct radeon_ib *ib)
 133 {
 134         int r = 0;
 135
 136         if (!ib->length_dw || !rdev->cp.ready) {
 137                 /* TODO: Nothings in the ib we should report. */
 138                 DRM_ERROR("radeon: couldn't schedule IB(%lu).\n", ib->idx);
 139                 return -EINVAL;
 140         }
 141
 142         /* 64 dwords should be enough for fence too */
 143         r = radeon_ring_lock(rdev, 64);
 144         if (r) {
 145                 DRM_ERROR("radeon: scheduling IB failled (%d).\n", r);
 146                 return r;
 147         }
 148         radeon_ring_ib_execute(rdev, ib);
 149         radeon_fence_emit(rdev, ib->fence);
 150         mutex_lock(&rdev->ib_pool.mutex);
 151         list_add_tail(&ib->list, &rdev->ib_pool.scheduled_ibs);
 152         mutex_unlock(&rdev->ib_pool.mutex);
 153         radeon_ring_unlock_commit(rdev);
 154         return 0;
 155 }
 156
 157 int radeon_ib_pool_init(struct radeon_device *rdev)
 158 {
 159         void *ptr;
 160         uint64_t gpu_addr;
 161         int i;
 162         int r = 0;
 163
 164         if (rdev->ib_pool.robj)
 165                 return 0;
 166         /* Allocate 1M object buffer */
 167         INIT_LIST_HEAD(&rdev->ib_pool.scheduled_ibs);
 168         r = radeon_object_create(rdev, NULL,  RADEON_IB_POOL_SIZE*64*1024,
 169                                  true, RADEON_GEM_DOMAIN_GTT,
 170                                  false, &rdev->ib_pool.robj);
 171         if (r) {
 172                 DRM_ERROR("radeon: failed to ib pool (%d).\n", r);
 173                 return r;
 174         }
 175         r = radeon_object_pin(rdev->ib_pool.robj, RADEON_GEM_DOMAIN_GTT, &gpu_addr);
 176         if (r) {
 177                 DRM_ERROR("radeon: failed to pin ib pool (%d).\n", r);
 178                 return r;
 179         }
 180         r = radeon_object_kmap(rdev->ib_pool.robj, &ptr);
 181         if (r) {
 182                 DRM_ERROR("radeon: failed to map ib poll (%d).\n", r);
 183                 return r;
 184         }
 185         for (i = 0; i < RADEON_IB_POOL_SIZE; i++) {
 186                 unsigned offset;
 187
 188                 offset = i * 64 * 1024;
 189                 rdev->ib_pool.ibs[i].gpu_addr = gpu_addr + offset;
 190                 rdev->ib_pool.ibs[i].ptr = ptr + offset;
 191                 rdev->ib_pool.ibs[i].idx = i;
 192                 rdev->ib_pool.ibs[i].length_dw = 0;
 193                 INIT_LIST_HEAD(&rdev->ib_pool.ibs[i].list);
 194         }
 195         bitmap_zero(rdev->ib_pool.alloc_bm, RADEON_IB_POOL_SIZE);
 196         rdev->ib_pool.ready = true;
 197         DRM_INFO("radeon: ib pool ready.\n");
 198         if (radeon_debugfs_ib_init(rdev)) {
 199                 DRM_ERROR("Failed to register debugfs file for IB !\n");
 200         }
 201         return r;
 202 }
 203
 204 void radeon_ib_pool_fini(struct radeon_device *rdev)
 205 {
 206         if (!rdev->ib_pool.ready) {
 207                 return;
 208         }
 209         mutex_lock(&rdev->ib_pool.mutex);
 210         bitmap_zero(rdev->ib_pool.alloc_bm, RADEON_IB_POOL_SIZE);
 211         if (rdev->ib_pool.robj) {
 212                 radeon_object_kunmap(rdev->ib_pool.robj);
 213                 radeon_object_unref(&rdev->ib_pool.robj);
 214                 rdev->ib_pool.robj = NULL;
 215         }
 216         mutex_unlock(&rdev->ib_pool.mutex);
 217 }
 218
 219
 220 /*
 221  * Ring.
 222  */
 223 void radeon_ring_free_size(struct radeon_device *rdev)
 224 {
 225         if (rdev->family >= CHIP_R600)
 226                 rdev->cp.rptr = RREG32(R600_CP_RB_RPTR);
 227         else
 228                 rdev->cp.rptr = RREG32(RADEON_CP_RB_RPTR);
 229         /* This works because ring_size is a power of 2 */
 230         rdev->cp.ring_free_dw = (rdev->cp.rptr + (rdev->cp.ring_size / 4));
 231         rdev->cp.ring_free_dw -= rdev->cp.wptr;
 232         rdev->cp.ring_free_dw &= rdev->cp.ptr_mask;
 233         if (!rdev->cp.ring_free_dw) {
 234                 rdev->cp.ring_free_dw = rdev->cp.ring_size / 4;
 235         }
 236 }
 237
 238 int radeon_ring_lock(struct radeon_device *rdev, unsigned ndw)
 239 {
 240         int r;
 241
 242         /* Align requested size with padding so unlock_commit can
 243          * pad safely */
 244         ndw = (ndw + rdev->cp.align_mask) & ~rdev->cp.align_mask;
 245         mutex_lock(&rdev->cp.mutex);
 246         while (ndw > (rdev->cp.ring_free_dw - 1)) {
 247                 radeon_ring_free_size(rdev);
 248                 if (ndw < rdev->cp.ring_free_dw) {
 249                         break;
 250                 }
 251                 r = radeon_fence_wait_next(rdev);
 252                 if (r) {
 253                         mutex_unlock(&rdev->cp.mutex);
 254                         return r;
 255                 }
 256         }
 257         rdev->cp.count_dw = ndw;
 258         rdev->cp.wptr_old = rdev->cp.wptr;
 259         return 0;
 260 }
 261
 262 void radeon_ring_unlock_commit(struct radeon_device *rdev)
 263 {
 264         unsigned count_dw_pad;
 265         unsigned i;
 266
 267         /* We pad to match fetch size */
 268         count_dw_pad = (rdev->cp.align_mask + 1) -
 269                        (rdev->cp.wptr & rdev->cp.align_mask);
 270         for (i = 0; i < count_dw_pad; i++) {
 271                 radeon_ring_write(rdev, 2 << 30);
 272         }
 273         DRM_MEMORYBARRIER();
 274         radeon_cp_commit(rdev);
 275         mutex_unlock(&rdev->cp.mutex);
 276 }
 277
 278 void radeon_ring_unlock_undo(struct radeon_device *rdev)
 279 {
 280         rdev->cp.wptr = rdev->cp.wptr_old;
 281         mutex_unlock(&rdev->cp.mutex);
 282 }
 283
 284 int radeon_ring_init(struct radeon_device *rdev, unsigned ring_size)
 285 {
 286         int r;
 287
 288         rdev->cp.ring_size = ring_size;
 289         /* Allocate ring buffer */
 290         if (rdev->cp.ring_obj == NULL) {
 291                 r = radeon_object_create(rdev, NULL, rdev->cp.ring_size,
 292                                          true,
 293                                          RADEON_GEM_DOMAIN_GTT,
 294                                          false,
 295                                          &rdev->cp.ring_obj);
 296                 if (r) {
 297                         DRM_ERROR("radeon: failed to create ring buffer (%d).\n", r);
 298                         mutex_unlock(&rdev->cp.mutex);
 299                         return r;
 300                 }
 301                 r = radeon_object_pin(rdev->cp.ring_obj,
 302                                       RADEON_GEM_DOMAIN_GTT,
 303                                       &rdev->cp.gpu_addr);
 304                 if (r) {
 305                         DRM_ERROR("radeon: failed to pin ring buffer (%d).\n", r);
 306                         mutex_unlock(&rdev->cp.mutex);
 307                         return r;
 308                 }
 309                 r = radeon_object_kmap(rdev->cp.ring_obj,
 310                                        (void **)&rdev->cp.ring);
 311                 if (r) {
 312                         DRM_ERROR("radeon: failed to map ring buffer (%d).\n", r);
 313                         mutex_unlock(&rdev->cp.mutex);
 314                         return r;
 315                 }
 316         }
 317         rdev->cp.ptr_mask = (rdev->cp.ring_size / 4) - 1;
 318         rdev->cp.ring_free_dw = rdev->cp.ring_size / 4;
 319         return 0;
 320 }
 321
 322 void radeon_ring_fini(struct radeon_device *rdev)
 323 {
 324         mutex_lock(&rdev->cp.mutex);
 325         if (rdev->cp.ring_obj) {
 326                 radeon_object_kunmap(rdev->cp.ring_obj);
 327                 radeon_object_unpin(rdev->cp.ring_obj);
 328                 radeon_object_unref(&rdev->cp.ring_obj);
 329                 rdev->cp.ring = NULL;
 330                 rdev->cp.ring_obj = NULL;
 331         }
 332         mutex_unlock(&rdev->cp.mutex);
 333 }
 334
 335
 336 /*
 337  * Debugfs info
 338  */
 339 #if defined(CONFIG_DEBUG_FS)
 340 static int radeon_debugfs_ib_info(struct seq_file *m, void *data)
 341 {
 342         struct drm_info_node *node = (struct drm_info_node *) m->private;
 343         struct radeon_ib *ib = node->info_ent->data;
 344         unsigned i;
 345
 346         if (ib == NULL) {
 347                 return 0;
 348         }
 349         seq_printf(m, "IB %04lu\n", ib->idx);
 350         seq_printf(m, "IB fence %p\n", ib->fence);
 351         seq_printf(m, "IB size %05u dwords\n", ib->length_dw);
 352         for (i = 0; i < ib->length_dw; i++) {
 353                 seq_printf(m, "[%05u]=0x%08X\n", i, ib->ptr[i]);
 354         }
 355         return 0;
 356 }
 357
 358 static struct drm_info_list radeon_debugfs_ib_list[RADEON_IB_POOL_SIZE];
 359 static char radeon_debugfs_ib_names[RADEON_IB_POOL_SIZE][32];
 360 #endif
 361
 362 int radeon_debugfs_ib_init(struct radeon_device *rdev)
 363 {
 364 #if defined(CONFIG_DEBUG_FS)
 365         unsigned i;
 366
 367         for (i = 0; i < RADEON_IB_POOL_SIZE; i++) {
 368                 sprintf(radeon_debugfs_ib_names[i], "radeon_ib_%04u", i);
 369                 radeon_debugfs_ib_list[i].name = radeon_debugfs_ib_names[i];
 370                 radeon_debugfs_ib_list[i].show = &radeon_debugfs_ib_info;
 371                 radeon_debugfs_ib_list[i].driver_features = 0;
 372                 radeon_debugfs_ib_list[i].data = &rdev->ib_pool.ibs[i];
 373         }
 374         return radeon_debugfs_add_files(rdev, radeon_debugfs_ib_list,
 375                                         RADEON_IB_POOL_SIZE);
 376 #else
 377         return 0;
 378 #endif
 379 }