drivers/gpu/drm/i915/i915_gem_stolen.c

   1 /*
   2  * Copyright © 2008-2012 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *    Chris Wilson <chris@chris-wilson.co.uk>
  26  *
  27  */
  28
  29 #include <drm/drmP.h>
  30 #include <drm/i915_drm.h>
  31 #include "i915_drv.h"
  32
  33 /*
  34  * The BIOS typically reserves some of the system's memory for the exclusive
  35  * use of the integrated graphics. This memory is no longer available for
  36  * use by the OS and so the user finds that his system has less memory
  37  * available than he put in. We refer to this memory as stolen.
  38  *
  39  * The BIOS will allocate its framebuffer from the stolen memory. Our
  40  * goal is try to reuse that object for our own fbcon which must always
  41  * be available for panics. Anything else we can reuse the stolen memory
  42  * for is a boon.
  43  */
  44
  45 int i915_gem_stolen_insert_node_in_range(struct drm_i915_private *dev_priv,
  46                                          struct drm_mm_node *node, u64 size,
  47                                          unsigned alignment, u64 start, u64 end)
  48 {
  49         int ret;
  50
  51         if (!drm_mm_initialized(&dev_priv->mm.stolen))
  52                 return -ENODEV;
  53
  54         mutex_lock(&dev_priv->mm.stolen_lock);
  55         ret = drm_mm_insert_node_in_range(&dev_priv->mm.stolen, node,
  56                                           size, alignment, 0,
  57                                           start, end, DRM_MM_INSERT_BEST);
  58         mutex_unlock(&dev_priv->mm.stolen_lock);
  59
  60         return ret;
  61 }
  62
  63 int i915_gem_stolen_insert_node(struct drm_i915_private *dev_priv,
  64                                 struct drm_mm_node *node, u64 size,
  65                                 unsigned alignment)
  66 {
  67         return i915_gem_stolen_insert_node_in_range(dev_priv, node, size,
  68                                                     alignment, 0, U64_MAX);
  69 }
  70
  71 void i915_gem_stolen_remove_node(struct drm_i915_private *dev_priv,
  72                                  struct drm_mm_node *node)
  73 {
  74         mutex_lock(&dev_priv->mm.stolen_lock);
  75         drm_mm_remove_node(node);
  76         mutex_unlock(&dev_priv->mm.stolen_lock);
  77 }
  78
  79 static int i915_adjust_stolen(struct drm_i915_private *dev_priv,
  80                               struct resource *dsm)
  81 {
  82         struct i915_ggtt *ggtt = &dev_priv->ggtt;
  83         struct resource *r;
  84
  85         if (dsm->start == 0 || dsm->end <= dsm->start)
  86                 return -EINVAL;
  87
  88         /*
  89          * TODO: We have yet too encounter the case where the GTT wasn't at the
  90          * end of stolen. With that assumption we could simplify this.
  91          */
  92
  93         /* Make sure we don't clobber the GTT if it's within stolen memory */
  94         if (INTEL_GEN(dev_priv) <= 4 &&
  95             !IS_G33(dev_priv) && !IS_PINEVIEW(dev_priv) && !IS_G4X(dev_priv)) {
  96                 struct resource stolen[2] = {*dsm, *dsm};
  97                 struct resource ggtt_res;
  98                 resource_size_t ggtt_start;
  99
 100                 ggtt_start = I915_READ(PGTBL_CTL);
 101                 if (IS_GEN4(dev_priv))
 102                         ggtt_start = (ggtt_start & PGTBL_ADDRESS_LO_MASK) |
 103                                      (ggtt_start & PGTBL_ADDRESS_HI_MASK) << 28;
 104                 else
 105                         ggtt_start &= PGTBL_ADDRESS_LO_MASK;
 106
 107                 ggtt_res =
 108                         (struct resource) DEFINE_RES_MEM(ggtt_start,
 109                                                          ggtt_total_entries(ggtt) * 4);
 110
 111                 if (ggtt_res.start >= stolen[0].start && ggtt_res.start < stolen[0].end)
 112                         stolen[0].end = ggtt_res.start;
 113                 if (ggtt_res.end > stolen[1].start && ggtt_res.end <= stolen[1].end)
 114                         stolen[1].start = ggtt_res.end;
 115
 116                 /* Pick the larger of the two chunks */
 117                 if (resource_size(&stolen[0]) > resource_size(&stolen[1]))
 118                         *dsm = stolen[0];
 119                 else
 120                         *dsm = stolen[1];
 121
 122                 if (stolen[0].start != stolen[1].start ||
 123                     stolen[0].end != stolen[1].end) {
 124                         DRM_DEBUG_KMS("GTT within stolen memory at %pR\n", &ggtt_res);
 125                         DRM_DEBUG_KMS("Stolen memory adjusted to %pR\n", dsm);
 126                 }
 127         }
 128
 129         /*
 130          * Verify that nothing else uses this physical address. Stolen
 131          * memory should be reserved by the BIOS and hidden from the
 132          * kernel. So if the region is already marked as busy, something
 133          * is seriously wrong.
 134          */
 135         r = devm_request_mem_region(dev_priv->drm.dev, dsm->start,
 136                                     resource_size(dsm),
 137                                     "Graphics Stolen Memory");
 138         if (r == NULL) {
 139                 /*
 140                  * One more attempt but this time requesting region from
 141                  * start + 1, as we have seen that this resolves the region
 142                  * conflict with the PCI Bus.
 143                  * This is a BIOS w/a: Some BIOS wrap stolen in the root
 144                  * PCI bus, but have an off-by-one error. Hence retry the
 145                  * reservation starting from 1 instead of 0.
 146                  * There's also BIOS with off-by-one on the other end.
 147                  */
 148                 r = devm_request_mem_region(dev_priv->drm.dev, dsm->start + 1,
 149                                             resource_size(dsm) - 2,
 150                                             "Graphics Stolen Memory");
 151                 /*
 152                  * GEN3 firmware likes to smash pci bridges into the stolen
 153                  * range. Apparently this works.
 154                  */
 155                 if (r == NULL && !IS_GEN3(dev_priv)) {
 156                         DRM_ERROR("conflict detected with stolen region: %pR\n",
 157                                   dsm);
 158
 159                         return -EBUSY;
 160                 }
 161         }
 162
 163         return 0;
 164 }
 165
 166 void i915_gem_cleanup_stolen(struct drm_device *dev)
 167 {
 168         struct drm_i915_private *dev_priv = to_i915(dev);
 169
 170         if (!drm_mm_initialized(&dev_priv->mm.stolen))
 171                 return;
 172
 173         drm_mm_takedown(&dev_priv->mm.stolen);
 174 }
 175
 176 static void g4x_get_stolen_reserved(struct drm_i915_private *dev_priv,
 177                                     resource_size_t *base, resource_size_t *size)
 178 {
 179         uint32_t reg_val = I915_READ(IS_GM45(dev_priv) ?
 180                                      CTG_STOLEN_RESERVED :
 181                                      ELK_STOLEN_RESERVED);
 182         resource_size_t stolen_top = dev_priv->dsm.end + 1;
 183
 184         if ((reg_val & G4X_STOLEN_RESERVED_ENABLE) == 0) {
 185                 *base = 0;
 186                 *size = 0;
 187                 return;
 188         }
 189
 190         /*
 191          * Whether ILK really reuses the ELK register for this is unclear.
 192          * Let's see if we catch anyone with this supposedly enabled on ILK.
 193          */
 194         WARN(IS_GEN5(dev_priv), "ILK stolen reserved found? 0x%08x\n", reg_val);
 195
 196         *base = (reg_val & G4X_STOLEN_RESERVED_ADDR2_MASK) << 16;
 197
 198         WARN_ON((reg_val & G4X_STOLEN_RESERVED_ADDR1_MASK) < *base);
 199
 200         /* On these platforms, the register doesn't have a size field, so the
 201          * size is the distance between the base and the top of the stolen
 202          * memory. We also have the genuine case where base is zero and there's
 203          * nothing reserved. */
 204         if (*base == 0)
 205                 *size = 0;
 206         else
 207                 *size = stolen_top - *base;
 208 }
 209
 210 static void gen6_get_stolen_reserved(struct drm_i915_private *dev_priv,
 211                                      resource_size_t *base, resource_size_t *size)
 212 {
 213         uint32_t reg_val = I915_READ(GEN6_STOLEN_RESERVED);
 214
 215         if ((reg_val & GEN6_STOLEN_RESERVED_ENABLE) == 0) {
 216                 *base = 0;
 217                 *size = 0;
 218                 return;
 219         }
 220
 221         *base = reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK;
 222
 223         switch (reg_val & GEN6_STOLEN_RESERVED_SIZE_MASK) {
 224         case GEN6_STOLEN_RESERVED_1M:
 225                 *size = 1024 * 1024;
 226                 break;
 227         case GEN6_STOLEN_RESERVED_512K:
 228                 *size = 512 * 1024;
 229                 break;
 230         case GEN6_STOLEN_RESERVED_256K:
 231                 *size = 256 * 1024;
 232                 break;
 233         case GEN6_STOLEN_RESERVED_128K:
 234                 *size = 128 * 1024;
 235                 break;
 236         default:
 237                 *size = 1024 * 1024;
 238                 MISSING_CASE(reg_val & GEN6_STOLEN_RESERVED_SIZE_MASK);
 239         }
 240 }
 241
 242 static void gen7_get_stolen_reserved(struct drm_i915_private *dev_priv,
 243                                      resource_size_t *base, resource_size_t *size)
 244 {
 245         uint32_t reg_val = I915_READ(GEN6_STOLEN_RESERVED);
 246
 247         if ((reg_val & GEN6_STOLEN_RESERVED_ENABLE) == 0) {
 248                 *base = 0;
 249                 *size = 0;
 250                 return;
 251         }
 252
 253         *base = reg_val & GEN7_STOLEN_RESERVED_ADDR_MASK;
 254
 255         switch (reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK) {
 256         case GEN7_STOLEN_RESERVED_1M:
 257                 *size = 1024 * 1024;
 258                 break;
 259         case GEN7_STOLEN_RESERVED_256K:
 260                 *size = 256 * 1024;
 261                 break;
 262         default:
 263                 *size = 1024 * 1024;
 264                 MISSING_CASE(reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK);
 265         }
 266 }
 267
 268 static void chv_get_stolen_reserved(struct drm_i915_private *dev_priv,
 269                                     resource_size_t *base, resource_size_t *size)
 270 {
 271         uint32_t reg_val = I915_READ(GEN6_STOLEN_RESERVED);
 272
 273         if ((reg_val & GEN6_STOLEN_RESERVED_ENABLE) == 0) {
 274                 *base = 0;
 275                 *size = 0;
 276                 return;
 277         }
 278
 279         *base = reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK;
 280
 281         switch (reg_val & GEN8_STOLEN_RESERVED_SIZE_MASK) {
 282         case GEN8_STOLEN_RESERVED_1M:
 283                 *size = 1024 * 1024;
 284                 break;
 285         case GEN8_STOLEN_RESERVED_2M:
 286                 *size = 2 * 1024 * 1024;
 287                 break;
 288         case GEN8_STOLEN_RESERVED_4M:
 289                 *size = 4 * 1024 * 1024;
 290                 break;
 291         case GEN8_STOLEN_RESERVED_8M:
 292                 *size = 8 * 1024 * 1024;
 293                 break;
 294         default:
 295                 *size = 8 * 1024 * 1024;
 296                 MISSING_CASE(reg_val & GEN8_STOLEN_RESERVED_SIZE_MASK);
 297         }
 298 }
 299
 300 static void bdw_get_stolen_reserved(struct drm_i915_private *dev_priv,
 301                                     resource_size_t *base, resource_size_t *size)
 302 {
 303         uint32_t reg_val = I915_READ(GEN6_STOLEN_RESERVED);
 304         resource_size_t stolen_top;
 305
 306         if ((reg_val & GEN6_STOLEN_RESERVED_ENABLE) == 0) {
 307                 *base = 0;
 308                 *size = 0;
 309                 return;
 310         }
 311
 312         stolen_top = dev_priv->dsm.end + 1;
 313
 314         *base = reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK;
 315
 316         /* On these platforms, the register doesn't have a size field, so the
 317          * size is the distance between the base and the top of the stolen
 318          * memory. We also have the genuine case where base is zero and there's
 319          * nothing reserved. */
 320         if (*base == 0)
 321                 *size = 0;
 322         else
 323                 *size = stolen_top - *base;
 324 }
 325
 326 int i915_gem_init_stolen(struct drm_i915_private *dev_priv)
 327 {
 328         resource_size_t reserved_base, stolen_top;
 329         resource_size_t reserved_total, reserved_size;
 330         resource_size_t stolen_usable_start;
 331
 332         mutex_init(&dev_priv->mm.stolen_lock);
 333
 334         if (intel_vgpu_active(dev_priv)) {
 335                 DRM_INFO("iGVT-g active, disabling use of stolen memory\n");
 336                 return 0;
 337         }
 338
 339         if (intel_vtd_active() && INTEL_GEN(dev_priv) < 8) {
 340                 DRM_INFO("DMAR active, disabling use of stolen memory\n");
 341                 return 0;
 342         }
 343
 344         if (resource_size(&intel_graphics_stolen_res) == 0)
 345                 return 0;
 346
 347         dev_priv->dsm = intel_graphics_stolen_res;
 348
 349         if (i915_adjust_stolen(dev_priv, &dev_priv->dsm))
 350                 return 0;
 351
 352         GEM_BUG_ON(dev_priv->dsm.start == 0);
 353         GEM_BUG_ON(dev_priv->dsm.end <= dev_priv->dsm.start);
 354
 355         stolen_top = dev_priv->dsm.end + 1;
 356         reserved_base = 0;
 357         reserved_size = 0;
 358
 359         switch (INTEL_INFO(dev_priv)->gen) {
 360         case 2:
 361         case 3:
 362                 break;
 363         case 4:
 364                 if (!IS_G4X(dev_priv))
 365                         break;
 366                 /* fall through */
 367         case 5:
 368                 g4x_get_stolen_reserved(dev_priv,
 369                                         &reserved_base, &reserved_size);
 370                 break;
 371         case 6:
 372                 gen6_get_stolen_reserved(dev_priv,
 373                                          &reserved_base, &reserved_size);
 374                 break;
 375         case 7:
 376                 gen7_get_stolen_reserved(dev_priv,
 377                                          &reserved_base, &reserved_size);
 378                 break;
 379         default:
 380                 if (IS_LP(dev_priv))
 381                         chv_get_stolen_reserved(dev_priv,
 382                                                 &reserved_base, &reserved_size);
 383                 else
 384                         bdw_get_stolen_reserved(dev_priv,
 385                                                 &reserved_base, &reserved_size);
 386                 break;
 387         }
 388
 389         /* It is possible for the reserved base to be zero, but the register
 390          * field for size doesn't have a zero option. */
 391         if (reserved_base == 0) {
 392                 reserved_size = 0;
 393                 reserved_base = stolen_top;
 394         }
 395
 396         dev_priv->dsm_reserved =
 397                 (struct resource) DEFINE_RES_MEM(reserved_base, reserved_size);
 398
 399         if (!resource_contains(&dev_priv->dsm, &dev_priv->dsm_reserved)) {
 400                 DRM_ERROR("Stolen reserved area %pR outside stolen memory %pR\n",
 401                           &dev_priv->dsm_reserved, &dev_priv->dsm);
 402                 return 0;
 403         }
 404
 405         /* It is possible for the reserved area to end before the end of stolen
 406          * memory, so just consider the start. */
 407         reserved_total = stolen_top - reserved_base;
 408
 409         DRM_DEBUG_KMS("Memory reserved for graphics device: %lluK, usable: %lluK\n",
 410                       (u64)resource_size(&dev_priv->dsm) >> 10,
 411                       ((u64)resource_size(&dev_priv->dsm) - reserved_total) >> 10);
 412
 413         stolen_usable_start = 0;
 414         /* WaSkipStolenMemoryFirstPage:bdw+ */
 415         if (INTEL_GEN(dev_priv) >= 8)
 416                 stolen_usable_start = 4096;
 417
 418         dev_priv->stolen_usable_size =
 419                 resource_size(&dev_priv->dsm) - reserved_total - stolen_usable_start;
 420
 421         /* Basic memrange allocator for stolen space. */
 422         drm_mm_init(&dev_priv->mm.stolen, stolen_usable_start,
 423                     dev_priv->stolen_usable_size);
 424
 425         return 0;
 426 }
 427
 428 static struct sg_table *
 429 i915_pages_create_for_stolen(struct drm_device *dev,
 430                              resource_size_t offset, resource_size_t size)
 431 {
 432         struct drm_i915_private *dev_priv = to_i915(dev);
 433         struct sg_table *st;
 434         struct scatterlist *sg;
 435
 436         GEM_BUG_ON(range_overflows(offset, size, resource_size(&dev_priv->dsm)));
 437
 438         /* We hide that we have no struct page backing our stolen object
 439          * by wrapping the contiguous physical allocation with a fake
 440          * dma mapping in a single scatterlist.
 441          */
 442
 443         st = kmalloc(sizeof(*st), GFP_KERNEL);
 444         if (st == NULL)
 445                 return ERR_PTR(-ENOMEM);
 446
 447         if (sg_alloc_table(st, 1, GFP_KERNEL)) {
 448                 kfree(st);
 449                 return ERR_PTR(-ENOMEM);
 450         }
 451
 452         sg = st->sgl;
 453         sg->offset = 0;
 454         sg->length = size;
 455
 456         sg_dma_address(sg) = (dma_addr_t)dev_priv->dsm.start + offset;
 457         sg_dma_len(sg) = size;
 458
 459         return st;
 460 }
 461
 462 static int i915_gem_object_get_pages_stolen(struct drm_i915_gem_object *obj)
 463 {
 464         struct sg_table *pages =
 465                 i915_pages_create_for_stolen(obj->base.dev,
 466                                              obj->stolen->start,
 467                                              obj->stolen->size);
 468         if (IS_ERR(pages))
 469                 return PTR_ERR(pages);
 470
 471         __i915_gem_object_set_pages(obj, pages, obj->stolen->size);
 472
 473         return 0;
 474 }
 475
 476 static void i915_gem_object_put_pages_stolen(struct drm_i915_gem_object *obj,
 477                                              struct sg_table *pages)
 478 {
 479         /* Should only be called from i915_gem_object_release_stolen() */
 480         sg_free_table(pages);
 481         kfree(pages);
 482 }
 483
 484 static void
 485 i915_gem_object_release_stolen(struct drm_i915_gem_object *obj)
 486 {
 487         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
 488         struct drm_mm_node *stolen = fetch_and_zero(&obj->stolen);
 489
 490         GEM_BUG_ON(!stolen);
 491
 492         __i915_gem_object_unpin_pages(obj);
 493
 494         i915_gem_stolen_remove_node(dev_priv, stolen);
 495         kfree(stolen);
 496 }
 497
 498 static const struct drm_i915_gem_object_ops i915_gem_object_stolen_ops = {
 499         .get_pages = i915_gem_object_get_pages_stolen,
 500         .put_pages = i915_gem_object_put_pages_stolen,
 501         .release = i915_gem_object_release_stolen,
 502 };
 503
 504 static struct drm_i915_gem_object *
 505 _i915_gem_object_create_stolen(struct drm_i915_private *dev_priv,
 506                                struct drm_mm_node *stolen)
 507 {
 508         struct drm_i915_gem_object *obj;
 509         unsigned int cache_level;
 510
 511         obj = i915_gem_object_alloc(dev_priv);
 512         if (obj == NULL)
 513                 return NULL;
 514
 515         drm_gem_private_object_init(&dev_priv->drm, &obj->base, stolen->size);
 516         i915_gem_object_init(obj, &i915_gem_object_stolen_ops);
 517
 518         obj->stolen = stolen;
 519         obj->base.read_domains = I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT;
 520         cache_level = HAS_LLC(dev_priv) ? I915_CACHE_LLC : I915_CACHE_NONE;
 521         i915_gem_object_set_cache_coherency(obj, cache_level);
 522
 523         if (i915_gem_object_pin_pages(obj))
 524                 goto cleanup;
 525
 526         return obj;
 527
 528 cleanup:
 529         i915_gem_object_free(obj);
 530         return NULL;
 531 }
 532
 533 struct drm_i915_gem_object *
 534 i915_gem_object_create_stolen(struct drm_i915_private *dev_priv,
 535                               resource_size_t size)
 536 {
 537         struct drm_i915_gem_object *obj;
 538         struct drm_mm_node *stolen;
 539         int ret;
 540
 541         if (!drm_mm_initialized(&dev_priv->mm.stolen))
 542                 return NULL;
 543
 544         if (size == 0)
 545                 return NULL;
 546
 547         stolen = kzalloc(sizeof(*stolen), GFP_KERNEL);
 548         if (!stolen)
 549                 return NULL;
 550
 551         ret = i915_gem_stolen_insert_node(dev_priv, stolen, size, 4096);
 552         if (ret) {
 553                 kfree(stolen);
 554                 return NULL;
 555         }
 556
 557         obj = _i915_gem_object_create_stolen(dev_priv, stolen);
 558         if (obj)
 559                 return obj;
 560
 561         i915_gem_stolen_remove_node(dev_priv, stolen);
 562         kfree(stolen);
 563         return NULL;
 564 }
 565
 566 struct drm_i915_gem_object *
 567 i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv,
 568                                                resource_size_t stolen_offset,
 569                                                resource_size_t gtt_offset,
 570                                                resource_size_t size)
 571 {
 572         struct i915_ggtt *ggtt = &dev_priv->ggtt;
 573         struct drm_i915_gem_object *obj;
 574         struct drm_mm_node *stolen;
 575         struct i915_vma *vma;
 576         int ret;
 577
 578         if (!drm_mm_initialized(&dev_priv->mm.stolen))
 579                 return NULL;
 580
 581         lockdep_assert_held(&dev_priv->drm.struct_mutex);
 582
 583         DRM_DEBUG_KMS("creating preallocated stolen object: stolen_offset=%pa, gtt_offset=%pa, size=%pa\n",
 584                         &stolen_offset, &gtt_offset, &size);
 585
 586         /* KISS and expect everything to be page-aligned */
 587         if (WARN_ON(size == 0) ||
 588             WARN_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE)) ||
 589             WARN_ON(!IS_ALIGNED(stolen_offset, I915_GTT_MIN_ALIGNMENT)))
 590                 return NULL;
 591
 592         stolen = kzalloc(sizeof(*stolen), GFP_KERNEL);
 593         if (!stolen)
 594                 return NULL;
 595
 596         stolen->start = stolen_offset;
 597         stolen->size = size;
 598         mutex_lock(&dev_priv->mm.stolen_lock);
 599         ret = drm_mm_reserve_node(&dev_priv->mm.stolen, stolen);
 600         mutex_unlock(&dev_priv->mm.stolen_lock);
 601         if (ret) {
 602                 DRM_DEBUG_KMS("failed to allocate stolen space\n");
 603                 kfree(stolen);
 604                 return NULL;
 605         }
 606
 607         obj = _i915_gem_object_create_stolen(dev_priv, stolen);
 608         if (obj == NULL) {
 609                 DRM_DEBUG_KMS("failed to allocate stolen object\n");
 610                 i915_gem_stolen_remove_node(dev_priv, stolen);
 611                 kfree(stolen);
 612                 return NULL;
 613         }
 614
 615         /* Some objects just need physical mem from stolen space */
 616         if (gtt_offset == I915_GTT_OFFSET_NONE)
 617                 return obj;
 618
 619         ret = i915_gem_object_pin_pages(obj);
 620         if (ret)
 621                 goto err;
 622
 623         vma = i915_vma_instance(obj, &ggtt->base, NULL);
 624         if (IS_ERR(vma)) {
 625                 ret = PTR_ERR(vma);
 626                 goto err_pages;
 627         }
 628
 629         /* To simplify the initialisation sequence between KMS and GTT,
 630          * we allow construction of the stolen object prior to
 631          * setting up the GTT space. The actual reservation will occur
 632          * later.
 633          */
 634         ret = i915_gem_gtt_reserve(&ggtt->base, &vma->node,
 635                                    size, gtt_offset, obj->cache_level,
 636                                    0);
 637         if (ret) {
 638                 DRM_DEBUG_KMS("failed to allocate stolen GTT space\n");
 639                 goto err_pages;
 640         }
 641
 642         GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
 643
 644         vma->pages = obj->mm.pages;
 645         vma->flags |= I915_VMA_GLOBAL_BIND;
 646         __i915_vma_set_map_and_fenceable(vma);
 647         list_move_tail(&vma->vm_link, &ggtt->base.inactive_list);
 648
 649         spin_lock(&dev_priv->mm.obj_lock);
 650         list_move_tail(&obj->mm.link, &dev_priv->mm.bound_list);
 651         obj->bind_count++;
 652         spin_unlock(&dev_priv->mm.obj_lock);
 653
 654         return obj;
 655
 656 err_pages:
 657         i915_gem_object_unpin_pages(obj);
 658 err:
 659         i915_gem_object_put(obj);
 660         return NULL;
 661 }