drivers/gpu/drm/i915/i915_gem_stolen.c

   1 /*
   2  * Copyright © 2008-2012 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *    Chris Wilson <chris@chris-wilson.co.uk>
  26  *
  27  */
  28
  29 #include <drm/drmP.h>
  30 #include <drm/i915_drm.h>
  31 #include "i915_drv.h"
  32
  33 /*
  34  * The BIOS typically reserves some of the system's memory for the exclusive
  35  * use of the integrated graphics. This memory is no longer available for
  36  * use by the OS and so the user finds that his system has less memory
  37  * available than he put in. We refer to this memory as stolen.
  38  *
  39  * The BIOS will allocate its framebuffer from the stolen memory. Our
  40  * goal is try to reuse that object for our own fbcon which must always
  41  * be available for panics. Anything else we can reuse the stolen memory
  42  * for is a boon.
  43  */
  44
  45 int i915_gem_stolen_insert_node_in_range(struct drm_i915_private *dev_priv,
  46                                          struct drm_mm_node *node, u64 size,
  47                                          unsigned alignment, u64 start, u64 end)
  48 {
  49         int ret;
  50
  51         if (!drm_mm_initialized(&dev_priv->mm.stolen))
  52                 return -ENODEV;
  53
  54         /* WaSkipStolenMemoryFirstPage:bdw+ */
  55         if (INTEL_GEN(dev_priv) >= 8 && start < 4096)
  56                 start = 4096;
  57
  58         mutex_lock(&dev_priv->mm.stolen_lock);
  59         ret = drm_mm_insert_node_in_range(&dev_priv->mm.stolen, node,
  60                                           size, alignment, 0,
  61                                           start, end, DRM_MM_INSERT_BEST);
  62         mutex_unlock(&dev_priv->mm.stolen_lock);
  63
  64         return ret;
  65 }
  66
  67 int i915_gem_stolen_insert_node(struct drm_i915_private *dev_priv,
  68                                 struct drm_mm_node *node, u64 size,
  69                                 unsigned alignment)
  70 {
  71         return i915_gem_stolen_insert_node_in_range(dev_priv, node, size,
  72                                                     alignment, 0, U64_MAX);
  73 }
  74
  75 void i915_gem_stolen_remove_node(struct drm_i915_private *dev_priv,
  76                                  struct drm_mm_node *node)
  77 {
  78         mutex_lock(&dev_priv->mm.stolen_lock);
  79         drm_mm_remove_node(node);
  80         mutex_unlock(&dev_priv->mm.stolen_lock);
  81 }
  82
  83 static int i915_adjust_stolen(struct drm_i915_private *dev_priv,
  84                               struct resource *dsm)
  85 {
  86         struct i915_ggtt *ggtt = &dev_priv->ggtt;
  87         struct resource *r;
  88
  89         if (dsm->start == 0 || dsm->end <= dsm->start)
  90                 return -EINVAL;
  91
  92         /*
  93          * TODO: We have yet too encounter the case where the GTT wasn't at the
  94          * end of stolen. With that assumption we could simplify this.
  95          */
  96
  97         /* Make sure we don't clobber the GTT if it's within stolen memory */
  98         if (INTEL_GEN(dev_priv) <= 4 &&
  99             !IS_G33(dev_priv) && !IS_PINEVIEW(dev_priv) && !IS_G4X(dev_priv)) {
 100                 struct resource stolen[2] = {*dsm, *dsm};
 101                 struct resource ggtt_res;
 102                 resource_size_t ggtt_start;
 103
 104                 ggtt_start = I915_READ(PGTBL_CTL);
 105                 if (IS_GEN4(dev_priv))
 106                         ggtt_start = (ggtt_start & PGTBL_ADDRESS_LO_MASK) |
 107                                      (ggtt_start & PGTBL_ADDRESS_HI_MASK) << 28;
 108                 else
 109                         ggtt_start &= PGTBL_ADDRESS_LO_MASK;
 110
 111                 ggtt_res =
 112                         (struct resource) DEFINE_RES_MEM(ggtt_start,
 113                                                          ggtt_total_entries(ggtt) * 4);
 114
 115                 if (ggtt_res.start >= stolen[0].start && ggtt_res.start < stolen[0].end)
 116                         stolen[0].end = ggtt_res.start;
 117                 if (ggtt_res.end > stolen[1].start && ggtt_res.end <= stolen[1].end)
 118                         stolen[1].start = ggtt_res.end;
 119
 120                 /* Pick the larger of the two chunks */
 121                 if (resource_size(&stolen[0]) > resource_size(&stolen[1]))
 122                         *dsm = stolen[0];
 123                 else
 124                         *dsm = stolen[1];
 125
 126                 if (stolen[0].start != stolen[1].start ||
 127                     stolen[0].end != stolen[1].end) {
 128                         DRM_DEBUG_DRIVER("GTT within stolen memory at %pR\n", &ggtt_res);
 129                         DRM_DEBUG_DRIVER("Stolen memory adjusted to %pR\n", dsm);
 130                 }
 131         }
 132
 133         /*
 134          * Verify that nothing else uses this physical address. Stolen
 135          * memory should be reserved by the BIOS and hidden from the
 136          * kernel. So if the region is already marked as busy, something
 137          * is seriously wrong.
 138          */
 139         r = devm_request_mem_region(dev_priv->drm.dev, dsm->start,
 140                                     resource_size(dsm),
 141                                     "Graphics Stolen Memory");
 142         if (r == NULL) {
 143                 /*
 144                  * One more attempt but this time requesting region from
 145                  * start + 1, as we have seen that this resolves the region
 146                  * conflict with the PCI Bus.
 147                  * This is a BIOS w/a: Some BIOS wrap stolen in the root
 148                  * PCI bus, but have an off-by-one error. Hence retry the
 149                  * reservation starting from 1 instead of 0.
 150                  * There's also BIOS with off-by-one on the other end.
 151                  */
 152                 r = devm_request_mem_region(dev_priv->drm.dev, dsm->start + 1,
 153                                             resource_size(dsm) - 2,
 154                                             "Graphics Stolen Memory");
 155                 /*
 156                  * GEN3 firmware likes to smash pci bridges into the stolen
 157                  * range. Apparently this works.
 158                  */
 159                 if (r == NULL && !IS_GEN3(dev_priv)) {
 160                         DRM_ERROR("conflict detected with stolen region: %pR\n",
 161                                   dsm);
 162
 163                         return -EBUSY;
 164                 }
 165         }
 166
 167         return 0;
 168 }
 169
 170 void i915_gem_cleanup_stolen(struct drm_device *dev)
 171 {
 172         struct drm_i915_private *dev_priv = to_i915(dev);
 173
 174         if (!drm_mm_initialized(&dev_priv->mm.stolen))
 175                 return;
 176
 177         drm_mm_takedown(&dev_priv->mm.stolen);
 178 }
 179
 180 static void g4x_get_stolen_reserved(struct drm_i915_private *dev_priv,
 181                                     resource_size_t *base,
 182                                     resource_size_t *size)
 183 {
 184         u32 reg_val = I915_READ(IS_GM45(dev_priv) ?
 185                                 CTG_STOLEN_RESERVED :
 186                                 ELK_STOLEN_RESERVED);
 187         resource_size_t stolen_top = dev_priv->dsm.end + 1;
 188
 189         DRM_DEBUG_DRIVER("%s_STOLEN_RESERVED = %08x\n",
 190                          IS_GM45(dev_priv) ? "CTG" : "ELK", reg_val);
 191
 192         if ((reg_val & G4X_STOLEN_RESERVED_ENABLE) == 0)
 193                 return;
 194
 195         /*
 196          * Whether ILK really reuses the ELK register for this is unclear.
 197          * Let's see if we catch anyone with this supposedly enabled on ILK.
 198          */
 199         WARN(IS_GEN5(dev_priv), "ILK stolen reserved found? 0x%08x\n", reg_val);
 200
 201         if (!(reg_val & G4X_STOLEN_RESERVED_ADDR2_MASK))
 202                 return;
 203
 204         *base = (reg_val & G4X_STOLEN_RESERVED_ADDR2_MASK) << 16;
 205         WARN_ON((reg_val & G4X_STOLEN_RESERVED_ADDR1_MASK) < *base);
 206
 207         *size = stolen_top - *base;
 208 }
 209
 210 static void gen6_get_stolen_reserved(struct drm_i915_private *dev_priv,
 211                                      resource_size_t *base,
 212                                      resource_size_t *size)
 213 {
 214         u32 reg_val = I915_READ(GEN6_STOLEN_RESERVED);
 215
 216         DRM_DEBUG_DRIVER("GEN6_STOLEN_RESERVED = %08x\n", reg_val);
 217
 218         if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE))
 219                 return;
 220
 221         *base = reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK;
 222
 223         switch (reg_val & GEN6_STOLEN_RESERVED_SIZE_MASK) {
 224         case GEN6_STOLEN_RESERVED_1M:
 225                 *size = 1024 * 1024;
 226                 break;
 227         case GEN6_STOLEN_RESERVED_512K:
 228                 *size = 512 * 1024;
 229                 break;
 230         case GEN6_STOLEN_RESERVED_256K:
 231                 *size = 256 * 1024;
 232                 break;
 233         case GEN6_STOLEN_RESERVED_128K:
 234                 *size = 128 * 1024;
 235                 break;
 236         default:
 237                 *size = 1024 * 1024;
 238                 MISSING_CASE(reg_val & GEN6_STOLEN_RESERVED_SIZE_MASK);
 239         }
 240 }
 241
 242 static void vlv_get_stolen_reserved(struct drm_i915_private *dev_priv,
 243                                     resource_size_t *base,
 244                                     resource_size_t *size)
 245 {
 246         u32 reg_val = I915_READ(GEN6_STOLEN_RESERVED);
 247         resource_size_t stolen_top = dev_priv->dsm.end + 1;
 248
 249         DRM_DEBUG_DRIVER("GEN6_STOLEN_RESERVED = %08x\n", reg_val);
 250
 251         if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE))
 252                 return;
 253
 254         switch (reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK) {
 255         default:
 256                 MISSING_CASE(reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK);
 257                 /* fall through */
 258         case GEN7_STOLEN_RESERVED_1M:
 259                 *size = 1024 * 1024;
 260                 break;
 261         }
 262
 263         /*
 264          * On vlv, the ADDR_MASK portion is left as 0 and HW deduces the
 265          * reserved location as (top - size).
 266          */
 267         *base = stolen_top - *size;
 268 }
 269
 270 static void gen7_get_stolen_reserved(struct drm_i915_private *dev_priv,
 271                                      resource_size_t *base,
 272                                      resource_size_t *size)
 273 {
 274         u32 reg_val = I915_READ(GEN6_STOLEN_RESERVED);
 275
 276         DRM_DEBUG_DRIVER("GEN6_STOLEN_RESERVED = %08x\n", reg_val);
 277
 278         if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE))
 279                 return;
 280
 281         *base = reg_val & GEN7_STOLEN_RESERVED_ADDR_MASK;
 282
 283         switch (reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK) {
 284         case GEN7_STOLEN_RESERVED_1M:
 285                 *size = 1024 * 1024;
 286                 break;
 287         case GEN7_STOLEN_RESERVED_256K:
 288                 *size = 256 * 1024;
 289                 break;
 290         default:
 291                 *size = 1024 * 1024;
 292                 MISSING_CASE(reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK);
 293         }
 294 }
 295
 296 static void chv_get_stolen_reserved(struct drm_i915_private *dev_priv,
 297                                     resource_size_t *base,
 298                                     resource_size_t *size)
 299 {
 300         u32 reg_val = I915_READ(GEN6_STOLEN_RESERVED);
 301
 302         DRM_DEBUG_DRIVER("GEN6_STOLEN_RESERVED = %08x\n", reg_val);
 303
 304         if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE))
 305                 return;
 306
 307         *base = reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK;
 308
 309         switch (reg_val & GEN8_STOLEN_RESERVED_SIZE_MASK) {
 310         case GEN8_STOLEN_RESERVED_1M:
 311                 *size = 1024 * 1024;
 312                 break;
 313         case GEN8_STOLEN_RESERVED_2M:
 314                 *size = 2 * 1024 * 1024;
 315                 break;
 316         case GEN8_STOLEN_RESERVED_4M:
 317                 *size = 4 * 1024 * 1024;
 318                 break;
 319         case GEN8_STOLEN_RESERVED_8M:
 320                 *size = 8 * 1024 * 1024;
 321                 break;
 322         default:
 323                 *size = 8 * 1024 * 1024;
 324                 MISSING_CASE(reg_val & GEN8_STOLEN_RESERVED_SIZE_MASK);
 325         }
 326 }
 327
 328 static void bdw_get_stolen_reserved(struct drm_i915_private *dev_priv,
 329                                     resource_size_t *base,
 330                                     resource_size_t *size)
 331 {
 332         u32 reg_val = I915_READ(GEN6_STOLEN_RESERVED);
 333         resource_size_t stolen_top = dev_priv->dsm.end + 1;
 334
 335         DRM_DEBUG_DRIVER("GEN6_STOLEN_RESERVED = %08x\n", reg_val);
 336
 337         if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE))
 338                 return;
 339
 340         if (!(reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK))
 341                 return;
 342
 343         *base = reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK;
 344         *size = stolen_top - *base;
 345 }
 346
 347 static void icl_get_stolen_reserved(struct drm_i915_private *dev_priv,
 348                                     resource_size_t *base,
 349                                     resource_size_t *size)
 350 {
 351         u64 reg_val = I915_READ64(GEN6_STOLEN_RESERVED);
 352
 353         DRM_DEBUG_DRIVER("GEN6_STOLEN_RESERVED = 0x%016llx\n", reg_val);
 354
 355         *base = reg_val & GEN11_STOLEN_RESERVED_ADDR_MASK;
 356
 357         switch (reg_val & GEN8_STOLEN_RESERVED_SIZE_MASK) {
 358         case GEN8_STOLEN_RESERVED_1M:
 359                 *size = 1024 * 1024;
 360                 break;
 361         case GEN8_STOLEN_RESERVED_2M:
 362                 *size = 2 * 1024 * 1024;
 363                 break;
 364         case GEN8_STOLEN_RESERVED_4M:
 365                 *size = 4 * 1024 * 1024;
 366                 break;
 367         case GEN8_STOLEN_RESERVED_8M:
 368                 *size = 8 * 1024 * 1024;
 369                 break;
 370         default:
 371                 *size = 8 * 1024 * 1024;
 372                 MISSING_CASE(reg_val & GEN8_STOLEN_RESERVED_SIZE_MASK);
 373         }
 374 }
 375
 376 int i915_gem_init_stolen(struct drm_i915_private *dev_priv)
 377 {
 378         resource_size_t reserved_base, stolen_top;
 379         resource_size_t reserved_total, reserved_size;
 380
 381         mutex_init(&dev_priv->mm.stolen_lock);
 382
 383         if (intel_vgpu_active(dev_priv)) {
 384                 DRM_INFO("iGVT-g active, disabling use of stolen memory\n");
 385                 return 0;
 386         }
 387
 388         if (intel_vtd_active() && INTEL_GEN(dev_priv) < 8) {
 389                 DRM_INFO("DMAR active, disabling use of stolen memory\n");
 390                 return 0;
 391         }
 392
 393         if (resource_size(&intel_graphics_stolen_res) == 0)
 394                 return 0;
 395
 396         dev_priv->dsm = intel_graphics_stolen_res;
 397
 398         if (i915_adjust_stolen(dev_priv, &dev_priv->dsm))
 399                 return 0;
 400
 401         GEM_BUG_ON(dev_priv->dsm.start == 0);
 402         GEM_BUG_ON(dev_priv->dsm.end <= dev_priv->dsm.start);
 403
 404         stolen_top = dev_priv->dsm.end + 1;
 405         reserved_base = stolen_top;
 406         reserved_size = 0;
 407
 408         switch (INTEL_GEN(dev_priv)) {
 409         case 2:
 410         case 3:
 411                 break;
 412         case 4:
 413                 if (!IS_G4X(dev_priv))
 414                         break;
 415                 /* fall through */
 416         case 5:
 417                 g4x_get_stolen_reserved(dev_priv,
 418                                         &reserved_base, &reserved_size);
 419                 break;
 420         case 6:
 421                 gen6_get_stolen_reserved(dev_priv,
 422                                          &reserved_base, &reserved_size);
 423                 break;
 424         case 7:
 425                 if (IS_VALLEYVIEW(dev_priv))
 426                         vlv_get_stolen_reserved(dev_priv,
 427                                                 &reserved_base, &reserved_size);
 428                 else
 429                         gen7_get_stolen_reserved(dev_priv,
 430                                                  &reserved_base, &reserved_size);
 431                 break;
 432         case 8:
 433         case 9:
 434         case 10:
 435                 if (IS_LP(dev_priv))
 436                         chv_get_stolen_reserved(dev_priv,
 437                                                 &reserved_base, &reserved_size);
 438                 else
 439                         bdw_get_stolen_reserved(dev_priv,
 440                                                 &reserved_base, &reserved_size);
 441                 break;
 442         case 11:
 443         default:
 444                 icl_get_stolen_reserved(dev_priv, &reserved_base,
 445                                         &reserved_size);
 446                 break;
 447         }
 448
 449         /*
 450          * Our expectation is that the reserved space is at the top of the
 451          * stolen region and *never* at the bottom. If we see !reserved_base,
 452          * it likely means we failed to read the registers correctly.
 453          */
 454         if (!reserved_base) {
 455                 DRM_ERROR("inconsistent reservation %pa + %pa; ignoring\n",
 456                           &reserved_base, &reserved_size);
 457                 reserved_base = stolen_top;
 458                 reserved_size = 0;
 459         }
 460
 461         dev_priv->dsm_reserved =
 462                 (struct resource) DEFINE_RES_MEM(reserved_base, reserved_size);
 463
 464         if (!resource_contains(&dev_priv->dsm, &dev_priv->dsm_reserved)) {
 465                 DRM_ERROR("Stolen reserved area %pR outside stolen memory %pR\n",
 466                           &dev_priv->dsm_reserved, &dev_priv->dsm);
 467                 return 0;
 468         }
 469
 470         /* It is possible for the reserved area to end before the end of stolen
 471          * memory, so just consider the start. */
 472         reserved_total = stolen_top - reserved_base;
 473
 474         DRM_DEBUG_DRIVER("Memory reserved for graphics device: %lluK, usable: %lluK\n",
 475                          (u64)resource_size(&dev_priv->dsm) >> 10,
 476                          ((u64)resource_size(&dev_priv->dsm) - reserved_total) >> 10);
 477
 478         dev_priv->stolen_usable_size =
 479                 resource_size(&dev_priv->dsm) - reserved_total;
 480
 481         /* Basic memrange allocator for stolen space. */
 482         drm_mm_init(&dev_priv->mm.stolen, 0, dev_priv->stolen_usable_size);
 483
 484         return 0;
 485 }
 486
 487 static struct sg_table *
 488 i915_pages_create_for_stolen(struct drm_device *dev,
 489                              resource_size_t offset, resource_size_t size)
 490 {
 491         struct drm_i915_private *dev_priv = to_i915(dev);
 492         struct sg_table *st;
 493         struct scatterlist *sg;
 494
 495         GEM_BUG_ON(range_overflows(offset, size, resource_size(&dev_priv->dsm)));
 496
 497         /* We hide that we have no struct page backing our stolen object
 498          * by wrapping the contiguous physical allocation with a fake
 499          * dma mapping in a single scatterlist.
 500          */
 501
 502         st = kmalloc(sizeof(*st), GFP_KERNEL);
 503         if (st == NULL)
 504                 return ERR_PTR(-ENOMEM);
 505
 506         if (sg_alloc_table(st, 1, GFP_KERNEL)) {
 507                 kfree(st);
 508                 return ERR_PTR(-ENOMEM);
 509         }
 510
 511         sg = st->sgl;
 512         sg->offset = 0;
 513         sg->length = size;
 514
 515         sg_dma_address(sg) = (dma_addr_t)dev_priv->dsm.start + offset;
 516         sg_dma_len(sg) = size;
 517
 518         return st;
 519 }
 520
 521 static int i915_gem_object_get_pages_stolen(struct drm_i915_gem_object *obj)
 522 {
 523         struct sg_table *pages =
 524                 i915_pages_create_for_stolen(obj->base.dev,
 525                                              obj->stolen->start,
 526                                              obj->stolen->size);
 527         if (IS_ERR(pages))
 528                 return PTR_ERR(pages);
 529
 530         __i915_gem_object_set_pages(obj, pages, obj->stolen->size);
 531
 532         return 0;
 533 }
 534
 535 static void i915_gem_object_put_pages_stolen(struct drm_i915_gem_object *obj,
 536                                              struct sg_table *pages)
 537 {
 538         /* Should only be called from i915_gem_object_release_stolen() */
 539         sg_free_table(pages);
 540         kfree(pages);
 541 }
 542
 543 static void
 544 i915_gem_object_release_stolen(struct drm_i915_gem_object *obj)
 545 {
 546         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
 547         struct drm_mm_node *stolen = fetch_and_zero(&obj->stolen);
 548
 549         GEM_BUG_ON(!stolen);
 550
 551         __i915_gem_object_unpin_pages(obj);
 552
 553         i915_gem_stolen_remove_node(dev_priv, stolen);
 554         kfree(stolen);
 555 }
 556
 557 static const struct drm_i915_gem_object_ops i915_gem_object_stolen_ops = {
 558         .get_pages = i915_gem_object_get_pages_stolen,
 559         .put_pages = i915_gem_object_put_pages_stolen,
 560         .release = i915_gem_object_release_stolen,
 561 };
 562
 563 static struct drm_i915_gem_object *
 564 _i915_gem_object_create_stolen(struct drm_i915_private *dev_priv,
 565                                struct drm_mm_node *stolen)
 566 {
 567         struct drm_i915_gem_object *obj;
 568         unsigned int cache_level;
 569
 570         obj = i915_gem_object_alloc(dev_priv);
 571         if (obj == NULL)
 572                 return NULL;
 573
 574         drm_gem_private_object_init(&dev_priv->drm, &obj->base, stolen->size);
 575         i915_gem_object_init(obj, &i915_gem_object_stolen_ops);
 576
 577         obj->stolen = stolen;
 578         obj->read_domains = I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT;
 579         cache_level = HAS_LLC(dev_priv) ? I915_CACHE_LLC : I915_CACHE_NONE;
 580         i915_gem_object_set_cache_coherency(obj, cache_level);
 581
 582         if (i915_gem_object_pin_pages(obj))
 583                 goto cleanup;
 584
 585         return obj;
 586
 587 cleanup:
 588         i915_gem_object_free(obj);
 589         return NULL;
 590 }
 591
 592 struct drm_i915_gem_object *
 593 i915_gem_object_create_stolen(struct drm_i915_private *dev_priv,
 594                               resource_size_t size)
 595 {
 596         struct drm_i915_gem_object *obj;
 597         struct drm_mm_node *stolen;
 598         int ret;
 599
 600         if (!drm_mm_initialized(&dev_priv->mm.stolen))
 601                 return NULL;
 602
 603         if (size == 0)
 604                 return NULL;
 605
 606         stolen = kzalloc(sizeof(*stolen), GFP_KERNEL);
 607         if (!stolen)
 608                 return NULL;
 609
 610         ret = i915_gem_stolen_insert_node(dev_priv, stolen, size, 4096);
 611         if (ret) {
 612                 kfree(stolen);
 613                 return NULL;
 614         }
 615
 616         obj = _i915_gem_object_create_stolen(dev_priv, stolen);
 617         if (obj)
 618                 return obj;
 619
 620         i915_gem_stolen_remove_node(dev_priv, stolen);
 621         kfree(stolen);
 622         return NULL;
 623 }
 624
 625 struct drm_i915_gem_object *
 626 i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv,
 627                                                resource_size_t stolen_offset,
 628                                                resource_size_t gtt_offset,
 629                                                resource_size_t size)
 630 {
 631         struct i915_ggtt *ggtt = &dev_priv->ggtt;
 632         struct drm_i915_gem_object *obj;
 633         struct drm_mm_node *stolen;
 634         struct i915_vma *vma;
 635         int ret;
 636
 637         if (!drm_mm_initialized(&dev_priv->mm.stolen))
 638                 return NULL;
 639
 640         lockdep_assert_held(&dev_priv->drm.struct_mutex);
 641
 642         DRM_DEBUG_DRIVER("creating preallocated stolen object: stolen_offset=%pa, gtt_offset=%pa, size=%pa\n",
 643                          &stolen_offset, &gtt_offset, &size);
 644
 645         /* KISS and expect everything to be page-aligned */
 646         if (WARN_ON(size == 0) ||
 647             WARN_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE)) ||
 648             WARN_ON(!IS_ALIGNED(stolen_offset, I915_GTT_MIN_ALIGNMENT)))
 649                 return NULL;
 650
 651         stolen = kzalloc(sizeof(*stolen), GFP_KERNEL);
 652         if (!stolen)
 653                 return NULL;
 654
 655         stolen->start = stolen_offset;
 656         stolen->size = size;
 657         mutex_lock(&dev_priv->mm.stolen_lock);
 658         ret = drm_mm_reserve_node(&dev_priv->mm.stolen, stolen);
 659         mutex_unlock(&dev_priv->mm.stolen_lock);
 660         if (ret) {
 661                 DRM_DEBUG_DRIVER("failed to allocate stolen space\n");
 662                 kfree(stolen);
 663                 return NULL;
 664         }
 665
 666         obj = _i915_gem_object_create_stolen(dev_priv, stolen);
 667         if (obj == NULL) {
 668                 DRM_DEBUG_DRIVER("failed to allocate stolen object\n");
 669                 i915_gem_stolen_remove_node(dev_priv, stolen);
 670                 kfree(stolen);
 671                 return NULL;
 672         }
 673
 674         /* Some objects just need physical mem from stolen space */
 675         if (gtt_offset == I915_GTT_OFFSET_NONE)
 676                 return obj;
 677
 678         ret = i915_gem_object_pin_pages(obj);
 679         if (ret)
 680                 goto err;
 681
 682         vma = i915_vma_instance(obj, &ggtt->vm, NULL);
 683         if (IS_ERR(vma)) {
 684                 ret = PTR_ERR(vma);
 685                 goto err_pages;
 686         }
 687
 688         /* To simplify the initialisation sequence between KMS and GTT,
 689          * we allow construction of the stolen object prior to
 690          * setting up the GTT space. The actual reservation will occur
 691          * later.
 692          */
 693         ret = i915_gem_gtt_reserve(&ggtt->vm, &vma->node,
 694                                    size, gtt_offset, obj->cache_level,
 695                                    0);
 696         if (ret) {
 697                 DRM_DEBUG_DRIVER("failed to allocate stolen GTT space\n");
 698                 goto err_pages;
 699         }
 700
 701         GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
 702
 703         vma->pages = obj->mm.pages;
 704         vma->flags |= I915_VMA_GLOBAL_BIND;
 705         __i915_vma_set_map_and_fenceable(vma);
 706         list_move_tail(&vma->vm_link, &ggtt->vm.inactive_list);
 707
 708         spin_lock(&dev_priv->mm.obj_lock);
 709         list_move_tail(&obj->mm.link, &dev_priv->mm.bound_list);
 710         obj->bind_count++;
 711         spin_unlock(&dev_priv->mm.obj_lock);
 712
 713         return obj;
 714
 715 err_pages:
 716         i915_gem_object_unpin_pages(obj);
 717 err:
 718         i915_gem_object_put(obj);
 719         return NULL;
 720 }