drivers/gpu/drm/i915/i915_gem_gtt.c

   1 /*
   2  * Copyright © 2010 Daniel Vetter
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include "drmP.h"
  26 #include "drm.h"
  27 #include "i915_drm.h"
  28 #include "i915_drv.h"
  29 #include "i915_trace.h"
  30 #include "intel_drv.h"
  31
  32 /* PPGTT support for Sandybdrige/Gen6 and later */
  33 static void i915_ppgtt_clear_range(struct i915_hw_ppgtt *ppgtt,
  34                                    unsigned first_entry,
  35                                    unsigned num_entries)
  36 {
  37         uint32_t *pt_vaddr;
  38         uint32_t scratch_pte;
  39         unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES;
  40         unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
  41         unsigned last_pte, i;
  42
  43         scratch_pte = GEN6_PTE_ADDR_ENCODE(ppgtt->scratch_page_dma_addr);
  44         scratch_pte |= GEN6_PTE_VALID | GEN6_PTE_CACHE_LLC;
  45
  46         while (num_entries) {
  47                 last_pte = first_pte + num_entries;
  48                 if (last_pte > I915_PPGTT_PT_ENTRIES)
  49                         last_pte = I915_PPGTT_PT_ENTRIES;
  50
  51                 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pd]);
  52
  53                 for (i = first_pte; i < last_pte; i++)
  54                         pt_vaddr[i] = scratch_pte;
  55
  56                 kunmap_atomic(pt_vaddr);
  57
  58                 num_entries -= last_pte - first_pte;
  59                 first_pte = 0;
  60                 act_pd++;
  61         }
  62 }
  63
  64 int i915_gem_init_aliasing_ppgtt(struct drm_device *dev)
  65 {
  66         struct drm_i915_private *dev_priv = dev->dev_private;
  67         struct i915_hw_ppgtt *ppgtt;
  68         unsigned first_pd_entry_in_global_pt;
  69         int i;
  70         int ret = -ENOMEM;
  71
  72         /* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024
  73          * entries. For aliasing ppgtt support we just steal them at the end for
  74          * now. */
  75         first_pd_entry_in_global_pt = dev_priv->mm.gtt->gtt_total_entries - I915_PPGTT_PD_ENTRIES;
  76
  77         ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
  78         if (!ppgtt)
  79                 return ret;
  80
  81         ppgtt->num_pd_entries = I915_PPGTT_PD_ENTRIES;
  82         ppgtt->pt_pages = kzalloc(sizeof(struct page *)*ppgtt->num_pd_entries,
  83                                   GFP_KERNEL);
  84         if (!ppgtt->pt_pages)
  85                 goto err_ppgtt;
  86
  87         for (i = 0; i < ppgtt->num_pd_entries; i++) {
  88                 ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL);
  89                 if (!ppgtt->pt_pages[i])
  90                         goto err_pt_alloc;
  91         }
  92
  93         if (dev_priv->mm.gtt->needs_dmar) {
  94                 ppgtt->pt_dma_addr = kzalloc(sizeof(dma_addr_t)
  95                                                 *ppgtt->num_pd_entries,
  96                                              GFP_KERNEL);
  97                 if (!ppgtt->pt_dma_addr)
  98                         goto err_pt_alloc;
  99
 100                 for (i = 0; i < ppgtt->num_pd_entries; i++) {
 101                         dma_addr_t pt_addr;
 102
 103                         pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i],
 104                                                0, 4096,
 105                                                PCI_DMA_BIDIRECTIONAL);
 106
 107                         if (pci_dma_mapping_error(dev->pdev,
 108                                                   pt_addr)) {
 109                                 ret = -EIO;
 110                                 goto err_pd_pin;
 111
 112                         }
 113                         ppgtt->pt_dma_addr[i] = pt_addr;
 114                 }
 115         }
 116
 117         ppgtt->scratch_page_dma_addr = dev_priv->mm.gtt->scratch_page_dma;
 118
 119         i915_ppgtt_clear_range(ppgtt, 0,
 120                                ppgtt->num_pd_entries*I915_PPGTT_PT_ENTRIES);
 121
 122         ppgtt->pd_offset = (first_pd_entry_in_global_pt)*sizeof(uint32_t);
 123
 124         dev_priv->mm.aliasing_ppgtt = ppgtt;
 125
 126         return 0;
 127
 128 err_pd_pin:
 129         if (ppgtt->pt_dma_addr) {
 130                 for (i--; i >= 0; i--)
 131                         pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i],
 132                                        4096, PCI_DMA_BIDIRECTIONAL);
 133         }
 134 err_pt_alloc:
 135         kfree(ppgtt->pt_dma_addr);
 136         for (i = 0; i < ppgtt->num_pd_entries; i++) {
 137                 if (ppgtt->pt_pages[i])
 138                         __free_page(ppgtt->pt_pages[i]);
 139         }
 140         kfree(ppgtt->pt_pages);
 141 err_ppgtt:
 142         kfree(ppgtt);
 143
 144         return ret;
 145 }
 146
 147 void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev)
 148 {
 149         struct drm_i915_private *dev_priv = dev->dev_private;
 150         struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
 151         int i;
 152
 153         if (!ppgtt)
 154                 return;
 155
 156         if (ppgtt->pt_dma_addr) {
 157                 for (i = 0; i < ppgtt->num_pd_entries; i++)
 158                         pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i],
 159                                        4096, PCI_DMA_BIDIRECTIONAL);
 160         }
 161
 162         kfree(ppgtt->pt_dma_addr);
 163         for (i = 0; i < ppgtt->num_pd_entries; i++)
 164                 __free_page(ppgtt->pt_pages[i]);
 165         kfree(ppgtt->pt_pages);
 166         kfree(ppgtt);
 167 }
 168
 169 static void i915_ppgtt_insert_sg_entries(struct i915_hw_ppgtt *ppgtt,
 170                                          struct scatterlist *sg_list,
 171                                          unsigned sg_len,
 172                                          unsigned first_entry,
 173                                          uint32_t pte_flags)
 174 {
 175         uint32_t *pt_vaddr, pte;
 176         unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES;
 177         unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
 178         unsigned i, j, m, segment_len;
 179         dma_addr_t page_addr;
 180         struct scatterlist *sg;
 181
 182         /* init sg walking */
 183         sg = sg_list;
 184         i = 0;
 185         segment_len = sg_dma_len(sg) >> PAGE_SHIFT;
 186         m = 0;
 187
 188         while (i < sg_len) {
 189                 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pd]);
 190
 191                 for (j = first_pte; j < I915_PPGTT_PT_ENTRIES; j++) {
 192                         page_addr = sg_dma_address(sg) + (m << PAGE_SHIFT);
 193                         pte = GEN6_PTE_ADDR_ENCODE(page_addr);
 194                         pt_vaddr[j] = pte | pte_flags;
 195
 196                         /* grab the next page */
 197                         m++;
 198                         if (m == segment_len) {
 199                                 sg = sg_next(sg);
 200                                 i++;
 201                                 if (i == sg_len)
 202                                         break;
 203
 204                                 segment_len = sg_dma_len(sg) >> PAGE_SHIFT;
 205                                 m = 0;
 206                         }
 207                 }
 208
 209                 kunmap_atomic(pt_vaddr);
 210
 211                 first_pte = 0;
 212                 act_pd++;
 213         }
 214 }
 215
 216 static void i915_ppgtt_insert_pages(struct i915_hw_ppgtt *ppgtt,
 217                                     unsigned first_entry, unsigned num_entries,
 218                                     struct page **pages, uint32_t pte_flags)
 219 {
 220         uint32_t *pt_vaddr, pte;
 221         unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES;
 222         unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
 223         unsigned last_pte, i;
 224         dma_addr_t page_addr;
 225
 226         while (num_entries) {
 227                 last_pte = first_pte + num_entries;
 228                 last_pte = min_t(unsigned, last_pte, I915_PPGTT_PT_ENTRIES);
 229
 230                 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pd]);
 231
 232                 for (i = first_pte; i < last_pte; i++) {
 233                         page_addr = page_to_phys(*pages);
 234                         pte = GEN6_PTE_ADDR_ENCODE(page_addr);
 235                         pt_vaddr[i] = pte | pte_flags;
 236
 237                         pages++;
 238                 }
 239
 240                 kunmap_atomic(pt_vaddr);
 241
 242                 num_entries -= last_pte - first_pte;
 243                 first_pte = 0;
 244                 act_pd++;
 245         }
 246 }
 247
 248 void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt,
 249                             struct drm_i915_gem_object *obj,
 250                             enum i915_cache_level cache_level)
 251 {
 252         struct drm_device *dev = obj->base.dev;
 253         struct drm_i915_private *dev_priv = dev->dev_private;
 254         uint32_t pte_flags = GEN6_PTE_VALID;
 255
 256         switch (cache_level) {
 257         case I915_CACHE_LLC_MLC:
 258                 pte_flags |= GEN6_PTE_CACHE_LLC_MLC;
 259                 break;
 260         case I915_CACHE_LLC:
 261                 pte_flags |= GEN6_PTE_CACHE_LLC;
 262                 break;
 263         case I915_CACHE_NONE:
 264                 if (IS_HASWELL(dev))
 265                         pte_flags |= HSW_PTE_UNCACHED;
 266                 else
 267                         pte_flags |= GEN6_PTE_UNCACHED;
 268                 break;
 269         default:
 270                 BUG();
 271         }
 272
 273         if (obj->sg_table) {
 274                 i915_ppgtt_insert_sg_entries(ppgtt,
 275                                              obj->sg_table->sgl,
 276                                              obj->sg_table->nents,
 277                                              obj->gtt_space->start >> PAGE_SHIFT,
 278                                              pte_flags);
 279         } else if (dev_priv->mm.gtt->needs_dmar) {
 280                 BUG_ON(!obj->sg_list);
 281
 282                 i915_ppgtt_insert_sg_entries(ppgtt,
 283                                              obj->sg_list,
 284                                              obj->num_sg,
 285                                              obj->gtt_space->start >> PAGE_SHIFT,
 286                                              pte_flags);
 287         } else
 288                 i915_ppgtt_insert_pages(ppgtt,
 289                                         obj->gtt_space->start >> PAGE_SHIFT,
 290                                         obj->base.size >> PAGE_SHIFT,
 291                                         obj->pages,
 292                                         pte_flags);
 293 }
 294
 295 void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt,
 296                               struct drm_i915_gem_object *obj)
 297 {
 298         i915_ppgtt_clear_range(ppgtt,
 299                                obj->gtt_space->start >> PAGE_SHIFT,
 300                                obj->base.size >> PAGE_SHIFT);
 301 }
 302
 303 /* XXX kill agp_type! */
 304 static unsigned int cache_level_to_agp_type(struct drm_device *dev,
 305                                             enum i915_cache_level cache_level)
 306 {
 307         switch (cache_level) {
 308         case I915_CACHE_LLC_MLC:
 309                 if (INTEL_INFO(dev)->gen >= 6)
 310                         return AGP_USER_CACHED_MEMORY_LLC_MLC;
 311                 /* Older chipsets do not have this extra level of CPU
 312                  * cacheing, so fallthrough and request the PTE simply
 313                  * as cached.
 314                  */
 315         case I915_CACHE_LLC:
 316                 return AGP_USER_CACHED_MEMORY;
 317         default:
 318         case I915_CACHE_NONE:
 319                 return AGP_USER_MEMORY;
 320         }
 321 }
 322
 323 static bool do_idling(struct drm_i915_private *dev_priv)
 324 {
 325         bool ret = dev_priv->mm.interruptible;
 326
 327         if (unlikely(dev_priv->mm.gtt->do_idle_maps)) {
 328                 dev_priv->mm.interruptible = false;
 329                 if (i915_gpu_idle(dev_priv->dev)) {
 330                         DRM_ERROR("Couldn't idle GPU\n");
 331                         /* Wait a bit, in hopes it avoids the hang */
 332                         udelay(10);
 333                 }
 334         }
 335
 336         return ret;
 337 }
 338
 339 static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible)
 340 {
 341         if (unlikely(dev_priv->mm.gtt->do_idle_maps))
 342                 dev_priv->mm.interruptible = interruptible;
 343 }
 344
 345 void i915_gem_restore_gtt_mappings(struct drm_device *dev)
 346 {
 347         struct drm_i915_private *dev_priv = dev->dev_private;
 348         struct drm_i915_gem_object *obj;
 349
 350         /* First fill our portion of the GTT with scratch pages */
 351         intel_gtt_clear_range(dev_priv->mm.gtt_start / PAGE_SIZE,
 352                               (dev_priv->mm.gtt_end - dev_priv->mm.gtt_start) / PAGE_SIZE);
 353
 354         list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
 355                 i915_gem_clflush_object(obj);
 356                 i915_gem_gtt_bind_object(obj, obj->cache_level);
 357         }
 358
 359         intel_gtt_chipset_flush();
 360 }
 361
 362 int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
 363 {
 364         struct drm_device *dev = obj->base.dev;
 365         struct drm_i915_private *dev_priv = dev->dev_private;
 366
 367         /* don't map imported dma buf objects */
 368         if (dev_priv->mm.gtt->needs_dmar && !obj->sg_table)
 369                 return intel_gtt_map_memory(obj->pages,
 370                                             obj->base.size >> PAGE_SHIFT,
 371                                             &obj->sg_list,
 372                                             &obj->num_sg);
 373         else
 374                 return 0;
 375 }
 376
 377 void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj,
 378                               enum i915_cache_level cache_level)
 379 {
 380         struct drm_device *dev = obj->base.dev;
 381         struct drm_i915_private *dev_priv = dev->dev_private;
 382         unsigned int agp_type = cache_level_to_agp_type(dev, cache_level);
 383
 384         if (obj->sg_table) {
 385                 intel_gtt_insert_sg_entries(obj->sg_table->sgl,
 386                                             obj->sg_table->nents,
 387                                             obj->gtt_space->start >> PAGE_SHIFT,
 388                                             agp_type);
 389         } else if (dev_priv->mm.gtt->needs_dmar) {
 390                 BUG_ON(!obj->sg_list);
 391
 392                 intel_gtt_insert_sg_entries(obj->sg_list,
 393                                             obj->num_sg,
 394                                             obj->gtt_space->start >> PAGE_SHIFT,
 395                                             agp_type);
 396         } else
 397                 intel_gtt_insert_pages(obj->gtt_space->start >> PAGE_SHIFT,
 398                                        obj->base.size >> PAGE_SHIFT,
 399                                        obj->pages,
 400                                        agp_type);
 401
 402         obj->has_global_gtt_mapping = 1;
 403 }
 404
 405 void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj)
 406 {
 407         intel_gtt_clear_range(obj->gtt_space->start >> PAGE_SHIFT,
 408                               obj->base.size >> PAGE_SHIFT);
 409
 410         obj->has_global_gtt_mapping = 0;
 411 }
 412
 413 void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
 414 {
 415         struct drm_device *dev = obj->base.dev;
 416         struct drm_i915_private *dev_priv = dev->dev_private;
 417         bool interruptible;
 418
 419         interruptible = do_idling(dev_priv);
 420
 421         if (obj->sg_list) {
 422                 intel_gtt_unmap_memory(obj->sg_list, obj->num_sg);
 423                 obj->sg_list = NULL;
 424         }
 425
 426         undo_idling(dev_priv, interruptible);
 427 }
 428
 429 void i915_gem_init_global_gtt(struct drm_device *dev,
 430                               unsigned long start,
 431                               unsigned long mappable_end,
 432                               unsigned long end)
 433 {
 434         drm_i915_private_t *dev_priv = dev->dev_private;
 435
 436         /* Substract the guard page ... */
 437         drm_mm_init(&dev_priv->mm.gtt_space, start, end - start - PAGE_SIZE);
 438
 439         dev_priv->mm.gtt_start = start;
 440         dev_priv->mm.gtt_mappable_end = mappable_end;
 441         dev_priv->mm.gtt_end = end;
 442         dev_priv->mm.gtt_total = end - start;
 443         dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start;
 444
 445         /* ... but ensure that we clear the entire range. */
 446         intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE);
 447 }