drivers/gpu/drm/i915/gem/i915_gem_domain.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2014-2016 Intel Corporation
   5  */
   6
   7 #include "display/intel_frontbuffer.h"
   8
   9 #include "i915_drv.h"
  10 #include "i915_gem_clflush.h"
  11 #include "i915_gem_gtt.h"
  12 #include "i915_gem_ioctls.h"
  13 #include "i915_gem_object.h"
  14 #include "i915_vma.h"
  15 #include "i915_gem_lmem.h"
  16 #include "i915_gem_mman.h"
  17
  18 static void __i915_gem_object_flush_for_display(struct drm_i915_gem_object *obj)
  19 {
  20         /*
  21          * We manually flush the CPU domain so that we can override and
  22          * force the flush for the display, and perform it asyncrhonously.
  23          */
  24         i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
  25         if (obj->cache_dirty)
  26                 i915_gem_clflush_object(obj, I915_CLFLUSH_FORCE);
  27         obj->write_domain = 0;
  28 }
  29
  30 void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj)
  31 {
  32         if (!i915_gem_object_is_framebuffer(obj))
  33                 return;
  34
  35         i915_gem_object_lock(obj);
  36         __i915_gem_object_flush_for_display(obj);
  37         i915_gem_object_unlock(obj);
  38 }
  39
  40 /**
  41  * Moves a single object to the WC read, and possibly write domain.
  42  * @obj: object to act on
  43  * @write: ask for write access or read only
  44  *
  45  * This function returns when the move is complete, including waiting on
  46  * flushes to occur.
  47  */
  48 int
  49 i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool write)
  50 {
  51         int ret;
  52
  53         assert_object_held(obj);
  54
  55         ret = i915_gem_object_wait(obj,
  56                                    I915_WAIT_INTERRUPTIBLE |
  57                                    (write ? I915_WAIT_ALL : 0),
  58                                    MAX_SCHEDULE_TIMEOUT);
  59         if (ret)
  60                 return ret;
  61
  62         if (obj->write_domain == I915_GEM_DOMAIN_WC)
  63                 return 0;
  64
  65         /* Flush and acquire obj->pages so that we are coherent through
  66          * direct access in memory with previous cached writes through
  67          * shmemfs and that our cache domain tracking remains valid.
  68          * For example, if the obj->filp was moved to swap without us
  69          * being notified and releasing the pages, we would mistakenly
  70          * continue to assume that the obj remained out of the CPU cached
  71          * domain.
  72          */
  73         ret = i915_gem_object_pin_pages(obj);
  74         if (ret)
  75                 return ret;
  76
  77         i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_WC);
  78
  79         /* Serialise direct access to this object with the barriers for
  80          * coherent writes from the GPU, by effectively invalidating the
  81          * WC domain upon first access.
  82          */
  83         if ((obj->read_domains & I915_GEM_DOMAIN_WC) == 0)
  84                 mb();
  85
  86         /* It should now be out of any other write domains, and we can update
  87          * the domain values for our changes.
  88          */
  89         GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_WC) != 0);
  90         obj->read_domains |= I915_GEM_DOMAIN_WC;
  91         if (write) {
  92                 obj->read_domains = I915_GEM_DOMAIN_WC;
  93                 obj->write_domain = I915_GEM_DOMAIN_WC;
  94                 obj->mm.dirty = true;
  95         }
  96
  97         i915_gem_object_unpin_pages(obj);
  98         return 0;
  99 }
 100
 101 /**
 102  * Moves a single object to the GTT read, and possibly write domain.
 103  * @obj: object to act on
 104  * @write: ask for write access or read only
 105  *
 106  * This function returns when the move is complete, including waiting on
 107  * flushes to occur.
 108  */
 109 int
 110 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
 111 {
 112         int ret;
 113
 114         assert_object_held(obj);
 115
 116         ret = i915_gem_object_wait(obj,
 117                                    I915_WAIT_INTERRUPTIBLE |
 118                                    (write ? I915_WAIT_ALL : 0),
 119                                    MAX_SCHEDULE_TIMEOUT);
 120         if (ret)
 121                 return ret;
 122
 123         if (obj->write_domain == I915_GEM_DOMAIN_GTT)
 124                 return 0;
 125
 126         /* Flush and acquire obj->pages so that we are coherent through
 127          * direct access in memory with previous cached writes through
 128          * shmemfs and that our cache domain tracking remains valid.
 129          * For example, if the obj->filp was moved to swap without us
 130          * being notified and releasing the pages, we would mistakenly
 131          * continue to assume that the obj remained out of the CPU cached
 132          * domain.
 133          */
 134         ret = i915_gem_object_pin_pages(obj);
 135         if (ret)
 136                 return ret;
 137
 138         i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_GTT);
 139
 140         /* Serialise direct access to this object with the barriers for
 141          * coherent writes from the GPU, by effectively invalidating the
 142          * GTT domain upon first access.
 143          */
 144         if ((obj->read_domains & I915_GEM_DOMAIN_GTT) == 0)
 145                 mb();
 146
 147         /* It should now be out of any other write domains, and we can update
 148          * the domain values for our changes.
 149          */
 150         GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
 151         obj->read_domains |= I915_GEM_DOMAIN_GTT;
 152         if (write) {
 153                 struct i915_vma *vma;
 154
 155                 obj->read_domains = I915_GEM_DOMAIN_GTT;
 156                 obj->write_domain = I915_GEM_DOMAIN_GTT;
 157                 obj->mm.dirty = true;
 158
 159                 spin_lock(&obj->vma.lock);
 160                 for_each_ggtt_vma(vma, obj)
 161                         if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND))
 162                                 i915_vma_set_ggtt_write(vma);
 163                 spin_unlock(&obj->vma.lock);
 164         }
 165
 166         i915_gem_object_unpin_pages(obj);
 167         return 0;
 168 }
 169
 170 /**
 171  * Changes the cache-level of an object across all VMA.
 172  * @obj: object to act on
 173  * @cache_level: new cache level to set for the object
 174  *
 175  * After this function returns, the object will be in the new cache-level
 176  * across all GTT and the contents of the backing storage will be coherent,
 177  * with respect to the new cache-level. In order to keep the backing storage
 178  * coherent for all users, we only allow a single cache level to be set
 179  * globally on the object and prevent it from being changed whilst the
 180  * hardware is reading from the object. That is if the object is currently
 181  * on the scanout it will be set to uncached (or equivalent display
 182  * cache coherency) and all non-MOCS GPU access will also be uncached so
 183  * that all direct access to the scanout remains coherent.
 184  */
 185 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
 186                                     enum i915_cache_level cache_level)
 187 {
 188         int ret;
 189
 190         if (obj->cache_level == cache_level)
 191                 return 0;
 192
 193         ret = i915_gem_object_wait(obj,
 194                                    I915_WAIT_INTERRUPTIBLE |
 195                                    I915_WAIT_ALL,
 196                                    MAX_SCHEDULE_TIMEOUT);
 197         if (ret)
 198                 return ret;
 199
 200         ret = i915_gem_object_lock_interruptible(obj);
 201         if (ret)
 202                 return ret;
 203
 204         /* Always invalidate stale cachelines */
 205         if (obj->cache_level != cache_level) {
 206                 i915_gem_object_set_cache_coherency(obj, cache_level);
 207                 obj->cache_dirty = true;
 208         }
 209
 210         i915_gem_object_unlock(obj);
 211
 212         /* The cache-level will be applied when each vma is rebound. */
 213         return i915_gem_object_unbind(obj,
 214                                       I915_GEM_OBJECT_UNBIND_ACTIVE |
 215                                       I915_GEM_OBJECT_UNBIND_BARRIER);
 216 }
 217
 218 int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
 219                                struct drm_file *file)
 220 {
 221         struct drm_i915_gem_caching *args = data;
 222         struct drm_i915_gem_object *obj;
 223         int err = 0;
 224
 225         rcu_read_lock();
 226         obj = i915_gem_object_lookup_rcu(file, args->handle);
 227         if (!obj) {
 228                 err = -ENOENT;
 229                 goto out;
 230         }
 231
 232         switch (obj->cache_level) {
 233         case I915_CACHE_LLC:
 234         case I915_CACHE_L3_LLC:
 235                 args->caching = I915_CACHING_CACHED;
 236                 break;
 237
 238         case I915_CACHE_WT:
 239                 args->caching = I915_CACHING_DISPLAY;
 240                 break;
 241
 242         default:
 243                 args->caching = I915_CACHING_NONE;
 244                 break;
 245         }
 246 out:
 247         rcu_read_unlock();
 248         return err;
 249 }
 250
 251 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
 252                                struct drm_file *file)
 253 {
 254         struct drm_i915_private *i915 = to_i915(dev);
 255         struct drm_i915_gem_caching *args = data;
 256         struct drm_i915_gem_object *obj;
 257         enum i915_cache_level level;
 258         int ret = 0;
 259
 260         switch (args->caching) {
 261         case I915_CACHING_NONE:
 262                 level = I915_CACHE_NONE;
 263                 break;
 264         case I915_CACHING_CACHED:
 265                 /*
 266                  * Due to a HW issue on BXT A stepping, GPU stores via a
 267                  * snooped mapping may leave stale data in a corresponding CPU
 268                  * cacheline, whereas normally such cachelines would get
 269                  * invalidated.
 270                  */
 271                 if (!HAS_LLC(i915) && !HAS_SNOOP(i915))
 272                         return -ENODEV;
 273
 274                 level = I915_CACHE_LLC;
 275                 break;
 276         case I915_CACHING_DISPLAY:
 277                 level = HAS_WT(i915) ? I915_CACHE_WT : I915_CACHE_NONE;
 278                 break;
 279         default:
 280                 return -EINVAL;
 281         }
 282
 283         obj = i915_gem_object_lookup(file, args->handle);
 284         if (!obj)
 285                 return -ENOENT;
 286
 287         /*
 288          * The caching mode of proxy object is handled by its generator, and
 289          * not allowed to be changed by userspace.
 290          */
 291         if (i915_gem_object_is_proxy(obj)) {
 292                 ret = -ENXIO;
 293                 goto out;
 294         }
 295
 296         ret = i915_gem_object_set_cache_level(obj, level);
 297
 298 out:
 299         i915_gem_object_put(obj);
 300         return ret;
 301 }
 302
 303 /*
 304  * Prepare buffer for display plane (scanout, cursors, etc). Can be called from
 305  * an uninterruptible phase (modesetting) and allows any flushes to be pipelined
 306  * (for pageflips). We only flush the caches while preparing the buffer for
 307  * display, the callers are responsible for frontbuffer flush.
 308  */
 309 struct i915_vma *
 310 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
 311                                      u32 alignment,
 312                                      const struct i915_ggtt_view *view,
 313                                      unsigned int flags)
 314 {
 315         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 316         struct i915_vma *vma;
 317         int ret;
 318
 319         /* Frame buffer must be in LMEM (no migration yet) */
 320         if (HAS_LMEM(i915) && !i915_gem_object_is_lmem(obj))
 321                 return ERR_PTR(-EINVAL);
 322
 323         /*
 324          * The display engine is not coherent with the LLC cache on gen6.  As
 325          * a result, we make sure that the pinning that is about to occur is
 326          * done with uncached PTEs. This is lowest common denominator for all
 327          * chipsets.
 328          *
 329          * However for gen6+, we could do better by using the GFDT bit instead
 330          * of uncaching, which would allow us to flush all the LLC-cached data
 331          * with that bit in the PTE to main memory with just one PIPE_CONTROL.
 332          */
 333         ret = i915_gem_object_set_cache_level(obj,
 334                                               HAS_WT(i915) ?
 335                                               I915_CACHE_WT : I915_CACHE_NONE);
 336         if (ret)
 337                 return ERR_PTR(ret);
 338
 339         /*
 340          * As the user may map the buffer once pinned in the display plane
 341          * (e.g. libkms for the bootup splash), we have to ensure that we
 342          * always use map_and_fenceable for all scanout buffers. However,
 343          * it may simply be too big to fit into mappable, in which case
 344          * put it anyway and hope that userspace can cope (but always first
 345          * try to preserve the existing ABI).
 346          */
 347         vma = ERR_PTR(-ENOSPC);
 348         if ((flags & PIN_MAPPABLE) == 0 &&
 349             (!view || view->type == I915_GGTT_VIEW_NORMAL))
 350                 vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment,
 351                                                flags |
 352                                                PIN_MAPPABLE |
 353                                                PIN_NONBLOCK);
 354         if (IS_ERR(vma))
 355                 vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment, flags);
 356         if (IS_ERR(vma))
 357                 return vma;
 358
 359         vma->display_alignment = max_t(u64, vma->display_alignment, alignment);
 360
 361         i915_gem_object_flush_if_display(obj);
 362
 363         return vma;
 364 }
 365
 366 static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj)
 367 {
 368         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 369         struct i915_vma *vma;
 370
 371         GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
 372         if (!atomic_read(&obj->bind_count))
 373                 return;
 374
 375         mutex_lock(&i915->ggtt.vm.mutex);
 376         spin_lock(&obj->vma.lock);
 377         for_each_ggtt_vma(vma, obj) {
 378                 if (!drm_mm_node_allocated(&vma->node))
 379                         continue;
 380
 381                 GEM_BUG_ON(vma->vm != &i915->ggtt.vm);
 382                 list_move_tail(&vma->vm_link, &vma->vm->bound_list);
 383         }
 384         spin_unlock(&obj->vma.lock);
 385         mutex_unlock(&i915->ggtt.vm.mutex);
 386
 387         if (i915_gem_object_is_shrinkable(obj)) {
 388                 unsigned long flags;
 389
 390                 spin_lock_irqsave(&i915->mm.obj_lock, flags);
 391
 392                 if (obj->mm.madv == I915_MADV_WILLNEED &&
 393                     !atomic_read(&obj->mm.shrink_pin))
 394                         list_move_tail(&obj->mm.link, &i915->mm.shrink_list);
 395
 396                 spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
 397         }
 398 }
 399
 400 void
 401 i915_gem_object_unpin_from_display_plane(struct i915_vma *vma)
 402 {
 403         struct drm_i915_gem_object *obj = vma->obj;
 404
 405         assert_object_held(obj);
 406
 407         /* Bump the LRU to try and avoid premature eviction whilst flipping  */
 408         i915_gem_object_bump_inactive_ggtt(obj);
 409
 410         i915_vma_unpin(vma);
 411 }
 412
 413 /**
 414  * Moves a single object to the CPU read, and possibly write domain.
 415  * @obj: object to act on
 416  * @write: requesting write or read-only access
 417  *
 418  * This function returns when the move is complete, including waiting on
 419  * flushes to occur.
 420  */
 421 int
 422 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
 423 {
 424         int ret;
 425
 426         assert_object_held(obj);
 427
 428         ret = i915_gem_object_wait(obj,
 429                                    I915_WAIT_INTERRUPTIBLE |
 430                                    (write ? I915_WAIT_ALL : 0),
 431                                    MAX_SCHEDULE_TIMEOUT);
 432         if (ret)
 433                 return ret;
 434
 435         i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
 436
 437         /* Flush the CPU cache if it's still invalid. */
 438         if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
 439                 i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC);
 440                 obj->read_domains |= I915_GEM_DOMAIN_CPU;
 441         }
 442
 443         /* It should now be out of any other write domains, and we can update
 444          * the domain values for our changes.
 445          */
 446         GEM_BUG_ON(obj->write_domain & ~I915_GEM_DOMAIN_CPU);
 447
 448         /* If we're writing through the CPU, then the GPU read domains will
 449          * need to be invalidated at next use.
 450          */
 451         if (write)
 452                 __start_cpu_write(obj);
 453
 454         return 0;
 455 }
 456
 457 /**
 458  * Called when user space prepares to use an object with the CPU, either
 459  * through the mmap ioctl's mapping or a GTT mapping.
 460  * @dev: drm device
 461  * @data: ioctl data blob
 462  * @file: drm file
 463  */
 464 int
 465 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 466                           struct drm_file *file)
 467 {
 468         struct drm_i915_gem_set_domain *args = data;
 469         struct drm_i915_gem_object *obj;
 470         u32 read_domains = args->read_domains;
 471         u32 write_domain = args->write_domain;
 472         int err;
 473
 474         /* Only handle setting domains to types used by the CPU. */
 475         if ((write_domain | read_domains) & I915_GEM_GPU_DOMAINS)
 476                 return -EINVAL;
 477
 478         /*
 479          * Having something in the write domain implies it's in the read
 480          * domain, and only that read domain.  Enforce that in the request.
 481          */
 482         if (write_domain && read_domains != write_domain)
 483                 return -EINVAL;
 484
 485         if (!read_domains)
 486                 return 0;
 487
 488         obj = i915_gem_object_lookup(file, args->handle);
 489         if (!obj)
 490                 return -ENOENT;
 491
 492         /*
 493          * Already in the desired write domain? Nothing for us to do!
 494          *
 495          * We apply a little bit of cunning here to catch a broader set of
 496          * no-ops. If obj->write_domain is set, we must be in the same
 497          * obj->read_domains, and only that domain. Therefore, if that
 498          * obj->write_domain matches the request read_domains, we are
 499          * already in the same read/write domain and can skip the operation,
 500          * without having to further check the requested write_domain.
 501          */
 502         if (READ_ONCE(obj->write_domain) == read_domains) {
 503                 err = 0;
 504                 goto out;
 505         }
 506
 507         /*
 508          * Try to flush the object off the GPU without holding the lock.
 509          * We will repeat the flush holding the lock in the normal manner
 510          * to catch cases where we are gazumped.
 511          */
 512         err = i915_gem_object_wait(obj,
 513                                    I915_WAIT_INTERRUPTIBLE |
 514                                    I915_WAIT_PRIORITY |
 515                                    (write_domain ? I915_WAIT_ALL : 0),
 516                                    MAX_SCHEDULE_TIMEOUT);
 517         if (err)
 518                 goto out;
 519
 520         /*
 521          * Proxy objects do not control access to the backing storage, ergo
 522          * they cannot be used as a means to manipulate the cache domain
 523          * tracking for that backing storage. The proxy object is always
 524          * considered to be outside of any cache domain.
 525          */
 526         if (i915_gem_object_is_proxy(obj)) {
 527                 err = -ENXIO;
 528                 goto out;
 529         }
 530
 531         /*
 532          * Flush and acquire obj->pages so that we are coherent through
 533          * direct access in memory with previous cached writes through
 534          * shmemfs and that our cache domain tracking remains valid.
 535          * For example, if the obj->filp was moved to swap without us
 536          * being notified and releasing the pages, we would mistakenly
 537          * continue to assume that the obj remained out of the CPU cached
 538          * domain.
 539          */
 540         err = i915_gem_object_pin_pages(obj);
 541         if (err)
 542                 goto out;
 543
 544         err = i915_gem_object_lock_interruptible(obj);
 545         if (err)
 546                 goto out_unpin;
 547
 548         if (read_domains & I915_GEM_DOMAIN_WC)
 549                 err = i915_gem_object_set_to_wc_domain(obj, write_domain);
 550         else if (read_domains & I915_GEM_DOMAIN_GTT)
 551                 err = i915_gem_object_set_to_gtt_domain(obj, write_domain);
 552         else
 553                 err = i915_gem_object_set_to_cpu_domain(obj, write_domain);
 554
 555         /* And bump the LRU for this access */
 556         i915_gem_object_bump_inactive_ggtt(obj);
 557
 558         i915_gem_object_unlock(obj);
 559
 560         if (write_domain)
 561                 i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
 562
 563 out_unpin:
 564         i915_gem_object_unpin_pages(obj);
 565 out:
 566         i915_gem_object_put(obj);
 567         return err;
 568 }
 569
 570 /*
 571  * Pins the specified object's pages and synchronizes the object with
 572  * GPU accesses. Sets needs_clflush to non-zero if the caller should
 573  * flush the object from the CPU cache.
 574  */
 575 int i915_gem_object_prepare_read(struct drm_i915_gem_object *obj,
 576                                  unsigned int *needs_clflush)
 577 {
 578         int ret;
 579
 580         *needs_clflush = 0;
 581         if (!i915_gem_object_has_struct_page(obj))
 582                 return -ENODEV;
 583
 584         ret = i915_gem_object_lock_interruptible(obj);
 585         if (ret)
 586                 return ret;
 587
 588         ret = i915_gem_object_wait(obj,
 589                                    I915_WAIT_INTERRUPTIBLE,
 590                                    MAX_SCHEDULE_TIMEOUT);
 591         if (ret)
 592                 goto err_unlock;
 593
 594         ret = i915_gem_object_pin_pages(obj);
 595         if (ret)
 596                 goto err_unlock;
 597
 598         if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ ||
 599             !static_cpu_has(X86_FEATURE_CLFLUSH)) {
 600                 ret = i915_gem_object_set_to_cpu_domain(obj, false);
 601                 if (ret)
 602                         goto err_unpin;
 603                 else
 604                         goto out;
 605         }
 606
 607         i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
 608
 609         /* If we're not in the cpu read domain, set ourself into the gtt
 610          * read domain and manually flush cachelines (if required). This
 611          * optimizes for the case when the gpu will dirty the data
 612          * anyway again before the next pread happens.
 613          */
 614         if (!obj->cache_dirty &&
 615             !(obj->read_domains & I915_GEM_DOMAIN_CPU))
 616                 *needs_clflush = CLFLUSH_BEFORE;
 617
 618 out:
 619         /* return with the pages pinned */
 620         return 0;
 621
 622 err_unpin:
 623         i915_gem_object_unpin_pages(obj);
 624 err_unlock:
 625         i915_gem_object_unlock(obj);
 626         return ret;
 627 }
 628
 629 int i915_gem_object_prepare_write(struct drm_i915_gem_object *obj,
 630                                   unsigned int *needs_clflush)
 631 {
 632         int ret;
 633
 634         *needs_clflush = 0;
 635         if (!i915_gem_object_has_struct_page(obj))
 636                 return -ENODEV;
 637
 638         ret = i915_gem_object_lock_interruptible(obj);
 639         if (ret)
 640                 return ret;
 641
 642         ret = i915_gem_object_wait(obj,
 643                                    I915_WAIT_INTERRUPTIBLE |
 644                                    I915_WAIT_ALL,
 645                                    MAX_SCHEDULE_TIMEOUT);
 646         if (ret)
 647                 goto err_unlock;
 648
 649         ret = i915_gem_object_pin_pages(obj);
 650         if (ret)
 651                 goto err_unlock;
 652
 653         if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE ||
 654             !static_cpu_has(X86_FEATURE_CLFLUSH)) {
 655                 ret = i915_gem_object_set_to_cpu_domain(obj, true);
 656                 if (ret)
 657                         goto err_unpin;
 658                 else
 659                         goto out;
 660         }
 661
 662         i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
 663
 664         /* If we're not in the cpu write domain, set ourself into the
 665          * gtt write domain and manually flush cachelines (as required).
 666          * This optimizes for the case when the gpu will use the data
 667          * right away and we therefore have to clflush anyway.
 668          */
 669         if (!obj->cache_dirty) {
 670                 *needs_clflush |= CLFLUSH_AFTER;
 671
 672                 /*
 673                  * Same trick applies to invalidate partially written
 674                  * cachelines read before writing.
 675                  */
 676                 if (!(obj->read_domains & I915_GEM_DOMAIN_CPU))
 677                         *needs_clflush |= CLFLUSH_BEFORE;
 678         }
 679
 680 out:
 681         i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
 682         obj->mm.dirty = true;
 683         /* return with the pages pinned */
 684         return 0;
 685
 686 err_unpin:
 687         i915_gem_object_unpin_pages(obj);
 688 err_unlock:
 689         i915_gem_object_unlock(obj);
 690         return ret;
 691 }