drivers/gpu/drm/i915/i915_memcpy.c

   1 /*
   2  * Copyright © 2016 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include <linux/kernel.h>
  26 #include <asm/fpu/api.h>
  27
  28 #include "i915_drv.h"
  29
  30 static DEFINE_STATIC_KEY_FALSE(has_movntdqa);
  31
  32 #ifdef CONFIG_AS_MOVNTDQA
  33 static void __memcpy_ntdqa(void *dst, const void *src, unsigned long len)
  34 {
  35         kernel_fpu_begin();
  36
  37         len >>= 4;
  38         while (len >= 4) {
  39                 asm("movntdqa   (%0), %%xmm0\n"
  40                     "movntdqa 16(%0), %%xmm1\n"
  41                     "movntdqa 32(%0), %%xmm2\n"
  42                     "movntdqa 48(%0), %%xmm3\n"
  43                     "movaps %%xmm0,   (%1)\n"
  44                     "movaps %%xmm1, 16(%1)\n"
  45                     "movaps %%xmm2, 32(%1)\n"
  46                     "movaps %%xmm3, 48(%1)\n"
  47                     :: "r" (src), "r" (dst) : "memory");
  48                 src += 64;
  49                 dst += 64;
  50                 len -= 4;
  51         }
  52         while (len--) {
  53                 asm("movntdqa (%0), %%xmm0\n"
  54                     "movaps %%xmm0, (%1)\n"
  55                     :: "r" (src), "r" (dst) : "memory");
  56                 src += 16;
  57                 dst += 16;
  58         }
  59
  60         kernel_fpu_end();
  61 }
  62 #endif
  63
  64 /**
  65  * i915_memcpy_from_wc: perform an accelerated *aligned* read from WC
  66  * @dst: destination pointer
  67  * @src: source pointer
  68  * @len: how many bytes to copy
  69  *
  70  * i915_memcpy_from_wc copies @len bytes from @src to @dst using
  71  * non-temporal instructions where available. Note that all arguments
  72  * (@src, @dst) must be aligned to 16 bytes and @len must be a multiple
  73  * of 16.
  74  *
  75  * To test whether accelerated reads from WC are supported, use
  76  * i915_memcpy_from_wc(NULL, NULL, 0);
  77  *
  78  * Returns true if the copy was successful, false if the preconditions
  79  * are not met.
  80  */
  81 bool i915_memcpy_from_wc(void *dst, const void *src, unsigned long len)
  82 {
  83         if (unlikely(((unsigned long)dst | (unsigned long)src | len) & 15))
  84                 return false;
  85
  86 #ifdef CONFIG_AS_MOVNTDQA
  87         if (static_branch_likely(&has_movntdqa)) {
  88                 if (likely(len))
  89                         __memcpy_ntdqa(dst, src, len);
  90                 return true;
  91         }
  92 #endif
  93
  94         return false;
  95 }
  96
  97 void i915_memcpy_init_early(struct drm_i915_private *dev_priv)
  98 {
  99         if (static_cpu_has(X86_FEATURE_XMM4_1))
 100                 static_branch_enable(&has_movntdqa);
 101 }