payloads/libpayload/arch/x86/cache.c

   1 /*
   2  *
   3  * Copyright 2022 Google LLC
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  * 3. The name of the author may not be used to endorse or promote products
  14  *    derived from this software without specific prior written permission.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  */
  28
  29 #include <stdint.h>
  30
  31 #include <arch/cache.h>
  32 #include <arch/cpuid.h>
  33
  34 unsigned int dcache_line_bytes(void)
  35 {
  36         /*
  37          * The value returned in EBX[15:8] is in 8-byte increments.
  38          * Cache line size is EBX[15:8] * 8
  39          */
  40         return (cpuid_ebx(1) & 0xff00) >> 5;
  41 }
  42
  43 static inline int cpu_supports_wbnoinvd(void)
  44 {
  45         return (cpuid_ebx(0x80000008) >> 9) & 1;
  46 }
  47
  48 static inline int cpu_supports_clwb(void)
  49 {
  50         return (cpuid_ebx(7) >> 24) & 1;
  51 }
  52
  53 static inline int cpu_supports_clflushopt(void)
  54 {
  55         return (cpuid_sub_leaf_ebx(7, 0) >> 23) & 1;
  56 }
  57
  58 static inline int cpu_supports_clflush(void)
  59 {
  60         return (cpuid_ebx(1) >> 19) & 1;
  61 }
  62
  63 inline void dcache_invalidate_all(void)
  64 {
  65         asm volatile("invd" ::: "memory");
  66 }
  67
  68 inline void dcache_clean_invalidate_all(void)
  69 {
  70         asm volatile("wbinvd" ::: "memory");
  71 }
  72
  73 inline void dcache_clean_all(void)
  74 {
  75         if (cpu_supports_wbnoinvd()) {
  76                 asm volatile(
  77                         "sfence\n\t"
  78                         "wbnoinvd\n\t"
  79                 ::: "memory");
  80         } else {
  81                 dcache_clean_invalidate_all();
  82         }
  83 }
  84
  85 void dcache_clean_by_mva(void const *addr, size_t len)
  86 {
  87         unsigned long line, linesize;
  88
  89         linesize = dcache_line_bytes();
  90         line = (uintptr_t)addr & ~(linesize - 1);
  91
  92         if (cpu_supports_clwb()) {
  93                 asm volatile("sfence");
  94                 while (line < (uintptr_t)addr + len) {
  95                         asm volatile("clwb (%0)" : : "r"(line) : "memory");
  96                         line += linesize;
  97                 }
  98         } else {
  99                 dcache_clean_invalidate_by_mva(addr, len);
 100         }
 101 }
 102
 103 void dcache_invalidate_by_mva(void const *addr, size_t len)
 104 {
 105         /*
 106          * x86 doesn't have a "invalidate without clean" for a cache line, fall
 107          * back to both.
 108          */
 109         dcache_clean_invalidate_by_mva(addr, len);
 110 }
 111
 112 void dcache_clean_invalidate_by_mva(void const *addr, size_t len)
 113 {
 114         unsigned long line, linesize;
 115
 116         linesize = dcache_line_bytes();
 117         line = (uintptr_t)addr & ~(linesize - 1);
 118
 119         if (cpu_supports_clflushopt()) {
 120                 asm volatile("sfence");
 121                 while (line < (uintptr_t)addr + len) {
 122                         asm volatile("clflushopt (%0)" ::"r"(line) : "memory");
 123                         line += linesize;
 124                 }
 125         } else if (cpu_supports_clflush()) {
 126                 asm volatile("sfence");
 127                 while (line < (uintptr_t)addr + len) {
 128                         asm volatile("clflush (%0)" : : "r"(line) : "memory");
 129                         line += linesize;
 130                 }
 131         } else {
 132                 dcache_clean_invalidate_all();
 133         }
 134 }