arch/arm64/kernel/io.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Based on arch/arm/kernel/io.c
   4  *
   5  * Copyright (C) 2012 ARM Ltd.
   6  */
   7
   8 #include <linux/export.h>
   9 #include <linux/types.h>
  10 #include <linux/io.h>
  11
  12 /*
  13  * Copy data from IO memory space to "real" memory space.
  14  */
  15 void __memcpy_fromio(void *to, const volatile void __iomem *from, size_t count)
  16 {
  17         while (count && !IS_ALIGNED((unsigned long)from, 8)) {
  18                 *(u8 *)to = __raw_readb(from);
  19                 from++;
  20                 to++;
  21                 count--;
  22         }
  23
  24         while (count >= 8) {
  25                 *(u64 *)to = __raw_readq(from);
  26                 from += 8;
  27                 to += 8;
  28                 count -= 8;
  29         }
  30
  31         while (count) {
  32                 *(u8 *)to = __raw_readb(from);
  33                 from++;
  34                 to++;
  35                 count--;
  36         }
  37 }
  38 EXPORT_SYMBOL(__memcpy_fromio);
  39
  40 /*
  41  * This generates a memcpy that works on a from/to address which is aligned to
  42  * bits. Count is in terms of the number of bits sized quantities to copy. It
  43  * optimizes to use the STR groupings when possible so that it is WC friendly.
  44  */
  45 #define memcpy_toio_aligned(to, from, count, bits)                        \
  46         ({                                                                \
  47                 volatile u##bits __iomem *_to = to;                       \
  48                 const u##bits *_from = from;                              \
  49                 size_t _count = count;                                    \
  50                 const u##bits *_end_from = _from + ALIGN_DOWN(_count, 8); \
  51                                                                           \
  52                 for (; _from < _end_from; _from += 8, _to += 8)           \
  53                         __const_memcpy_toio_aligned##bits(_to, _from, 8); \
  54                 if ((_count % 8) >= 4) {                                  \
  55                         __const_memcpy_toio_aligned##bits(_to, _from, 4); \
  56                         _from += 4;                                       \
  57                         _to += 4;                                         \
  58                 }                                                         \
  59                 if ((_count % 4) >= 2) {                                  \
  60                         __const_memcpy_toio_aligned##bits(_to, _from, 2); \
  61                         _from += 2;                                       \
  62                         _to += 2;                                         \
  63                 }                                                         \
  64                 if (_count % 2)                                           \
  65                         __const_memcpy_toio_aligned##bits(_to, _from, 1); \
  66         })
  67
  68 void __iowrite64_copy_full(void __iomem *to, const void *from, size_t count)
  69 {
  70         memcpy_toio_aligned(to, from, count, 64);
  71         dgh();
  72 }
  73 EXPORT_SYMBOL(__iowrite64_copy_full);
  74
  75 void __iowrite32_copy_full(void __iomem *to, const void *from, size_t count)
  76 {
  77         memcpy_toio_aligned(to, from, count, 32);
  78         dgh();
  79 }
  80 EXPORT_SYMBOL(__iowrite32_copy_full);
  81
  82 /*
  83  * Copy data from "real" memory space to IO memory space.
  84  */
  85 void __memcpy_toio(volatile void __iomem *to, const void *from, size_t count)
  86 {
  87         while (count && !IS_ALIGNED((unsigned long)to, 8)) {
  88                 __raw_writeb(*(u8 *)from, to);
  89                 from++;
  90                 to++;
  91                 count--;
  92         }
  93
  94         while (count >= 8) {
  95                 __raw_writeq(*(u64 *)from, to);
  96                 from += 8;
  97                 to += 8;
  98                 count -= 8;
  99         }
 100
 101         while (count) {
 102                 __raw_writeb(*(u8 *)from, to);
 103                 from++;
 104                 to++;
 105                 count--;
 106         }
 107 }
 108 EXPORT_SYMBOL(__memcpy_toio);
 109
 110 /*
 111  * "memset" on IO memory space.
 112  */
 113 void __memset_io(volatile void __iomem *dst, int c, size_t count)
 114 {
 115         u64 qc = (u8)c;
 116
 117         qc |= qc << 8;
 118         qc |= qc << 16;
 119         qc |= qc << 32;
 120
 121         while (count && !IS_ALIGNED((unsigned long)dst, 8)) {
 122                 __raw_writeb(c, dst);
 123                 dst++;
 124                 count--;
 125         }
 126
 127         while (count >= 8) {
 128                 __raw_writeq(qc, dst);
 129                 dst += 8;
 130                 count -= 8;
 131         }
 132
 133         while (count) {
 134                 __raw_writeb(c, dst);
 135                 dst++;
 136                 count--;
 137         }
 138 }
 139 EXPORT_SYMBOL(__memset_io);