arch/arm64/lib/csum.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 // Copyright (C) 2019-2020 Arm Ltd.
   3
   4 #include <linux/compiler.h>
   5 #include <linux/kasan-checks.h>
   6 #include <linux/kernel.h>
   7
   8 #include <net/checksum.h>
   9
  10 /* Looks dumb, but generates nice-ish code */
  11 static u64 accumulate(u64 sum, u64 data)
  12 {
  13         __uint128_t tmp = (__uint128_t)sum + data;
  14         return tmp + (tmp >> 64);
  15 }
  16
  17 /*
  18  * We over-read the buffer and this makes KASAN unhappy. Instead, disable
  19  * instrumentation and call kasan explicitly.
  20  */
  21 unsigned int __no_sanitize_address do_csum(const unsigned char *buff, int len)
  22 {
  23         unsigned int offset, shift, sum;
  24         const u64 *ptr;
  25         u64 data, sum64 = 0;
  26
  27         if (unlikely(len == 0))
  28                 return 0;
  29
  30         offset = (unsigned long)buff & 7;
  31         /*
  32          * This is to all intents and purposes safe, since rounding down cannot
  33          * result in a different page or cache line being accessed, and @buff
  34          * should absolutely not be pointing to anything read-sensitive. We do,
  35          * however, have to be careful not to piss off KASAN, which means using
  36          * unchecked reads to accommodate the head and tail, for which we'll
  37          * compensate with an explicit check up-front.
  38          */
  39         kasan_check_read(buff, len);
  40         ptr = (u64 *)(buff - offset);
  41         len = len + offset - 8;
  42
  43         /*
  44          * Head: zero out any excess leading bytes. Shifting back by the same
  45          * amount should be at least as fast as any other way of handling the
  46          * odd/even alignment, and means we can ignore it until the very end.
  47          */
  48         shift = offset * 8;
  49         data = *ptr++;
  50 #ifdef __LITTLE_ENDIAN
  51         data = (data >> shift) << shift;
  52 #else
  53         data = (data << shift) >> shift;
  54 #endif
  55
  56         /*
  57          * Body: straightforward aligned loads from here on (the paired loads
  58          * underlying the quadword type still only need dword alignment). The
  59          * main loop strictly excludes the tail, so the second loop will always
  60          * run at least once.
  61          */
  62         while (unlikely(len > 64)) {
  63                 __uint128_t tmp1, tmp2, tmp3, tmp4;
  64
  65                 tmp1 = *(__uint128_t *)ptr;
  66                 tmp2 = *(__uint128_t *)(ptr + 2);
  67                 tmp3 = *(__uint128_t *)(ptr + 4);
  68                 tmp4 = *(__uint128_t *)(ptr + 6);
  69
  70                 len -= 64;
  71                 ptr += 8;
  72
  73                 /* This is the "don't dump the carry flag into a GPR" idiom */
  74                 tmp1 += (tmp1 >> 64) | (tmp1 << 64);
  75                 tmp2 += (tmp2 >> 64) | (tmp2 << 64);
  76                 tmp3 += (tmp3 >> 64) | (tmp3 << 64);
  77                 tmp4 += (tmp4 >> 64) | (tmp4 << 64);
  78                 tmp1 = ((tmp1 >> 64) << 64) | (tmp2 >> 64);
  79                 tmp1 += (tmp1 >> 64) | (tmp1 << 64);
  80                 tmp3 = ((tmp3 >> 64) << 64) | (tmp4 >> 64);
  81                 tmp3 += (tmp3 >> 64) | (tmp3 << 64);
  82                 tmp1 = ((tmp1 >> 64) << 64) | (tmp3 >> 64);
  83                 tmp1 += (tmp1 >> 64) | (tmp1 << 64);
  84                 tmp1 = ((tmp1 >> 64) << 64) | sum64;
  85                 tmp1 += (tmp1 >> 64) | (tmp1 << 64);
  86                 sum64 = tmp1 >> 64;
  87         }
  88         while (len > 8) {
  89                 __uint128_t tmp;
  90
  91                 sum64 = accumulate(sum64, data);
  92                 tmp = *(__uint128_t *)ptr;
  93
  94                 len -= 16;
  95                 ptr += 2;
  96
  97 #ifdef __LITTLE_ENDIAN
  98                 data = tmp >> 64;
  99                 sum64 = accumulate(sum64, tmp);
 100 #else
 101                 data = tmp;
 102                 sum64 = accumulate(sum64, tmp >> 64);
 103 #endif
 104         }
 105         if (len > 0) {
 106                 sum64 = accumulate(sum64, data);
 107                 data = *ptr;
 108                 len -= 8;
 109         }
 110         /*
 111          * Tail: zero any over-read bytes similarly to the head, again
 112          * preserving odd/even alignment.
 113          */
 114         shift = len * -8;
 115 #ifdef __LITTLE_ENDIAN
 116         data = (data << shift) >> shift;
 117 #else
 118         data = (data >> shift) << shift;
 119 #endif
 120         sum64 = accumulate(sum64, data);
 121
 122         /* Finally, folding */
 123         sum64 += (sum64 >> 32) | (sum64 << 32);
 124         sum = sum64 >> 32;
 125         sum += (sum >> 16) | (sum << 16);
 126         if (offset & 1)
 127                 return (u16)swab32(sum);
 128
 129         return sum >> 16;
 130 }
 131
 132 __sum16 csum_ipv6_magic(const struct in6_addr *saddr,
 133                         const struct in6_addr *daddr,
 134                         __u32 len, __u8 proto, __wsum csum)
 135 {
 136         __uint128_t src, dst;
 137         u64 sum = (__force u64)csum;
 138
 139         src = *(const __uint128_t *)saddr->s6_addr;
 140         dst = *(const __uint128_t *)daddr->s6_addr;
 141
 142         sum += (__force u32)htonl(len);
 143 #ifdef __LITTLE_ENDIAN
 144         sum += (u32)proto << 24;
 145 #else
 146         sum += proto;
 147 #endif
 148         src += (src >> 64) | (src << 64);
 149         dst += (dst >> 64) | (dst << 64);
 150
 151         sum = accumulate(sum, src >> 64);
 152         sum = accumulate(sum, dst >> 64);
 153
 154         sum += ((sum >> 32) | (sum << 32));
 155         return csum_fold((__force __wsum)(sum >> 32));
 156 }
 157 EXPORT_SYMBOL(csum_ipv6_magic);