include/asm-generic/bitops/instrumented-non-atomic.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2
   3 /*
   4  * This file provides wrappers with sanitizer instrumentation for non-atomic
   5  * bit operations.
   6  *
   7  * To use this functionality, an arch's bitops.h file needs to define each of
   8  * the below bit operations with an arch_ prefix (e.g. arch_set_bit(),
   9  * arch___set_bit(), etc.).
  10  */
  11 #ifndef _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H
  12 #define _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H
  13
  14 #include <linux/instrumented.h>
  15
  16 /**
  17  * ___set_bit - Set a bit in memory
  18  * @nr: the bit to set
  19  * @addr: the address to start counting from
  20  *
  21  * Unlike set_bit(), this function is non-atomic. If it is called on the same
  22  * region of memory concurrently, the effect may be that only one operation
  23  * succeeds.
  24  */
  25 static __always_inline void
  26 ___set_bit(unsigned long nr, volatile unsigned long *addr)
  27 {
  28         instrument_write(addr + BIT_WORD(nr), sizeof(long));
  29         arch___set_bit(nr, addr);
  30 }
  31
  32 /**
  33  * ___clear_bit - Clears a bit in memory
  34  * @nr: the bit to clear
  35  * @addr: the address to start counting from
  36  *
  37  * Unlike clear_bit(), this function is non-atomic. If it is called on the same
  38  * region of memory concurrently, the effect may be that only one operation
  39  * succeeds.
  40  */
  41 static __always_inline void
  42 ___clear_bit(unsigned long nr, volatile unsigned long *addr)
  43 {
  44         instrument_write(addr + BIT_WORD(nr), sizeof(long));
  45         arch___clear_bit(nr, addr);
  46 }
  47
  48 /**
  49  * ___change_bit - Toggle a bit in memory
  50  * @nr: the bit to change
  51  * @addr: the address to start counting from
  52  *
  53  * Unlike change_bit(), this function is non-atomic. If it is called on the same
  54  * region of memory concurrently, the effect may be that only one operation
  55  * succeeds.
  56  */
  57 static __always_inline void
  58 ___change_bit(unsigned long nr, volatile unsigned long *addr)
  59 {
  60         instrument_write(addr + BIT_WORD(nr), sizeof(long));
  61         arch___change_bit(nr, addr);
  62 }
  63
  64 static __always_inline void __instrument_read_write_bitop(long nr, volatile unsigned long *addr)
  65 {
  66         if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC)) {
  67                 /*
  68                  * We treat non-atomic read-write bitops a little more special.
  69                  * Given the operations here only modify a single bit, assuming
  70                  * non-atomicity of the writer is sufficient may be reasonable
  71                  * for certain usage (and follows the permissible nature of the
  72                  * assume-plain-writes-atomic rule):
  73                  * 1. report read-modify-write races -> check read;
  74                  * 2. do not report races with marked readers, but do report
  75                  *    races with unmarked readers -> check "atomic" write.
  76                  */
  77                 kcsan_check_read(addr + BIT_WORD(nr), sizeof(long));
  78                 /*
  79                  * Use generic write instrumentation, in case other sanitizers
  80                  * or tools are enabled alongside KCSAN.
  81                  */
  82                 instrument_write(addr + BIT_WORD(nr), sizeof(long));
  83         } else {
  84                 instrument_read_write(addr + BIT_WORD(nr), sizeof(long));
  85         }
  86 }
  87
  88 /**
  89  * ___test_and_set_bit - Set a bit and return its old value
  90  * @nr: Bit to set
  91  * @addr: Address to count from
  92  *
  93  * This operation is non-atomic. If two instances of this operation race, one
  94  * can appear to succeed but actually fail.
  95  */
  96 static __always_inline bool
  97 ___test_and_set_bit(unsigned long nr, volatile unsigned long *addr)
  98 {
  99         __instrument_read_write_bitop(nr, addr);
 100         return arch___test_and_set_bit(nr, addr);
 101 }
 102
 103 /**
 104  * ___test_and_clear_bit - Clear a bit and return its old value
 105  * @nr: Bit to clear
 106  * @addr: Address to count from
 107  *
 108  * This operation is non-atomic. If two instances of this operation race, one
 109  * can appear to succeed but actually fail.
 110  */
 111 static __always_inline bool
 112 ___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr)
 113 {
 114         __instrument_read_write_bitop(nr, addr);
 115         return arch___test_and_clear_bit(nr, addr);
 116 }
 117
 118 /**
 119  * ___test_and_change_bit - Change a bit and return its old value
 120  * @nr: Bit to change
 121  * @addr: Address to count from
 122  *
 123  * This operation is non-atomic. If two instances of this operation race, one
 124  * can appear to succeed but actually fail.
 125  */
 126 static __always_inline bool
 127 ___test_and_change_bit(unsigned long nr, volatile unsigned long *addr)
 128 {
 129         __instrument_read_write_bitop(nr, addr);
 130         return arch___test_and_change_bit(nr, addr);
 131 }
 132
 133 /**
 134  * _test_bit - Determine whether a bit is set
 135  * @nr: bit number to test
 136  * @addr: Address to start counting from
 137  */
 138 static __always_inline bool
 139 _test_bit(unsigned long nr, const volatile unsigned long *addr)
 140 {
 141         instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long));
 142         return arch_test_bit(nr, addr);
 143 }
 144
 145 /**
 146  * _test_bit_acquire - Determine, with acquire semantics, whether a bit is set
 147  * @nr: bit number to test
 148  * @addr: Address to start counting from
 149  */
 150 static __always_inline bool
 151 _test_bit_acquire(unsigned long nr, const volatile unsigned long *addr)
 152 {
 153         instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long));
 154         return arch_test_bit_acquire(nr, addr);
 155 }
 156
 157 #endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H */