include/vdso/helpers.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef __VDSO_HELPERS_H
   3 #define __VDSO_HELPERS_H
   4
   5 #ifndef __ASSEMBLY__
   6
   7 #include <vdso/datapage.h>
   8
   9 static __always_inline u32 vdso_read_begin(const struct vdso_data *vd)
  10 {
  11         u32 seq;
  12
  13         while (unlikely((seq = READ_ONCE(vd->seq)) & 1))
  14                 cpu_relax();
  15
  16         smp_rmb();
  17         return seq;
  18 }
  19
  20 static __always_inline u32 vdso_read_retry(const struct vdso_data *vd,
  21                                            u32 start)
  22 {
  23         u32 seq;
  24
  25         smp_rmb();
  26         seq = READ_ONCE(vd->seq);
  27         return seq != start;
  28 }
  29
  30 static __always_inline void vdso_write_begin(struct vdso_data *vd)
  31 {
  32         /*
  33          * WRITE_ONCE it is required otherwise the compiler can validly tear
  34          * updates to vd[x].seq and it is possible that the value seen by the
  35          * reader it is inconsistent.
  36          */
  37         WRITE_ONCE(vd[CS_HRES_COARSE].seq, vd[CS_HRES_COARSE].seq + 1);
  38         WRITE_ONCE(vd[CS_RAW].seq, vd[CS_RAW].seq + 1);
  39         smp_wmb();
  40 }
  41
  42 static __always_inline void vdso_write_end(struct vdso_data *vd)
  43 {
  44         smp_wmb();
  45         /*
  46          * WRITE_ONCE it is required otherwise the compiler can validly tear
  47          * updates to vd[x].seq and it is possible that the value seen by the
  48          * reader it is inconsistent.
  49          */
  50         WRITE_ONCE(vd[CS_HRES_COARSE].seq, vd[CS_HRES_COARSE].seq + 1);
  51         WRITE_ONCE(vd[CS_RAW].seq, vd[CS_RAW].seq + 1);
  52 }
  53
  54 #endif /* !__ASSEMBLY__ */
  55
  56 #endif /* __VDSO_HELPERS_H */