kernel/time/vsyscall.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright 2019 ARM Ltd.
   4  *
   5  * Generic implementation of update_vsyscall and update_vsyscall_tz.
   6  *
   7  * Based on the x86 specific implementation.
   8  */
   9
  10 #include <linux/hrtimer.h>
  11 #include <linux/timekeeper_internal.h>
  12 #include <vdso/datapage.h>
  13 #include <vdso/helpers.h>
  14 #include <vdso/vsyscall.h>
  15
  16 #include "timekeeping_internal.h"
  17
  18 static inline void update_vdso_data(struct vdso_data *vdata,
  19                                     struct timekeeper *tk)
  20 {
  21         struct vdso_timestamp *vdso_ts;
  22         u64 nsec, sec;
  23
  24         vdata[CS_HRES_COARSE].cycle_last        = tk->tkr_mono.cycle_last;
  25 #ifdef CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT
  26         vdata[CS_HRES_COARSE].max_cycles        = tk->tkr_mono.clock->max_cycles;
  27 #endif
  28         vdata[CS_HRES_COARSE].mask              = tk->tkr_mono.mask;
  29         vdata[CS_HRES_COARSE].mult              = tk->tkr_mono.mult;
  30         vdata[CS_HRES_COARSE].shift             = tk->tkr_mono.shift;
  31         vdata[CS_RAW].cycle_last                = tk->tkr_raw.cycle_last;
  32 #ifdef CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT
  33         vdata[CS_RAW].max_cycles                = tk->tkr_raw.clock->max_cycles;
  34 #endif
  35         vdata[CS_RAW].mask                      = tk->tkr_raw.mask;
  36         vdata[CS_RAW].mult                      = tk->tkr_raw.mult;
  37         vdata[CS_RAW].shift                     = tk->tkr_raw.shift;
  38
  39         /* CLOCK_MONOTONIC */
  40         vdso_ts         = &vdata[CS_HRES_COARSE].basetime[CLOCK_MONOTONIC];
  41         vdso_ts->sec    = tk->xtime_sec + tk->wall_to_monotonic.tv_sec;
  42
  43         nsec = tk->tkr_mono.xtime_nsec;
  44         nsec += ((u64)tk->wall_to_monotonic.tv_nsec << tk->tkr_mono.shift);
  45         while (nsec >= (((u64)NSEC_PER_SEC) << tk->tkr_mono.shift)) {
  46                 nsec -= (((u64)NSEC_PER_SEC) << tk->tkr_mono.shift);
  47                 vdso_ts->sec++;
  48         }
  49         vdso_ts->nsec   = nsec;
  50
  51         /* Copy MONOTONIC time for BOOTTIME */
  52         sec     = vdso_ts->sec;
  53         /* Add the boot offset */
  54         sec     += tk->monotonic_to_boot.tv_sec;
  55         nsec    += (u64)tk->monotonic_to_boot.tv_nsec << tk->tkr_mono.shift;
  56
  57         /* CLOCK_BOOTTIME */
  58         vdso_ts         = &vdata[CS_HRES_COARSE].basetime[CLOCK_BOOTTIME];
  59         vdso_ts->sec    = sec;
  60
  61         while (nsec >= (((u64)NSEC_PER_SEC) << tk->tkr_mono.shift)) {
  62                 nsec -= (((u64)NSEC_PER_SEC) << tk->tkr_mono.shift);
  63                 vdso_ts->sec++;
  64         }
  65         vdso_ts->nsec   = nsec;
  66
  67         /* CLOCK_MONOTONIC_RAW */
  68         vdso_ts         = &vdata[CS_RAW].basetime[CLOCK_MONOTONIC_RAW];
  69         vdso_ts->sec    = tk->raw_sec;
  70         vdso_ts->nsec   = tk->tkr_raw.xtime_nsec;
  71
  72         /* CLOCK_TAI */
  73         vdso_ts         = &vdata[CS_HRES_COARSE].basetime[CLOCK_TAI];
  74         vdso_ts->sec    = tk->xtime_sec + (s64)tk->tai_offset;
  75         vdso_ts->nsec   = tk->tkr_mono.xtime_nsec;
  76 }
  77
  78 void update_vsyscall(struct timekeeper *tk)
  79 {
  80         struct vdso_data *vdata = __arch_get_k_vdso_data();
  81         struct vdso_timestamp *vdso_ts;
  82         s32 clock_mode;
  83         u64 nsec;
  84
  85         /* copy vsyscall data */
  86         vdso_write_begin(vdata);
  87
  88         clock_mode = tk->tkr_mono.clock->vdso_clock_mode;
  89         vdata[CS_HRES_COARSE].clock_mode        = clock_mode;
  90         vdata[CS_RAW].clock_mode                = clock_mode;
  91
  92         /* CLOCK_REALTIME also required for time() */
  93         vdso_ts         = &vdata[CS_HRES_COARSE].basetime[CLOCK_REALTIME];
  94         vdso_ts->sec    = tk->xtime_sec;
  95         vdso_ts->nsec   = tk->tkr_mono.xtime_nsec;
  96
  97         /* CLOCK_REALTIME_COARSE */
  98         vdso_ts         = &vdata[CS_HRES_COARSE].basetime[CLOCK_REALTIME_COARSE];
  99         vdso_ts->sec    = tk->xtime_sec;
 100         vdso_ts->nsec   = tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift;
 101
 102         /* CLOCK_MONOTONIC_COARSE */
 103         vdso_ts         = &vdata[CS_HRES_COARSE].basetime[CLOCK_MONOTONIC_COARSE];
 104         vdso_ts->sec    = tk->xtime_sec + tk->wall_to_monotonic.tv_sec;
 105         nsec            = tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift;
 106         nsec            = nsec + tk->wall_to_monotonic.tv_nsec;
 107         vdso_ts->sec    += __iter_div_u64_rem(nsec, NSEC_PER_SEC, &vdso_ts->nsec);
 108
 109         /*
 110          * Read without the seqlock held by clock_getres().
 111          * Note: No need to have a second copy.
 112          */
 113         WRITE_ONCE(vdata[CS_HRES_COARSE].hrtimer_res, hrtimer_resolution);
 114
 115         /*
 116          * If the current clocksource is not VDSO capable, then spare the
 117          * update of the high resolution parts.
 118          */
 119         if (clock_mode != VDSO_CLOCKMODE_NONE)
 120                 update_vdso_data(vdata, tk);
 121
 122         __arch_update_vsyscall(vdata);
 123
 124         vdso_write_end(vdata);
 125
 126         __arch_sync_vdso_data(vdata);
 127 }
 128
 129 void update_vsyscall_tz(void)
 130 {
 131         struct vdso_data *vdata = __arch_get_k_vdso_data();
 132
 133         vdata[CS_HRES_COARSE].tz_minuteswest = sys_tz.tz_minuteswest;
 134         vdata[CS_HRES_COARSE].tz_dsttime = sys_tz.tz_dsttime;
 135
 136         __arch_sync_vdso_data(vdata);
 137 }
 138
 139 /**
 140  * vdso_update_begin - Start of a VDSO update section
 141  *
 142  * Allows architecture code to safely update the architecture specific VDSO
 143  * data. Disables interrupts, acquires timekeeper lock to serialize against
 144  * concurrent updates from timekeeping and invalidates the VDSO data
 145  * sequence counter to prevent concurrent readers from accessing
 146  * inconsistent data.
 147  *
 148  * Returns: Saved interrupt flags which need to be handed in to
 149  * vdso_update_end().
 150  */
 151 unsigned long vdso_update_begin(void)
 152 {
 153         struct vdso_data *vdata = __arch_get_k_vdso_data();
 154         unsigned long flags = timekeeper_lock_irqsave();
 155
 156         vdso_write_begin(vdata);
 157         return flags;
 158 }
 159
 160 /**
 161  * vdso_update_end - End of a VDSO update section
 162  * @flags:      Interrupt flags as returned from vdso_update_begin()
 163  *
 164  * Pairs with vdso_update_begin(). Marks vdso data consistent, invokes data
 165  * synchronization if the architecture requires it, drops timekeeper lock
 166  * and restores interrupt flags.
 167  */
 168 void vdso_update_end(unsigned long flags)
 169 {
 170         struct vdso_data *vdata = __arch_get_k_vdso_data();
 171
 172         vdso_write_end(vdata);
 173         __arch_sync_vdso_data(vdata);
 174         timekeeper_unlock_irqrestore(flags);
 175 }