kernel/sched/pelt.h

   1 #ifdef CONFIG_SMP
   2 #include "sched-pelt.h"
   3
   4 int __update_load_avg_blocked_se(u64 now, struct sched_entity *se);
   5 int __update_load_avg_se(u64 now, struct cfs_rq *cfs_rq, struct sched_entity *se);
   6 int __update_load_avg_cfs_rq(u64 now, struct cfs_rq *cfs_rq);
   7 int update_rt_rq_load_avg(u64 now, struct rq *rq, int running);
   8 int update_dl_rq_load_avg(u64 now, struct rq *rq, int running);
   9 bool update_other_load_avgs(struct rq *rq);
  10
  11 #ifdef CONFIG_SCHED_HW_PRESSURE
  12 int update_hw_load_avg(u64 now, struct rq *rq, u64 capacity);
  13
  14 static inline u64 hw_load_avg(struct rq *rq)
  15 {
  16         return READ_ONCE(rq->avg_hw.load_avg);
  17 }
  18 #else
  19 static inline int
  20 update_hw_load_avg(u64 now, struct rq *rq, u64 capacity)
  21 {
  22         return 0;
  23 }
  24
  25 static inline u64 hw_load_avg(struct rq *rq)
  26 {
  27         return 0;
  28 }
  29 #endif
  30
  31 #ifdef CONFIG_HAVE_SCHED_AVG_IRQ
  32 int update_irq_load_avg(struct rq *rq, u64 running);
  33 #else
  34 static inline int
  35 update_irq_load_avg(struct rq *rq, u64 running)
  36 {
  37         return 0;
  38 }
  39 #endif
  40
  41 #define PELT_MIN_DIVIDER        (LOAD_AVG_MAX - 1024)
  42
  43 static inline u32 get_pelt_divider(struct sched_avg *avg)
  44 {
  45         return PELT_MIN_DIVIDER + avg->period_contrib;
  46 }
  47
  48 static inline void cfs_se_util_change(struct sched_avg *avg)
  49 {
  50         unsigned int enqueued;
  51
  52         if (!sched_feat(UTIL_EST))
  53                 return;
  54
  55         /* Avoid store if the flag has been already reset */
  56         enqueued = avg->util_est;
  57         if (!(enqueued & UTIL_AVG_UNCHANGED))
  58                 return;
  59
  60         /* Reset flag to report util_avg has been updated */
  61         enqueued &= ~UTIL_AVG_UNCHANGED;
  62         WRITE_ONCE(avg->util_est, enqueued);
  63 }
  64
  65 static inline u64 rq_clock_pelt(struct rq *rq)
  66 {
  67         lockdep_assert_rq_held(rq);
  68         assert_clock_updated(rq);
  69
  70         return rq->clock_pelt - rq->lost_idle_time;
  71 }
  72
  73 /* The rq is idle, we can sync to clock_task */
  74 static inline void _update_idle_rq_clock_pelt(struct rq *rq)
  75 {
  76         rq->clock_pelt  = rq_clock_task(rq);
  77
  78         u64_u32_store(rq->clock_idle, rq_clock(rq));
  79         /* Paired with smp_rmb in migrate_se_pelt_lag() */
  80         smp_wmb();
  81         u64_u32_store(rq->clock_pelt_idle, rq_clock_pelt(rq));
  82 }
  83
  84 /*
  85  * The clock_pelt scales the time to reflect the effective amount of
  86  * computation done during the running delta time but then sync back to
  87  * clock_task when rq is idle.
  88  *
  89  *
  90  * absolute time   | 1| 2| 3| 4| 5| 6| 7| 8| 9|10|11|12|13|14|15|16
  91  * @ max capacity  ------******---------------******---------------
  92  * @ half capacity ------************---------************---------
  93  * clock pelt      | 1| 2|    3|    4| 7| 8| 9|   10|   11|14|15|16
  94  *
  95  */
  96 static inline void update_rq_clock_pelt(struct rq *rq, s64 delta)
  97 {
  98         if (unlikely(is_idle_task(rq->curr))) {
  99                 _update_idle_rq_clock_pelt(rq);
 100                 return;
 101         }
 102
 103         /*
 104          * When a rq runs at a lower compute capacity, it will need
 105          * more time to do the same amount of work than at max
 106          * capacity. In order to be invariant, we scale the delta to
 107          * reflect how much work has been really done.
 108          * Running longer results in stealing idle time that will
 109          * disturb the load signal compared to max capacity. This
 110          * stolen idle time will be automatically reflected when the
 111          * rq will be idle and the clock will be synced with
 112          * rq_clock_task.
 113          */
 114
 115         /*
 116          * Scale the elapsed time to reflect the real amount of
 117          * computation
 118          */
 119         delta = cap_scale(delta, arch_scale_cpu_capacity(cpu_of(rq)));
 120         delta = cap_scale(delta, arch_scale_freq_capacity(cpu_of(rq)));
 121
 122         rq->clock_pelt += delta;
 123 }
 124
 125 /*
 126  * When rq becomes idle, we have to check if it has lost idle time
 127  * because it was fully busy. A rq is fully used when the /Sum util_sum
 128  * is greater or equal to:
 129  * (LOAD_AVG_MAX - 1024 + rq->cfs.avg.period_contrib) << SCHED_CAPACITY_SHIFT;
 130  * For optimization and computing rounding purpose, we don't take into account
 131  * the position in the current window (period_contrib) and we use the higher
 132  * bound of util_sum to decide.
 133  */
 134 static inline void update_idle_rq_clock_pelt(struct rq *rq)
 135 {
 136         u32 divider = ((LOAD_AVG_MAX - 1024) << SCHED_CAPACITY_SHIFT) - LOAD_AVG_MAX;
 137         u32 util_sum = rq->cfs.avg.util_sum;
 138         util_sum += rq->avg_rt.util_sum;
 139         util_sum += rq->avg_dl.util_sum;
 140
 141         /*
 142          * Reflecting stolen time makes sense only if the idle
 143          * phase would be present at max capacity. As soon as the
 144          * utilization of a rq has reached the maximum value, it is
 145          * considered as an always running rq without idle time to
 146          * steal. This potential idle time is considered as lost in
 147          * this case. We keep track of this lost idle time compare to
 148          * rq's clock_task.
 149          */
 150         if (util_sum >= divider)
 151                 rq->lost_idle_time += rq_clock_task(rq) - rq->clock_pelt;
 152
 153         _update_idle_rq_clock_pelt(rq);
 154 }
 155
 156 #ifdef CONFIG_CFS_BANDWIDTH
 157 static inline void update_idle_cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
 158 {
 159         u64 throttled;
 160
 161         if (unlikely(cfs_rq->throttle_count))
 162                 throttled = U64_MAX;
 163         else
 164                 throttled = cfs_rq->throttled_clock_pelt_time;
 165
 166         u64_u32_store(cfs_rq->throttled_pelt_idle, throttled);
 167 }
 168
 169 /* rq->task_clock normalized against any time this cfs_rq has spent throttled */
 170 static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
 171 {
 172         if (unlikely(cfs_rq->throttle_count))
 173                 return cfs_rq->throttled_clock_pelt - cfs_rq->throttled_clock_pelt_time;
 174
 175         return rq_clock_pelt(rq_of(cfs_rq)) - cfs_rq->throttled_clock_pelt_time;
 176 }
 177 #else
 178 static inline void update_idle_cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) { }
 179 static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
 180 {
 181         return rq_clock_pelt(rq_of(cfs_rq));
 182 }
 183 #endif
 184
 185 #else
 186
 187 static inline int
 188 update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq)
 189 {
 190         return 0;
 191 }
 192
 193 static inline int
 194 update_rt_rq_load_avg(u64 now, struct rq *rq, int running)
 195 {
 196         return 0;
 197 }
 198
 199 static inline int
 200 update_dl_rq_load_avg(u64 now, struct rq *rq, int running)
 201 {
 202         return 0;
 203 }
 204
 205 static inline int
 206 update_hw_load_avg(u64 now, struct rq *rq, u64 capacity)
 207 {
 208         return 0;
 209 }
 210
 211 static inline u64 hw_load_avg(struct rq *rq)
 212 {
 213         return 0;
 214 }
 215
 216 static inline int
 217 update_irq_load_avg(struct rq *rq, u64 running)
 218 {
 219         return 0;
 220 }
 221
 222 static inline u64 rq_clock_pelt(struct rq *rq)
 223 {
 224         return rq_clock_task(rq);
 225 }
 226
 227 static inline void
 228 update_rq_clock_pelt(struct rq *rq, s64 delta) { }
 229
 230 static inline void
 231 update_idle_rq_clock_pelt(struct rq *rq) { }
 232
 233 static inline void update_idle_cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) { }
 234 #endif
 235
 236