kernel/time/timer_migration.h

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 #ifndef _KERNEL_TIME_MIGRATION_H
   3 #define _KERNEL_TIME_MIGRATION_H
   4
   5 /* Per group capacity. Must be a power of 2! */
   6 #define TMIGR_CHILDREN_PER_GROUP 8
   7
   8 /**
   9  * struct tmigr_event - a timer event associated to a CPU
  10  * @nextevt:    The node to enqueue an event in the parent group queue
  11  * @cpu:        The CPU to which this event belongs
  12  * @ignore:     Hint whether the event could be ignored; it is set when
  13  *              CPU or group is active;
  14  */
  15 struct tmigr_event {
  16         struct timerqueue_node  nextevt;
  17         unsigned int            cpu;
  18         bool                    ignore;
  19 };
  20
  21 /**
  22  * struct tmigr_group - timer migration hierarchy group
  23  * @lock:               Lock protecting the event information and group hierarchy
  24  *                      information during setup
  25  * @parent:             Pointer to the parent group. Pointer is updated when a
  26  *                      new hierarchy level is added because of a CPU coming
  27  *                      online the first time. Once it is set, the pointer will
  28  *                      not be removed or updated. When accessing parent pointer
  29  *                      lock less to decide whether to abort a propagation or
  30  *                      not, it is not a problem. The worst outcome is an
  31  *                      unnecessary/early CPU wake up. But do not access parent
  32  *                      pointer several times in the same 'action' (like
  33  *                      activation, deactivation, check for remote expiry,...)
  34  *                      without holding the lock as it is not ensured that value
  35  *                      will not change.
  36  * @groupevt:           Next event of the group which is only used when the
  37  *                      group is !active. The group event is then queued into
  38  *                      the parent timer queue.
  39  *                      Ignore bit of @groupevt is set when the group is active.
  40  * @next_expiry:        Base monotonic expiry time of the next event of the
  41  *                      group; It is used for the racy lockless check whether a
  42  *                      remote expiry is required; it is always reliable
  43  * @events:             Timer queue for child events queued in the group
  44  * @migr_state:         State of the group (see union tmigr_state)
  45  * @level:              Hierarchy level of the group; Required during setup
  46  * @numa_node:          Required for setup only to make sure CPU and low level
  47  *                      group information is NUMA local. It is set to NUMA node
  48  *                      as long as the group level is per NUMA node (level <
  49  *                      tmigr_crossnode_level); otherwise it is set to
  50  *                      NUMA_NO_NODE
  51  * @num_children:       Counter of group children to make sure the group is only
  52  *                      filled with TMIGR_CHILDREN_PER_GROUP; Required for setup
  53  *                      only
  54  * @groupmask:          mask of the group in the parent group; is set during
  55  *                      setup and will never change; can be read lockless
  56  * @list:               List head that is added to the per level
  57  *                      tmigr_level_list; is required during setup when a
  58  *                      new group needs to be connected to the existing
  59  *                      hierarchy groups
  60  */
  61 struct tmigr_group {
  62         raw_spinlock_t          lock;
  63         struct tmigr_group      *parent;
  64         struct tmigr_event      groupevt;
  65         u64                     next_expiry;
  66         struct timerqueue_head  events;
  67         atomic_t                migr_state;
  68         unsigned int            level;
  69         int                     numa_node;
  70         unsigned int            num_children;
  71         u8                      groupmask;
  72         struct list_head        list;
  73 };
  74
  75 /**
  76  * struct tmigr_cpu - timer migration per CPU group
  77  * @lock:               Lock protecting the tmigr_cpu group information
  78  * @online:             Indicates whether the CPU is online; In deactivate path
  79  *                      it is required to know whether the migrator in the top
  80  *                      level group is to be set offline, while a timer is
  81  *                      pending. Then another online CPU needs to be notified to
  82  *                      take over the migrator role. Furthermore the information
  83  *                      is required in CPU hotplug path as the CPU is able to go
  84  *                      idle before the timer migration hierarchy hotplug AP is
  85  *                      reached. During this phase, the CPU has to handle the
  86  *                      global timers on its own and must not act as a migrator.
  87  * @idle:               Indicates whether the CPU is idle in the timer migration
  88  *                      hierarchy
  89  * @remote:             Is set when timers of the CPU are expired remotely
  90  * @tmgroup:            Pointer to the parent group
  91  * @groupmask:          mask of tmigr_cpu in the parent group
  92  * @wakeup:             Stores the first timer when the timer migration
  93  *                      hierarchy is completely idle and remote expiry was done;
  94  *                      is returned to timer code in the idle path and is only
  95  *                      used in idle path.
  96  * @cpuevt:             CPU event which could be enqueued into the parent group
  97  */
  98 struct tmigr_cpu {
  99         raw_spinlock_t          lock;
 100         bool                    online;
 101         bool                    idle;
 102         bool                    remote;
 103         struct tmigr_group      *tmgroup;
 104         u8                      groupmask;
 105         u64                     wakeup;
 106         struct tmigr_event      cpuevt;
 107 };
 108
 109 /**
 110  * union tmigr_state - state of tmigr_group
 111  * @state:      Combined version of the state - only used for atomic
 112  *              read/cmpxchg function
 113  * @struct:     Split version of the state - only use the struct members to
 114  *              update information to stay independent of endianness
 115  */
 116 union tmigr_state {
 117         u32 state;
 118         /**
 119          * struct - split state of tmigr_group
 120          * @active:     Contains each mask bit of the active children
 121          * @migrator:   Contains mask of the child which is migrator
 122          * @seq:        Sequence counter needs to be increased when an update
 123          *              to the tmigr_state is done. It prevents a race when
 124          *              updates in the child groups are propagated in changed
 125          *              order. Detailed information about the scenario is
 126          *              given in the documentation at the begin of
 127          *              timer_migration.c.
 128          */
 129         struct {
 130                 u8      active;
 131                 u8      migrator;
 132                 u16     seq;
 133         } __packed;
 134 };
 135
 136 #if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
 137 extern void tmigr_handle_remote(void);
 138 extern bool tmigr_requires_handle_remote(void);
 139 extern void tmigr_cpu_activate(void);
 140 extern u64 tmigr_cpu_deactivate(u64 nextevt);
 141 extern u64 tmigr_cpu_new_timer(u64 nextevt);
 142 extern u64 tmigr_quick_check(u64 nextevt);
 143 #else
 144 static inline void tmigr_handle_remote(void) { }
 145 static inline bool tmigr_requires_handle_remote(void) { return false; }
 146 static inline void tmigr_cpu_activate(void) { }
 147 #endif
 148
 149 #endif