arch/x86/kernel/cpu/intel_rdt.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _ASM_X86_INTEL_RDT_H
   3 #define _ASM_X86_INTEL_RDT_H
   4
   5 #include <linux/sched.h>
   6 #include <linux/kernfs.h>
   7 #include <linux/jump_label.h>
   8
   9 #define IA32_L3_QOS_CFG         0xc81
  10 #define IA32_L3_CBM_BASE        0xc90
  11 #define IA32_L2_CBM_BASE        0xd10
  12 #define IA32_MBA_THRTL_BASE     0xd50
  13
  14 #define L3_QOS_CDP_ENABLE       0x01ULL
  15
  16 /*
  17  * Event IDs are used to program IA32_QM_EVTSEL before reading event
  18  * counter from IA32_QM_CTR
  19  */
  20 #define QOS_L3_OCCUP_EVENT_ID           0x01
  21 #define QOS_L3_MBM_TOTAL_EVENT_ID       0x02
  22 #define QOS_L3_MBM_LOCAL_EVENT_ID       0x03
  23
  24 #define CQM_LIMBOCHECK_INTERVAL 1000
  25
  26 #define MBM_CNTR_WIDTH                  24
  27 #define MBM_OVERFLOW_INTERVAL           1000
  28
  29 #define RMID_VAL_ERROR                  BIT_ULL(63)
  30 #define RMID_VAL_UNAVAIL                BIT_ULL(62)
  31
  32 DECLARE_STATIC_KEY_FALSE(rdt_enable_key);
  33
  34 /**
  35  * struct mon_evt - Entry in the event list of a resource
  36  * @evtid:              event id
  37  * @name:               name of the event
  38  */
  39 struct mon_evt {
  40         u32                     evtid;
  41         char                    *name;
  42         struct list_head        list;
  43 };
  44
  45 /**
  46  * struct mon_data_bits - Monitoring details for each event file
  47  * @rid:               Resource id associated with the event file.
  48  * @evtid:             Event id associated with the event file
  49  * @domid:             The domain to which the event file belongs
  50  */
  51 union mon_data_bits {
  52         void *priv;
  53         struct {
  54                 unsigned int rid        : 10;
  55                 unsigned int evtid      : 8;
  56                 unsigned int domid      : 14;
  57         } u;
  58 };
  59
  60 struct rmid_read {
  61         struct rdtgroup         *rgrp;
  62         struct rdt_domain       *d;
  63         int                     evtid;
  64         bool                    first;
  65         u64                     val;
  66 };
  67
  68 extern unsigned int intel_cqm_threshold;
  69 extern bool rdt_alloc_capable;
  70 extern bool rdt_mon_capable;
  71 extern unsigned int rdt_mon_features;
  72
  73 enum rdt_group_type {
  74         RDTCTRL_GROUP = 0,
  75         RDTMON_GROUP,
  76         RDT_NUM_GROUP,
  77 };
  78
  79 /**
  80  * struct mongroup - store mon group's data in resctrl fs.
  81  * @mon_data_kn         kernlfs node for the mon_data directory
  82  * @parent:                     parent rdtgrp
  83  * @crdtgrp_list:               child rdtgroup node list
  84  * @rmid:                       rmid for this rdtgroup
  85  */
  86 struct mongroup {
  87         struct kernfs_node      *mon_data_kn;
  88         struct rdtgroup         *parent;
  89         struct list_head        crdtgrp_list;
  90         u32                     rmid;
  91 };
  92
  93 /**
  94  * struct rdtgroup - store rdtgroup's data in resctrl file system.
  95  * @kn:                         kernfs node
  96  * @rdtgroup_list:              linked list for all rdtgroups
  97  * @closid:                     closid for this rdtgroup
  98  * @cpu_mask:                   CPUs assigned to this rdtgroup
  99  * @flags:                      status bits
 100  * @waitcount:                  how many cpus expect to find this
 101  *                              group when they acquire rdtgroup_mutex
 102  * @type:                       indicates type of this rdtgroup - either
 103  *                              monitor only or ctrl_mon group
 104  * @mon:                        mongroup related data
 105  */
 106 struct rdtgroup {
 107         struct kernfs_node      *kn;
 108         struct list_head        rdtgroup_list;
 109         u32                     closid;
 110         struct cpumask          cpu_mask;
 111         int                     flags;
 112         atomic_t                waitcount;
 113         enum rdt_group_type     type;
 114         struct mongroup         mon;
 115 };
 116
 117 /* rdtgroup.flags */
 118 #define RDT_DELETED             1
 119
 120 /* rftype.flags */
 121 #define RFTYPE_FLAGS_CPUS_LIST  1
 122
 123 /*
 124  * Define the file type flags for base and info directories.
 125  */
 126 #define RFTYPE_INFO                     BIT(0)
 127 #define RFTYPE_BASE                     BIT(1)
 128 #define RF_CTRLSHIFT                    4
 129 #define RF_MONSHIFT                     5
 130 #define RFTYPE_CTRL                     BIT(RF_CTRLSHIFT)
 131 #define RFTYPE_MON                      BIT(RF_MONSHIFT)
 132 #define RFTYPE_RES_CACHE                BIT(8)
 133 #define RFTYPE_RES_MB                   BIT(9)
 134 #define RF_CTRL_INFO                    (RFTYPE_INFO | RFTYPE_CTRL)
 135 #define RF_MON_INFO                     (RFTYPE_INFO | RFTYPE_MON)
 136 #define RF_CTRL_BASE                    (RFTYPE_BASE | RFTYPE_CTRL)
 137
 138 /* List of all resource groups */
 139 extern struct list_head rdt_all_groups;
 140
 141 extern int max_name_width, max_data_width;
 142
 143 int __init rdtgroup_init(void);
 144
 145 /**
 146  * struct rftype - describe each file in the resctrl file system
 147  * @name:       File name
 148  * @mode:       Access mode
 149  * @kf_ops:     File operations
 150  * @flags:      File specific RFTYPE_FLAGS_* flags
 151  * @fflags:     File specific RF_* or RFTYPE_* flags
 152  * @seq_show:   Show content of the file
 153  * @write:      Write to the file
 154  */
 155 struct rftype {
 156         char                    *name;
 157         umode_t                 mode;
 158         struct kernfs_ops       *kf_ops;
 159         unsigned long           flags;
 160         unsigned long           fflags;
 161
 162         int (*seq_show)(struct kernfs_open_file *of,
 163                         struct seq_file *sf, void *v);
 164         /*
 165          * write() is the generic write callback which maps directly to
 166          * kernfs write operation and overrides all other operations.
 167          * Maximum write size is determined by ->max_write_len.
 168          */
 169         ssize_t (*write)(struct kernfs_open_file *of,
 170                          char *buf, size_t nbytes, loff_t off);
 171 };
 172
 173 /**
 174  * struct mbm_state - status for each MBM counter in each domain
 175  * @chunks:     Total data moved (multiply by rdt_group.mon_scale to get bytes)
 176  * @prev_msr    Value of IA32_QM_CTR for this RMID last time we read it
 177  */
 178 struct mbm_state {
 179         u64     chunks;
 180         u64     prev_msr;
 181 };
 182
 183 /**
 184  * struct rdt_domain - group of cpus sharing an RDT resource
 185  * @list:       all instances of this resource
 186  * @id:         unique id for this instance
 187  * @cpu_mask:   which cpus share this resource
 188  * @rmid_busy_llc:
 189  *              bitmap of which limbo RMIDs are above threshold
 190  * @mbm_total:  saved state for MBM total bandwidth
 191  * @mbm_local:  saved state for MBM local bandwidth
 192  * @mbm_over:   worker to periodically read MBM h/w counters
 193  * @cqm_limbo:  worker to periodically read CQM h/w counters
 194  * @mbm_work_cpu:
 195  *              worker cpu for MBM h/w counters
 196  * @cqm_work_cpu:
 197  *              worker cpu for CQM h/w counters
 198  * @ctrl_val:   array of cache or mem ctrl values (indexed by CLOSID)
 199  * @new_ctrl:   new ctrl value to be loaded
 200  * @have_new_ctrl: did user provide new_ctrl for this domain
 201  */
 202 struct rdt_domain {
 203         struct list_head        list;
 204         int                     id;
 205         struct cpumask          cpu_mask;
 206         unsigned long           *rmid_busy_llc;
 207         struct mbm_state        *mbm_total;
 208         struct mbm_state        *mbm_local;
 209         struct delayed_work     mbm_over;
 210         struct delayed_work     cqm_limbo;
 211         int                     mbm_work_cpu;
 212         int                     cqm_work_cpu;
 213         u32                     *ctrl_val;
 214         u32                     new_ctrl;
 215         bool                    have_new_ctrl;
 216 };
 217
 218 /**
 219  * struct msr_param - set a range of MSRs from a domain
 220  * @res:       The resource to use
 221  * @low:       Beginning index from base MSR
 222  * @high:      End index
 223  */
 224 struct msr_param {
 225         struct rdt_resource     *res;
 226         int                     low;
 227         int                     high;
 228 };
 229
 230 /**
 231  * struct rdt_cache - Cache allocation related data
 232  * @cbm_len:            Length of the cache bit mask
 233  * @min_cbm_bits:       Minimum number of consecutive bits to be set
 234  * @cbm_idx_mult:       Multiplier of CBM index
 235  * @cbm_idx_offset:     Offset of CBM index. CBM index is computed by:
 236  *                      closid * cbm_idx_multi + cbm_idx_offset
 237  *                      in a cache bit mask
 238  * @shareable_bits:     Bitmask of shareable resource with other
 239  *                      executing entities
 240  */
 241 struct rdt_cache {
 242         unsigned int    cbm_len;
 243         unsigned int    min_cbm_bits;
 244         unsigned int    cbm_idx_mult;
 245         unsigned int    cbm_idx_offset;
 246         unsigned int    shareable_bits;
 247 };
 248
 249 /**
 250  * struct rdt_membw - Memory bandwidth allocation related data
 251  * @max_delay:          Max throttle delay. Delay is the hardware
 252  *                      representation for memory bandwidth.
 253  * @min_bw:             Minimum memory bandwidth percentage user can request
 254  * @bw_gran:            Granularity at which the memory bandwidth is allocated
 255  * @delay_linear:       True if memory B/W delay is in linear scale
 256  * @mb_map:             Mapping of memory B/W percentage to memory B/W delay
 257  */
 258 struct rdt_membw {
 259         u32             max_delay;
 260         u32             min_bw;
 261         u32             bw_gran;
 262         u32             delay_linear;
 263         u32             *mb_map;
 264 };
 265
 266 static inline bool is_llc_occupancy_enabled(void)
 267 {
 268         return (rdt_mon_features & (1 << QOS_L3_OCCUP_EVENT_ID));
 269 }
 270
 271 static inline bool is_mbm_total_enabled(void)
 272 {
 273         return (rdt_mon_features & (1 << QOS_L3_MBM_TOTAL_EVENT_ID));
 274 }
 275
 276 static inline bool is_mbm_local_enabled(void)
 277 {
 278         return (rdt_mon_features & (1 << QOS_L3_MBM_LOCAL_EVENT_ID));
 279 }
 280
 281 static inline bool is_mbm_enabled(void)
 282 {
 283         return (is_mbm_total_enabled() || is_mbm_local_enabled());
 284 }
 285
 286 static inline bool is_mbm_event(int e)
 287 {
 288         return (e >= QOS_L3_MBM_TOTAL_EVENT_ID &&
 289                 e <= QOS_L3_MBM_LOCAL_EVENT_ID);
 290 }
 291
 292 /**
 293  * struct rdt_resource - attributes of an RDT resource
 294  * @rid:                The index of the resource
 295  * @alloc_enabled:      Is allocation enabled on this machine
 296  * @mon_enabled:                Is monitoring enabled for this feature
 297  * @alloc_capable:      Is allocation available on this machine
 298  * @mon_capable:                Is monitor feature available on this machine
 299  * @name:               Name to use in "schemata" file
 300  * @num_closid:         Number of CLOSIDs available
 301  * @cache_level:        Which cache level defines scope of this resource
 302  * @default_ctrl:       Specifies default cache cbm or memory B/W percent.
 303  * @msr_base:           Base MSR address for CBMs
 304  * @msr_update:         Function pointer to update QOS MSRs
 305  * @data_width:         Character width of data when displaying
 306  * @domains:            All domains for this resource
 307  * @cache:              Cache allocation related data
 308  * @format_str:         Per resource format string to show domain value
 309  * @parse_ctrlval:      Per resource function pointer to parse control values
 310  * @evt_list:                   List of monitoring events
 311  * @num_rmid:                   Number of RMIDs available
 312  * @mon_scale:                  cqm counter * mon_scale = occupancy in bytes
 313  * @fflags:                     flags to choose base and info files
 314  */
 315 struct rdt_resource {
 316         int                     rid;
 317         bool                    alloc_enabled;
 318         bool                    mon_enabled;
 319         bool                    alloc_capable;
 320         bool                    mon_capable;
 321         char                    *name;
 322         int                     num_closid;
 323         int                     cache_level;
 324         u32                     default_ctrl;
 325         unsigned int            msr_base;
 326         void (*msr_update)      (struct rdt_domain *d, struct msr_param *m,
 327                                  struct rdt_resource *r);
 328         int                     data_width;
 329         struct list_head        domains;
 330         struct rdt_cache        cache;
 331         struct rdt_membw        membw;
 332         const char              *format_str;
 333         int (*parse_ctrlval)    (char *buf, struct rdt_resource *r,
 334                                  struct rdt_domain *d);
 335         struct list_head        evt_list;
 336         int                     num_rmid;
 337         unsigned int            mon_scale;
 338         unsigned long           fflags;
 339 };
 340
 341 int parse_cbm(char *buf, struct rdt_resource *r, struct rdt_domain *d);
 342 int parse_bw(char *buf, struct rdt_resource *r,  struct rdt_domain *d);
 343
 344 extern struct mutex rdtgroup_mutex;
 345
 346 extern struct rdt_resource rdt_resources_all[];
 347 extern struct rdtgroup rdtgroup_default;
 348 DECLARE_STATIC_KEY_FALSE(rdt_alloc_enable_key);
 349
 350 int __init rdtgroup_init(void);
 351
 352 enum {
 353         RDT_RESOURCE_L3,
 354         RDT_RESOURCE_L3DATA,
 355         RDT_RESOURCE_L3CODE,
 356         RDT_RESOURCE_L2,
 357         RDT_RESOURCE_MBA,
 358
 359         /* Must be the last */
 360         RDT_NUM_RESOURCES,
 361 };
 362
 363 #define for_each_capable_rdt_resource(r)                                      \
 364         for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\
 365              r++)                                                             \
 366                 if (r->alloc_capable || r->mon_capable)
 367
 368 #define for_each_alloc_capable_rdt_resource(r)                                \
 369         for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\
 370              r++)                                                             \
 371                 if (r->alloc_capable)
 372
 373 #define for_each_mon_capable_rdt_resource(r)                                  \
 374         for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\
 375              r++)                                                             \
 376                 if (r->mon_capable)
 377
 378 #define for_each_alloc_enabled_rdt_resource(r)                                \
 379         for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\
 380              r++)                                                             \
 381                 if (r->alloc_enabled)
 382
 383 #define for_each_mon_enabled_rdt_resource(r)                                  \
 384         for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\
 385              r++)                                                             \
 386                 if (r->mon_enabled)
 387
 388 /* CPUID.(EAX=10H, ECX=ResID=1).EAX */
 389 union cpuid_0x10_1_eax {
 390         struct {
 391                 unsigned int cbm_len:5;
 392         } split;
 393         unsigned int full;
 394 };
 395
 396 /* CPUID.(EAX=10H, ECX=ResID=3).EAX */
 397 union cpuid_0x10_3_eax {
 398         struct {
 399                 unsigned int max_delay:12;
 400         } split;
 401         unsigned int full;
 402 };
 403
 404 /* CPUID.(EAX=10H, ECX=ResID).EDX */
 405 union cpuid_0x10_x_edx {
 406         struct {
 407                 unsigned int cos_max:16;
 408         } split;
 409         unsigned int full;
 410 };
 411
 412 void rdt_ctrl_update(void *arg);
 413 struct rdtgroup *rdtgroup_kn_lock_live(struct kernfs_node *kn);
 414 void rdtgroup_kn_unlock(struct kernfs_node *kn);
 415 struct rdt_domain *rdt_find_domain(struct rdt_resource *r, int id,
 416                                    struct list_head **pos);
 417 ssize_t rdtgroup_schemata_write(struct kernfs_open_file *of,
 418                                 char *buf, size_t nbytes, loff_t off);
 419 int rdtgroup_schemata_show(struct kernfs_open_file *of,
 420                            struct seq_file *s, void *v);
 421 struct rdt_domain *get_domain_from_cpu(int cpu, struct rdt_resource *r);
 422 int alloc_rmid(void);
 423 void free_rmid(u32 rmid);
 424 int rdt_get_mon_l3_config(struct rdt_resource *r);
 425 void mon_event_count(void *info);
 426 int rdtgroup_mondata_show(struct seq_file *m, void *arg);
 427 void rmdir_mondata_subdir_allrdtgrp(struct rdt_resource *r,
 428                                     unsigned int dom_id);
 429 void mkdir_mondata_subdir_allrdtgrp(struct rdt_resource *r,
 430                                     struct rdt_domain *d);
 431 void mon_event_read(struct rmid_read *rr, struct rdt_domain *d,
 432                     struct rdtgroup *rdtgrp, int evtid, int first);
 433 void mbm_setup_overflow_handler(struct rdt_domain *dom,
 434                                 unsigned long delay_ms);
 435 void mbm_handle_overflow(struct work_struct *work);
 436 void cqm_setup_limbo_handler(struct rdt_domain *dom, unsigned long delay_ms);
 437 void cqm_handle_limbo(struct work_struct *work);
 438 bool has_busy_rmid(struct rdt_resource *r, struct rdt_domain *d);
 439 void __check_limbo(struct rdt_domain *d, bool force_free);
 440
 441 #endif /* _ASM_X86_INTEL_RDT_H */