arch/x86/kernel/cpu/intel_rdt.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _ASM_X86_INTEL_RDT_H
   3 #define _ASM_X86_INTEL_RDT_H
   4
   5 #include <linux/sched.h>
   6 #include <linux/kernfs.h>
   7 #include <linux/jump_label.h>
   8
   9 #define IA32_L3_QOS_CFG         0xc81
  10 #define IA32_L2_QOS_CFG         0xc82
  11 #define IA32_L3_CBM_BASE        0xc90
  12 #define IA32_L2_CBM_BASE        0xd10
  13 #define IA32_MBA_THRTL_BASE     0xd50
  14
  15 #define L3_QOS_CDP_ENABLE       0x01ULL
  16
  17 #define L2_QOS_CDP_ENABLE       0x01ULL
  18
  19 /*
  20  * Event IDs are used to program IA32_QM_EVTSEL before reading event
  21  * counter from IA32_QM_CTR
  22  */
  23 #define QOS_L3_OCCUP_EVENT_ID           0x01
  24 #define QOS_L3_MBM_TOTAL_EVENT_ID       0x02
  25 #define QOS_L3_MBM_LOCAL_EVENT_ID       0x03
  26
  27 #define CQM_LIMBOCHECK_INTERVAL 1000
  28
  29 #define MBM_CNTR_WIDTH                  24
  30 #define MBM_OVERFLOW_INTERVAL           1000
  31 #define MAX_MBA_BW                      100u
  32
  33 #define RMID_VAL_ERROR                  BIT_ULL(63)
  34 #define RMID_VAL_UNAVAIL                BIT_ULL(62)
  35
  36 DECLARE_STATIC_KEY_FALSE(rdt_enable_key);
  37
  38 /**
  39  * struct mon_evt - Entry in the event list of a resource
  40  * @evtid:              event id
  41  * @name:               name of the event
  42  */
  43 struct mon_evt {
  44         u32                     evtid;
  45         char                    *name;
  46         struct list_head        list;
  47 };
  48
  49 /**
  50  * struct mon_data_bits - Monitoring details for each event file
  51  * @rid:               Resource id associated with the event file.
  52  * @evtid:             Event id associated with the event file
  53  * @domid:             The domain to which the event file belongs
  54  */
  55 union mon_data_bits {
  56         void *priv;
  57         struct {
  58                 unsigned int rid        : 10;
  59                 unsigned int evtid      : 8;
  60                 unsigned int domid      : 14;
  61         } u;
  62 };
  63
  64 struct rmid_read {
  65         struct rdtgroup         *rgrp;
  66         struct rdt_domain       *d;
  67         int                     evtid;
  68         bool                    first;
  69         u64                     val;
  70 };
  71
  72 extern unsigned int intel_cqm_threshold;
  73 extern bool rdt_alloc_capable;
  74 extern bool rdt_mon_capable;
  75 extern unsigned int rdt_mon_features;
  76
  77 enum rdt_group_type {
  78         RDTCTRL_GROUP = 0,
  79         RDTMON_GROUP,
  80         RDT_NUM_GROUP,
  81 };
  82
  83 /**
  84  * struct mongroup - store mon group's data in resctrl fs.
  85  * @mon_data_kn         kernlfs node for the mon_data directory
  86  * @parent:                     parent rdtgrp
  87  * @crdtgrp_list:               child rdtgroup node list
  88  * @rmid:                       rmid for this rdtgroup
  89  */
  90 struct mongroup {
  91         struct kernfs_node      *mon_data_kn;
  92         struct rdtgroup         *parent;
  93         struct list_head        crdtgrp_list;
  94         u32                     rmid;
  95 };
  96
  97 /**
  98  * struct rdtgroup - store rdtgroup's data in resctrl file system.
  99  * @kn:                         kernfs node
 100  * @rdtgroup_list:              linked list for all rdtgroups
 101  * @closid:                     closid for this rdtgroup
 102  * @cpu_mask:                   CPUs assigned to this rdtgroup
 103  * @flags:                      status bits
 104  * @waitcount:                  how many cpus expect to find this
 105  *                              group when they acquire rdtgroup_mutex
 106  * @type:                       indicates type of this rdtgroup - either
 107  *                              monitor only or ctrl_mon group
 108  * @mon:                        mongroup related data
 109  */
 110 struct rdtgroup {
 111         struct kernfs_node      *kn;
 112         struct list_head        rdtgroup_list;
 113         u32                     closid;
 114         struct cpumask          cpu_mask;
 115         int                     flags;
 116         atomic_t                waitcount;
 117         enum rdt_group_type     type;
 118         struct mongroup         mon;
 119 };
 120
 121 /* rdtgroup.flags */
 122 #define RDT_DELETED             1
 123
 124 /* rftype.flags */
 125 #define RFTYPE_FLAGS_CPUS_LIST  1
 126
 127 /*
 128  * Define the file type flags for base and info directories.
 129  */
 130 #define RFTYPE_INFO                     BIT(0)
 131 #define RFTYPE_BASE                     BIT(1)
 132 #define RF_CTRLSHIFT                    4
 133 #define RF_MONSHIFT                     5
 134 #define RF_TOPSHIFT                     6
 135 #define RFTYPE_CTRL                     BIT(RF_CTRLSHIFT)
 136 #define RFTYPE_MON                      BIT(RF_MONSHIFT)
 137 #define RFTYPE_TOP                      BIT(RF_TOPSHIFT)
 138 #define RFTYPE_RES_CACHE                BIT(8)
 139 #define RFTYPE_RES_MB                   BIT(9)
 140 #define RF_CTRL_INFO                    (RFTYPE_INFO | RFTYPE_CTRL)
 141 #define RF_MON_INFO                     (RFTYPE_INFO | RFTYPE_MON)
 142 #define RF_TOP_INFO                     (RFTYPE_INFO | RFTYPE_TOP)
 143 #define RF_CTRL_BASE                    (RFTYPE_BASE | RFTYPE_CTRL)
 144
 145 /* List of all resource groups */
 146 extern struct list_head rdt_all_groups;
 147
 148 extern int max_name_width, max_data_width;
 149
 150 int __init rdtgroup_init(void);
 151
 152 /**
 153  * struct rftype - describe each file in the resctrl file system
 154  * @name:       File name
 155  * @mode:       Access mode
 156  * @kf_ops:     File operations
 157  * @flags:      File specific RFTYPE_FLAGS_* flags
 158  * @fflags:     File specific RF_* or RFTYPE_* flags
 159  * @seq_show:   Show content of the file
 160  * @write:      Write to the file
 161  */
 162 struct rftype {
 163         char                    *name;
 164         umode_t                 mode;
 165         struct kernfs_ops       *kf_ops;
 166         unsigned long           flags;
 167         unsigned long           fflags;
 168
 169         int (*seq_show)(struct kernfs_open_file *of,
 170                         struct seq_file *sf, void *v);
 171         /*
 172          * write() is the generic write callback which maps directly to
 173          * kernfs write operation and overrides all other operations.
 174          * Maximum write size is determined by ->max_write_len.
 175          */
 176         ssize_t (*write)(struct kernfs_open_file *of,
 177                          char *buf, size_t nbytes, loff_t off);
 178 };
 179
 180 /**
 181  * struct mbm_state - status for each MBM counter in each domain
 182  * @chunks:     Total data moved (multiply by rdt_group.mon_scale to get bytes)
 183  * @prev_msr    Value of IA32_QM_CTR for this RMID last time we read it
 184  * @chunks_bw   Total local data moved. Used for bandwidth calculation
 185  * @prev_bw_msr:Value of previous IA32_QM_CTR for bandwidth counting
 186  * @prev_bw     The most recent bandwidth in MBps
 187  * @delta_bw    Difference between the current and previous bandwidth
 188  * @delta_comp  Indicates whether to compute the delta_bw
 189  */
 190 struct mbm_state {
 191         u64     chunks;
 192         u64     prev_msr;
 193         u64     chunks_bw;
 194         u64     prev_bw_msr;
 195         u32     prev_bw;
 196         u32     delta_bw;
 197         bool    delta_comp;
 198 };
 199
 200 /**
 201  * struct rdt_domain - group of cpus sharing an RDT resource
 202  * @list:       all instances of this resource
 203  * @id:         unique id for this instance
 204  * @cpu_mask:   which cpus share this resource
 205  * @rmid_busy_llc:
 206  *              bitmap of which limbo RMIDs are above threshold
 207  * @mbm_total:  saved state for MBM total bandwidth
 208  * @mbm_local:  saved state for MBM local bandwidth
 209  * @mbm_over:   worker to periodically read MBM h/w counters
 210  * @cqm_limbo:  worker to periodically read CQM h/w counters
 211  * @mbm_work_cpu:
 212  *              worker cpu for MBM h/w counters
 213  * @cqm_work_cpu:
 214  *              worker cpu for CQM h/w counters
 215  * @ctrl_val:   array of cache or mem ctrl values (indexed by CLOSID)
 216  * @mbps_val:   When mba_sc is enabled, this holds the bandwidth in MBps
 217  * @new_ctrl:   new ctrl value to be loaded
 218  * @have_new_ctrl: did user provide new_ctrl for this domain
 219  */
 220 struct rdt_domain {
 221         struct list_head        list;
 222         int                     id;
 223         struct cpumask          cpu_mask;
 224         unsigned long           *rmid_busy_llc;
 225         struct mbm_state        *mbm_total;
 226         struct mbm_state        *mbm_local;
 227         struct delayed_work     mbm_over;
 228         struct delayed_work     cqm_limbo;
 229         int                     mbm_work_cpu;
 230         int                     cqm_work_cpu;
 231         u32                     *ctrl_val;
 232         u32                     *mbps_val;
 233         u32                     new_ctrl;
 234         bool                    have_new_ctrl;
 235 };
 236
 237 /**
 238  * struct msr_param - set a range of MSRs from a domain
 239  * @res:       The resource to use
 240  * @low:       Beginning index from base MSR
 241  * @high:      End index
 242  */
 243 struct msr_param {
 244         struct rdt_resource     *res;
 245         int                     low;
 246         int                     high;
 247 };
 248
 249 /**
 250  * struct rdt_cache - Cache allocation related data
 251  * @cbm_len:            Length of the cache bit mask
 252  * @min_cbm_bits:       Minimum number of consecutive bits to be set
 253  * @cbm_idx_mult:       Multiplier of CBM index
 254  * @cbm_idx_offset:     Offset of CBM index. CBM index is computed by:
 255  *                      closid * cbm_idx_multi + cbm_idx_offset
 256  *                      in a cache bit mask
 257  * @shareable_bits:     Bitmask of shareable resource with other
 258  *                      executing entities
 259  */
 260 struct rdt_cache {
 261         unsigned int    cbm_len;
 262         unsigned int    min_cbm_bits;
 263         unsigned int    cbm_idx_mult;
 264         unsigned int    cbm_idx_offset;
 265         unsigned int    shareable_bits;
 266 };
 267
 268 /**
 269  * struct rdt_membw - Memory bandwidth allocation related data
 270  * @max_delay:          Max throttle delay. Delay is the hardware
 271  *                      representation for memory bandwidth.
 272  * @min_bw:             Minimum memory bandwidth percentage user can request
 273  * @bw_gran:            Granularity at which the memory bandwidth is allocated
 274  * @delay_linear:       True if memory B/W delay is in linear scale
 275  * @mba_sc:             True if MBA software controller(mba_sc) is enabled
 276  * @mb_map:             Mapping of memory B/W percentage to memory B/W delay
 277  */
 278 struct rdt_membw {
 279         u32             max_delay;
 280         u32             min_bw;
 281         u32             bw_gran;
 282         u32             delay_linear;
 283         bool            mba_sc;
 284         u32             *mb_map;
 285 };
 286
 287 static inline bool is_llc_occupancy_enabled(void)
 288 {
 289         return (rdt_mon_features & (1 << QOS_L3_OCCUP_EVENT_ID));
 290 }
 291
 292 static inline bool is_mbm_total_enabled(void)
 293 {
 294         return (rdt_mon_features & (1 << QOS_L3_MBM_TOTAL_EVENT_ID));
 295 }
 296
 297 static inline bool is_mbm_local_enabled(void)
 298 {
 299         return (rdt_mon_features & (1 << QOS_L3_MBM_LOCAL_EVENT_ID));
 300 }
 301
 302 static inline bool is_mbm_enabled(void)
 303 {
 304         return (is_mbm_total_enabled() || is_mbm_local_enabled());
 305 }
 306
 307 static inline bool is_mbm_event(int e)
 308 {
 309         return (e >= QOS_L3_MBM_TOTAL_EVENT_ID &&
 310                 e <= QOS_L3_MBM_LOCAL_EVENT_ID);
 311 }
 312
 313 /**
 314  * struct rdt_resource - attributes of an RDT resource
 315  * @rid:                The index of the resource
 316  * @alloc_enabled:      Is allocation enabled on this machine
 317  * @mon_enabled:                Is monitoring enabled for this feature
 318  * @alloc_capable:      Is allocation available on this machine
 319  * @mon_capable:                Is monitor feature available on this machine
 320  * @name:               Name to use in "schemata" file
 321  * @num_closid:         Number of CLOSIDs available
 322  * @cache_level:        Which cache level defines scope of this resource
 323  * @default_ctrl:       Specifies default cache cbm or memory B/W percent.
 324  * @msr_base:           Base MSR address for CBMs
 325  * @msr_update:         Function pointer to update QOS MSRs
 326  * @data_width:         Character width of data when displaying
 327  * @domains:            All domains for this resource
 328  * @cache:              Cache allocation related data
 329  * @format_str:         Per resource format string to show domain value
 330  * @parse_ctrlval:      Per resource function pointer to parse control values
 331  * @evt_list:                   List of monitoring events
 332  * @num_rmid:                   Number of RMIDs available
 333  * @mon_scale:                  cqm counter * mon_scale = occupancy in bytes
 334  * @fflags:                     flags to choose base and info files
 335  */
 336 struct rdt_resource {
 337         int                     rid;
 338         bool                    alloc_enabled;
 339         bool                    mon_enabled;
 340         bool                    alloc_capable;
 341         bool                    mon_capable;
 342         char                    *name;
 343         int                     num_closid;
 344         int                     cache_level;
 345         u32                     default_ctrl;
 346         unsigned int            msr_base;
 347         void (*msr_update)      (struct rdt_domain *d, struct msr_param *m,
 348                                  struct rdt_resource *r);
 349         int                     data_width;
 350         struct list_head        domains;
 351         struct rdt_cache        cache;
 352         struct rdt_membw        membw;
 353         const char              *format_str;
 354         int (*parse_ctrlval)    (char *buf, struct rdt_resource *r,
 355                                  struct rdt_domain *d);
 356         struct list_head        evt_list;
 357         int                     num_rmid;
 358         unsigned int            mon_scale;
 359         unsigned long           fflags;
 360 };
 361
 362 int parse_cbm(char *buf, struct rdt_resource *r, struct rdt_domain *d);
 363 int parse_bw(char *buf, struct rdt_resource *r,  struct rdt_domain *d);
 364
 365 extern struct mutex rdtgroup_mutex;
 366
 367 extern struct rdt_resource rdt_resources_all[];
 368 extern struct rdtgroup rdtgroup_default;
 369 DECLARE_STATIC_KEY_FALSE(rdt_alloc_enable_key);
 370
 371 int __init rdtgroup_init(void);
 372
 373 enum {
 374         RDT_RESOURCE_L3,
 375         RDT_RESOURCE_L3DATA,
 376         RDT_RESOURCE_L3CODE,
 377         RDT_RESOURCE_L2,
 378         RDT_RESOURCE_L2DATA,
 379         RDT_RESOURCE_L2CODE,
 380         RDT_RESOURCE_MBA,
 381
 382         /* Must be the last */
 383         RDT_NUM_RESOURCES,
 384 };
 385
 386 #define for_each_capable_rdt_resource(r)                                      \
 387         for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\
 388              r++)                                                             \
 389                 if (r->alloc_capable || r->mon_capable)
 390
 391 #define for_each_alloc_capable_rdt_resource(r)                                \
 392         for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\
 393              r++)                                                             \
 394                 if (r->alloc_capable)
 395
 396 #define for_each_mon_capable_rdt_resource(r)                                  \
 397         for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\
 398              r++)                                                             \
 399                 if (r->mon_capable)
 400
 401 #define for_each_alloc_enabled_rdt_resource(r)                                \
 402         for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\
 403              r++)                                                             \
 404                 if (r->alloc_enabled)
 405
 406 #define for_each_mon_enabled_rdt_resource(r)                                  \
 407         for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\
 408              r++)                                                             \
 409                 if (r->mon_enabled)
 410
 411 /* CPUID.(EAX=10H, ECX=ResID=1).EAX */
 412 union cpuid_0x10_1_eax {
 413         struct {
 414                 unsigned int cbm_len:5;
 415         } split;
 416         unsigned int full;
 417 };
 418
 419 /* CPUID.(EAX=10H, ECX=ResID=3).EAX */
 420 union cpuid_0x10_3_eax {
 421         struct {
 422                 unsigned int max_delay:12;
 423         } split;
 424         unsigned int full;
 425 };
 426
 427 /* CPUID.(EAX=10H, ECX=ResID).EDX */
 428 union cpuid_0x10_x_edx {
 429         struct {
 430                 unsigned int cos_max:16;
 431         } split;
 432         unsigned int full;
 433 };
 434
 435 void rdt_last_cmd_clear(void);
 436 void rdt_last_cmd_puts(const char *s);
 437 void rdt_last_cmd_printf(const char *fmt, ...);
 438
 439 void rdt_ctrl_update(void *arg);
 440 struct rdtgroup *rdtgroup_kn_lock_live(struct kernfs_node *kn);
 441 void rdtgroup_kn_unlock(struct kernfs_node *kn);
 442 struct rdt_domain *rdt_find_domain(struct rdt_resource *r, int id,
 443                                    struct list_head **pos);
 444 ssize_t rdtgroup_schemata_write(struct kernfs_open_file *of,
 445                                 char *buf, size_t nbytes, loff_t off);
 446 int rdtgroup_schemata_show(struct kernfs_open_file *of,
 447                            struct seq_file *s, void *v);
 448 struct rdt_domain *get_domain_from_cpu(int cpu, struct rdt_resource *r);
 449 int alloc_rmid(void);
 450 void free_rmid(u32 rmid);
 451 int rdt_get_mon_l3_config(struct rdt_resource *r);
 452 void mon_event_count(void *info);
 453 int rdtgroup_mondata_show(struct seq_file *m, void *arg);
 454 void rmdir_mondata_subdir_allrdtgrp(struct rdt_resource *r,
 455                                     unsigned int dom_id);
 456 void mkdir_mondata_subdir_allrdtgrp(struct rdt_resource *r,
 457                                     struct rdt_domain *d);
 458 void mon_event_read(struct rmid_read *rr, struct rdt_domain *d,
 459                     struct rdtgroup *rdtgrp, int evtid, int first);
 460 void mbm_setup_overflow_handler(struct rdt_domain *dom,
 461                                 unsigned long delay_ms);
 462 void mbm_handle_overflow(struct work_struct *work);
 463 bool is_mba_sc(struct rdt_resource *r);
 464 void setup_default_ctrlval(struct rdt_resource *r, u32 *dc, u32 *dm);
 465 u32 delay_bw_map(unsigned long bw, struct rdt_resource *r);
 466 void cqm_setup_limbo_handler(struct rdt_domain *dom, unsigned long delay_ms);
 467 void cqm_handle_limbo(struct work_struct *work);
 468 bool has_busy_rmid(struct rdt_resource *r, struct rdt_domain *d);
 469 void __check_limbo(struct rdt_domain *d, bool force_free);
 470
 471 #endif /* _ASM_X86_INTEL_RDT_H */