include/kvm/arm_vgic.h

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /*
   3  * Copyright (C) 2015, 2016 ARM Ltd.
   4  */
   5 #ifndef __KVM_ARM_VGIC_H
   6 #define __KVM_ARM_VGIC_H
   7
   8 #include <linux/bits.h>
   9 #include <linux/kvm.h>
  10 #include <linux/irqreturn.h>
  11 #include <linux/kref.h>
  12 #include <linux/mutex.h>
  13 #include <linux/spinlock.h>
  14 #include <linux/static_key.h>
  15 #include <linux/types.h>
  16 #include <linux/xarray.h>
  17 #include <kvm/iodev.h>
  18 #include <linux/list.h>
  19 #include <linux/jump_label.h>
  20
  21 #include <linux/irqchip/arm-gic-v4.h>
  22
  23 #define VGIC_V3_MAX_CPUS        512
  24 #define VGIC_V2_MAX_CPUS        8
  25 #define VGIC_NR_IRQS_LEGACY     256
  26 #define VGIC_NR_SGIS            16
  27 #define VGIC_NR_PPIS            16
  28 #define VGIC_NR_PRIVATE_IRQS    (VGIC_NR_SGIS + VGIC_NR_PPIS)
  29 #define VGIC_MAX_PRIVATE        (VGIC_NR_PRIVATE_IRQS - 1)
  30 #define VGIC_MAX_SPI            1019
  31 #define VGIC_MAX_RESERVED       1023
  32 #define VGIC_MIN_LPI            8192
  33 #define KVM_IRQCHIP_NUM_PINS    (1020 - 32)
  34
  35 #define irq_is_ppi(irq) ((irq) >= VGIC_NR_SGIS && (irq) < VGIC_NR_PRIVATE_IRQS)
  36 #define irq_is_spi(irq) ((irq) >= VGIC_NR_PRIVATE_IRQS && \
  37                          (irq) <= VGIC_MAX_SPI)
  38
  39 enum vgic_type {
  40         VGIC_V2,                /* Good ol' GICv2 */
  41         VGIC_V3,                /* New fancy GICv3 */
  42 };
  43
  44 /* same for all guests, as depending only on the _host's_ GIC model */
  45 struct vgic_global {
  46         /* type of the host GIC */
  47         enum vgic_type          type;
  48
  49         /* Physical address of vgic virtual cpu interface */
  50         phys_addr_t             vcpu_base;
  51
  52         /* GICV mapping, kernel VA */
  53         void __iomem            *vcpu_base_va;
  54         /* GICV mapping, HYP VA */
  55         void __iomem            *vcpu_hyp_va;
  56
  57         /* virtual control interface mapping, kernel VA */
  58         void __iomem            *vctrl_base;
  59         /* virtual control interface mapping, HYP VA */
  60         void __iomem            *vctrl_hyp;
  61
  62         /* Number of implemented list registers */
  63         int                     nr_lr;
  64
  65         /* Maintenance IRQ number */
  66         unsigned int            maint_irq;
  67
  68         /* maximum number of VCPUs allowed (GICv2 limits us to 8) */
  69         int                     max_gic_vcpus;
  70
  71         /* Only needed for the legacy KVM_CREATE_IRQCHIP */
  72         bool                    can_emulate_gicv2;
  73
  74         /* Hardware has GICv4? */
  75         bool                    has_gicv4;
  76         bool                    has_gicv4_1;
  77
  78         /* Pseudo GICv3 from outer space */
  79         bool                    no_hw_deactivation;
  80
  81         /* GIC system register CPU interface */
  82         struct static_key_false gicv3_cpuif;
  83
  84         u32                     ich_vtr_el2;
  85 };
  86
  87 extern struct vgic_global kvm_vgic_global_state;
  88
  89 #define VGIC_V2_MAX_LRS         (1 << 6)
  90 #define VGIC_V3_MAX_LRS         16
  91 #define VGIC_V3_LR_INDEX(lr)    (VGIC_V3_MAX_LRS - 1 - lr)
  92
  93 enum vgic_irq_config {
  94         VGIC_CONFIG_EDGE = 0,
  95         VGIC_CONFIG_LEVEL
  96 };
  97
  98 /*
  99  * Per-irq ops overriding some common behavious.
 100  *
 101  * Always called in non-preemptible section and the functions can use
 102  * kvm_arm_get_running_vcpu() to get the vcpu pointer for private IRQs.
 103  */
 104 struct irq_ops {
 105         /* Per interrupt flags for special-cased interrupts */
 106         unsigned long flags;
 107
 108 #define VGIC_IRQ_SW_RESAMPLE    BIT(0)  /* Clear the active state for resampling */
 109
 110         /*
 111          * Callback function pointer to in-kernel devices that can tell us the
 112          * state of the input level of mapped level-triggered IRQ faster than
 113          * peaking into the physical GIC.
 114          */
 115         bool (*get_input_level)(int vintid);
 116 };
 117
 118 struct vgic_irq {
 119         raw_spinlock_t irq_lock;        /* Protects the content of the struct */
 120         struct rcu_head rcu;
 121         struct list_head ap_list;
 122
 123         struct kvm_vcpu *vcpu;          /* SGIs and PPIs: The VCPU
 124                                          * SPIs and LPIs: The VCPU whose ap_list
 125                                          * this is queued on.
 126                                          */
 127
 128         struct kvm_vcpu *target_vcpu;   /* The VCPU that this interrupt should
 129                                          * be sent to, as a result of the
 130                                          * targets reg (v2) or the
 131                                          * affinity reg (v3).
 132                                          */
 133
 134         u32 intid;                      /* Guest visible INTID */
 135         bool line_level;                /* Level only */
 136         bool pending_latch;             /* The pending latch state used to calculate
 137                                          * the pending state for both level
 138                                          * and edge triggered IRQs. */
 139         bool active;                    /* not used for LPIs */
 140         bool enabled;
 141         bool hw;                        /* Tied to HW IRQ */
 142         struct kref refcount;           /* Used for LPIs */
 143         u32 hwintid;                    /* HW INTID number */
 144         unsigned int host_irq;          /* linux irq corresponding to hwintid */
 145         union {
 146                 u8 targets;                     /* GICv2 target VCPUs mask */
 147                 u32 mpidr;                      /* GICv3 target VCPU */
 148         };
 149         u8 source;                      /* GICv2 SGIs only */
 150         u8 active_source;               /* GICv2 SGIs only */
 151         u8 priority;
 152         u8 group;                       /* 0 == group 0, 1 == group 1 */
 153         enum vgic_irq_config config;    /* Level or edge */
 154
 155         struct irq_ops *ops;
 156
 157         void *owner;                    /* Opaque pointer to reserve an interrupt
 158                                            for in-kernel devices. */
 159 };
 160
 161 static inline bool vgic_irq_needs_resampling(struct vgic_irq *irq)
 162 {
 163         return irq->ops && (irq->ops->flags & VGIC_IRQ_SW_RESAMPLE);
 164 }
 165
 166 struct vgic_register_region;
 167 struct vgic_its;
 168
 169 enum iodev_type {
 170         IODEV_CPUIF,
 171         IODEV_DIST,
 172         IODEV_REDIST,
 173         IODEV_ITS
 174 };
 175
 176 struct vgic_io_device {
 177         gpa_t base_addr;
 178         union {
 179                 struct kvm_vcpu *redist_vcpu;
 180                 struct vgic_its *its;
 181         };
 182         const struct vgic_register_region *regions;
 183         enum iodev_type iodev_type;
 184         int nr_regions;
 185         struct kvm_io_device dev;
 186 };
 187
 188 struct vgic_its {
 189         /* The base address of the ITS control register frame */
 190         gpa_t                   vgic_its_base;
 191
 192         bool                    enabled;
 193         struct vgic_io_device   iodev;
 194         struct kvm_device       *dev;
 195
 196         /* These registers correspond to GITS_BASER{0,1} */
 197         u64                     baser_device_table;
 198         u64                     baser_coll_table;
 199
 200         /* Protects the command queue */
 201         struct mutex            cmd_lock;
 202         u64                     cbaser;
 203         u32                     creadr;
 204         u32                     cwriter;
 205
 206         /* migration ABI revision in use */
 207         u32                     abi_rev;
 208
 209         /* Protects the device and collection lists */
 210         struct mutex            its_lock;
 211         struct list_head        device_list;
 212         struct list_head        collection_list;
 213
 214         /*
 215          * Caches the (device_id, event_id) -> vgic_irq translation for
 216          * LPIs that are mapped and enabled.
 217          */
 218         struct xarray           translation_cache;
 219 };
 220
 221 struct vgic_state_iter;
 222
 223 struct vgic_redist_region {
 224         u32 index;
 225         gpa_t base;
 226         u32 count; /* number of redistributors or 0 if single region */
 227         u32 free_index; /* index of the next free redistributor */
 228         struct list_head list;
 229 };
 230
 231 struct vgic_dist {
 232         bool                    in_kernel;
 233         bool                    ready;
 234         bool                    initialized;
 235
 236         /* vGIC model the kernel emulates for the guest (GICv2 or GICv3) */
 237         u32                     vgic_model;
 238
 239         /* Implementation revision as reported in the GICD_IIDR */
 240         u32                     implementation_rev;
 241 #define KVM_VGIC_IMP_REV_2      2 /* GICv2 restorable groups */
 242 #define KVM_VGIC_IMP_REV_3      3 /* GICv3 GICR_CTLR.{IW,CES,RWP} */
 243 #define KVM_VGIC_IMP_REV_LATEST KVM_VGIC_IMP_REV_3
 244
 245         /* Userspace can write to GICv2 IGROUPR */
 246         bool                    v2_groups_user_writable;
 247
 248         /* Do injected MSIs require an additional device ID? */
 249         bool                    msis_require_devid;
 250
 251         int                     nr_spis;
 252
 253         /* base addresses in guest physical address space: */
 254         gpa_t                   vgic_dist_base;         /* distributor */
 255         union {
 256                 /* either a GICv2 CPU interface */
 257                 gpa_t                   vgic_cpu_base;
 258                 /* or a number of GICv3 redistributor regions */
 259                 struct list_head rd_regions;
 260         };
 261
 262         /* distributor enabled */
 263         bool                    enabled;
 264
 265         /* Wants SGIs without active state */
 266         bool                    nassgireq;
 267
 268         struct vgic_irq         *spis;
 269
 270         struct vgic_io_device   dist_iodev;
 271
 272         bool                    has_its;
 273         bool                    table_write_in_progress;
 274
 275         /*
 276          * Contains the attributes and gpa of the LPI configuration table.
 277          * Since we report GICR_TYPER.CommonLPIAff as 0b00, we can share
 278          * one address across all redistributors.
 279          * GICv3 spec: IHI 0069E 6.1.1 "LPI Configuration tables"
 280          */
 281         u64                     propbaser;
 282
 283 #define LPI_XA_MARK_DEBUG_ITER  XA_MARK_0
 284         struct xarray           lpi_xa;
 285
 286         /* used by vgic-debug */
 287         struct vgic_state_iter *iter;
 288
 289         /*
 290          * GICv4 ITS per-VM data, containing the IRQ domain, the VPE
 291          * array, the property table pointer as well as allocation
 292          * data. This essentially ties the Linux IRQ core and ITS
 293          * together, and avoids leaking KVM's data structures anywhere
 294          * else.
 295          */
 296         struct its_vm           its_vm;
 297 };
 298
 299 struct vgic_v2_cpu_if {
 300         u32             vgic_hcr;
 301         u32             vgic_vmcr;
 302         u32             vgic_apr;
 303         u32             vgic_lr[VGIC_V2_MAX_LRS];
 304
 305         unsigned int used_lrs;
 306 };
 307
 308 struct vgic_v3_cpu_if {
 309         u32             vgic_hcr;
 310         u32             vgic_vmcr;
 311         u32             vgic_sre;       /* Restored only, change ignored */
 312         u32             vgic_ap0r[4];
 313         u32             vgic_ap1r[4];
 314         u64             vgic_lr[VGIC_V3_MAX_LRS];
 315
 316         /*
 317          * GICv4 ITS per-VPE data, containing the doorbell IRQ, the
 318          * pending table pointer, the its_vm pointer and a few other
 319          * HW specific things. As for the its_vm structure, this is
 320          * linking the Linux IRQ subsystem and the ITS together.
 321          */
 322         struct its_vpe  its_vpe;
 323
 324         unsigned int used_lrs;
 325 };
 326
 327 struct vgic_cpu {
 328         /* CPU vif control registers for world switch */
 329         union {
 330                 struct vgic_v2_cpu_if   vgic_v2;
 331                 struct vgic_v3_cpu_if   vgic_v3;
 332         };
 333
 334         struct vgic_irq *private_irqs;
 335
 336         raw_spinlock_t ap_list_lock;    /* Protects the ap_list */
 337
 338         /*
 339          * List of IRQs that this VCPU should consider because they are either
 340          * Active or Pending (hence the name; AP list), or because they recently
 341          * were one of the two and need to be migrated off this list to another
 342          * VCPU.
 343          */
 344         struct list_head ap_list_head;
 345
 346         /*
 347          * Members below are used with GICv3 emulation only and represent
 348          * parts of the redistributor.
 349          */
 350         struct vgic_io_device   rd_iodev;
 351         struct vgic_redist_region *rdreg;
 352         u32 rdreg_index;
 353         atomic_t syncr_busy;
 354
 355         /* Contains the attributes and gpa of the LPI pending tables. */
 356         u64 pendbaser;
 357         /* GICR_CTLR.{ENABLE_LPIS,RWP} */
 358         atomic_t ctlr;
 359
 360         /* Cache guest priority bits */
 361         u32 num_pri_bits;
 362
 363         /* Cache guest interrupt ID bits */
 364         u32 num_id_bits;
 365 };
 366
 367 extern struct static_key_false vgic_v2_cpuif_trap;
 368 extern struct static_key_false vgic_v3_cpuif_trap;
 369
 370 int kvm_set_legacy_vgic_v2_addr(struct kvm *kvm, struct kvm_arm_device_addr *dev_addr);
 371 void kvm_vgic_early_init(struct kvm *kvm);
 372 int kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu);
 373 int kvm_vgic_create(struct kvm *kvm, u32 type);
 374 void kvm_vgic_destroy(struct kvm *kvm);
 375 void kvm_vgic_vcpu_destroy(struct kvm_vcpu *vcpu);
 376 int kvm_vgic_map_resources(struct kvm *kvm);
 377 int kvm_vgic_hyp_init(void);
 378 void kvm_vgic_init_cpu_hardware(void);
 379
 380 int kvm_vgic_inject_irq(struct kvm *kvm, struct kvm_vcpu *vcpu,
 381                         unsigned int intid, bool level, void *owner);
 382 int kvm_vgic_map_phys_irq(struct kvm_vcpu *vcpu, unsigned int host_irq,
 383                           u32 vintid, struct irq_ops *ops);
 384 int kvm_vgic_unmap_phys_irq(struct kvm_vcpu *vcpu, unsigned int vintid);
 385 int kvm_vgic_get_map(struct kvm_vcpu *vcpu, unsigned int vintid);
 386 bool kvm_vgic_map_is_active(struct kvm_vcpu *vcpu, unsigned int vintid);
 387
 388 int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu);
 389
 390 void kvm_vgic_load(struct kvm_vcpu *vcpu);
 391 void kvm_vgic_put(struct kvm_vcpu *vcpu);
 392
 393 #define irqchip_in_kernel(k)    (!!((k)->arch.vgic.in_kernel))
 394 #define vgic_initialized(k)     ((k)->arch.vgic.initialized)
 395 #define vgic_ready(k)           ((k)->arch.vgic.ready)
 396 #define vgic_valid_spi(k, i)    (((i) >= VGIC_NR_PRIVATE_IRQS) && \
 397                         ((i) < (k)->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS))
 398
 399 bool kvm_vcpu_has_pending_irqs(struct kvm_vcpu *vcpu);
 400 void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu);
 401 void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu);
 402 void kvm_vgic_reset_mapped_irq(struct kvm_vcpu *vcpu, u32 vintid);
 403
 404 void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg, bool allow_group1);
 405
 406 /**
 407  * kvm_vgic_get_max_vcpus - Get the maximum number of VCPUs allowed by HW
 408  *
 409  * The host's GIC naturally limits the maximum amount of VCPUs a guest
 410  * can use.
 411  */
 412 static inline int kvm_vgic_get_max_vcpus(void)
 413 {
 414         return kvm_vgic_global_state.max_gic_vcpus;
 415 }
 416
 417 /**
 418  * kvm_vgic_setup_default_irq_routing:
 419  * Setup a default flat gsi routing table mapping all SPIs
 420  */
 421 int kvm_vgic_setup_default_irq_routing(struct kvm *kvm);
 422
 423 int kvm_vgic_set_owner(struct kvm_vcpu *vcpu, unsigned int intid, void *owner);
 424
 425 struct kvm_kernel_irq_routing_entry;
 426
 427 int kvm_vgic_v4_set_forwarding(struct kvm *kvm, int irq,
 428                                struct kvm_kernel_irq_routing_entry *irq_entry);
 429
 430 int kvm_vgic_v4_unset_forwarding(struct kvm *kvm, int irq,
 431                                  struct kvm_kernel_irq_routing_entry *irq_entry);
 432
 433 int vgic_v4_load(struct kvm_vcpu *vcpu);
 434 void vgic_v4_commit(struct kvm_vcpu *vcpu);
 435 int vgic_v4_put(struct kvm_vcpu *vcpu);
 436
 437 /* CPU HP callbacks */
 438 void kvm_vgic_cpu_up(void);
 439 void kvm_vgic_cpu_down(void);
 440
 441 #endif /* __KVM_ARM_VGIC_H */