arch/arm64/kvm/hyp/vgic-v3-sr.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2012-2015 - ARM Ltd
   4  * Author: Marc Zyngier <marc.zyngier@arm.com>
   5  */
   6
   7 #include <hyp/adjust_pc.h>
   8
   9 #include <linux/compiler.h>
  10 #include <linux/irqchip/arm-gic-v3.h>
  11 #include <linux/kvm_host.h>
  12
  13 #include <asm/kvm_emulate.h>
  14 #include <asm/kvm_hyp.h>
  15 #include <asm/kvm_mmu.h>
  16
  17 #define vtr_to_max_lr_idx(v)            ((v) & 0xf)
  18 #define vtr_to_nr_pre_bits(v)           ((((u32)(v) >> 26) & 7) + 1)
  19 #define vtr_to_nr_apr_regs(v)           (1 << (vtr_to_nr_pre_bits(v) - 5))
  20
  21 static u64 __gic_v3_get_lr(unsigned int lr)
  22 {
  23         switch (lr & 0xf) {
  24         case 0:
  25                 return read_gicreg(ICH_LR0_EL2);
  26         case 1:
  27                 return read_gicreg(ICH_LR1_EL2);
  28         case 2:
  29                 return read_gicreg(ICH_LR2_EL2);
  30         case 3:
  31                 return read_gicreg(ICH_LR3_EL2);
  32         case 4:
  33                 return read_gicreg(ICH_LR4_EL2);
  34         case 5:
  35                 return read_gicreg(ICH_LR5_EL2);
  36         case 6:
  37                 return read_gicreg(ICH_LR6_EL2);
  38         case 7:
  39                 return read_gicreg(ICH_LR7_EL2);
  40         case 8:
  41                 return read_gicreg(ICH_LR8_EL2);
  42         case 9:
  43                 return read_gicreg(ICH_LR9_EL2);
  44         case 10:
  45                 return read_gicreg(ICH_LR10_EL2);
  46         case 11:
  47                 return read_gicreg(ICH_LR11_EL2);
  48         case 12:
  49                 return read_gicreg(ICH_LR12_EL2);
  50         case 13:
  51                 return read_gicreg(ICH_LR13_EL2);
  52         case 14:
  53                 return read_gicreg(ICH_LR14_EL2);
  54         case 15:
  55                 return read_gicreg(ICH_LR15_EL2);
  56         }
  57
  58         unreachable();
  59 }
  60
  61 static void __gic_v3_set_lr(u64 val, int lr)
  62 {
  63         switch (lr & 0xf) {
  64         case 0:
  65                 write_gicreg(val, ICH_LR0_EL2);
  66                 break;
  67         case 1:
  68                 write_gicreg(val, ICH_LR1_EL2);
  69                 break;
  70         case 2:
  71                 write_gicreg(val, ICH_LR2_EL2);
  72                 break;
  73         case 3:
  74                 write_gicreg(val, ICH_LR3_EL2);
  75                 break;
  76         case 4:
  77                 write_gicreg(val, ICH_LR4_EL2);
  78                 break;
  79         case 5:
  80                 write_gicreg(val, ICH_LR5_EL2);
  81                 break;
  82         case 6:
  83                 write_gicreg(val, ICH_LR6_EL2);
  84                 break;
  85         case 7:
  86                 write_gicreg(val, ICH_LR7_EL2);
  87                 break;
  88         case 8:
  89                 write_gicreg(val, ICH_LR8_EL2);
  90                 break;
  91         case 9:
  92                 write_gicreg(val, ICH_LR9_EL2);
  93                 break;
  94         case 10:
  95                 write_gicreg(val, ICH_LR10_EL2);
  96                 break;
  97         case 11:
  98                 write_gicreg(val, ICH_LR11_EL2);
  99                 break;
 100         case 12:
 101                 write_gicreg(val, ICH_LR12_EL2);
 102                 break;
 103         case 13:
 104                 write_gicreg(val, ICH_LR13_EL2);
 105                 break;
 106         case 14:
 107                 write_gicreg(val, ICH_LR14_EL2);
 108                 break;
 109         case 15:
 110                 write_gicreg(val, ICH_LR15_EL2);
 111                 break;
 112         }
 113 }
 114
 115 static void __vgic_v3_write_ap0rn(u32 val, int n)
 116 {
 117         switch (n) {
 118         case 0:
 119                 write_gicreg(val, ICH_AP0R0_EL2);
 120                 break;
 121         case 1:
 122                 write_gicreg(val, ICH_AP0R1_EL2);
 123                 break;
 124         case 2:
 125                 write_gicreg(val, ICH_AP0R2_EL2);
 126                 break;
 127         case 3:
 128                 write_gicreg(val, ICH_AP0R3_EL2);
 129                 break;
 130         }
 131 }
 132
 133 static void __vgic_v3_write_ap1rn(u32 val, int n)
 134 {
 135         switch (n) {
 136         case 0:
 137                 write_gicreg(val, ICH_AP1R0_EL2);
 138                 break;
 139         case 1:
 140                 write_gicreg(val, ICH_AP1R1_EL2);
 141                 break;
 142         case 2:
 143                 write_gicreg(val, ICH_AP1R2_EL2);
 144                 break;
 145         case 3:
 146                 write_gicreg(val, ICH_AP1R3_EL2);
 147                 break;
 148         }
 149 }
 150
 151 static u32 __vgic_v3_read_ap0rn(int n)
 152 {
 153         u32 val;
 154
 155         switch (n) {
 156         case 0:
 157                 val = read_gicreg(ICH_AP0R0_EL2);
 158                 break;
 159         case 1:
 160                 val = read_gicreg(ICH_AP0R1_EL2);
 161                 break;
 162         case 2:
 163                 val = read_gicreg(ICH_AP0R2_EL2);
 164                 break;
 165         case 3:
 166                 val = read_gicreg(ICH_AP0R3_EL2);
 167                 break;
 168         default:
 169                 unreachable();
 170         }
 171
 172         return val;
 173 }
 174
 175 static u32 __vgic_v3_read_ap1rn(int n)
 176 {
 177         u32 val;
 178
 179         switch (n) {
 180         case 0:
 181                 val = read_gicreg(ICH_AP1R0_EL2);
 182                 break;
 183         case 1:
 184                 val = read_gicreg(ICH_AP1R1_EL2);
 185                 break;
 186         case 2:
 187                 val = read_gicreg(ICH_AP1R2_EL2);
 188                 break;
 189         case 3:
 190                 val = read_gicreg(ICH_AP1R3_EL2);
 191                 break;
 192         default:
 193                 unreachable();
 194         }
 195
 196         return val;
 197 }
 198
 199 void __vgic_v3_save_state(struct vgic_v3_cpu_if *cpu_if)
 200 {
 201         u64 used_lrs = cpu_if->used_lrs;
 202
 203         /*
 204          * Make sure stores to the GIC via the memory mapped interface
 205          * are now visible to the system register interface when reading the
 206          * LRs, and when reading back the VMCR on non-VHE systems.
 207          */
 208         if (used_lrs || !has_vhe()) {
 209                 if (!cpu_if->vgic_sre) {
 210                         dsb(sy);
 211                         isb();
 212                 }
 213         }
 214
 215         if (used_lrs || cpu_if->its_vpe.its_vm) {
 216                 int i;
 217                 u32 elrsr;
 218
 219                 elrsr = read_gicreg(ICH_ELRSR_EL2);
 220
 221                 write_gicreg(cpu_if->vgic_hcr & ~ICH_HCR_EN, ICH_HCR_EL2);
 222
 223                 for (i = 0; i < used_lrs; i++) {
 224                         if (elrsr & (1 << i))
 225                                 cpu_if->vgic_lr[i] &= ~ICH_LR_STATE;
 226                         else
 227                                 cpu_if->vgic_lr[i] = __gic_v3_get_lr(i);
 228
 229                         __gic_v3_set_lr(0, i);
 230                 }
 231         }
 232 }
 233
 234 void __vgic_v3_restore_state(struct vgic_v3_cpu_if *cpu_if)
 235 {
 236         u64 used_lrs = cpu_if->used_lrs;
 237         int i;
 238
 239         if (used_lrs || cpu_if->its_vpe.its_vm) {
 240                 write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2);
 241
 242                 for (i = 0; i < used_lrs; i++)
 243                         __gic_v3_set_lr(cpu_if->vgic_lr[i], i);
 244         }
 245
 246         /*
 247          * Ensure that writes to the LRs, and on non-VHE systems ensure that
 248          * the write to the VMCR in __vgic_v3_activate_traps(), will have
 249          * reached the (re)distributors. This ensure the guest will read the
 250          * correct values from the memory-mapped interface.
 251          */
 252         if (used_lrs || !has_vhe()) {
 253                 if (!cpu_if->vgic_sre) {
 254                         isb();
 255                         dsb(sy);
 256                 }
 257         }
 258 }
 259
 260 void __vgic_v3_activate_traps(struct vgic_v3_cpu_if *cpu_if)
 261 {
 262         /*
 263          * VFIQEn is RES1 if ICC_SRE_EL1.SRE is 1. This causes a
 264          * Group0 interrupt (as generated in GICv2 mode) to be
 265          * delivered as a FIQ to the guest, with potentially fatal
 266          * consequences. So we must make sure that ICC_SRE_EL1 has
 267          * been actually programmed with the value we want before
 268          * starting to mess with the rest of the GIC, and VMCR_EL2 in
 269          * particular.  This logic must be called before
 270          * __vgic_v3_restore_state().
 271          *
 272          * However, if the vgic is disabled (ICH_HCR_EL2.EN==0), no GIC is
 273          * provisioned at all. In order to prevent illegal accesses to the
 274          * system registers to trap to EL1 (duh), force ICC_SRE_EL1.SRE to 1
 275          * so that the trap bits can take effect. Yes, we *loves* the GIC.
 276          */
 277         if (!(cpu_if->vgic_hcr & ICH_HCR_EN)) {
 278                 write_gicreg(ICC_SRE_EL1_SRE, ICC_SRE_EL1);
 279                 isb();
 280         } else if (!cpu_if->vgic_sre) {
 281                 write_gicreg(0, ICC_SRE_EL1);
 282                 isb();
 283                 write_gicreg(cpu_if->vgic_vmcr, ICH_VMCR_EL2);
 284
 285
 286                 if (has_vhe()) {
 287                         /*
 288                          * Ensure that the write to the VMCR will have reached
 289                          * the (re)distributors. This ensure the guest will
 290                          * read the correct values from the memory-mapped
 291                          * interface.
 292                          */
 293                         isb();
 294                         dsb(sy);
 295                 }
 296         }
 297
 298         /*
 299          * Prevent the guest from touching the ICC_SRE_EL1 system
 300          * register. Note that this may not have any effect, as
 301          * ICC_SRE_EL2.Enable being RAO/WI is a valid implementation.
 302          */
 303         write_gicreg(read_gicreg(ICC_SRE_EL2) & ~ICC_SRE_EL2_ENABLE,
 304                      ICC_SRE_EL2);
 305
 306         /*
 307          * If we need to trap system registers, we must write
 308          * ICH_HCR_EL2 anyway, even if no interrupts are being
 309          * injected. Note that this also applies if we don't expect
 310          * any system register access (no vgic at all).
 311          */
 312         if (static_branch_unlikely(&vgic_v3_cpuif_trap) ||
 313             cpu_if->its_vpe.its_vm || !cpu_if->vgic_sre)
 314                 write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2);
 315 }
 316
 317 void __vgic_v3_deactivate_traps(struct vgic_v3_cpu_if *cpu_if)
 318 {
 319         u64 val;
 320
 321         if (!cpu_if->vgic_sre) {
 322                 cpu_if->vgic_vmcr = read_gicreg(ICH_VMCR_EL2);
 323         }
 324
 325         val = read_gicreg(ICC_SRE_EL2);
 326         write_gicreg(val | ICC_SRE_EL2_ENABLE, ICC_SRE_EL2);
 327
 328         if (!cpu_if->vgic_sre) {
 329                 /* Make sure ENABLE is set at EL2 before setting SRE at EL1 */
 330                 isb();
 331                 write_gicreg(1, ICC_SRE_EL1);
 332         }
 333
 334         /*
 335          * If we were trapping system registers, we enabled the VGIC even if
 336          * no interrupts were being injected, and we disable it again here.
 337          */
 338         if (static_branch_unlikely(&vgic_v3_cpuif_trap) ||
 339             cpu_if->its_vpe.its_vm || !cpu_if->vgic_sre)
 340                 write_gicreg(0, ICH_HCR_EL2);
 341 }
 342
 343 static void __vgic_v3_save_aprs(struct vgic_v3_cpu_if *cpu_if)
 344 {
 345         u64 val;
 346         u32 nr_pre_bits;
 347
 348         val = read_gicreg(ICH_VTR_EL2);
 349         nr_pre_bits = vtr_to_nr_pre_bits(val);
 350
 351         switch (nr_pre_bits) {
 352         case 7:
 353                 cpu_if->vgic_ap0r[3] = __vgic_v3_read_ap0rn(3);
 354                 cpu_if->vgic_ap0r[2] = __vgic_v3_read_ap0rn(2);
 355                 fallthrough;
 356         case 6:
 357                 cpu_if->vgic_ap0r[1] = __vgic_v3_read_ap0rn(1);
 358                 fallthrough;
 359         default:
 360                 cpu_if->vgic_ap0r[0] = __vgic_v3_read_ap0rn(0);
 361         }
 362
 363         switch (nr_pre_bits) {
 364         case 7:
 365                 cpu_if->vgic_ap1r[3] = __vgic_v3_read_ap1rn(3);
 366                 cpu_if->vgic_ap1r[2] = __vgic_v3_read_ap1rn(2);
 367                 fallthrough;
 368         case 6:
 369                 cpu_if->vgic_ap1r[1] = __vgic_v3_read_ap1rn(1);
 370                 fallthrough;
 371         default:
 372                 cpu_if->vgic_ap1r[0] = __vgic_v3_read_ap1rn(0);
 373         }
 374 }
 375
 376 static void __vgic_v3_restore_aprs(struct vgic_v3_cpu_if *cpu_if)
 377 {
 378         u64 val;
 379         u32 nr_pre_bits;
 380
 381         val = read_gicreg(ICH_VTR_EL2);
 382         nr_pre_bits = vtr_to_nr_pre_bits(val);
 383
 384         switch (nr_pre_bits) {
 385         case 7:
 386                 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[3], 3);
 387                 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[2], 2);
 388                 fallthrough;
 389         case 6:
 390                 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[1], 1);
 391                 fallthrough;
 392         default:
 393                 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[0], 0);
 394         }
 395
 396         switch (nr_pre_bits) {
 397         case 7:
 398                 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[3], 3);
 399                 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[2], 2);
 400                 fallthrough;
 401         case 6:
 402                 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[1], 1);
 403                 fallthrough;
 404         default:
 405                 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[0], 0);
 406         }
 407 }
 408
 409 void __vgic_v3_init_lrs(void)
 410 {
 411         int max_lr_idx = vtr_to_max_lr_idx(read_gicreg(ICH_VTR_EL2));
 412         int i;
 413
 414         for (i = 0; i <= max_lr_idx; i++)
 415                 __gic_v3_set_lr(0, i);
 416 }
 417
 418 /*
 419  * Return the GIC CPU configuration:
 420  * - [31:0]  ICH_VTR_EL2
 421  * - [62:32] RES0
 422  * - [63]    MMIO (GICv2) capable
 423  */
 424 u64 __vgic_v3_get_gic_config(void)
 425 {
 426         u64 val, sre = read_gicreg(ICC_SRE_EL1);
 427         unsigned long flags = 0;
 428
 429         /*
 430          * To check whether we have a MMIO-based (GICv2 compatible)
 431          * CPU interface, we need to disable the system register
 432          * view. To do that safely, we have to prevent any interrupt
 433          * from firing (which would be deadly).
 434          *
 435          * Note that this only makes sense on VHE, as interrupts are
 436          * already masked for nVHE as part of the exception entry to
 437          * EL2.
 438          */
 439         if (has_vhe())
 440                 flags = local_daif_save();
 441
 442         /*
 443          * Table 11-2 "Permitted ICC_SRE_ELx.SRE settings" indicates
 444          * that to be able to set ICC_SRE_EL1.SRE to 0, all the
 445          * interrupt overrides must be set. You've got to love this.
 446          */
 447         sysreg_clear_set(hcr_el2, 0, HCR_AMO | HCR_FMO | HCR_IMO);
 448         isb();
 449         write_gicreg(0, ICC_SRE_EL1);
 450         isb();
 451
 452         val = read_gicreg(ICC_SRE_EL1);
 453
 454         write_gicreg(sre, ICC_SRE_EL1);
 455         isb();
 456         sysreg_clear_set(hcr_el2, HCR_AMO | HCR_FMO | HCR_IMO, 0);
 457         isb();
 458
 459         if (has_vhe())
 460                 local_daif_restore(flags);
 461
 462         val  = (val & ICC_SRE_EL1_SRE) ? 0 : (1ULL << 63);
 463         val |= read_gicreg(ICH_VTR_EL2);
 464
 465         return val;
 466 }
 467
 468 static u64 __vgic_v3_read_vmcr(void)
 469 {
 470         return read_gicreg(ICH_VMCR_EL2);
 471 }
 472
 473 static void __vgic_v3_write_vmcr(u32 vmcr)
 474 {
 475         write_gicreg(vmcr, ICH_VMCR_EL2);
 476 }
 477
 478 void __vgic_v3_save_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if)
 479 {
 480         __vgic_v3_save_aprs(cpu_if);
 481         if (cpu_if->vgic_sre)
 482                 cpu_if->vgic_vmcr = __vgic_v3_read_vmcr();
 483 }
 484
 485 void __vgic_v3_restore_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if)
 486 {
 487         /*
 488          * If dealing with a GICv2 emulation on GICv3, VMCR_EL2.VFIQen
 489          * is dependent on ICC_SRE_EL1.SRE, and we have to perform the
 490          * VMCR_EL2 save/restore in the world switch.
 491          */
 492         if (cpu_if->vgic_sre)
 493                 __vgic_v3_write_vmcr(cpu_if->vgic_vmcr);
 494         __vgic_v3_restore_aprs(cpu_if);
 495 }
 496
 497 static int __vgic_v3_bpr_min(void)
 498 {
 499         /* See Pseudocode for VPriorityGroup */
 500         return 8 - vtr_to_nr_pre_bits(read_gicreg(ICH_VTR_EL2));
 501 }
 502
 503 static int __vgic_v3_get_group(struct kvm_vcpu *vcpu)
 504 {
 505         u64 esr = kvm_vcpu_get_esr(vcpu);
 506         u8 crm = (esr & ESR_ELx_SYS64_ISS_CRM_MASK) >> ESR_ELx_SYS64_ISS_CRM_SHIFT;
 507
 508         return crm != 8;
 509 }
 510
 511 #define GICv3_IDLE_PRIORITY     0xff
 512
 513 static int __vgic_v3_highest_priority_lr(struct kvm_vcpu *vcpu, u32 vmcr,
 514                                          u64 *lr_val)
 515 {
 516         unsigned int used_lrs = vcpu->arch.vgic_cpu.vgic_v3.used_lrs;
 517         u8 priority = GICv3_IDLE_PRIORITY;
 518         int i, lr = -1;
 519
 520         for (i = 0; i < used_lrs; i++) {
 521                 u64 val = __gic_v3_get_lr(i);
 522                 u8 lr_prio = (val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
 523
 524                 /* Not pending in the state? */
 525                 if ((val & ICH_LR_STATE) != ICH_LR_PENDING_BIT)
 526                         continue;
 527
 528                 /* Group-0 interrupt, but Group-0 disabled? */
 529                 if (!(val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG0_MASK))
 530                         continue;
 531
 532                 /* Group-1 interrupt, but Group-1 disabled? */
 533                 if ((val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG1_MASK))
 534                         continue;
 535
 536                 /* Not the highest priority? */
 537                 if (lr_prio >= priority)
 538                         continue;
 539
 540                 /* This is a candidate */
 541                 priority = lr_prio;
 542                 *lr_val = val;
 543                 lr = i;
 544         }
 545
 546         if (lr == -1)
 547                 *lr_val = ICC_IAR1_EL1_SPURIOUS;
 548
 549         return lr;
 550 }
 551
 552 static int __vgic_v3_find_active_lr(struct kvm_vcpu *vcpu, int intid,
 553                                     u64 *lr_val)
 554 {
 555         unsigned int used_lrs = vcpu->arch.vgic_cpu.vgic_v3.used_lrs;
 556         int i;
 557
 558         for (i = 0; i < used_lrs; i++) {
 559                 u64 val = __gic_v3_get_lr(i);
 560
 561                 if ((val & ICH_LR_VIRTUAL_ID_MASK) == intid &&
 562                     (val & ICH_LR_ACTIVE_BIT)) {
 563                         *lr_val = val;
 564                         return i;
 565                 }
 566         }
 567
 568         *lr_val = ICC_IAR1_EL1_SPURIOUS;
 569         return -1;
 570 }
 571
 572 static int __vgic_v3_get_highest_active_priority(void)
 573 {
 574         u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2));
 575         u32 hap = 0;
 576         int i;
 577
 578         for (i = 0; i < nr_apr_regs; i++) {
 579                 u32 val;
 580
 581                 /*
 582                  * The ICH_AP0Rn_EL2 and ICH_AP1Rn_EL2 registers
 583                  * contain the active priority levels for this VCPU
 584                  * for the maximum number of supported priority
 585                  * levels, and we return the full priority level only
 586                  * if the BPR is programmed to its minimum, otherwise
 587                  * we return a combination of the priority level and
 588                  * subpriority, as determined by the setting of the
 589                  * BPR, but without the full subpriority.
 590                  */
 591                 val  = __vgic_v3_read_ap0rn(i);
 592                 val |= __vgic_v3_read_ap1rn(i);
 593                 if (!val) {
 594                         hap += 32;
 595                         continue;
 596                 }
 597
 598                 return (hap + __ffs(val)) << __vgic_v3_bpr_min();
 599         }
 600
 601         return GICv3_IDLE_PRIORITY;
 602 }
 603
 604 static unsigned int __vgic_v3_get_bpr0(u32 vmcr)
 605 {
 606         return (vmcr & ICH_VMCR_BPR0_MASK) >> ICH_VMCR_BPR0_SHIFT;
 607 }
 608
 609 static unsigned int __vgic_v3_get_bpr1(u32 vmcr)
 610 {
 611         unsigned int bpr;
 612
 613         if (vmcr & ICH_VMCR_CBPR_MASK) {
 614                 bpr = __vgic_v3_get_bpr0(vmcr);
 615                 if (bpr < 7)
 616                         bpr++;
 617         } else {
 618                 bpr = (vmcr & ICH_VMCR_BPR1_MASK) >> ICH_VMCR_BPR1_SHIFT;
 619         }
 620
 621         return bpr;
 622 }
 623
 624 /*
 625  * Convert a priority to a preemption level, taking the relevant BPR
 626  * into account by zeroing the sub-priority bits.
 627  */
 628 static u8 __vgic_v3_pri_to_pre(u8 pri, u32 vmcr, int grp)
 629 {
 630         unsigned int bpr;
 631
 632         if (!grp)
 633                 bpr = __vgic_v3_get_bpr0(vmcr) + 1;
 634         else
 635                 bpr = __vgic_v3_get_bpr1(vmcr);
 636
 637         return pri & (GENMASK(7, 0) << bpr);
 638 }
 639
 640 /*
 641  * The priority value is independent of any of the BPR values, so we
 642  * normalize it using the minimal BPR value. This guarantees that no
 643  * matter what the guest does with its BPR, we can always set/get the
 644  * same value of a priority.
 645  */
 646 static void __vgic_v3_set_active_priority(u8 pri, u32 vmcr, int grp)
 647 {
 648         u8 pre, ap;
 649         u32 val;
 650         int apr;
 651
 652         pre = __vgic_v3_pri_to_pre(pri, vmcr, grp);
 653         ap = pre >> __vgic_v3_bpr_min();
 654         apr = ap / 32;
 655
 656         if (!grp) {
 657                 val = __vgic_v3_read_ap0rn(apr);
 658                 __vgic_v3_write_ap0rn(val | BIT(ap % 32), apr);
 659         } else {
 660                 val = __vgic_v3_read_ap1rn(apr);
 661                 __vgic_v3_write_ap1rn(val | BIT(ap % 32), apr);
 662         }
 663 }
 664
 665 static int __vgic_v3_clear_highest_active_priority(void)
 666 {
 667         u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2));
 668         u32 hap = 0;
 669         int i;
 670
 671         for (i = 0; i < nr_apr_regs; i++) {
 672                 u32 ap0, ap1;
 673                 int c0, c1;
 674
 675                 ap0 = __vgic_v3_read_ap0rn(i);
 676                 ap1 = __vgic_v3_read_ap1rn(i);
 677                 if (!ap0 && !ap1) {
 678                         hap += 32;
 679                         continue;
 680                 }
 681
 682                 c0 = ap0 ? __ffs(ap0) : 32;
 683                 c1 = ap1 ? __ffs(ap1) : 32;
 684
 685                 /* Always clear the LSB, which is the highest priority */
 686                 if (c0 < c1) {
 687                         ap0 &= ~BIT(c0);
 688                         __vgic_v3_write_ap0rn(ap0, i);
 689                         hap += c0;
 690                 } else {
 691                         ap1 &= ~BIT(c1);
 692                         __vgic_v3_write_ap1rn(ap1, i);
 693                         hap += c1;
 694                 }
 695
 696                 /* Rescale to 8 bits of priority */
 697                 return hap << __vgic_v3_bpr_min();
 698         }
 699
 700         return GICv3_IDLE_PRIORITY;
 701 }
 702
 703 static void __vgic_v3_read_iar(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 704 {
 705         u64 lr_val;
 706         u8 lr_prio, pmr;
 707         int lr, grp;
 708
 709         grp = __vgic_v3_get_group(vcpu);
 710
 711         lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val);
 712         if (lr < 0)
 713                 goto spurious;
 714
 715         if (grp != !!(lr_val & ICH_LR_GROUP))
 716                 goto spurious;
 717
 718         pmr = (vmcr & ICH_VMCR_PMR_MASK) >> ICH_VMCR_PMR_SHIFT;
 719         lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
 720         if (pmr <= lr_prio)
 721                 goto spurious;
 722
 723         if (__vgic_v3_get_highest_active_priority() <= __vgic_v3_pri_to_pre(lr_prio, vmcr, grp))
 724                 goto spurious;
 725
 726         lr_val &= ~ICH_LR_STATE;
 727         lr_val |= ICH_LR_ACTIVE_BIT;
 728         __gic_v3_set_lr(lr_val, lr);
 729         __vgic_v3_set_active_priority(lr_prio, vmcr, grp);
 730         vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK);
 731         return;
 732
 733 spurious:
 734         vcpu_set_reg(vcpu, rt, ICC_IAR1_EL1_SPURIOUS);
 735 }
 736
 737 static void __vgic_v3_clear_active_lr(int lr, u64 lr_val)
 738 {
 739         lr_val &= ~ICH_LR_ACTIVE_BIT;
 740         if (lr_val & ICH_LR_HW) {
 741                 u32 pid;
 742
 743                 pid = (lr_val & ICH_LR_PHYS_ID_MASK) >> ICH_LR_PHYS_ID_SHIFT;
 744                 gic_write_dir(pid);
 745         }
 746
 747         __gic_v3_set_lr(lr_val, lr);
 748 }
 749
 750 static void __vgic_v3_bump_eoicount(void)
 751 {
 752         u32 hcr;
 753
 754         hcr = read_gicreg(ICH_HCR_EL2);
 755         hcr += 1 << ICH_HCR_EOIcount_SHIFT;
 756         write_gicreg(hcr, ICH_HCR_EL2);
 757 }
 758
 759 static void __vgic_v3_write_dir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 760 {
 761         u32 vid = vcpu_get_reg(vcpu, rt);
 762         u64 lr_val;
 763         int lr;
 764
 765         /* EOImode == 0, nothing to be done here */
 766         if (!(vmcr & ICH_VMCR_EOIM_MASK))
 767                 return;
 768
 769         /* No deactivate to be performed on an LPI */
 770         if (vid >= VGIC_MIN_LPI)
 771                 return;
 772
 773         lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val);
 774         if (lr == -1) {
 775                 __vgic_v3_bump_eoicount();
 776                 return;
 777         }
 778
 779         __vgic_v3_clear_active_lr(lr, lr_val);
 780 }
 781
 782 static void __vgic_v3_write_eoir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 783 {
 784         u32 vid = vcpu_get_reg(vcpu, rt);
 785         u64 lr_val;
 786         u8 lr_prio, act_prio;
 787         int lr, grp;
 788
 789         grp = __vgic_v3_get_group(vcpu);
 790
 791         /* Drop priority in any case */
 792         act_prio = __vgic_v3_clear_highest_active_priority();
 793
 794         lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val);
 795         if (lr == -1) {
 796                 /* Do not bump EOIcount for LPIs that aren't in the LRs */
 797                 if (!(vid >= VGIC_MIN_LPI))
 798                         __vgic_v3_bump_eoicount();
 799                 return;
 800         }
 801
 802         /* EOImode == 1 and not an LPI, nothing to be done here */
 803         if ((vmcr & ICH_VMCR_EOIM_MASK) && !(vid >= VGIC_MIN_LPI))
 804                 return;
 805
 806         lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
 807
 808         /* If priorities or group do not match, the guest has fscked-up. */
 809         if (grp != !!(lr_val & ICH_LR_GROUP) ||
 810             __vgic_v3_pri_to_pre(lr_prio, vmcr, grp) != act_prio)
 811                 return;
 812
 813         /* Let's now perform the deactivation */
 814         __vgic_v3_clear_active_lr(lr, lr_val);
 815 }
 816
 817 static void __vgic_v3_read_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 818 {
 819         vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG0_MASK));
 820 }
 821
 822 static void __vgic_v3_read_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 823 {
 824         vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG1_MASK));
 825 }
 826
 827 static void __vgic_v3_write_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 828 {
 829         u64 val = vcpu_get_reg(vcpu, rt);
 830
 831         if (val & 1)
 832                 vmcr |= ICH_VMCR_ENG0_MASK;
 833         else
 834                 vmcr &= ~ICH_VMCR_ENG0_MASK;
 835
 836         __vgic_v3_write_vmcr(vmcr);
 837 }
 838
 839 static void __vgic_v3_write_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 840 {
 841         u64 val = vcpu_get_reg(vcpu, rt);
 842
 843         if (val & 1)
 844                 vmcr |= ICH_VMCR_ENG1_MASK;
 845         else
 846                 vmcr &= ~ICH_VMCR_ENG1_MASK;
 847
 848         __vgic_v3_write_vmcr(vmcr);
 849 }
 850
 851 static void __vgic_v3_read_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 852 {
 853         vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr0(vmcr));
 854 }
 855
 856 static void __vgic_v3_read_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 857 {
 858         vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr1(vmcr));
 859 }
 860
 861 static void __vgic_v3_write_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 862 {
 863         u64 val = vcpu_get_reg(vcpu, rt);
 864         u8 bpr_min = __vgic_v3_bpr_min() - 1;
 865
 866         /* Enforce BPR limiting */
 867         if (val < bpr_min)
 868                 val = bpr_min;
 869
 870         val <<= ICH_VMCR_BPR0_SHIFT;
 871         val &= ICH_VMCR_BPR0_MASK;
 872         vmcr &= ~ICH_VMCR_BPR0_MASK;
 873         vmcr |= val;
 874
 875         __vgic_v3_write_vmcr(vmcr);
 876 }
 877
 878 static void __vgic_v3_write_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 879 {
 880         u64 val = vcpu_get_reg(vcpu, rt);
 881         u8 bpr_min = __vgic_v3_bpr_min();
 882
 883         if (vmcr & ICH_VMCR_CBPR_MASK)
 884                 return;
 885
 886         /* Enforce BPR limiting */
 887         if (val < bpr_min)
 888                 val = bpr_min;
 889
 890         val <<= ICH_VMCR_BPR1_SHIFT;
 891         val &= ICH_VMCR_BPR1_MASK;
 892         vmcr &= ~ICH_VMCR_BPR1_MASK;
 893         vmcr |= val;
 894
 895         __vgic_v3_write_vmcr(vmcr);
 896 }
 897
 898 static void __vgic_v3_read_apxrn(struct kvm_vcpu *vcpu, int rt, int n)
 899 {
 900         u32 val;
 901
 902         if (!__vgic_v3_get_group(vcpu))
 903                 val = __vgic_v3_read_ap0rn(n);
 904         else
 905                 val = __vgic_v3_read_ap1rn(n);
 906
 907         vcpu_set_reg(vcpu, rt, val);
 908 }
 909
 910 static void __vgic_v3_write_apxrn(struct kvm_vcpu *vcpu, int rt, int n)
 911 {
 912         u32 val = vcpu_get_reg(vcpu, rt);
 913
 914         if (!__vgic_v3_get_group(vcpu))
 915                 __vgic_v3_write_ap0rn(val, n);
 916         else
 917                 __vgic_v3_write_ap1rn(val, n);
 918 }
 919
 920 static void __vgic_v3_read_apxr0(struct kvm_vcpu *vcpu,
 921                                             u32 vmcr, int rt)
 922 {
 923         __vgic_v3_read_apxrn(vcpu, rt, 0);
 924 }
 925
 926 static void __vgic_v3_read_apxr1(struct kvm_vcpu *vcpu,
 927                                             u32 vmcr, int rt)
 928 {
 929         __vgic_v3_read_apxrn(vcpu, rt, 1);
 930 }
 931
 932 static void __vgic_v3_read_apxr2(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 933 {
 934         __vgic_v3_read_apxrn(vcpu, rt, 2);
 935 }
 936
 937 static void __vgic_v3_read_apxr3(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 938 {
 939         __vgic_v3_read_apxrn(vcpu, rt, 3);
 940 }
 941
 942 static void __vgic_v3_write_apxr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 943 {
 944         __vgic_v3_write_apxrn(vcpu, rt, 0);
 945 }
 946
 947 static void __vgic_v3_write_apxr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 948 {
 949         __vgic_v3_write_apxrn(vcpu, rt, 1);
 950 }
 951
 952 static void __vgic_v3_write_apxr2(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 953 {
 954         __vgic_v3_write_apxrn(vcpu, rt, 2);
 955 }
 956
 957 static void __vgic_v3_write_apxr3(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 958 {
 959         __vgic_v3_write_apxrn(vcpu, rt, 3);
 960 }
 961
 962 static void __vgic_v3_read_hppir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 963 {
 964         u64 lr_val;
 965         int lr, lr_grp, grp;
 966
 967         grp = __vgic_v3_get_group(vcpu);
 968
 969         lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val);
 970         if (lr == -1)
 971                 goto spurious;
 972
 973         lr_grp = !!(lr_val & ICH_LR_GROUP);
 974         if (lr_grp != grp)
 975                 lr_val = ICC_IAR1_EL1_SPURIOUS;
 976
 977 spurious:
 978         vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK);
 979 }
 980
 981 static void __vgic_v3_read_pmr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 982 {
 983         vmcr &= ICH_VMCR_PMR_MASK;
 984         vmcr >>= ICH_VMCR_PMR_SHIFT;
 985         vcpu_set_reg(vcpu, rt, vmcr);
 986 }
 987
 988 static void __vgic_v3_write_pmr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 989 {
 990         u32 val = vcpu_get_reg(vcpu, rt);
 991
 992         val <<= ICH_VMCR_PMR_SHIFT;
 993         val &= ICH_VMCR_PMR_MASK;
 994         vmcr &= ~ICH_VMCR_PMR_MASK;
 995         vmcr |= val;
 996
 997         write_gicreg(vmcr, ICH_VMCR_EL2);
 998 }
 999
1000 static void __vgic_v3_read_rpr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
1001 {
1002         u32 val = __vgic_v3_get_highest_active_priority();
1003         vcpu_set_reg(vcpu, rt, val);
1004 }
1005
1006 static void __vgic_v3_read_ctlr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
1007 {
1008         u32 vtr, val;
1009
1010         vtr = read_gicreg(ICH_VTR_EL2);
1011         /* PRIbits */
1012         val = ((vtr >> 29) & 7) << ICC_CTLR_EL1_PRI_BITS_SHIFT;
1013         /* IDbits */
1014         val |= ((vtr >> 23) & 7) << ICC_CTLR_EL1_ID_BITS_SHIFT;
1015         /* A3V */
1016         val |= ((vtr >> 21) & 1) << ICC_CTLR_EL1_A3V_SHIFT;
1017         /* EOImode */
1018         val |= ((vmcr & ICH_VMCR_EOIM_MASK) >> ICH_VMCR_EOIM_SHIFT) << ICC_CTLR_EL1_EOImode_SHIFT;
1019         /* CBPR */
1020         val |= (vmcr & ICH_VMCR_CBPR_MASK) >> ICH_VMCR_CBPR_SHIFT;
1021
1022         vcpu_set_reg(vcpu, rt, val);
1023 }
1024
1025 static void __vgic_v3_write_ctlr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
1026 {
1027         u32 val = vcpu_get_reg(vcpu, rt);
1028
1029         if (val & ICC_CTLR_EL1_CBPR_MASK)
1030                 vmcr |= ICH_VMCR_CBPR_MASK;
1031         else
1032                 vmcr &= ~ICH_VMCR_CBPR_MASK;
1033
1034         if (val & ICC_CTLR_EL1_EOImode_MASK)
1035                 vmcr |= ICH_VMCR_EOIM_MASK;
1036         else
1037                 vmcr &= ~ICH_VMCR_EOIM_MASK;
1038
1039         write_gicreg(vmcr, ICH_VMCR_EL2);
1040 }
1041
1042 static bool __vgic_v3_check_trap_forwarding(struct kvm_vcpu *vcpu,
1043                                             u32 sysreg, bool is_read)
1044 {
1045         u64 ich_hcr;
1046
1047         if (!vcpu_has_nv(vcpu) || is_hyp_ctxt(vcpu))
1048                 return false;
1049
1050         ich_hcr = __vcpu_sys_reg(vcpu, ICH_HCR_EL2);
1051
1052         switch (sysreg) {
1053         case SYS_ICC_IGRPEN0_EL1:
1054                 if (is_read &&
1055                     (__vcpu_sys_reg(vcpu, HFGRTR_EL2) & HFGxTR_EL2_ICC_IGRPENn_EL1))
1056                         return true;
1057
1058                 if (!is_read &&
1059                     (__vcpu_sys_reg(vcpu, HFGWTR_EL2) & HFGxTR_EL2_ICC_IGRPENn_EL1))
1060                         return true;
1061
1062                 fallthrough;
1063
1064         case SYS_ICC_AP0Rn_EL1(0):
1065         case SYS_ICC_AP0Rn_EL1(1):
1066         case SYS_ICC_AP0Rn_EL1(2):
1067         case SYS_ICC_AP0Rn_EL1(3):
1068         case SYS_ICC_BPR0_EL1:
1069         case SYS_ICC_EOIR0_EL1:
1070         case SYS_ICC_HPPIR0_EL1:
1071         case SYS_ICC_IAR0_EL1:
1072                 return ich_hcr & ICH_HCR_TALL0;
1073
1074         case SYS_ICC_IGRPEN1_EL1:
1075                 if (is_read &&
1076                     (__vcpu_sys_reg(vcpu, HFGRTR_EL2) & HFGxTR_EL2_ICC_IGRPENn_EL1))
1077                         return true;
1078
1079                 if (!is_read &&
1080                     (__vcpu_sys_reg(vcpu, HFGWTR_EL2) & HFGxTR_EL2_ICC_IGRPENn_EL1))
1081                         return true;
1082
1083                 fallthrough;
1084
1085         case SYS_ICC_AP1Rn_EL1(0):
1086         case SYS_ICC_AP1Rn_EL1(1):
1087         case SYS_ICC_AP1Rn_EL1(2):
1088         case SYS_ICC_AP1Rn_EL1(3):
1089         case SYS_ICC_BPR1_EL1:
1090         case SYS_ICC_EOIR1_EL1:
1091         case SYS_ICC_HPPIR1_EL1:
1092         case SYS_ICC_IAR1_EL1:
1093                 return ich_hcr & ICH_HCR_TALL1;
1094
1095         case SYS_ICC_DIR_EL1:
1096                 if (ich_hcr & ICH_HCR_TDIR)
1097                         return true;
1098
1099                 fallthrough;
1100
1101         case SYS_ICC_RPR_EL1:
1102         case SYS_ICC_CTLR_EL1:
1103         case SYS_ICC_PMR_EL1:
1104                 return ich_hcr & ICH_HCR_TC;
1105
1106         default:
1107                 return false;
1108         }
1109 }
1110
1111 int __vgic_v3_perform_cpuif_access(struct kvm_vcpu *vcpu)
1112 {
1113         int rt;
1114         u64 esr;
1115         u32 vmcr;
1116         void (*fn)(struct kvm_vcpu *, u32, int);
1117         bool is_read;
1118         u32 sysreg;
1119
1120         if (kern_hyp_va(vcpu->kvm)->arch.vgic.vgic_model != KVM_DEV_TYPE_ARM_VGIC_V3)
1121                 return 0;
1122
1123         esr = kvm_vcpu_get_esr(vcpu);
1124         if (vcpu_mode_is_32bit(vcpu)) {
1125                 if (!kvm_condition_valid(vcpu)) {
1126                         __kvm_skip_instr(vcpu);
1127                         return 1;
1128                 }
1129
1130                 sysreg = esr_cp15_to_sysreg(esr);
1131         } else {
1132                 sysreg = esr_sys64_to_sysreg(esr);
1133         }
1134
1135         is_read = (esr & ESR_ELx_SYS64_ISS_DIR_MASK) == ESR_ELx_SYS64_ISS_DIR_READ;
1136
1137         if (__vgic_v3_check_trap_forwarding(vcpu, sysreg, is_read))
1138                 return 0;
1139
1140         switch (sysreg) {
1141         case SYS_ICC_IAR0_EL1:
1142         case SYS_ICC_IAR1_EL1:
1143                 if (unlikely(!is_read))
1144                         return 0;
1145                 fn = __vgic_v3_read_iar;
1146                 break;
1147         case SYS_ICC_EOIR0_EL1:
1148         case SYS_ICC_EOIR1_EL1:
1149                 if (unlikely(is_read))
1150                         return 0;
1151                 fn = __vgic_v3_write_eoir;
1152                 break;
1153         case SYS_ICC_IGRPEN1_EL1:
1154                 if (is_read)
1155                         fn = __vgic_v3_read_igrpen1;
1156                 else
1157                         fn = __vgic_v3_write_igrpen1;
1158                 break;
1159         case SYS_ICC_BPR1_EL1:
1160                 if (is_read)
1161                         fn = __vgic_v3_read_bpr1;
1162                 else
1163                         fn = __vgic_v3_write_bpr1;
1164                 break;
1165         case SYS_ICC_AP0Rn_EL1(0):
1166         case SYS_ICC_AP1Rn_EL1(0):
1167                 if (is_read)
1168                         fn = __vgic_v3_read_apxr0;
1169                 else
1170                         fn = __vgic_v3_write_apxr0;
1171                 break;
1172         case SYS_ICC_AP0Rn_EL1(1):
1173         case SYS_ICC_AP1Rn_EL1(1):
1174                 if (is_read)
1175                         fn = __vgic_v3_read_apxr1;
1176                 else
1177                         fn = __vgic_v3_write_apxr1;
1178                 break;
1179         case SYS_ICC_AP0Rn_EL1(2):
1180         case SYS_ICC_AP1Rn_EL1(2):
1181                 if (is_read)
1182                         fn = __vgic_v3_read_apxr2;
1183                 else
1184                         fn = __vgic_v3_write_apxr2;
1185                 break;
1186         case SYS_ICC_AP0Rn_EL1(3):
1187         case SYS_ICC_AP1Rn_EL1(3):
1188                 if (is_read)
1189                         fn = __vgic_v3_read_apxr3;
1190                 else
1191                         fn = __vgic_v3_write_apxr3;
1192                 break;
1193         case SYS_ICC_HPPIR0_EL1:
1194         case SYS_ICC_HPPIR1_EL1:
1195                 if (unlikely(!is_read))
1196                         return 0;
1197                 fn = __vgic_v3_read_hppir;
1198                 break;
1199         case SYS_ICC_IGRPEN0_EL1:
1200                 if (is_read)
1201                         fn = __vgic_v3_read_igrpen0;
1202                 else
1203                         fn = __vgic_v3_write_igrpen0;
1204                 break;
1205         case SYS_ICC_BPR0_EL1:
1206                 if (is_read)
1207                         fn = __vgic_v3_read_bpr0;
1208                 else
1209                         fn = __vgic_v3_write_bpr0;
1210                 break;
1211         case SYS_ICC_DIR_EL1:
1212                 if (unlikely(is_read))
1213                         return 0;
1214                 fn = __vgic_v3_write_dir;
1215                 break;
1216         case SYS_ICC_RPR_EL1:
1217                 if (unlikely(!is_read))
1218                         return 0;
1219                 fn = __vgic_v3_read_rpr;
1220                 break;
1221         case SYS_ICC_CTLR_EL1:
1222                 if (is_read)
1223                         fn = __vgic_v3_read_ctlr;
1224                 else
1225                         fn = __vgic_v3_write_ctlr;
1226                 break;
1227         case SYS_ICC_PMR_EL1:
1228                 if (is_read)
1229                         fn = __vgic_v3_read_pmr;
1230                 else
1231                         fn = __vgic_v3_write_pmr;
1232                 break;
1233         default:
1234                 return 0;
1235         }
1236
1237         vmcr = __vgic_v3_read_vmcr();
1238         rt = kvm_vcpu_sys_get_rt(vcpu);
1239         fn(vcpu, vmcr, rt);
1240
1241         __kvm_skip_instr(vcpu);
1242
1243         return 1;
1244 }