virt/kvm/arm/hyp/vgic-v3-sr.c

   1 /*
   2  * Copyright (C) 2012-2015 - ARM Ltd
   3  * Author: Marc Zyngier <marc.zyngier@arm.com>
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License version 2 as
   7  * published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16  */
  17
  18 #include <linux/compiler.h>
  19 #include <linux/irqchip/arm-gic-v3.h>
  20 #include <linux/kvm_host.h>
  21
  22 #include <asm/kvm_emulate.h>
  23 #include <asm/kvm_hyp.h>
  24
  25 #define vtr_to_max_lr_idx(v)            ((v) & 0xf)
  26 #define vtr_to_nr_pre_bits(v)           ((((u32)(v) >> 26) & 7) + 1)
  27 #define vtr_to_nr_apr_regs(v)           (1 << (vtr_to_nr_pre_bits(v) - 5))
  28
  29 static u64 __hyp_text __gic_v3_get_lr(unsigned int lr)
  30 {
  31         switch (lr & 0xf) {
  32         case 0:
  33                 return read_gicreg(ICH_LR0_EL2);
  34         case 1:
  35                 return read_gicreg(ICH_LR1_EL2);
  36         case 2:
  37                 return read_gicreg(ICH_LR2_EL2);
  38         case 3:
  39                 return read_gicreg(ICH_LR3_EL2);
  40         case 4:
  41                 return read_gicreg(ICH_LR4_EL2);
  42         case 5:
  43                 return read_gicreg(ICH_LR5_EL2);
  44         case 6:
  45                 return read_gicreg(ICH_LR6_EL2);
  46         case 7:
  47                 return read_gicreg(ICH_LR7_EL2);
  48         case 8:
  49                 return read_gicreg(ICH_LR8_EL2);
  50         case 9:
  51                 return read_gicreg(ICH_LR9_EL2);
  52         case 10:
  53                 return read_gicreg(ICH_LR10_EL2);
  54         case 11:
  55                 return read_gicreg(ICH_LR11_EL2);
  56         case 12:
  57                 return read_gicreg(ICH_LR12_EL2);
  58         case 13:
  59                 return read_gicreg(ICH_LR13_EL2);
  60         case 14:
  61                 return read_gicreg(ICH_LR14_EL2);
  62         case 15:
  63                 return read_gicreg(ICH_LR15_EL2);
  64         }
  65
  66         unreachable();
  67 }
  68
  69 static void __hyp_text __gic_v3_set_lr(u64 val, int lr)
  70 {
  71         switch (lr & 0xf) {
  72         case 0:
  73                 write_gicreg(val, ICH_LR0_EL2);
  74                 break;
  75         case 1:
  76                 write_gicreg(val, ICH_LR1_EL2);
  77                 break;
  78         case 2:
  79                 write_gicreg(val, ICH_LR2_EL2);
  80                 break;
  81         case 3:
  82                 write_gicreg(val, ICH_LR3_EL2);
  83                 break;
  84         case 4:
  85                 write_gicreg(val, ICH_LR4_EL2);
  86                 break;
  87         case 5:
  88                 write_gicreg(val, ICH_LR5_EL2);
  89                 break;
  90         case 6:
  91                 write_gicreg(val, ICH_LR6_EL2);
  92                 break;
  93         case 7:
  94                 write_gicreg(val, ICH_LR7_EL2);
  95                 break;
  96         case 8:
  97                 write_gicreg(val, ICH_LR8_EL2);
  98                 break;
  99         case 9:
 100                 write_gicreg(val, ICH_LR9_EL2);
 101                 break;
 102         case 10:
 103                 write_gicreg(val, ICH_LR10_EL2);
 104                 break;
 105         case 11:
 106                 write_gicreg(val, ICH_LR11_EL2);
 107                 break;
 108         case 12:
 109                 write_gicreg(val, ICH_LR12_EL2);
 110                 break;
 111         case 13:
 112                 write_gicreg(val, ICH_LR13_EL2);
 113                 break;
 114         case 14:
 115                 write_gicreg(val, ICH_LR14_EL2);
 116                 break;
 117         case 15:
 118                 write_gicreg(val, ICH_LR15_EL2);
 119                 break;
 120         }
 121 }
 122
 123 static void __hyp_text __vgic_v3_write_ap0rn(u32 val, int n)
 124 {
 125         switch (n) {
 126         case 0:
 127                 write_gicreg(val, ICH_AP0R0_EL2);
 128                 break;
 129         case 1:
 130                 write_gicreg(val, ICH_AP0R1_EL2);
 131                 break;
 132         case 2:
 133                 write_gicreg(val, ICH_AP0R2_EL2);
 134                 break;
 135         case 3:
 136                 write_gicreg(val, ICH_AP0R3_EL2);
 137                 break;
 138         }
 139 }
 140
 141 static void __hyp_text __vgic_v3_write_ap1rn(u32 val, int n)
 142 {
 143         switch (n) {
 144         case 0:
 145                 write_gicreg(val, ICH_AP1R0_EL2);
 146                 break;
 147         case 1:
 148                 write_gicreg(val, ICH_AP1R1_EL2);
 149                 break;
 150         case 2:
 151                 write_gicreg(val, ICH_AP1R2_EL2);
 152                 break;
 153         case 3:
 154                 write_gicreg(val, ICH_AP1R3_EL2);
 155                 break;
 156         }
 157 }
 158
 159 static u32 __hyp_text __vgic_v3_read_ap0rn(int n)
 160 {
 161         u32 val;
 162
 163         switch (n) {
 164         case 0:
 165                 val = read_gicreg(ICH_AP0R0_EL2);
 166                 break;
 167         case 1:
 168                 val = read_gicreg(ICH_AP0R1_EL2);
 169                 break;
 170         case 2:
 171                 val = read_gicreg(ICH_AP0R2_EL2);
 172                 break;
 173         case 3:
 174                 val = read_gicreg(ICH_AP0R3_EL2);
 175                 break;
 176         default:
 177                 unreachable();
 178         }
 179
 180         return val;
 181 }
 182
 183 static u32 __hyp_text __vgic_v3_read_ap1rn(int n)
 184 {
 185         u32 val;
 186
 187         switch (n) {
 188         case 0:
 189                 val = read_gicreg(ICH_AP1R0_EL2);
 190                 break;
 191         case 1:
 192                 val = read_gicreg(ICH_AP1R1_EL2);
 193                 break;
 194         case 2:
 195                 val = read_gicreg(ICH_AP1R2_EL2);
 196                 break;
 197         case 3:
 198                 val = read_gicreg(ICH_AP1R3_EL2);
 199                 break;
 200         default:
 201                 unreachable();
 202         }
 203
 204         return val;
 205 }
 206
 207 void __hyp_text __vgic_v3_save_state(struct kvm_vcpu *vcpu)
 208 {
 209         struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3;
 210         u64 used_lrs = vcpu->arch.vgic_cpu.used_lrs;
 211         u64 val;
 212
 213         /*
 214          * Make sure stores to the GIC via the memory mapped interface
 215          * are now visible to the system register interface.
 216          */
 217         if (!cpu_if->vgic_sre) {
 218                 dsb(st);
 219                 cpu_if->vgic_vmcr = read_gicreg(ICH_VMCR_EL2);
 220         }
 221
 222         if (used_lrs) {
 223                 int i;
 224                 u32 nr_pre_bits;
 225
 226                 cpu_if->vgic_elrsr = read_gicreg(ICH_ELSR_EL2);
 227
 228                 write_gicreg(0, ICH_HCR_EL2);
 229                 val = read_gicreg(ICH_VTR_EL2);
 230                 nr_pre_bits = vtr_to_nr_pre_bits(val);
 231
 232                 for (i = 0; i < used_lrs; i++) {
 233                         if (cpu_if->vgic_elrsr & (1 << i))
 234                                 cpu_if->vgic_lr[i] &= ~ICH_LR_STATE;
 235                         else
 236                                 cpu_if->vgic_lr[i] = __gic_v3_get_lr(i);
 237
 238                         __gic_v3_set_lr(0, i);
 239                 }
 240
 241                 switch (nr_pre_bits) {
 242                 case 7:
 243                         cpu_if->vgic_ap0r[3] = __vgic_v3_read_ap0rn(3);
 244                         cpu_if->vgic_ap0r[2] = __vgic_v3_read_ap0rn(2);
 245                 case 6:
 246                         cpu_if->vgic_ap0r[1] = __vgic_v3_read_ap0rn(1);
 247                 default:
 248                         cpu_if->vgic_ap0r[0] = __vgic_v3_read_ap0rn(0);
 249                 }
 250
 251                 switch (nr_pre_bits) {
 252                 case 7:
 253                         cpu_if->vgic_ap1r[3] = __vgic_v3_read_ap1rn(3);
 254                         cpu_if->vgic_ap1r[2] = __vgic_v3_read_ap1rn(2);
 255                 case 6:
 256                         cpu_if->vgic_ap1r[1] = __vgic_v3_read_ap1rn(1);
 257                 default:
 258                         cpu_if->vgic_ap1r[0] = __vgic_v3_read_ap1rn(0);
 259                 }
 260         } else {
 261                 if (static_branch_unlikely(&vgic_v3_cpuif_trap) ||
 262                     cpu_if->its_vpe.its_vm)
 263                         write_gicreg(0, ICH_HCR_EL2);
 264
 265                 cpu_if->vgic_elrsr = 0xffff;
 266                 cpu_if->vgic_ap0r[0] = 0;
 267                 cpu_if->vgic_ap0r[1] = 0;
 268                 cpu_if->vgic_ap0r[2] = 0;
 269                 cpu_if->vgic_ap0r[3] = 0;
 270                 cpu_if->vgic_ap1r[0] = 0;
 271                 cpu_if->vgic_ap1r[1] = 0;
 272                 cpu_if->vgic_ap1r[2] = 0;
 273                 cpu_if->vgic_ap1r[3] = 0;
 274         }
 275
 276         val = read_gicreg(ICC_SRE_EL2);
 277         write_gicreg(val | ICC_SRE_EL2_ENABLE, ICC_SRE_EL2);
 278
 279         if (!cpu_if->vgic_sre) {
 280                 /* Make sure ENABLE is set at EL2 before setting SRE at EL1 */
 281                 isb();
 282                 write_gicreg(1, ICC_SRE_EL1);
 283         }
 284 }
 285
 286 void __hyp_text __vgic_v3_restore_state(struct kvm_vcpu *vcpu)
 287 {
 288         struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3;
 289         u64 used_lrs = vcpu->arch.vgic_cpu.used_lrs;
 290         u64 val;
 291         u32 nr_pre_bits;
 292         int i;
 293
 294         /*
 295          * VFIQEn is RES1 if ICC_SRE_EL1.SRE is 1. This causes a
 296          * Group0 interrupt (as generated in GICv2 mode) to be
 297          * delivered as a FIQ to the guest, with potentially fatal
 298          * consequences. So we must make sure that ICC_SRE_EL1 has
 299          * been actually programmed with the value we want before
 300          * starting to mess with the rest of the GIC, and VMCR_EL2 in
 301          * particular.
 302          */
 303         if (!cpu_if->vgic_sre) {
 304                 write_gicreg(0, ICC_SRE_EL1);
 305                 isb();
 306                 write_gicreg(cpu_if->vgic_vmcr, ICH_VMCR_EL2);
 307         }
 308
 309         val = read_gicreg(ICH_VTR_EL2);
 310         nr_pre_bits = vtr_to_nr_pre_bits(val);
 311
 312         if (used_lrs) {
 313                 write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2);
 314
 315                 switch (nr_pre_bits) {
 316                 case 7:
 317                         __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[3], 3);
 318                         __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[2], 2);
 319                 case 6:
 320                         __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[1], 1);
 321                 default:
 322                         __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[0], 0);
 323                 }
 324
 325                 switch (nr_pre_bits) {
 326                 case 7:
 327                         __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[3], 3);
 328                         __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[2], 2);
 329                 case 6:
 330                         __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[1], 1);
 331                 default:
 332                         __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[0], 0);
 333                 }
 334
 335                 for (i = 0; i < used_lrs; i++)
 336                         __gic_v3_set_lr(cpu_if->vgic_lr[i], i);
 337         } else {
 338                 /*
 339                  * If we need to trap system registers, we must write
 340                  * ICH_HCR_EL2 anyway, even if no interrupts are being
 341                  * injected. Same thing if GICv4 is used, as VLPI
 342                  * delivery is gated by ICH_HCR_EL2.En.
 343                  */
 344                 if (static_branch_unlikely(&vgic_v3_cpuif_trap) ||
 345                     cpu_if->its_vpe.its_vm)
 346                         write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2);
 347         }
 348
 349         /*
 350          * Ensures that the above will have reached the
 351          * (re)distributors. This ensure the guest will read the
 352          * correct values from the memory-mapped interface.
 353          */
 354         if (!cpu_if->vgic_sre) {
 355                 isb();
 356                 dsb(sy);
 357         }
 358
 359         /*
 360          * Prevent the guest from touching the GIC system registers if
 361          * SRE isn't enabled for GICv3 emulation.
 362          */
 363         write_gicreg(read_gicreg(ICC_SRE_EL2) & ~ICC_SRE_EL2_ENABLE,
 364                      ICC_SRE_EL2);
 365 }
 366
 367 void __hyp_text __vgic_v3_init_lrs(void)
 368 {
 369         int max_lr_idx = vtr_to_max_lr_idx(read_gicreg(ICH_VTR_EL2));
 370         int i;
 371
 372         for (i = 0; i <= max_lr_idx; i++)
 373                 __gic_v3_set_lr(0, i);
 374 }
 375
 376 u64 __hyp_text __vgic_v3_get_ich_vtr_el2(void)
 377 {
 378         return read_gicreg(ICH_VTR_EL2);
 379 }
 380
 381 u64 __hyp_text __vgic_v3_read_vmcr(void)
 382 {
 383         return read_gicreg(ICH_VMCR_EL2);
 384 }
 385
 386 void __hyp_text __vgic_v3_write_vmcr(u32 vmcr)
 387 {
 388         write_gicreg(vmcr, ICH_VMCR_EL2);
 389 }
 390
 391 #ifdef CONFIG_ARM64
 392
 393 static int __hyp_text __vgic_v3_bpr_min(void)
 394 {
 395         /* See Pseudocode for VPriorityGroup */
 396         return 8 - vtr_to_nr_pre_bits(read_gicreg(ICH_VTR_EL2));
 397 }
 398
 399 static int __hyp_text __vgic_v3_get_group(struct kvm_vcpu *vcpu)
 400 {
 401         u32 esr = kvm_vcpu_get_hsr(vcpu);
 402         u8 crm = (esr & ESR_ELx_SYS64_ISS_CRM_MASK) >> ESR_ELx_SYS64_ISS_CRM_SHIFT;
 403
 404         return crm != 8;
 405 }
 406
 407 #define GICv3_IDLE_PRIORITY     0xff
 408
 409 static int __hyp_text __vgic_v3_highest_priority_lr(struct kvm_vcpu *vcpu,
 410                                                     u32 vmcr,
 411                                                     u64 *lr_val)
 412 {
 413         unsigned int used_lrs = vcpu->arch.vgic_cpu.used_lrs;
 414         u8 priority = GICv3_IDLE_PRIORITY;
 415         int i, lr = -1;
 416
 417         for (i = 0; i < used_lrs; i++) {
 418                 u64 val = __gic_v3_get_lr(i);
 419                 u8 lr_prio = (val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
 420
 421                 /* Not pending in the state? */
 422                 if ((val & ICH_LR_STATE) != ICH_LR_PENDING_BIT)
 423                         continue;
 424
 425                 /* Group-0 interrupt, but Group-0 disabled? */
 426                 if (!(val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG0_MASK))
 427                         continue;
 428
 429                 /* Group-1 interrupt, but Group-1 disabled? */
 430                 if ((val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG1_MASK))
 431                         continue;
 432
 433                 /* Not the highest priority? */
 434                 if (lr_prio >= priority)
 435                         continue;
 436
 437                 /* This is a candidate */
 438                 priority = lr_prio;
 439                 *lr_val = val;
 440                 lr = i;
 441         }
 442
 443         if (lr == -1)
 444                 *lr_val = ICC_IAR1_EL1_SPURIOUS;
 445
 446         return lr;
 447 }
 448
 449 static int __hyp_text __vgic_v3_find_active_lr(struct kvm_vcpu *vcpu,
 450                                                int intid, u64 *lr_val)
 451 {
 452         unsigned int used_lrs = vcpu->arch.vgic_cpu.used_lrs;
 453         int i;
 454
 455         for (i = 0; i < used_lrs; i++) {
 456                 u64 val = __gic_v3_get_lr(i);
 457
 458                 if ((val & ICH_LR_VIRTUAL_ID_MASK) == intid &&
 459                     (val & ICH_LR_ACTIVE_BIT)) {
 460                         *lr_val = val;
 461                         return i;
 462                 }
 463         }
 464
 465         *lr_val = ICC_IAR1_EL1_SPURIOUS;
 466         return -1;
 467 }
 468
 469 static int __hyp_text __vgic_v3_get_highest_active_priority(void)
 470 {
 471         u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2));
 472         u32 hap = 0;
 473         int i;
 474
 475         for (i = 0; i < nr_apr_regs; i++) {
 476                 u32 val;
 477
 478                 /*
 479                  * The ICH_AP0Rn_EL2 and ICH_AP1Rn_EL2 registers
 480                  * contain the active priority levels for this VCPU
 481                  * for the maximum number of supported priority
 482                  * levels, and we return the full priority level only
 483                  * if the BPR is programmed to its minimum, otherwise
 484                  * we return a combination of the priority level and
 485                  * subpriority, as determined by the setting of the
 486                  * BPR, but without the full subpriority.
 487                  */
 488                 val  = __vgic_v3_read_ap0rn(i);
 489                 val |= __vgic_v3_read_ap1rn(i);
 490                 if (!val) {
 491                         hap += 32;
 492                         continue;
 493                 }
 494
 495                 return (hap + __ffs(val)) << __vgic_v3_bpr_min();
 496         }
 497
 498         return GICv3_IDLE_PRIORITY;
 499 }
 500
 501 static unsigned int __hyp_text __vgic_v3_get_bpr0(u32 vmcr)
 502 {
 503         return (vmcr & ICH_VMCR_BPR0_MASK) >> ICH_VMCR_BPR0_SHIFT;
 504 }
 505
 506 static unsigned int __hyp_text __vgic_v3_get_bpr1(u32 vmcr)
 507 {
 508         unsigned int bpr;
 509
 510         if (vmcr & ICH_VMCR_CBPR_MASK) {
 511                 bpr = __vgic_v3_get_bpr0(vmcr);
 512                 if (bpr < 7)
 513                         bpr++;
 514         } else {
 515                 bpr = (vmcr & ICH_VMCR_BPR1_MASK) >> ICH_VMCR_BPR1_SHIFT;
 516         }
 517
 518         return bpr;
 519 }
 520
 521 /*
 522  * Convert a priority to a preemption level, taking the relevant BPR
 523  * into account by zeroing the sub-priority bits.
 524  */
 525 static u8 __hyp_text __vgic_v3_pri_to_pre(u8 pri, u32 vmcr, int grp)
 526 {
 527         unsigned int bpr;
 528
 529         if (!grp)
 530                 bpr = __vgic_v3_get_bpr0(vmcr) + 1;
 531         else
 532                 bpr = __vgic_v3_get_bpr1(vmcr);
 533
 534         return pri & (GENMASK(7, 0) << bpr);
 535 }
 536
 537 /*
 538  * The priority value is independent of any of the BPR values, so we
 539  * normalize it using the minumal BPR value. This guarantees that no
 540  * matter what the guest does with its BPR, we can always set/get the
 541  * same value of a priority.
 542  */
 543 static void __hyp_text __vgic_v3_set_active_priority(u8 pri, u32 vmcr, int grp)
 544 {
 545         u8 pre, ap;
 546         u32 val;
 547         int apr;
 548
 549         pre = __vgic_v3_pri_to_pre(pri, vmcr, grp);
 550         ap = pre >> __vgic_v3_bpr_min();
 551         apr = ap / 32;
 552
 553         if (!grp) {
 554                 val = __vgic_v3_read_ap0rn(apr);
 555                 __vgic_v3_write_ap0rn(val | BIT(ap % 32), apr);
 556         } else {
 557                 val = __vgic_v3_read_ap1rn(apr);
 558                 __vgic_v3_write_ap1rn(val | BIT(ap % 32), apr);
 559         }
 560 }
 561
 562 static int __hyp_text __vgic_v3_clear_highest_active_priority(void)
 563 {
 564         u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2));
 565         u32 hap = 0;
 566         int i;
 567
 568         for (i = 0; i < nr_apr_regs; i++) {
 569                 u32 ap0, ap1;
 570                 int c0, c1;
 571
 572                 ap0 = __vgic_v3_read_ap0rn(i);
 573                 ap1 = __vgic_v3_read_ap1rn(i);
 574                 if (!ap0 && !ap1) {
 575                         hap += 32;
 576                         continue;
 577                 }
 578
 579                 c0 = ap0 ? __ffs(ap0) : 32;
 580                 c1 = ap1 ? __ffs(ap1) : 32;
 581
 582                 /* Always clear the LSB, which is the highest priority */
 583                 if (c0 < c1) {
 584                         ap0 &= ~BIT(c0);
 585                         __vgic_v3_write_ap0rn(ap0, i);
 586                         hap += c0;
 587                 } else {
 588                         ap1 &= ~BIT(c1);
 589                         __vgic_v3_write_ap1rn(ap1, i);
 590                         hap += c1;
 591                 }
 592
 593                 /* Rescale to 8 bits of priority */
 594                 return hap << __vgic_v3_bpr_min();
 595         }
 596
 597         return GICv3_IDLE_PRIORITY;
 598 }
 599
 600 static void __hyp_text __vgic_v3_read_iar(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 601 {
 602         u64 lr_val;
 603         u8 lr_prio, pmr;
 604         int lr, grp;
 605
 606         grp = __vgic_v3_get_group(vcpu);
 607
 608         lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val);
 609         if (lr < 0)
 610                 goto spurious;
 611
 612         if (grp != !!(lr_val & ICH_LR_GROUP))
 613                 goto spurious;
 614
 615         pmr = (vmcr & ICH_VMCR_PMR_MASK) >> ICH_VMCR_PMR_SHIFT;
 616         lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
 617         if (pmr <= lr_prio)
 618                 goto spurious;
 619
 620         if (__vgic_v3_get_highest_active_priority() <= __vgic_v3_pri_to_pre(lr_prio, vmcr, grp))
 621                 goto spurious;
 622
 623         lr_val &= ~ICH_LR_STATE;
 624         /* No active state for LPIs */
 625         if ((lr_val & ICH_LR_VIRTUAL_ID_MASK) <= VGIC_MAX_SPI)
 626                 lr_val |= ICH_LR_ACTIVE_BIT;
 627         __gic_v3_set_lr(lr_val, lr);
 628         __vgic_v3_set_active_priority(lr_prio, vmcr, grp);
 629         vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK);
 630         return;
 631
 632 spurious:
 633         vcpu_set_reg(vcpu, rt, ICC_IAR1_EL1_SPURIOUS);
 634 }
 635
 636 static void __hyp_text __vgic_v3_clear_active_lr(int lr, u64 lr_val)
 637 {
 638         lr_val &= ~ICH_LR_ACTIVE_BIT;
 639         if (lr_val & ICH_LR_HW) {
 640                 u32 pid;
 641
 642                 pid = (lr_val & ICH_LR_PHYS_ID_MASK) >> ICH_LR_PHYS_ID_SHIFT;
 643                 gic_write_dir(pid);
 644         }
 645
 646         __gic_v3_set_lr(lr_val, lr);
 647 }
 648
 649 static void __hyp_text __vgic_v3_bump_eoicount(void)
 650 {
 651         u32 hcr;
 652
 653         hcr = read_gicreg(ICH_HCR_EL2);
 654         hcr += 1 << ICH_HCR_EOIcount_SHIFT;
 655         write_gicreg(hcr, ICH_HCR_EL2);
 656 }
 657
 658 static void __hyp_text __vgic_v3_write_dir(struct kvm_vcpu *vcpu,
 659                                            u32 vmcr, int rt)
 660 {
 661         u32 vid = vcpu_get_reg(vcpu, rt);
 662         u64 lr_val;
 663         int lr;
 664
 665         /* EOImode == 0, nothing to be done here */
 666         if (!(vmcr & ICH_VMCR_EOIM_MASK))
 667                 return;
 668
 669         /* No deactivate to be performed on an LPI */
 670         if (vid >= VGIC_MIN_LPI)
 671                 return;
 672
 673         lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val);
 674         if (lr == -1) {
 675                 __vgic_v3_bump_eoicount();
 676                 return;
 677         }
 678
 679         __vgic_v3_clear_active_lr(lr, lr_val);
 680 }
 681
 682 static void __hyp_text __vgic_v3_write_eoir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 683 {
 684         u32 vid = vcpu_get_reg(vcpu, rt);
 685         u64 lr_val;
 686         u8 lr_prio, act_prio;
 687         int lr, grp;
 688
 689         grp = __vgic_v3_get_group(vcpu);
 690
 691         /* Drop priority in any case */
 692         act_prio = __vgic_v3_clear_highest_active_priority();
 693
 694         /* If EOIing an LPI, no deactivate to be performed */
 695         if (vid >= VGIC_MIN_LPI)
 696                 return;
 697
 698         /* EOImode == 1, nothing to be done here */
 699         if (vmcr & ICH_VMCR_EOIM_MASK)
 700                 return;
 701
 702         lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val);
 703         if (lr == -1) {
 704                 __vgic_v3_bump_eoicount();
 705                 return;
 706         }
 707
 708         lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
 709
 710         /* If priorities or group do not match, the guest has fscked-up. */
 711         if (grp != !!(lr_val & ICH_LR_GROUP) ||
 712             __vgic_v3_pri_to_pre(lr_prio, vmcr, grp) != act_prio)
 713                 return;
 714
 715         /* Let's now perform the deactivation */
 716         __vgic_v3_clear_active_lr(lr, lr_val);
 717 }
 718
 719 static void __hyp_text __vgic_v3_read_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 720 {
 721         vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG0_MASK));
 722 }
 723
 724 static void __hyp_text __vgic_v3_read_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 725 {
 726         vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG1_MASK));
 727 }
 728
 729 static void __hyp_text __vgic_v3_write_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 730 {
 731         u64 val = vcpu_get_reg(vcpu, rt);
 732
 733         if (val & 1)
 734                 vmcr |= ICH_VMCR_ENG0_MASK;
 735         else
 736                 vmcr &= ~ICH_VMCR_ENG0_MASK;
 737
 738         __vgic_v3_write_vmcr(vmcr);
 739 }
 740
 741 static void __hyp_text __vgic_v3_write_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 742 {
 743         u64 val = vcpu_get_reg(vcpu, rt);
 744
 745         if (val & 1)
 746                 vmcr |= ICH_VMCR_ENG1_MASK;
 747         else
 748                 vmcr &= ~ICH_VMCR_ENG1_MASK;
 749
 750         __vgic_v3_write_vmcr(vmcr);
 751 }
 752
 753 static void __hyp_text __vgic_v3_read_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 754 {
 755         vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr0(vmcr));
 756 }
 757
 758 static void __hyp_text __vgic_v3_read_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 759 {
 760         vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr1(vmcr));
 761 }
 762
 763 static void __hyp_text __vgic_v3_write_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 764 {
 765         u64 val = vcpu_get_reg(vcpu, rt);
 766         u8 bpr_min = __vgic_v3_bpr_min() - 1;
 767
 768         /* Enforce BPR limiting */
 769         if (val < bpr_min)
 770                 val = bpr_min;
 771
 772         val <<= ICH_VMCR_BPR0_SHIFT;
 773         val &= ICH_VMCR_BPR0_MASK;
 774         vmcr &= ~ICH_VMCR_BPR0_MASK;
 775         vmcr |= val;
 776
 777         __vgic_v3_write_vmcr(vmcr);
 778 }
 779
 780 static void __hyp_text __vgic_v3_write_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 781 {
 782         u64 val = vcpu_get_reg(vcpu, rt);
 783         u8 bpr_min = __vgic_v3_bpr_min();
 784
 785         if (vmcr & ICH_VMCR_CBPR_MASK)
 786                 return;
 787
 788         /* Enforce BPR limiting */
 789         if (val < bpr_min)
 790                 val = bpr_min;
 791
 792         val <<= ICH_VMCR_BPR1_SHIFT;
 793         val &= ICH_VMCR_BPR1_MASK;
 794         vmcr &= ~ICH_VMCR_BPR1_MASK;
 795         vmcr |= val;
 796
 797         __vgic_v3_write_vmcr(vmcr);
 798 }
 799
 800 static void __hyp_text __vgic_v3_read_apxrn(struct kvm_vcpu *vcpu, int rt, int n)
 801 {
 802         u32 val;
 803
 804         if (!__vgic_v3_get_group(vcpu))
 805                 val = __vgic_v3_read_ap0rn(n);
 806         else
 807                 val = __vgic_v3_read_ap1rn(n);
 808
 809         vcpu_set_reg(vcpu, rt, val);
 810 }
 811
 812 static void __hyp_text __vgic_v3_write_apxrn(struct kvm_vcpu *vcpu, int rt, int n)
 813 {
 814         u32 val = vcpu_get_reg(vcpu, rt);
 815
 816         if (!__vgic_v3_get_group(vcpu))
 817                 __vgic_v3_write_ap0rn(val, n);
 818         else
 819                 __vgic_v3_write_ap1rn(val, n);
 820 }
 821
 822 static void __hyp_text __vgic_v3_read_apxr0(struct kvm_vcpu *vcpu,
 823                                             u32 vmcr, int rt)
 824 {
 825         __vgic_v3_read_apxrn(vcpu, rt, 0);
 826 }
 827
 828 static void __hyp_text __vgic_v3_read_apxr1(struct kvm_vcpu *vcpu,
 829                                             u32 vmcr, int rt)
 830 {
 831         __vgic_v3_read_apxrn(vcpu, rt, 1);
 832 }
 833
 834 static void __hyp_text __vgic_v3_read_apxr2(struct kvm_vcpu *vcpu,
 835                                             u32 vmcr, int rt)
 836 {
 837         __vgic_v3_read_apxrn(vcpu, rt, 2);
 838 }
 839
 840 static void __hyp_text __vgic_v3_read_apxr3(struct kvm_vcpu *vcpu,
 841                                             u32 vmcr, int rt)
 842 {
 843         __vgic_v3_read_apxrn(vcpu, rt, 3);
 844 }
 845
 846 static void __hyp_text __vgic_v3_write_apxr0(struct kvm_vcpu *vcpu,
 847                                              u32 vmcr, int rt)
 848 {
 849         __vgic_v3_write_apxrn(vcpu, rt, 0);
 850 }
 851
 852 static void __hyp_text __vgic_v3_write_apxr1(struct kvm_vcpu *vcpu,
 853                                              u32 vmcr, int rt)
 854 {
 855         __vgic_v3_write_apxrn(vcpu, rt, 1);
 856 }
 857
 858 static void __hyp_text __vgic_v3_write_apxr2(struct kvm_vcpu *vcpu,
 859                                              u32 vmcr, int rt)
 860 {
 861         __vgic_v3_write_apxrn(vcpu, rt, 2);
 862 }
 863
 864 static void __hyp_text __vgic_v3_write_apxr3(struct kvm_vcpu *vcpu,
 865                                              u32 vmcr, int rt)
 866 {
 867         __vgic_v3_write_apxrn(vcpu, rt, 3);
 868 }
 869
 870 static void __hyp_text __vgic_v3_read_hppir(struct kvm_vcpu *vcpu,
 871                                             u32 vmcr, int rt)
 872 {
 873         u64 lr_val;
 874         int lr, lr_grp, grp;
 875
 876         grp = __vgic_v3_get_group(vcpu);
 877
 878         lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val);
 879         if (lr == -1)
 880                 goto spurious;
 881
 882         lr_grp = !!(lr_val & ICH_LR_GROUP);
 883         if (lr_grp != grp)
 884                 lr_val = ICC_IAR1_EL1_SPURIOUS;
 885
 886 spurious:
 887         vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK);
 888 }
 889
 890 static void __hyp_text __vgic_v3_read_pmr(struct kvm_vcpu *vcpu,
 891                                           u32 vmcr, int rt)
 892 {
 893         vmcr &= ICH_VMCR_PMR_MASK;
 894         vmcr >>= ICH_VMCR_PMR_SHIFT;
 895         vcpu_set_reg(vcpu, rt, vmcr);
 896 }
 897
 898 static void __hyp_text __vgic_v3_write_pmr(struct kvm_vcpu *vcpu,
 899                                            u32 vmcr, int rt)
 900 {
 901         u32 val = vcpu_get_reg(vcpu, rt);
 902
 903         val <<= ICH_VMCR_PMR_SHIFT;
 904         val &= ICH_VMCR_PMR_MASK;
 905         vmcr &= ~ICH_VMCR_PMR_MASK;
 906         vmcr |= val;
 907
 908         write_gicreg(vmcr, ICH_VMCR_EL2);
 909 }
 910
 911 static void __hyp_text __vgic_v3_read_rpr(struct kvm_vcpu *vcpu,
 912                                           u32 vmcr, int rt)
 913 {
 914         u32 val = __vgic_v3_get_highest_active_priority();
 915         vcpu_set_reg(vcpu, rt, val);
 916 }
 917
 918 static void __hyp_text __vgic_v3_read_ctlr(struct kvm_vcpu *vcpu,
 919                                            u32 vmcr, int rt)
 920 {
 921         u32 vtr, val;
 922
 923         vtr = read_gicreg(ICH_VTR_EL2);
 924         /* PRIbits */
 925         val = ((vtr >> 29) & 7) << ICC_CTLR_EL1_PRI_BITS_SHIFT;
 926         /* IDbits */
 927         val |= ((vtr >> 23) & 7) << ICC_CTLR_EL1_ID_BITS_SHIFT;
 928         /* SEIS */
 929         val |= ((vtr >> 22) & 1) << ICC_CTLR_EL1_SEIS_SHIFT;
 930         /* A3V */
 931         val |= ((vtr >> 21) & 1) << ICC_CTLR_EL1_A3V_SHIFT;
 932         /* EOImode */
 933         val |= ((vmcr & ICH_VMCR_EOIM_MASK) >> ICH_VMCR_EOIM_SHIFT) << ICC_CTLR_EL1_EOImode_SHIFT;
 934         /* CBPR */
 935         val |= (vmcr & ICH_VMCR_CBPR_MASK) >> ICH_VMCR_CBPR_SHIFT;
 936
 937         vcpu_set_reg(vcpu, rt, val);
 938 }
 939
 940 static void __hyp_text __vgic_v3_write_ctlr(struct kvm_vcpu *vcpu,
 941                                             u32 vmcr, int rt)
 942 {
 943         u32 val = vcpu_get_reg(vcpu, rt);
 944
 945         if (val & ICC_CTLR_EL1_CBPR_MASK)
 946                 vmcr |= ICH_VMCR_CBPR_MASK;
 947         else
 948                 vmcr &= ~ICH_VMCR_CBPR_MASK;
 949
 950         if (val & ICC_CTLR_EL1_EOImode_MASK)
 951                 vmcr |= ICH_VMCR_EOIM_MASK;
 952         else
 953                 vmcr &= ~ICH_VMCR_EOIM_MASK;
 954
 955         write_gicreg(vmcr, ICH_VMCR_EL2);
 956 }
 957
 958 int __hyp_text __vgic_v3_perform_cpuif_access(struct kvm_vcpu *vcpu)
 959 {
 960         int rt;
 961         u32 esr;
 962         u32 vmcr;
 963         void (*fn)(struct kvm_vcpu *, u32, int);
 964         bool is_read;
 965         u32 sysreg;
 966
 967         esr = kvm_vcpu_get_hsr(vcpu);
 968         if (vcpu_mode_is_32bit(vcpu)) {
 969                 if (!kvm_condition_valid(vcpu))
 970                         return 1;
 971
 972                 sysreg = esr_cp15_to_sysreg(esr);
 973         } else {
 974                 sysreg = esr_sys64_to_sysreg(esr);
 975         }
 976
 977         is_read = (esr & ESR_ELx_SYS64_ISS_DIR_MASK) == ESR_ELx_SYS64_ISS_DIR_READ;
 978
 979         switch (sysreg) {
 980         case SYS_ICC_IAR0_EL1:
 981         case SYS_ICC_IAR1_EL1:
 982                 if (unlikely(!is_read))
 983                         return 0;
 984                 fn = __vgic_v3_read_iar;
 985                 break;
 986         case SYS_ICC_EOIR0_EL1:
 987         case SYS_ICC_EOIR1_EL1:
 988                 if (unlikely(is_read))
 989                         return 0;
 990                 fn = __vgic_v3_write_eoir;
 991                 break;
 992         case SYS_ICC_IGRPEN1_EL1:
 993                 if (is_read)
 994                         fn = __vgic_v3_read_igrpen1;
 995                 else
 996                         fn = __vgic_v3_write_igrpen1;
 997                 break;
 998         case SYS_ICC_BPR1_EL1:
 999                 if (is_read)
1000                         fn = __vgic_v3_read_bpr1;
1001                 else
1002                         fn = __vgic_v3_write_bpr1;
1003                 break;
1004         case SYS_ICC_AP0Rn_EL1(0):
1005         case SYS_ICC_AP1Rn_EL1(0):
1006                 if (is_read)
1007                         fn = __vgic_v3_read_apxr0;
1008                 else
1009                         fn = __vgic_v3_write_apxr0;
1010                 break;
1011         case SYS_ICC_AP0Rn_EL1(1):
1012         case SYS_ICC_AP1Rn_EL1(1):
1013                 if (is_read)
1014                         fn = __vgic_v3_read_apxr1;
1015                 else
1016                         fn = __vgic_v3_write_apxr1;
1017                 break;
1018         case SYS_ICC_AP0Rn_EL1(2):
1019         case SYS_ICC_AP1Rn_EL1(2):
1020                 if (is_read)
1021                         fn = __vgic_v3_read_apxr2;
1022                 else
1023                         fn = __vgic_v3_write_apxr2;
1024                 break;
1025         case SYS_ICC_AP0Rn_EL1(3):
1026         case SYS_ICC_AP1Rn_EL1(3):
1027                 if (is_read)
1028                         fn = __vgic_v3_read_apxr3;
1029                 else
1030                         fn = __vgic_v3_write_apxr3;
1031                 break;
1032         case SYS_ICC_HPPIR0_EL1:
1033         case SYS_ICC_HPPIR1_EL1:
1034                 if (unlikely(!is_read))
1035                         return 0;
1036                 fn = __vgic_v3_read_hppir;
1037                 break;
1038         case SYS_ICC_IGRPEN0_EL1:
1039                 if (is_read)
1040                         fn = __vgic_v3_read_igrpen0;
1041                 else
1042                         fn = __vgic_v3_write_igrpen0;
1043                 break;
1044         case SYS_ICC_BPR0_EL1:
1045                 if (is_read)
1046                         fn = __vgic_v3_read_bpr0;
1047                 else
1048                         fn = __vgic_v3_write_bpr0;
1049                 break;
1050         case SYS_ICC_DIR_EL1:
1051                 if (unlikely(is_read))
1052                         return 0;
1053                 fn = __vgic_v3_write_dir;
1054                 break;
1055         case SYS_ICC_RPR_EL1:
1056                 if (unlikely(!is_read))
1057                         return 0;
1058                 fn = __vgic_v3_read_rpr;
1059                 break;
1060         case SYS_ICC_CTLR_EL1:
1061                 if (is_read)
1062                         fn = __vgic_v3_read_ctlr;
1063                 else
1064                         fn = __vgic_v3_write_ctlr;
1065                 break;
1066         case SYS_ICC_PMR_EL1:
1067                 if (is_read)
1068                         fn = __vgic_v3_read_pmr;
1069                 else
1070                         fn = __vgic_v3_write_pmr;
1071                 break;
1072         default:
1073                 return 0;
1074         }
1075
1076         vmcr = __vgic_v3_read_vmcr();
1077         rt = kvm_vcpu_sys_get_rt(vcpu);
1078         fn(vcpu, vmcr, rt);
1079
1080         return 1;
1081 }
1082
1083 #endif