2 * irq_comm.c: Common API for in kernel interrupt controller
3 * Copyright (c) 2007, Intel Corporation.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
16 * Place - Suite 330, Boston, MA 02111-1307 USA.
18 * Yaozu (Eddie) Dong <Eddie.dong@intel.com>
20 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
23 #include <linux/kvm_host.h>
24 #include <linux/slab.h>
25 #include <linux/export.h>
26 #include <linux/rculist.h>
28 #include <trace/events/kvm.h>
30 #include <asm/msidef.h>
41 static int kvm_set_pic_irq(struct kvm_kernel_irq_routing_entry
*e
,
42 struct kvm
*kvm
, int irq_source_id
, int level
,
45 struct kvm_pic
*pic
= kvm
->arch
.vpic
;
46 return kvm_pic_set_irq(pic
, e
->irqchip
.pin
, irq_source_id
, level
);
49 static int kvm_set_ioapic_irq(struct kvm_kernel_irq_routing_entry
*e
,
50 struct kvm
*kvm
, int irq_source_id
, int level
,
53 struct kvm_ioapic
*ioapic
= kvm
->arch
.vioapic
;
54 return kvm_ioapic_set_irq(ioapic
, e
->irqchip
.pin
, irq_source_id
, level
,
58 int kvm_irq_delivery_to_apic(struct kvm
*kvm
, struct kvm_lapic
*src
,
59 struct kvm_lapic_irq
*irq
, struct dest_map
*dest_map
)
62 struct kvm_vcpu
*vcpu
, *lowest
= NULL
;
63 unsigned long dest_vcpu_bitmap
[BITS_TO_LONGS(KVM_MAX_VCPUS
)];
64 unsigned int dest_vcpus
= 0;
66 if (irq
->dest_mode
== 0 && irq
->dest_id
== 0xff &&
67 kvm_lowest_prio_delivery(irq
)) {
68 printk(KERN_INFO
"kvm: apic: phys broadcast and lowest prio\n");
69 irq
->delivery_mode
= APIC_DM_FIXED
;
72 if (kvm_irq_delivery_to_apic_fast(kvm
, src
, irq
, &r
, dest_map
))
75 memset(dest_vcpu_bitmap
, 0, sizeof(dest_vcpu_bitmap
));
77 kvm_for_each_vcpu(i
, vcpu
, kvm
) {
78 if (!kvm_apic_present(vcpu
))
81 if (!kvm_apic_match_dest(vcpu
, src
, irq
->shorthand
,
82 irq
->dest_id
, irq
->dest_mode
))
85 if (!kvm_lowest_prio_delivery(irq
)) {
88 r
+= kvm_apic_set_irq(vcpu
, irq
, dest_map
);
89 } else if (kvm_lapic_enabled(vcpu
)) {
90 if (!kvm_vector_hashing_enabled()) {
93 else if (kvm_apic_compare_prio(vcpu
, lowest
) < 0)
96 __set_bit(i
, dest_vcpu_bitmap
);
102 if (dest_vcpus
!= 0) {
103 int idx
= kvm_vector_to_index(irq
->vector
, dest_vcpus
,
104 dest_vcpu_bitmap
, KVM_MAX_VCPUS
);
106 lowest
= kvm_get_vcpu(kvm
, idx
);
110 r
= kvm_apic_set_irq(lowest
, irq
, dest_map
);
115 void kvm_set_msi_irq(struct kvm
*kvm
, struct kvm_kernel_irq_routing_entry
*e
,
116 struct kvm_lapic_irq
*irq
)
118 trace_kvm_msi_set_irq(e
->msi
.address_lo
| (kvm
->arch
.x2apic_format
?
119 (u64
)e
->msi
.address_hi
<< 32 : 0),
122 irq
->dest_id
= (e
->msi
.address_lo
&
123 MSI_ADDR_DEST_ID_MASK
) >> MSI_ADDR_DEST_ID_SHIFT
;
124 if (kvm
->arch
.x2apic_format
)
125 irq
->dest_id
|= MSI_ADDR_EXT_DEST_ID(e
->msi
.address_hi
);
126 irq
->vector
= (e
->msi
.data
&
127 MSI_DATA_VECTOR_MASK
) >> MSI_DATA_VECTOR_SHIFT
;
128 irq
->dest_mode
= (1 << MSI_ADDR_DEST_MODE_SHIFT
) & e
->msi
.address_lo
;
129 irq
->trig_mode
= (1 << MSI_DATA_TRIGGER_SHIFT
) & e
->msi
.data
;
130 irq
->delivery_mode
= e
->msi
.data
& 0x700;
131 irq
->msi_redir_hint
= ((e
->msi
.address_lo
132 & MSI_ADDR_REDIRECTION_LOWPRI
) > 0);
136 EXPORT_SYMBOL_GPL(kvm_set_msi_irq
);
138 static inline bool kvm_msi_route_invalid(struct kvm
*kvm
,
139 struct kvm_kernel_irq_routing_entry
*e
)
141 return kvm
->arch
.x2apic_format
&& (e
->msi
.address_hi
& 0xff);
144 int kvm_set_msi(struct kvm_kernel_irq_routing_entry
*e
,
145 struct kvm
*kvm
, int irq_source_id
, int level
, bool line_status
)
147 struct kvm_lapic_irq irq
;
149 if (kvm_msi_route_invalid(kvm
, e
))
155 kvm_set_msi_irq(kvm
, e
, &irq
);
157 return kvm_irq_delivery_to_apic(kvm
, NULL
, &irq
, NULL
);
161 static int kvm_hv_set_sint(struct kvm_kernel_irq_routing_entry
*e
,
162 struct kvm
*kvm
, int irq_source_id
, int level
,
168 return kvm_hv_synic_set_irq(kvm
, e
->hv_sint
.vcpu
, e
->hv_sint
.sint
);
171 int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry
*e
,
172 struct kvm
*kvm
, int irq_source_id
, int level
,
175 struct kvm_lapic_irq irq
;
179 case KVM_IRQ_ROUTING_HV_SINT
:
180 return kvm_hv_set_sint(e
, kvm
, irq_source_id
, level
,
183 case KVM_IRQ_ROUTING_MSI
:
184 if (kvm_msi_route_invalid(kvm
, e
))
187 kvm_set_msi_irq(kvm
, e
, &irq
);
189 if (kvm_irq_delivery_to_apic_fast(kvm
, NULL
, &irq
, &r
, NULL
))
200 int kvm_request_irq_source_id(struct kvm
*kvm
)
202 unsigned long *bitmap
= &kvm
->arch
.irq_sources_bitmap
;
205 mutex_lock(&kvm
->irq_lock
);
206 irq_source_id
= find_first_zero_bit(bitmap
, BITS_PER_LONG
);
208 if (irq_source_id
>= BITS_PER_LONG
) {
209 printk(KERN_WARNING
"kvm: exhaust allocatable IRQ sources!\n");
210 irq_source_id
= -EFAULT
;
214 ASSERT(irq_source_id
!= KVM_USERSPACE_IRQ_SOURCE_ID
);
215 ASSERT(irq_source_id
!= KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID
);
216 set_bit(irq_source_id
, bitmap
);
218 mutex_unlock(&kvm
->irq_lock
);
220 return irq_source_id
;
223 void kvm_free_irq_source_id(struct kvm
*kvm
, int irq_source_id
)
225 ASSERT(irq_source_id
!= KVM_USERSPACE_IRQ_SOURCE_ID
);
226 ASSERT(irq_source_id
!= KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID
);
228 mutex_lock(&kvm
->irq_lock
);
229 if (irq_source_id
< 0 ||
230 irq_source_id
>= BITS_PER_LONG
) {
231 printk(KERN_ERR
"kvm: IRQ source ID out of range!\n");
234 clear_bit(irq_source_id
, &kvm
->arch
.irq_sources_bitmap
);
235 if (!irqchip_kernel(kvm
))
238 kvm_ioapic_clear_all(kvm
->arch
.vioapic
, irq_source_id
);
239 kvm_pic_clear_all(kvm
->arch
.vpic
, irq_source_id
);
241 mutex_unlock(&kvm
->irq_lock
);
244 void kvm_register_irq_mask_notifier(struct kvm
*kvm
, int irq
,
245 struct kvm_irq_mask_notifier
*kimn
)
247 mutex_lock(&kvm
->irq_lock
);
249 hlist_add_head_rcu(&kimn
->link
, &kvm
->arch
.mask_notifier_list
);
250 mutex_unlock(&kvm
->irq_lock
);
253 void kvm_unregister_irq_mask_notifier(struct kvm
*kvm
, int irq
,
254 struct kvm_irq_mask_notifier
*kimn
)
256 mutex_lock(&kvm
->irq_lock
);
257 hlist_del_rcu(&kimn
->link
);
258 mutex_unlock(&kvm
->irq_lock
);
259 synchronize_srcu(&kvm
->irq_srcu
);
262 void kvm_fire_mask_notifiers(struct kvm
*kvm
, unsigned irqchip
, unsigned pin
,
265 struct kvm_irq_mask_notifier
*kimn
;
268 idx
= srcu_read_lock(&kvm
->irq_srcu
);
269 gsi
= kvm_irq_map_chip_pin(kvm
, irqchip
, pin
);
271 hlist_for_each_entry_rcu(kimn
, &kvm
->arch
.mask_notifier_list
, link
)
272 if (kimn
->irq
== gsi
)
273 kimn
->func(kimn
, mask
);
274 srcu_read_unlock(&kvm
->irq_srcu
, idx
);
277 bool kvm_arch_can_set_irq_routing(struct kvm
*kvm
)
279 return irqchip_in_kernel(kvm
);
282 int kvm_set_routing_entry(struct kvm
*kvm
,
283 struct kvm_kernel_irq_routing_entry
*e
,
284 const struct kvm_irq_routing_entry
*ue
)
286 /* We can't check irqchip_in_kernel() here as some callers are
287 * currently inititalizing the irqchip. Other callers should therefore
288 * check kvm_arch_can_set_irq_routing() before calling this function.
291 case KVM_IRQ_ROUTING_IRQCHIP
:
292 if (irqchip_split(kvm
))
294 e
->irqchip
.pin
= ue
->u
.irqchip
.pin
;
295 switch (ue
->u
.irqchip
.irqchip
) {
296 case KVM_IRQCHIP_PIC_SLAVE
:
297 e
->irqchip
.pin
+= PIC_NUM_PINS
/ 2;
299 case KVM_IRQCHIP_PIC_MASTER
:
300 if (ue
->u
.irqchip
.pin
>= PIC_NUM_PINS
/ 2)
302 e
->set
= kvm_set_pic_irq
;
304 case KVM_IRQCHIP_IOAPIC
:
305 if (ue
->u
.irqchip
.pin
>= KVM_IOAPIC_NUM_PINS
)
307 e
->set
= kvm_set_ioapic_irq
;
312 e
->irqchip
.irqchip
= ue
->u
.irqchip
.irqchip
;
314 case KVM_IRQ_ROUTING_MSI
:
315 e
->set
= kvm_set_msi
;
316 e
->msi
.address_lo
= ue
->u
.msi
.address_lo
;
317 e
->msi
.address_hi
= ue
->u
.msi
.address_hi
;
318 e
->msi
.data
= ue
->u
.msi
.data
;
320 if (kvm_msi_route_invalid(kvm
, e
))
323 case KVM_IRQ_ROUTING_HV_SINT
:
324 e
->set
= kvm_hv_set_sint
;
325 e
->hv_sint
.vcpu
= ue
->u
.hv_sint
.vcpu
;
326 e
->hv_sint
.sint
= ue
->u
.hv_sint
.sint
;
335 bool kvm_intr_is_single_vcpu(struct kvm
*kvm
, struct kvm_lapic_irq
*irq
,
336 struct kvm_vcpu
**dest_vcpu
)
339 struct kvm_vcpu
*vcpu
;
341 if (kvm_intr_is_single_vcpu_fast(kvm
, irq
, dest_vcpu
))
344 kvm_for_each_vcpu(i
, vcpu
, kvm
) {
345 if (!kvm_apic_present(vcpu
))
348 if (!kvm_apic_match_dest(vcpu
, NULL
, irq
->shorthand
,
349 irq
->dest_id
, irq
->dest_mode
))
360 EXPORT_SYMBOL_GPL(kvm_intr_is_single_vcpu
);
362 #define IOAPIC_ROUTING_ENTRY(irq) \
363 { .gsi = irq, .type = KVM_IRQ_ROUTING_IRQCHIP, \
364 .u.irqchip = { .irqchip = KVM_IRQCHIP_IOAPIC, .pin = (irq) } }
365 #define ROUTING_ENTRY1(irq) IOAPIC_ROUTING_ENTRY(irq)
367 #define PIC_ROUTING_ENTRY(irq) \
368 { .gsi = irq, .type = KVM_IRQ_ROUTING_IRQCHIP, \
369 .u.irqchip = { .irqchip = SELECT_PIC(irq), .pin = (irq) % 8 } }
370 #define ROUTING_ENTRY2(irq) \
371 IOAPIC_ROUTING_ENTRY(irq), PIC_ROUTING_ENTRY(irq)
373 static const struct kvm_irq_routing_entry default_routing
[] = {
374 ROUTING_ENTRY2(0), ROUTING_ENTRY2(1),
375 ROUTING_ENTRY2(2), ROUTING_ENTRY2(3),
376 ROUTING_ENTRY2(4), ROUTING_ENTRY2(5),
377 ROUTING_ENTRY2(6), ROUTING_ENTRY2(7),
378 ROUTING_ENTRY2(8), ROUTING_ENTRY2(9),
379 ROUTING_ENTRY2(10), ROUTING_ENTRY2(11),
380 ROUTING_ENTRY2(12), ROUTING_ENTRY2(13),
381 ROUTING_ENTRY2(14), ROUTING_ENTRY2(15),
382 ROUTING_ENTRY1(16), ROUTING_ENTRY1(17),
383 ROUTING_ENTRY1(18), ROUTING_ENTRY1(19),
384 ROUTING_ENTRY1(20), ROUTING_ENTRY1(21),
385 ROUTING_ENTRY1(22), ROUTING_ENTRY1(23),
388 int kvm_setup_default_irq_routing(struct kvm
*kvm
)
390 return kvm_set_irq_routing(kvm
, default_routing
,
391 ARRAY_SIZE(default_routing
), 0);
394 static const struct kvm_irq_routing_entry empty_routing
[] = {};
396 int kvm_setup_empty_irq_routing(struct kvm
*kvm
)
398 return kvm_set_irq_routing(kvm
, empty_routing
, 0, 0);
401 void kvm_arch_post_irq_routing_update(struct kvm
*kvm
)
403 if (!irqchip_split(kvm
))
405 kvm_make_scan_ioapic_request(kvm
);
408 void kvm_scan_ioapic_routes(struct kvm_vcpu
*vcpu
,
409 ulong
*ioapic_handled_vectors
)
411 struct kvm
*kvm
= vcpu
->kvm
;
412 struct kvm_kernel_irq_routing_entry
*entry
;
413 struct kvm_irq_routing_table
*table
;
414 u32 i
, nr_ioapic_pins
;
417 idx
= srcu_read_lock(&kvm
->irq_srcu
);
418 table
= srcu_dereference(kvm
->irq_routing
, &kvm
->irq_srcu
);
419 nr_ioapic_pins
= min_t(u32
, table
->nr_rt_entries
,
420 kvm
->arch
.nr_reserved_ioapic_pins
);
421 for (i
= 0; i
< nr_ioapic_pins
; ++i
) {
422 hlist_for_each_entry(entry
, &table
->map
[i
], link
) {
423 struct kvm_lapic_irq irq
;
425 if (entry
->type
!= KVM_IRQ_ROUTING_MSI
)
428 kvm_set_msi_irq(vcpu
->kvm
, entry
, &irq
);
430 if (irq
.level
&& kvm_apic_match_dest(vcpu
, NULL
, 0,
431 irq
.dest_id
, irq
.dest_mode
))
432 __set_bit(irq
.vector
, ioapic_handled_vectors
);
435 srcu_read_unlock(&kvm
->irq_srcu
, idx
);
438 void kvm_arch_irq_routing_update(struct kvm
*kvm
)
440 kvm_hv_irq_routing_update(kvm
);