2 * irq_comm.c: Common API for in kernel interrupt controller
3 * Copyright (c) 2007, Intel Corporation.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
16 * Place - Suite 330, Boston, MA 02111-1307 USA.
18 * Yaozu (Eddie) Dong <Eddie.dong@intel.com>
20 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
23 #include <linux/kvm_host.h>
24 #include <linux/slab.h>
25 #include <linux/export.h>
26 #include <trace/events/kvm.h>
28 #include <asm/msidef.h>
39 static int kvm_set_pic_irq(struct kvm_kernel_irq_routing_entry
*e
,
40 struct kvm
*kvm
, int irq_source_id
, int level
,
43 struct kvm_pic
*pic
= pic_irqchip(kvm
);
44 return kvm_pic_set_irq(pic
, e
->irqchip
.pin
, irq_source_id
, level
);
47 static int kvm_set_ioapic_irq(struct kvm_kernel_irq_routing_entry
*e
,
48 struct kvm
*kvm
, int irq_source_id
, int level
,
51 struct kvm_ioapic
*ioapic
= kvm
->arch
.vioapic
;
52 return kvm_ioapic_set_irq(ioapic
, e
->irqchip
.pin
, irq_source_id
, level
,
56 int kvm_irq_delivery_to_apic(struct kvm
*kvm
, struct kvm_lapic
*src
,
57 struct kvm_lapic_irq
*irq
, struct dest_map
*dest_map
)
60 struct kvm_vcpu
*vcpu
, *lowest
= NULL
;
61 unsigned long dest_vcpu_bitmap
[BITS_TO_LONGS(KVM_MAX_VCPUS
)];
62 unsigned int dest_vcpus
= 0;
64 if (irq
->dest_mode
== 0 && irq
->dest_id
== 0xff &&
65 kvm_lowest_prio_delivery(irq
)) {
66 printk(KERN_INFO
"kvm: apic: phys broadcast and lowest prio\n");
67 irq
->delivery_mode
= APIC_DM_FIXED
;
70 if (kvm_irq_delivery_to_apic_fast(kvm
, src
, irq
, &r
, dest_map
))
73 memset(dest_vcpu_bitmap
, 0, sizeof(dest_vcpu_bitmap
));
75 kvm_for_each_vcpu(i
, vcpu
, kvm
) {
76 if (!kvm_apic_present(vcpu
))
79 if (!kvm_apic_match_dest(vcpu
, src
, irq
->shorthand
,
80 irq
->dest_id
, irq
->dest_mode
))
83 if (!kvm_lowest_prio_delivery(irq
)) {
86 r
+= kvm_apic_set_irq(vcpu
, irq
, dest_map
);
87 } else if (kvm_lapic_enabled(vcpu
)) {
88 if (!kvm_vector_hashing_enabled()) {
91 else if (kvm_apic_compare_prio(vcpu
, lowest
) < 0)
94 __set_bit(i
, dest_vcpu_bitmap
);
100 if (dest_vcpus
!= 0) {
101 int idx
= kvm_vector_to_index(irq
->vector
, dest_vcpus
,
102 dest_vcpu_bitmap
, KVM_MAX_VCPUS
);
104 lowest
= kvm_get_vcpu(kvm
, idx
);
108 r
= kvm_apic_set_irq(lowest
, irq
, dest_map
);
113 void kvm_set_msi_irq(struct kvm_kernel_irq_routing_entry
*e
,
114 struct kvm_lapic_irq
*irq
)
116 trace_kvm_msi_set_irq(e
->msi
.address_lo
, e
->msi
.data
);
118 irq
->dest_id
= (e
->msi
.address_lo
&
119 MSI_ADDR_DEST_ID_MASK
) >> MSI_ADDR_DEST_ID_SHIFT
;
120 irq
->vector
= (e
->msi
.data
&
121 MSI_DATA_VECTOR_MASK
) >> MSI_DATA_VECTOR_SHIFT
;
122 irq
->dest_mode
= (1 << MSI_ADDR_DEST_MODE_SHIFT
) & e
->msi
.address_lo
;
123 irq
->trig_mode
= (1 << MSI_DATA_TRIGGER_SHIFT
) & e
->msi
.data
;
124 irq
->delivery_mode
= e
->msi
.data
& 0x700;
125 irq
->msi_redir_hint
= ((e
->msi
.address_lo
126 & MSI_ADDR_REDIRECTION_LOWPRI
) > 0);
130 EXPORT_SYMBOL_GPL(kvm_set_msi_irq
);
132 int kvm_set_msi(struct kvm_kernel_irq_routing_entry
*e
,
133 struct kvm
*kvm
, int irq_source_id
, int level
, bool line_status
)
135 struct kvm_lapic_irq irq
;
140 kvm_set_msi_irq(e
, &irq
);
142 return kvm_irq_delivery_to_apic(kvm
, NULL
, &irq
, NULL
);
146 int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry
*e
,
147 struct kvm
*kvm
, int irq_source_id
, int level
,
150 struct kvm_lapic_irq irq
;
153 if (unlikely(e
->type
!= KVM_IRQ_ROUTING_MSI
))
156 kvm_set_msi_irq(e
, &irq
);
158 if (kvm_irq_delivery_to_apic_fast(kvm
, NULL
, &irq
, &r
, NULL
))
164 int kvm_request_irq_source_id(struct kvm
*kvm
)
166 unsigned long *bitmap
= &kvm
->arch
.irq_sources_bitmap
;
169 mutex_lock(&kvm
->irq_lock
);
170 irq_source_id
= find_first_zero_bit(bitmap
, BITS_PER_LONG
);
172 if (irq_source_id
>= BITS_PER_LONG
) {
173 printk(KERN_WARNING
"kvm: exhaust allocatable IRQ sources!\n");
174 irq_source_id
= -EFAULT
;
178 ASSERT(irq_source_id
!= KVM_USERSPACE_IRQ_SOURCE_ID
);
179 ASSERT(irq_source_id
!= KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID
);
180 set_bit(irq_source_id
, bitmap
);
182 mutex_unlock(&kvm
->irq_lock
);
184 return irq_source_id
;
187 void kvm_free_irq_source_id(struct kvm
*kvm
, int irq_source_id
)
189 ASSERT(irq_source_id
!= KVM_USERSPACE_IRQ_SOURCE_ID
);
190 ASSERT(irq_source_id
!= KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID
);
192 mutex_lock(&kvm
->irq_lock
);
193 if (irq_source_id
< 0 ||
194 irq_source_id
>= BITS_PER_LONG
) {
195 printk(KERN_ERR
"kvm: IRQ source ID out of range!\n");
198 clear_bit(irq_source_id
, &kvm
->arch
.irq_sources_bitmap
);
199 if (!ioapic_in_kernel(kvm
))
202 kvm_ioapic_clear_all(kvm
->arch
.vioapic
, irq_source_id
);
203 kvm_pic_clear_all(pic_irqchip(kvm
), irq_source_id
);
205 mutex_unlock(&kvm
->irq_lock
);
208 void kvm_register_irq_mask_notifier(struct kvm
*kvm
, int irq
,
209 struct kvm_irq_mask_notifier
*kimn
)
211 mutex_lock(&kvm
->irq_lock
);
213 hlist_add_head_rcu(&kimn
->link
, &kvm
->arch
.mask_notifier_list
);
214 mutex_unlock(&kvm
->irq_lock
);
217 void kvm_unregister_irq_mask_notifier(struct kvm
*kvm
, int irq
,
218 struct kvm_irq_mask_notifier
*kimn
)
220 mutex_lock(&kvm
->irq_lock
);
221 hlist_del_rcu(&kimn
->link
);
222 mutex_unlock(&kvm
->irq_lock
);
223 synchronize_srcu(&kvm
->irq_srcu
);
226 void kvm_fire_mask_notifiers(struct kvm
*kvm
, unsigned irqchip
, unsigned pin
,
229 struct kvm_irq_mask_notifier
*kimn
;
232 idx
= srcu_read_lock(&kvm
->irq_srcu
);
233 gsi
= kvm_irq_map_chip_pin(kvm
, irqchip
, pin
);
235 hlist_for_each_entry_rcu(kimn
, &kvm
->arch
.mask_notifier_list
, link
)
236 if (kimn
->irq
== gsi
)
237 kimn
->func(kimn
, mask
);
238 srcu_read_unlock(&kvm
->irq_srcu
, idx
);
241 static int kvm_hv_set_sint(struct kvm_kernel_irq_routing_entry
*e
,
242 struct kvm
*kvm
, int irq_source_id
, int level
,
248 return kvm_hv_synic_set_irq(kvm
, e
->hv_sint
.vcpu
, e
->hv_sint
.sint
);
251 int kvm_set_routing_entry(struct kvm_kernel_irq_routing_entry
*e
,
252 const struct kvm_irq_routing_entry
*ue
)
259 case KVM_IRQ_ROUTING_IRQCHIP
:
261 switch (ue
->u
.irqchip
.irqchip
) {
262 case KVM_IRQCHIP_PIC_MASTER
:
263 e
->set
= kvm_set_pic_irq
;
264 max_pin
= PIC_NUM_PINS
;
266 case KVM_IRQCHIP_PIC_SLAVE
:
267 e
->set
= kvm_set_pic_irq
;
268 max_pin
= PIC_NUM_PINS
;
271 case KVM_IRQCHIP_IOAPIC
:
272 max_pin
= KVM_IOAPIC_NUM_PINS
;
273 e
->set
= kvm_set_ioapic_irq
;
278 e
->irqchip
.irqchip
= ue
->u
.irqchip
.irqchip
;
279 e
->irqchip
.pin
= ue
->u
.irqchip
.pin
+ delta
;
280 if (e
->irqchip
.pin
>= max_pin
)
283 case KVM_IRQ_ROUTING_MSI
:
284 e
->set
= kvm_set_msi
;
285 e
->msi
.address_lo
= ue
->u
.msi
.address_lo
;
286 e
->msi
.address_hi
= ue
->u
.msi
.address_hi
;
287 e
->msi
.data
= ue
->u
.msi
.data
;
289 case KVM_IRQ_ROUTING_HV_SINT
:
290 e
->set
= kvm_hv_set_sint
;
291 e
->hv_sint
.vcpu
= ue
->u
.hv_sint
.vcpu
;
292 e
->hv_sint
.sint
= ue
->u
.hv_sint
.sint
;
303 bool kvm_intr_is_single_vcpu(struct kvm
*kvm
, struct kvm_lapic_irq
*irq
,
304 struct kvm_vcpu
**dest_vcpu
)
307 struct kvm_vcpu
*vcpu
;
309 if (kvm_intr_is_single_vcpu_fast(kvm
, irq
, dest_vcpu
))
312 kvm_for_each_vcpu(i
, vcpu
, kvm
) {
313 if (!kvm_apic_present(vcpu
))
316 if (!kvm_apic_match_dest(vcpu
, NULL
, irq
->shorthand
,
317 irq
->dest_id
, irq
->dest_mode
))
328 EXPORT_SYMBOL_GPL(kvm_intr_is_single_vcpu
);
330 #define IOAPIC_ROUTING_ENTRY(irq) \
331 { .gsi = irq, .type = KVM_IRQ_ROUTING_IRQCHIP, \
332 .u.irqchip = { .irqchip = KVM_IRQCHIP_IOAPIC, .pin = (irq) } }
333 #define ROUTING_ENTRY1(irq) IOAPIC_ROUTING_ENTRY(irq)
335 #define PIC_ROUTING_ENTRY(irq) \
336 { .gsi = irq, .type = KVM_IRQ_ROUTING_IRQCHIP, \
337 .u.irqchip = { .irqchip = SELECT_PIC(irq), .pin = (irq) % 8 } }
338 #define ROUTING_ENTRY2(irq) \
339 IOAPIC_ROUTING_ENTRY(irq), PIC_ROUTING_ENTRY(irq)
341 static const struct kvm_irq_routing_entry default_routing
[] = {
342 ROUTING_ENTRY2(0), ROUTING_ENTRY2(1),
343 ROUTING_ENTRY2(2), ROUTING_ENTRY2(3),
344 ROUTING_ENTRY2(4), ROUTING_ENTRY2(5),
345 ROUTING_ENTRY2(6), ROUTING_ENTRY2(7),
346 ROUTING_ENTRY2(8), ROUTING_ENTRY2(9),
347 ROUTING_ENTRY2(10), ROUTING_ENTRY2(11),
348 ROUTING_ENTRY2(12), ROUTING_ENTRY2(13),
349 ROUTING_ENTRY2(14), ROUTING_ENTRY2(15),
350 ROUTING_ENTRY1(16), ROUTING_ENTRY1(17),
351 ROUTING_ENTRY1(18), ROUTING_ENTRY1(19),
352 ROUTING_ENTRY1(20), ROUTING_ENTRY1(21),
353 ROUTING_ENTRY1(22), ROUTING_ENTRY1(23),
356 int kvm_setup_default_irq_routing(struct kvm
*kvm
)
358 return kvm_set_irq_routing(kvm
, default_routing
,
359 ARRAY_SIZE(default_routing
), 0);
362 static const struct kvm_irq_routing_entry empty_routing
[] = {};
364 int kvm_setup_empty_irq_routing(struct kvm
*kvm
)
366 return kvm_set_irq_routing(kvm
, empty_routing
, 0, 0);
369 void kvm_arch_post_irq_routing_update(struct kvm
*kvm
)
371 if (ioapic_in_kernel(kvm
) || !irqchip_in_kernel(kvm
))
373 kvm_make_scan_ioapic_request(kvm
);
376 void kvm_scan_ioapic_routes(struct kvm_vcpu
*vcpu
,
377 ulong
*ioapic_handled_vectors
)
379 struct kvm
*kvm
= vcpu
->kvm
;
380 struct kvm_kernel_irq_routing_entry
*entry
;
381 struct kvm_irq_routing_table
*table
;
382 u32 i
, nr_ioapic_pins
;
385 /* kvm->irq_routing must be read after clearing
386 * KVM_SCAN_IOAPIC. */
388 idx
= srcu_read_lock(&kvm
->irq_srcu
);
389 table
= srcu_dereference(kvm
->irq_routing
, &kvm
->irq_srcu
);
390 nr_ioapic_pins
= min_t(u32
, table
->nr_rt_entries
,
391 kvm
->arch
.nr_reserved_ioapic_pins
);
392 for (i
= 0; i
< nr_ioapic_pins
; ++i
) {
393 hlist_for_each_entry(entry
, &table
->map
[i
], link
) {
394 u32 dest_id
, dest_mode
;
397 if (entry
->type
!= KVM_IRQ_ROUTING_MSI
)
399 dest_id
= (entry
->msi
.address_lo
>> 12) & 0xff;
400 dest_mode
= (entry
->msi
.address_lo
>> 2) & 0x1;
401 level
= entry
->msi
.data
& MSI_DATA_TRIGGER_LEVEL
;
402 if (level
&& kvm_apic_match_dest(vcpu
, NULL
, 0,
403 dest_id
, dest_mode
)) {
404 u32 vector
= entry
->msi
.data
& 0xff;
407 ioapic_handled_vectors
);
411 srcu_read_unlock(&kvm
->irq_srcu
, idx
);
414 int kvm_arch_set_irq(struct kvm_kernel_irq_routing_entry
*irq
, struct kvm
*kvm
,
415 int irq_source_id
, int level
, bool line_status
)
418 case KVM_IRQ_ROUTING_HV_SINT
:
419 return kvm_hv_set_sint(irq
, kvm
, irq_source_id
, level
,
426 void kvm_arch_irq_routing_update(struct kvm
*kvm
)
428 kvm_hv_irq_routing_update(kvm
);