?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
[PATCH] W1: w1_netlink: New init/fini netlink callbacks.
[linux-2.6/verdex.git] / drivers / pci / msi.c
blob532f73bb2224af802af190aa0a38ea66242aa188
1 /*
2 * File: msi.c
3 * Purpose: PCI Message Signaled Interrupt (MSI)
4 *
5 * Copyright (C) 2003-2004 Intel
6 * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com)
7 */
8
9 #include <linux/mm.h>
10 #include <linux/irq.h>
11 #include <linux/interrupt.h>
12 #include <linux/init.h>
13 #include <linux/config.h>
14 #include <linux/ioport.h>
15 #include <linux/smp_lock.h>
16 #include <linux/pci.h>
17 #include <linux/proc_fs.h>
18
19 #include <asm/errno.h>
20 #include <asm/io.h>
21 #include <asm/smp.h>
22
23 #include "pci.h"
24 #include "msi.h"
25
26 static DEFINE_SPINLOCK(msi_lock);
27 static struct msi_desc* msi_desc[NR_IRQS] = { [0 ... NR_IRQS-1] = NULL };
28 static kmem_cache_t* msi_cachep;
29
30 static int pci_msi_enable = 1;
31 static int last_alloc_vector;
32 static int nr_released_vectors;
33 static int nr_reserved_vectors = NR_HP_RESERVED_VECTORS;
34 static int nr_msix_devices;
35
36 #ifndef CONFIG_X86_IO_APIC
37 int vector_irq[NR_VECTORS] = { [0 ... NR_VECTORS - 1] = -1};
38 u8 irq_vector[NR_IRQ_VECTORS] = { FIRST_DEVICE_VECTOR , 0 };
39 #endif
40
41 static void msi_cache_ctor(void *p, kmem_cache_t *cache, unsigned long flags)
42 {
43 memset(p, 0, NR_IRQS * sizeof(struct msi_desc));
44 }
45
46 static int msi_cache_init(void)
47 {
48 msi_cachep = kmem_cache_create("msi_cache",
49 NR_IRQS * sizeof(struct msi_desc),
50 0, SLAB_HWCACHE_ALIGN, msi_cache_ctor, NULL);
51 if (!msi_cachep)
52 return -ENOMEM;
53
54 return 0;
55 }
56
57 static void msi_set_mask_bit(unsigned int vector, int flag)
58 {
59 struct msi_desc *entry;
60
61 entry = (struct msi_desc *)msi_desc[vector];
62 if (!entry || !entry->dev || !entry->mask_base)
63 return;
64 switch (entry->msi_attrib.type) {
65 case PCI_CAP_ID_MSI:
66 {
67 int pos;
68 u32 mask_bits;
69
70 pos = (long)entry->mask_base;
71 pci_read_config_dword(entry->dev, pos, &mask_bits);
72 mask_bits &= ~(1);
73 mask_bits |= flag;
74 pci_write_config_dword(entry->dev, pos, mask_bits);
75 break;
76 }
77 case PCI_CAP_ID_MSIX:
78 {
79 int offset = entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE +
80 PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET;
81 writel(flag, entry->mask_base + offset);
82 break;
83 }
84 default:
85 break;
86 }
87 }
88
89 #ifdef CONFIG_SMP
90 static void set_msi_affinity(unsigned int vector, cpumask_t cpu_mask)
91 {
92 struct msi_desc *entry;
93 struct msg_address address;
94 unsigned int irq = vector;
95
96 entry = (struct msi_desc *)msi_desc[vector];
97 if (!entry || !entry->dev)
98 return;
99
100 switch (entry->msi_attrib.type) {
101 case PCI_CAP_ID_MSI:
102 {
103 int pos;
104
105 if (!(pos = pci_find_capability(entry->dev, PCI_CAP_ID_MSI)))
106 return;
107
108 pci_read_config_dword(entry->dev, msi_lower_address_reg(pos),
109 &address.lo_address.value);
110 address.lo_address.value &= MSI_ADDRESS_DEST_ID_MASK;
111 address.lo_address.value |= (cpu_mask_to_apicid(cpu_mask) <<
112 MSI_TARGET_CPU_SHIFT);
113 entry->msi_attrib.current_cpu = cpu_mask_to_apicid(cpu_mask);
114 pci_write_config_dword(entry->dev, msi_lower_address_reg(pos),
115 address.lo_address.value);
116 set_native_irq_info(irq, cpu_mask);
117 break;
118 }
119 case PCI_CAP_ID_MSIX:
120 {
121 int offset = entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE +
122 PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET;
123
124 address.lo_address.value = readl(entry->mask_base + offset);
125 address.lo_address.value &= MSI_ADDRESS_DEST_ID_MASK;
126 address.lo_address.value |= (cpu_mask_to_apicid(cpu_mask) <<
127 MSI_TARGET_CPU_SHIFT);
128 entry->msi_attrib.current_cpu = cpu_mask_to_apicid(cpu_mask);
129 writel(address.lo_address.value, entry->mask_base + offset);
130 set_native_irq_info(irq, cpu_mask);
131 break;
132 }
133 default:
134 break;
135 }
136 }
137 #endif /* CONFIG_SMP */
138
139 static void mask_MSI_irq(unsigned int vector)
140 {
141 msi_set_mask_bit(vector, 1);
142 }
143
144 static void unmask_MSI_irq(unsigned int vector)
145 {
146 msi_set_mask_bit(vector, 0);
147 }
148
149 static unsigned int startup_msi_irq_wo_maskbit(unsigned int vector)
150 {
151 struct msi_desc *entry;
152 unsigned long flags;
153
154 spin_lock_irqsave(&msi_lock, flags);
155 entry = msi_desc[vector];
156 if (!entry || !entry->dev) {
157 spin_unlock_irqrestore(&msi_lock, flags);
158 return 0;
159 }
160 entry->msi_attrib.state = 1; /* Mark it active */
161 spin_unlock_irqrestore(&msi_lock, flags);
162
163 return 0; /* never anything pending */
164 }
165
166 static unsigned int startup_msi_irq_w_maskbit(unsigned int vector)
167 {
168 startup_msi_irq_wo_maskbit(vector);
169 unmask_MSI_irq(vector);
170 return 0; /* never anything pending */
171 }
172
173 static void shutdown_msi_irq(unsigned int vector)
174 {
175 struct msi_desc *entry;
176 unsigned long flags;
177
178 spin_lock_irqsave(&msi_lock, flags);
179 entry = msi_desc[vector];
180 if (entry && entry->dev)
181 entry->msi_attrib.state = 0; /* Mark it not active */
182 spin_unlock_irqrestore(&msi_lock, flags);
183 }
184
185 static void end_msi_irq_wo_maskbit(unsigned int vector)
186 {
187 move_native_irq(vector);
188 ack_APIC_irq();
189 }
190
191 static void end_msi_irq_w_maskbit(unsigned int vector)
192 {
193 move_native_irq(vector);
194 unmask_MSI_irq(vector);
195 ack_APIC_irq();
196 }
197
198 static void do_nothing(unsigned int vector)
199 {
200 }
201
202 /*
203 * Interrupt Type for MSI-X PCI/PCI-X/PCI-Express Devices,
204 * which implement the MSI-X Capability Structure.
205 */
206 static struct hw_interrupt_type msix_irq_type = {
207 .typename = "PCI-MSI-X",
208 .startup = startup_msi_irq_w_maskbit,
209 .shutdown = shutdown_msi_irq,
210 .enable = unmask_MSI_irq,
211 .disable = mask_MSI_irq,
212 .ack = mask_MSI_irq,
213 .end = end_msi_irq_w_maskbit,
214 .set_affinity = set_msi_irq_affinity
215 };
216
217 /*
218 * Interrupt Type for MSI PCI/PCI-X/PCI-Express Devices,
219 * which implement the MSI Capability Structure with
220 * Mask-and-Pending Bits.
221 */
222 static struct hw_interrupt_type msi_irq_w_maskbit_type = {
223 .typename = "PCI-MSI",
224 .startup = startup_msi_irq_w_maskbit,
225 .shutdown = shutdown_msi_irq,
226 .enable = unmask_MSI_irq,
227 .disable = mask_MSI_irq,
228 .ack = mask_MSI_irq,
229 .end = end_msi_irq_w_maskbit,
230 .set_affinity = set_msi_irq_affinity
231 };
232
233 /*
234 * Interrupt Type for MSI PCI/PCI-X/PCI-Express Devices,
235 * which implement the MSI Capability Structure without
236 * Mask-and-Pending Bits.
237 */
238 static struct hw_interrupt_type msi_irq_wo_maskbit_type = {
239 .typename = "PCI-MSI",
240 .startup = startup_msi_irq_wo_maskbit,
241 .shutdown = shutdown_msi_irq,
242 .enable = do_nothing,
243 .disable = do_nothing,
244 .ack = do_nothing,
245 .end = end_msi_irq_wo_maskbit,
246 .set_affinity = set_msi_irq_affinity
247 };
248
249 static void msi_data_init(struct msg_data *msi_data,
250 unsigned int vector)
251 {
252 memset(msi_data, 0, sizeof(struct msg_data));
253 msi_data->vector = (u8)vector;
254 msi_data->delivery_mode = MSI_DELIVERY_MODE;
255 msi_data->level = MSI_LEVEL_MODE;
256 msi_data->trigger = MSI_TRIGGER_MODE;
257 }
258
259 static void msi_address_init(struct msg_address *msi_address)
260 {
261 unsigned int dest_id;
262
263 memset(msi_address, 0, sizeof(struct msg_address));
264 msi_address->hi_address = (u32)0;
265 dest_id = (MSI_ADDRESS_HEADER << MSI_ADDRESS_HEADER_SHIFT);
266 msi_address->lo_address.u.dest_mode = MSI_DEST_MODE;
267 msi_address->lo_address.u.redirection_hint = MSI_REDIRECTION_HINT_MODE;
268 msi_address->lo_address.u.dest_id = dest_id;
269 msi_address->lo_address.value |= (MSI_TARGET_CPU << MSI_TARGET_CPU_SHIFT);
270 }
271
272 static int msi_free_vector(struct pci_dev* dev, int vector, int reassign);
273 static int assign_msi_vector(void)
274 {
275 static int new_vector_avail = 1;
276 int vector;
277 unsigned long flags;
278
279 /*
280 * msi_lock is provided to ensure that successful allocation of MSI
281 * vector is assigned unique among drivers.
282 */
283 spin_lock_irqsave(&msi_lock, flags);
284
285 if (!new_vector_avail) {
286 int free_vector = 0;
287
288 /*
289 * vector_irq[] = -1 indicates that this specific vector is:
290 * - assigned for MSI (since MSI have no associated IRQ) or
291 * - assigned for legacy if less than 16, or
292 * - having no corresponding 1:1 vector-to-IOxAPIC IRQ mapping
293 * vector_irq[] = 0 indicates that this vector, previously
294 * assigned for MSI, is freed by hotplug removed operations.
295 * This vector will be reused for any subsequent hotplug added
296 * operations.
297 * vector_irq[] > 0 indicates that this vector is assigned for
298 * IOxAPIC IRQs. This vector and its value provides a 1-to-1
299 * vector-to-IOxAPIC IRQ mapping.
300 */
301 for (vector = FIRST_DEVICE_VECTOR; vector < NR_IRQS; vector++) {
302 if (vector_irq[vector] != 0)
303 continue;
304 free_vector = vector;
305 if (!msi_desc[vector])
306 break;
307 else
308 continue;
309 }
310 if (!free_vector) {
311 spin_unlock_irqrestore(&msi_lock, flags);
312 return -EBUSY;
313 }
314 vector_irq[free_vector] = -1;
315 nr_released_vectors--;
316 spin_unlock_irqrestore(&msi_lock, flags);
317 if (msi_desc[free_vector] != NULL) {
318 struct pci_dev *dev;
319 int tail;
320
321 /* free all linked vectors before re-assign */
322 do {
323 spin_lock_irqsave(&msi_lock, flags);
324 dev = msi_desc[free_vector]->dev;
325 tail = msi_desc[free_vector]->link.tail;
326 spin_unlock_irqrestore(&msi_lock, flags);
327 msi_free_vector(dev, tail, 1);
328 } while (free_vector != tail);
329 }
330
331 return free_vector;
332 }
333 vector = assign_irq_vector(AUTO_ASSIGN);
334 last_alloc_vector = vector;
335 if (vector == LAST_DEVICE_VECTOR)
336 new_vector_avail = 0;
337
338 spin_unlock_irqrestore(&msi_lock, flags);
339 return vector;
340 }
341
342 static int get_new_vector(void)
343 {
344 int vector;
345
346 if ((vector = assign_msi_vector()) > 0)
347 set_intr_gate(vector, interrupt[vector]);
348
349 return vector;
350 }
351
352 static int msi_init(void)
353 {
354 static int status = -ENOMEM;
355
356 if (!status)
357 return status;
358
359 if (pci_msi_quirk) {
360 pci_msi_enable = 0;
361 printk(KERN_WARNING "PCI: MSI quirk detected. MSI disabled.\n");
362 status = -EINVAL;
363 return status;
364 }
365
366 if ((status = msi_cache_init()) < 0) {
367 pci_msi_enable = 0;
368 printk(KERN_WARNING "PCI: MSI cache init failed\n");
369 return status;
370 }
371 last_alloc_vector = assign_irq_vector(AUTO_ASSIGN);
372 if (last_alloc_vector < 0) {
373 pci_msi_enable = 0;
374 printk(KERN_WARNING "PCI: No interrupt vectors available for MSI\n");
375 status = -EBUSY;
376 return status;
377 }
378 vector_irq[last_alloc_vector] = 0;
379 nr_released_vectors++;
380
381 return status;
382 }
383
384 static int get_msi_vector(struct pci_dev *dev)
385 {
386 return get_new_vector();
387 }
388
389 static struct msi_desc* alloc_msi_entry(void)
390 {
391 struct msi_desc *entry;
392
393 entry = kmem_cache_alloc(msi_cachep, SLAB_KERNEL);
394 if (!entry)
395 return NULL;
396
397 memset(entry, 0, sizeof(struct msi_desc));
398 entry->link.tail = entry->link.head = 0; /* single message */
399 entry->dev = NULL;
400
401 return entry;
402 }
403
404 static void attach_msi_entry(struct msi_desc *entry, int vector)
405 {
406 unsigned long flags;
407
408 spin_lock_irqsave(&msi_lock, flags);
409 msi_desc[vector] = entry;
410 spin_unlock_irqrestore(&msi_lock, flags);
411 }
412
413 static void irq_handler_init(int cap_id, int pos, int mask)
414 {
415 spin_lock(&irq_desc[pos].lock);
416 if (cap_id == PCI_CAP_ID_MSIX)
417 irq_desc[pos].handler = &msix_irq_type;
418 else {
419 if (!mask)
420 irq_desc[pos].handler = &msi_irq_wo_maskbit_type;
421 else
422 irq_desc[pos].handler = &msi_irq_w_maskbit_type;
423 }
424 spin_unlock(&irq_desc[pos].lock);
425 }
426
427 static void enable_msi_mode(struct pci_dev *dev, int pos, int type)
428 {
429 u16 control;
430
431 pci_read_config_word(dev, msi_control_reg(pos), &control);
432 if (type == PCI_CAP_ID_MSI) {
433 /* Set enabled bits to single MSI & enable MSI_enable bit */
434 msi_enable(control, 1);
435 pci_write_config_word(dev, msi_control_reg(pos), control);
436 } else {
437 msix_enable(control);
438 pci_write_config_word(dev, msi_control_reg(pos), control);
439 }
440 if (pci_find_capability(dev, PCI_CAP_ID_EXP)) {
441 /* PCI Express Endpoint device detected */
442 u16 cmd;
443 pci_read_config_word(dev, PCI_COMMAND, &cmd);
444 cmd |= PCI_COMMAND_INTX_DISABLE;
445 pci_write_config_word(dev, PCI_COMMAND, cmd);
446 }
447 }
448
449 void disable_msi_mode(struct pci_dev *dev, int pos, int type)
450 {
451 u16 control;
452
453 pci_read_config_word(dev, msi_control_reg(pos), &control);
454 if (type == PCI_CAP_ID_MSI) {
455 /* Set enabled bits to single MSI & enable MSI_enable bit */
456 msi_disable(control);
457 pci_write_config_word(dev, msi_control_reg(pos), control);
458 } else {
459 msix_disable(control);
460 pci_write_config_word(dev, msi_control_reg(pos), control);
461 }
462 if (pci_find_capability(dev, PCI_CAP_ID_EXP)) {
463 /* PCI Express Endpoint device detected */
464 u16 cmd;
465 pci_read_config_word(dev, PCI_COMMAND, &cmd);
466 cmd &= ~PCI_COMMAND_INTX_DISABLE;
467 pci_write_config_word(dev, PCI_COMMAND, cmd);
468 }
469 }
470
471 static int msi_lookup_vector(struct pci_dev *dev, int type)
472 {
473 int vector;
474 unsigned long flags;
475
476 spin_lock_irqsave(&msi_lock, flags);
477 for (vector = FIRST_DEVICE_VECTOR; vector < NR_IRQS; vector++) {
478 if (!msi_desc[vector] || msi_desc[vector]->dev != dev ||
479 msi_desc[vector]->msi_attrib.type != type ||
480 msi_desc[vector]->msi_attrib.default_vector != dev->irq)
481 continue;
482 spin_unlock_irqrestore(&msi_lock, flags);
483 /* This pre-assigned MSI vector for this device
484 already exits. Override dev->irq with this vector */
485 dev->irq = vector;
486 return 0;
487 }
488 spin_unlock_irqrestore(&msi_lock, flags);
489
490 return -EACCES;
491 }
492
493 void pci_scan_msi_device(struct pci_dev *dev)
494 {
495 if (!dev)
496 return;
497
498 if (pci_find_capability(dev, PCI_CAP_ID_MSIX) > 0)
499 nr_msix_devices++;
500 else if (pci_find_capability(dev, PCI_CAP_ID_MSI) > 0)
501 nr_reserved_vectors++;
502 }
503
504 /**
505 * msi_capability_init - configure device's MSI capability structure
506 * @dev: pointer to the pci_dev data structure of MSI device function
507 *
508 * Setup the MSI capability structure of device function with a single
509 * MSI vector, regardless of device function is capable of handling
510 * multiple messages. A return of zero indicates the successful setup
511 * of an entry zero with the new MSI vector or non-zero for otherwise.
512 **/
513 static int msi_capability_init(struct pci_dev *dev)
514 {
515 struct msi_desc *entry;
516 struct msg_address address;
517 struct msg_data data;
518 int pos, vector;
519 u16 control;
520
521 pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
522 pci_read_config_word(dev, msi_control_reg(pos), &control);
523 /* MSI Entry Initialization */
524 if (!(entry = alloc_msi_entry()))
525 return -ENOMEM;
526
527 if ((vector = get_msi_vector(dev)) < 0) {
528 kmem_cache_free(msi_cachep, entry);
529 return -EBUSY;
530 }
531 entry->link.head = vector;
532 entry->link.tail = vector;
533 entry->msi_attrib.type = PCI_CAP_ID_MSI;
534 entry->msi_attrib.state = 0; /* Mark it not active */
535 entry->msi_attrib.entry_nr = 0;
536 entry->msi_attrib.maskbit = is_mask_bit_support(control);
537 entry->msi_attrib.default_vector = dev->irq; /* Save IOAPIC IRQ */
538 dev->irq = vector;
539 entry->dev = dev;
540 if (is_mask_bit_support(control)) {
541 entry->mask_base = (void __iomem *)(long)msi_mask_bits_reg(pos,
542 is_64bit_address(control));
543 }
544 /* Replace with MSI handler */
545 irq_handler_init(PCI_CAP_ID_MSI, vector, entry->msi_attrib.maskbit);
546 /* Configure MSI capability structure */
547 msi_address_init(&address);
548 msi_data_init(&data, vector);
549 entry->msi_attrib.current_cpu = ((address.lo_address.u.dest_id >>
550 MSI_TARGET_CPU_SHIFT) & MSI_TARGET_CPU_MASK);
551 pci_write_config_dword(dev, msi_lower_address_reg(pos),
552 address.lo_address.value);
553 if (is_64bit_address(control)) {
554 pci_write_config_dword(dev,
555 msi_upper_address_reg(pos), address.hi_address);
556 pci_write_config_word(dev,
557 msi_data_reg(pos, 1), *((u32*)&data));
558 } else
559 pci_write_config_word(dev,
560 msi_data_reg(pos, 0), *((u32*)&data));
561 if (entry->msi_attrib.maskbit) {
562 unsigned int maskbits, temp;
563 /* All MSIs are unmasked by default, Mask them all */
564 pci_read_config_dword(dev,
565 msi_mask_bits_reg(pos, is_64bit_address(control)),
566 &maskbits);
567 temp = (1 << multi_msi_capable(control));
568 temp = ((temp - 1) & ~temp);
569 maskbits |= temp;
570 pci_write_config_dword(dev,
571 msi_mask_bits_reg(pos, is_64bit_address(control)),
572 maskbits);
573 }
574 attach_msi_entry(entry, vector);
575 /* Set MSI enabled bits */
576 enable_msi_mode(dev, pos, PCI_CAP_ID_MSI);
577
578 return 0;
579 }
580
581 /**
582 * msix_capability_init - configure device's MSI-X capability
583 * @dev: pointer to the pci_dev data structure of MSI-X device function
584 *
585 * Setup the MSI-X capability structure of device function with a
586 * single MSI-X vector. A return of zero indicates the successful setup of
587 * requested MSI-X entries with allocated vectors or non-zero for otherwise.
588 **/
589 static int msix_capability_init(struct pci_dev *dev,
590 struct msix_entry *entries, int nvec)
591 {
592 struct msi_desc *head = NULL, *tail = NULL, *entry = NULL;
593 struct msg_address address;
594 struct msg_data data;
595 int vector, pos, i, j, nr_entries, temp = 0;
596 u32 phys_addr, table_offset;
597 u16 control;
598 u8 bir;
599 void __iomem *base;
600
601 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
602 /* Request & Map MSI-X table region */
603 pci_read_config_word(dev, msi_control_reg(pos), &control);
604 nr_entries = multi_msix_capable(control);
605 pci_read_config_dword(dev, msix_table_offset_reg(pos),
606 &table_offset);
607 bir = (u8)(table_offset & PCI_MSIX_FLAGS_BIRMASK);
608 phys_addr = pci_resource_start (dev, bir);
609 phys_addr += (u32)(table_offset & ~PCI_MSIX_FLAGS_BIRMASK);
610 base = ioremap_nocache(phys_addr, nr_entries * PCI_MSIX_ENTRY_SIZE);
611 if (base == NULL)
612 return -ENOMEM;
613
614 /* MSI-X Table Initialization */
615 for (i = 0; i < nvec; i++) {
616 entry = alloc_msi_entry();
617 if (!entry)
618 break;
619 if ((vector = get_msi_vector(dev)) < 0)
620 break;
621
622 j = entries[i].entry;
623 entries[i].vector = vector;
624 entry->msi_attrib.type = PCI_CAP_ID_MSIX;
625 entry->msi_attrib.state = 0; /* Mark it not active */
626 entry->msi_attrib.entry_nr = j;
627 entry->msi_attrib.maskbit = 1;
628 entry->msi_attrib.default_vector = dev->irq;
629 entry->dev = dev;
630 entry->mask_base = base;
631 if (!head) {
632 entry->link.head = vector;
633 entry->link.tail = vector;
634 head = entry;
635 } else {
636 entry->link.head = temp;
637 entry->link.tail = tail->link.tail;
638 tail->link.tail = vector;
639 head->link.head = vector;
640 }
641 temp = vector;
642 tail = entry;
643 /* Replace with MSI-X handler */
644 irq_handler_init(PCI_CAP_ID_MSIX, vector, 1);
645 /* Configure MSI-X capability structure */
646 msi_address_init(&address);
647 msi_data_init(&data, vector);
648 entry->msi_attrib.current_cpu =
649 ((address.lo_address.u.dest_id >>
650 MSI_TARGET_CPU_SHIFT) & MSI_TARGET_CPU_MASK);
651 writel(address.lo_address.value,
652 base + j * PCI_MSIX_ENTRY_SIZE +
653 PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
654 writel(address.hi_address,
655 base + j * PCI_MSIX_ENTRY_SIZE +
656 PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET);
657 writel(*(u32*)&data,
658 base + j * PCI_MSIX_ENTRY_SIZE +
659 PCI_MSIX_ENTRY_DATA_OFFSET);
660 attach_msi_entry(entry, vector);
661 }
662 if (i != nvec) {
663 i--;
664 for (; i >= 0; i--) {
665 vector = (entries + i)->vector;
666 msi_free_vector(dev, vector, 0);
667 (entries + i)->vector = 0;
668 }
669 return -EBUSY;
670 }
671 /* Set MSI-X enabled bits */
672 enable_msi_mode(dev, pos, PCI_CAP_ID_MSIX);
673
674 return 0;
675 }
676
677 /**
678 * pci_enable_msi - configure device's MSI capability structure
679 * @dev: pointer to the pci_dev data structure of MSI device function
680 *
681 * Setup the MSI capability structure of device function with
682 * a single MSI vector upon its software driver call to request for
683 * MSI mode enabled on its hardware device function. A return of zero
684 * indicates the successful setup of an entry zero with the new MSI
685 * vector or non-zero for otherwise.
686 **/
687 int pci_enable_msi(struct pci_dev* dev)
688 {
689 int pos, temp, status = -EINVAL;
690 u16 control;
691
692 if (!pci_msi_enable || !dev)
693 return status;
694
695 if (dev->no_msi)
696 return status;
697
698 temp = dev->irq;
699
700 if ((status = msi_init()) < 0)
701 return status;
702
703 if (!(pos = pci_find_capability(dev, PCI_CAP_ID_MSI)))
704 return -EINVAL;
705
706 pci_read_config_word(dev, msi_control_reg(pos), &control);
707 if (control & PCI_MSI_FLAGS_ENABLE)
708 return 0; /* Already in MSI mode */
709
710 if (!msi_lookup_vector(dev, PCI_CAP_ID_MSI)) {
711 /* Lookup Sucess */
712 unsigned long flags;
713
714 spin_lock_irqsave(&msi_lock, flags);
715 if (!vector_irq[dev->irq]) {
716 msi_desc[dev->irq]->msi_attrib.state = 0;
717 vector_irq[dev->irq] = -1;
718 nr_released_vectors--;
719 spin_unlock_irqrestore(&msi_lock, flags);
720 enable_msi_mode(dev, pos, PCI_CAP_ID_MSI);
721 return 0;
722 }
723 spin_unlock_irqrestore(&msi_lock, flags);
724 dev->irq = temp;
725 }
726 /* Check whether driver already requested for MSI-X vectors */
727 if ((pos = pci_find_capability(dev, PCI_CAP_ID_MSIX)) > 0 &&
728 !msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) {
729 printk(KERN_INFO "PCI: %s: Can't enable MSI. "
730 "Device already has MSI-X vectors assigned\n",
731 pci_name(dev));
732 dev->irq = temp;
733 return -EINVAL;
734 }
735 status = msi_capability_init(dev);
736 if (!status) {
737 if (!pos)
738 nr_reserved_vectors--; /* Only MSI capable */
739 else if (nr_msix_devices > 0)
740 nr_msix_devices--; /* Both MSI and MSI-X capable,
741 but choose enabling MSI */
742 }
743
744 return status;
745 }
746
747 void pci_disable_msi(struct pci_dev* dev)
748 {
749 struct msi_desc *entry;
750 int pos, default_vector;
751 u16 control;
752 unsigned long flags;
753
754 if (!dev || !(pos = pci_find_capability(dev, PCI_CAP_ID_MSI)))
755 return;
756
757 pci_read_config_word(dev, msi_control_reg(pos), &control);
758 if (!(control & PCI_MSI_FLAGS_ENABLE))
759 return;
760
761 spin_lock_irqsave(&msi_lock, flags);
762 entry = msi_desc[dev->irq];
763 if (!entry || !entry->dev || entry->msi_attrib.type != PCI_CAP_ID_MSI) {
764 spin_unlock_irqrestore(&msi_lock, flags);
765 return;
766 }
767 if (entry->msi_attrib.state) {
768 spin_unlock_irqrestore(&msi_lock, flags);
769 printk(KERN_WARNING "PCI: %s: pci_disable_msi() called without "
770 "free_irq() on MSI vector %d\n",
771 pci_name(dev), dev->irq);
772 BUG_ON(entry->msi_attrib.state > 0);
773 } else {
774 vector_irq[dev->irq] = 0; /* free it */
775 nr_released_vectors++;
776 default_vector = entry->msi_attrib.default_vector;
777 spin_unlock_irqrestore(&msi_lock, flags);
778 /* Restore dev->irq to its default pin-assertion vector */
779 dev->irq = default_vector;
780 disable_msi_mode(dev, pci_find_capability(dev, PCI_CAP_ID_MSI),
781 PCI_CAP_ID_MSI);
782 }
783 }
784
785 static int msi_free_vector(struct pci_dev* dev, int vector, int reassign)
786 {
787 struct msi_desc *entry;
788 int head, entry_nr, type;
789 void __iomem *base;
790 unsigned long flags;
791
792 spin_lock_irqsave(&msi_lock, flags);
793 entry = msi_desc[vector];
794 if (!entry || entry->dev != dev) {
795 spin_unlock_irqrestore(&msi_lock, flags);
796 return -EINVAL;
797 }
798 type = entry->msi_attrib.type;
799 entry_nr = entry->msi_attrib.entry_nr;
800 head = entry->link.head;
801 base = entry->mask_base;
802 msi_desc[entry->link.head]->link.tail = entry->link.tail;
803 msi_desc[entry->link.tail]->link.head = entry->link.head;
804 entry->dev = NULL;
805 if (!reassign) {
806 vector_irq[vector] = 0;
807 nr_released_vectors++;
808 }
809 msi_desc[vector] = NULL;
810 spin_unlock_irqrestore(&msi_lock, flags);
811
812 kmem_cache_free(msi_cachep, entry);
813
814 if (type == PCI_CAP_ID_MSIX) {
815 if (!reassign)
816 writel(1, base +
817 entry_nr * PCI_MSIX_ENTRY_SIZE +
818 PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET);
819
820 if (head == vector) {
821 /*
822 * Detect last MSI-X vector to be released.
823 * Release the MSI-X memory-mapped table.
824 */
825 int pos, nr_entries;
826 u32 phys_addr, table_offset;
827 u16 control;
828 u8 bir;
829
830 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
831 pci_read_config_word(dev, msi_control_reg(pos),
832 &control);
833 nr_entries = multi_msix_capable(control);
834 pci_read_config_dword(dev, msix_table_offset_reg(pos),
835 &table_offset);
836 bir = (u8)(table_offset & PCI_MSIX_FLAGS_BIRMASK);
837 phys_addr = pci_resource_start (dev, bir);
838 phys_addr += (u32)(table_offset &
839 ~PCI_MSIX_FLAGS_BIRMASK);
840 iounmap(base);
841 }
842 }
843
844 return 0;
845 }
846
847 static int reroute_msix_table(int head, struct msix_entry *entries, int *nvec)
848 {
849 int vector = head, tail = 0;
850 int i, j = 0, nr_entries = 0;
851 void __iomem *base;
852 unsigned long flags;
853
854 spin_lock_irqsave(&msi_lock, flags);
855 while (head != tail) {
856 nr_entries++;
857 tail = msi_desc[vector]->link.tail;
858 if (entries[0].entry == msi_desc[vector]->msi_attrib.entry_nr)
859 j = vector;
860 vector = tail;
861 }
862 if (*nvec > nr_entries) {
863 spin_unlock_irqrestore(&msi_lock, flags);
864 *nvec = nr_entries;
865 return -EINVAL;
866 }
867 vector = ((j > 0) ? j : head);
868 for (i = 0; i < *nvec; i++) {
869 j = msi_desc[vector]->msi_attrib.entry_nr;
870 msi_desc[vector]->msi_attrib.state = 0; /* Mark it not active */
871 vector_irq[vector] = -1; /* Mark it busy */
872 nr_released_vectors--;
873 entries[i].vector = vector;
874 if (j != (entries + i)->entry) {
875 base = msi_desc[vector]->mask_base;
876 msi_desc[vector]->msi_attrib.entry_nr =
877 (entries + i)->entry;
878 writel( readl(base + j * PCI_MSIX_ENTRY_SIZE +
879 PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET), base +
880 (entries + i)->entry * PCI_MSIX_ENTRY_SIZE +
881 PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
882 writel( readl(base + j * PCI_MSIX_ENTRY_SIZE +
883 PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET), base +
884 (entries + i)->entry * PCI_MSIX_ENTRY_SIZE +
885 PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET);
886 writel( (readl(base + j * PCI_MSIX_ENTRY_SIZE +
887 PCI_MSIX_ENTRY_DATA_OFFSET) & 0xff00) | vector,
888 base + (entries+i)->entry*PCI_MSIX_ENTRY_SIZE +
889 PCI_MSIX_ENTRY_DATA_OFFSET);
890 }
891 vector = msi_desc[vector]->link.tail;
892 }
893 spin_unlock_irqrestore(&msi_lock, flags);
894
895 return 0;
896 }
897
898 /**
899 * pci_enable_msix - configure device's MSI-X capability structure
900 * @dev: pointer to the pci_dev data structure of MSI-X device function
901 * @entries: pointer to an array of MSI-X entries
902 * @nvec: number of MSI-X vectors requested for allocation by device driver
903 *
904 * Setup the MSI-X capability structure of device function with the number
905 * of requested vectors upon its software driver call to request for
906 * MSI-X mode enabled on its hardware device function. A return of zero
907 * indicates the successful configuration of MSI-X capability structure
908 * with new allocated MSI-X vectors. A return of < 0 indicates a failure.
909 * Or a return of > 0 indicates that driver request is exceeding the number
910 * of vectors available. Driver should use the returned value to re-send
911 * its request.
912 **/
913 int pci_enable_msix(struct pci_dev* dev, struct msix_entry *entries, int nvec)
914 {
915 int status, pos, nr_entries, free_vectors;
916 int i, j, temp;
917 u16 control;
918 unsigned long flags;
919
920 if (!pci_msi_enable || !dev || !entries)
921 return -EINVAL;
922
923 if ((status = msi_init()) < 0)
924 return status;
925
926 if (!(pos = pci_find_capability(dev, PCI_CAP_ID_MSIX)))
927 return -EINVAL;
928
929 pci_read_config_word(dev, msi_control_reg(pos), &control);
930 if (control & PCI_MSIX_FLAGS_ENABLE)
931 return -EINVAL; /* Already in MSI-X mode */
932
933 nr_entries = multi_msix_capable(control);
934 if (nvec > nr_entries)
935 return -EINVAL;
936
937 /* Check for any invalid entries */
938 for (i = 0; i < nvec; i++) {
939 if (entries[i].entry >= nr_entries)
940 return -EINVAL; /* invalid entry */
941 for (j = i + 1; j < nvec; j++) {
942 if (entries[i].entry == entries[j].entry)
943 return -EINVAL; /* duplicate entry */
944 }
945 }
946 temp = dev->irq;
947 if (!msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) {
948 /* Lookup Sucess */
949 nr_entries = nvec;
950 /* Reroute MSI-X table */
951 if (reroute_msix_table(dev->irq, entries, &nr_entries)) {
952 /* #requested > #previous-assigned */
953 dev->irq = temp;
954 return nr_entries;
955 }
956 dev->irq = temp;
957 enable_msi_mode(dev, pos, PCI_CAP_ID_MSIX);
958 return 0;
959 }
960 /* Check whether driver already requested for MSI vector */
961 if (pci_find_capability(dev, PCI_CAP_ID_MSI) > 0 &&
962 !msi_lookup_vector(dev, PCI_CAP_ID_MSI)) {
963 printk(KERN_INFO "PCI: %s: Can't enable MSI-X. "
964 "Device already has an MSI vector assigned\n",
965 pci_name(dev));
966 dev->irq = temp;
967 return -EINVAL;
968 }
969
970 spin_lock_irqsave(&msi_lock, flags);
971 /*
972 * msi_lock is provided to ensure that enough vectors resources are
973 * available before granting.
974 */
975 free_vectors = pci_vector_resources(last_alloc_vector,
976 nr_released_vectors);
977 /* Ensure that each MSI/MSI-X device has one vector reserved by
978 default to avoid any MSI-X driver to take all available
979 resources */
980 free_vectors -= nr_reserved_vectors;
981 /* Find the average of free vectors among MSI-X devices */
982 if (nr_msix_devices > 0)
983 free_vectors /= nr_msix_devices;
984 spin_unlock_irqrestore(&msi_lock, flags);
985
986 if (nvec > free_vectors) {
987 if (free_vectors > 0)
988 return free_vectors;
989 else
990 return -EBUSY;
991 }
992
993 status = msix_capability_init(dev, entries, nvec);
994 if (!status && nr_msix_devices > 0)
995 nr_msix_devices--;
996
997 return status;
998 }
999
1000 void pci_disable_msix(struct pci_dev* dev)
1001 {
1002 int pos, temp;
1003 u16 control;
1004
1005 if (!dev || !(pos = pci_find_capability(dev, PCI_CAP_ID_MSIX)))
1006 return;
1007
1008 pci_read_config_word(dev, msi_control_reg(pos), &control);
1009 if (!(control & PCI_MSIX_FLAGS_ENABLE))
1010 return;
1011
1012 temp = dev->irq;
1013 if (!msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) {
1014 int state, vector, head, tail = 0, warning = 0;
1015 unsigned long flags;
1016
1017 vector = head = dev->irq;
1018 spin_lock_irqsave(&msi_lock, flags);
1019 while (head != tail) {
1020 state = msi_desc[vector]->msi_attrib.state;
1021 if (state)
1022 warning = 1;
1023 else {
1024 vector_irq[vector] = 0; /* free it */
1025 nr_released_vectors++;
1026 }
1027 tail = msi_desc[vector]->link.tail;
1028 vector = tail;
1029 }
1030 spin_unlock_irqrestore(&msi_lock, flags);
1031 if (warning) {
1032 dev->irq = temp;
1033 printk(KERN_WARNING "PCI: %s: pci_disable_msix() called without "
1034 "free_irq() on all MSI-X vectors\n",
1035 pci_name(dev));
1036 BUG_ON(warning > 0);
1037 } else {
1038 dev->irq = temp;
1039 disable_msi_mode(dev,
1040 pci_find_capability(dev, PCI_CAP_ID_MSIX),
1041 PCI_CAP_ID_MSIX);
1042
1043 }
1044 }
1045 }
1046
1047 /**
1048 * msi_remove_pci_irq_vectors - reclaim MSI(X) vectors to unused state
1049 * @dev: pointer to the pci_dev data structure of MSI(X) device function
1050 *
1051 * Being called during hotplug remove, from which the device function
1052 * is hot-removed. All previous assigned MSI/MSI-X vectors, if
1053 * allocated for this device function, are reclaimed to unused state,
1054 * which may be used later on.
1055 **/
1056 void msi_remove_pci_irq_vectors(struct pci_dev* dev)
1057 {
1058 int state, pos, temp;
1059 unsigned long flags;
1060
1061 if (!pci_msi_enable || !dev)
1062 return;
1063
1064 temp = dev->irq; /* Save IOAPIC IRQ */
1065 if ((pos = pci_find_capability(dev, PCI_CAP_ID_MSI)) > 0 &&
1066 !msi_lookup_vector(dev, PCI_CAP_ID_MSI)) {
1067 spin_lock_irqsave(&msi_lock, flags);
1068 state = msi_desc[dev->irq]->msi_attrib.state;
1069 spin_unlock_irqrestore(&msi_lock, flags);
1070 if (state) {
1071 printk(KERN_WARNING "PCI: %s: msi_remove_pci_irq_vectors() "
1072 "called without free_irq() on MSI vector %d\n",
1073 pci_name(dev), dev->irq);
1074 BUG_ON(state > 0);
1075 } else /* Release MSI vector assigned to this device */
1076 msi_free_vector(dev, dev->irq, 0);
1077 dev->irq = temp; /* Restore IOAPIC IRQ */
1078 }
1079 if ((pos = pci_find_capability(dev, PCI_CAP_ID_MSIX)) > 0 &&
1080 !msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) {
1081 int vector, head, tail = 0, warning = 0;
1082 void __iomem *base = NULL;
1083
1084 vector = head = dev->irq;
1085 while (head != tail) {
1086 spin_lock_irqsave(&msi_lock, flags);
1087 state = msi_desc[vector]->msi_attrib.state;
1088 tail = msi_desc[vector]->link.tail;
1089 base = msi_desc[vector]->mask_base;
1090 spin_unlock_irqrestore(&msi_lock, flags);
1091 if (state)
1092 warning = 1;
1093 else if (vector != head) /* Release MSI-X vector */
1094 msi_free_vector(dev, vector, 0);
1095 vector = tail;
1096 }
1097 msi_free_vector(dev, vector, 0);
1098 if (warning) {
1099 /* Force to release the MSI-X memory-mapped table */
1100 u32 phys_addr, table_offset;
1101 u16 control;
1102 u8 bir;
1103
1104 pci_read_config_word(dev, msi_control_reg(pos),
1105 &control);
1106 pci_read_config_dword(dev, msix_table_offset_reg(pos),
1107 &table_offset);
1108 bir = (u8)(table_offset & PCI_MSIX_FLAGS_BIRMASK);
1109 phys_addr = pci_resource_start (dev, bir);
1110 phys_addr += (u32)(table_offset &
1111 ~PCI_MSIX_FLAGS_BIRMASK);
1112 iounmap(base);
1113 printk(KERN_WARNING "PCI: %s: msi_remove_pci_irq_vectors() "
1114 "called without free_irq() on all MSI-X vectors\n",
1115 pci_name(dev));
1116 BUG_ON(warning > 0);
1117 }
1118 dev->irq = temp; /* Restore IOAPIC IRQ */
1119 }
1120 }
1121
1122 EXPORT_SYMBOL(pci_enable_msi);
1123 EXPORT_SYMBOL(pci_disable_msi);
1124 EXPORT_SYMBOL(pci_enable_msix);
1125 EXPORT_SYMBOL(pci_disable_msix);
Verdex Pro support