dccp: do not assume DCCP code is non preemptible
[linux/fpc-iii.git] / arch / alpha / kernel / pci_iommu.c
blob8969bf2dfe3a0d4ff797888d2ce0a4a8785103dc
1 /*
2 * linux/arch/alpha/kernel/pci_iommu.c
3 */
5 #include <linux/kernel.h>
6 #include <linux/mm.h>
7 #include <linux/pci.h>
8 #include <linux/gfp.h>
9 #include <linux/bootmem.h>
10 #include <linux/export.h>
11 #include <linux/scatterlist.h>
12 #include <linux/log2.h>
13 #include <linux/dma-mapping.h>
14 #include <linux/iommu-helper.h>
16 #include <asm/io.h>
17 #include <asm/hwrpb.h>
19 #include "proto.h"
20 #include "pci_impl.h"
23 #define DEBUG_ALLOC 0
24 #if DEBUG_ALLOC > 0
25 # define DBGA(args...) printk(KERN_DEBUG args)
26 #else
27 # define DBGA(args...)
28 #endif
29 #if DEBUG_ALLOC > 1
30 # define DBGA2(args...) printk(KERN_DEBUG args)
31 #else
32 # define DBGA2(args...)
33 #endif
35 #define DEBUG_NODIRECT 0
37 #define ISA_DMA_MASK 0x00ffffff
39 static inline unsigned long
40 mk_iommu_pte(unsigned long paddr)
42 return (paddr >> (PAGE_SHIFT-1)) | 1;
45 /* Return the minimum of MAX or the first power of two larger
46 than main memory. */
48 unsigned long
49 size_for_memory(unsigned long max)
51 unsigned long mem = max_low_pfn << PAGE_SHIFT;
52 if (mem < max)
53 max = roundup_pow_of_two(mem);
54 return max;
57 struct pci_iommu_arena * __init
58 iommu_arena_new_node(int nid, struct pci_controller *hose, dma_addr_t base,
59 unsigned long window_size, unsigned long align)
61 unsigned long mem_size;
62 struct pci_iommu_arena *arena;
64 mem_size = window_size / (PAGE_SIZE / sizeof(unsigned long));
66 /* Note that the TLB lookup logic uses bitwise concatenation,
67 not addition, so the required arena alignment is based on
68 the size of the window. Retain the align parameter so that
69 particular systems can over-align the arena. */
70 if (align < mem_size)
71 align = mem_size;
74 #ifdef CONFIG_DISCONTIGMEM
76 arena = alloc_bootmem_node(NODE_DATA(nid), sizeof(*arena));
77 if (!NODE_DATA(nid) || !arena) {
78 printk("%s: couldn't allocate arena from node %d\n"
79 " falling back to system-wide allocation\n",
80 __func__, nid);
81 arena = alloc_bootmem(sizeof(*arena));
84 arena->ptes = __alloc_bootmem_node(NODE_DATA(nid), mem_size, align, 0);
85 if (!NODE_DATA(nid) || !arena->ptes) {
86 printk("%s: couldn't allocate arena ptes from node %d\n"
87 " falling back to system-wide allocation\n",
88 __func__, nid);
89 arena->ptes = __alloc_bootmem(mem_size, align, 0);
92 #else /* CONFIG_DISCONTIGMEM */
94 arena = alloc_bootmem(sizeof(*arena));
95 arena->ptes = __alloc_bootmem(mem_size, align, 0);
97 #endif /* CONFIG_DISCONTIGMEM */
99 spin_lock_init(&arena->lock);
100 arena->hose = hose;
101 arena->dma_base = base;
102 arena->size = window_size;
103 arena->next_entry = 0;
105 /* Align allocations to a multiple of a page size. Not needed
106 unless there are chip bugs. */
107 arena->align_entry = 1;
109 return arena;
112 struct pci_iommu_arena * __init
113 iommu_arena_new(struct pci_controller *hose, dma_addr_t base,
114 unsigned long window_size, unsigned long align)
116 return iommu_arena_new_node(0, hose, base, window_size, align);
119 /* Must be called with the arena lock held */
120 static long
121 iommu_arena_find_pages(struct device *dev, struct pci_iommu_arena *arena,
122 long n, long mask)
124 unsigned long *ptes;
125 long i, p, nent;
126 int pass = 0;
127 unsigned long base;
128 unsigned long boundary_size;
130 base = arena->dma_base >> PAGE_SHIFT;
131 if (dev) {
132 boundary_size = dma_get_seg_boundary(dev) + 1;
133 boundary_size >>= PAGE_SHIFT;
134 } else {
135 boundary_size = 1UL << (32 - PAGE_SHIFT);
138 /* Search forward for the first mask-aligned sequence of N free ptes */
139 ptes = arena->ptes;
140 nent = arena->size >> PAGE_SHIFT;
141 p = ALIGN(arena->next_entry, mask + 1);
142 i = 0;
144 again:
145 while (i < n && p+i < nent) {
146 if (!i && iommu_is_span_boundary(p, n, base, boundary_size)) {
147 p = ALIGN(p + 1, mask + 1);
148 goto again;
151 if (ptes[p+i])
152 p = ALIGN(p + i + 1, mask + 1), i = 0;
153 else
154 i = i + 1;
157 if (i < n) {
158 if (pass < 1) {
160 * Reached the end. Flush the TLB and restart
161 * the search from the beginning.
163 alpha_mv.mv_pci_tbi(arena->hose, 0, -1);
165 pass++;
166 p = 0;
167 i = 0;
168 goto again;
169 } else
170 return -1;
173 /* Success. It's the responsibility of the caller to mark them
174 in use before releasing the lock */
175 return p;
178 static long
179 iommu_arena_alloc(struct device *dev, struct pci_iommu_arena *arena, long n,
180 unsigned int align)
182 unsigned long flags;
183 unsigned long *ptes;
184 long i, p, mask;
186 spin_lock_irqsave(&arena->lock, flags);
188 /* Search for N empty ptes */
189 ptes = arena->ptes;
190 mask = max(align, arena->align_entry) - 1;
191 p = iommu_arena_find_pages(dev, arena, n, mask);
192 if (p < 0) {
193 spin_unlock_irqrestore(&arena->lock, flags);
194 return -1;
197 /* Success. Mark them all in use, ie not zero and invalid
198 for the iommu tlb that could load them from under us.
199 The chip specific bits will fill this in with something
200 kosher when we return. */
201 for (i = 0; i < n; ++i)
202 ptes[p+i] = IOMMU_INVALID_PTE;
204 arena->next_entry = p + n;
205 spin_unlock_irqrestore(&arena->lock, flags);
207 return p;
210 static void
211 iommu_arena_free(struct pci_iommu_arena *arena, long ofs, long n)
213 unsigned long *p;
214 long i;
216 p = arena->ptes + ofs;
217 for (i = 0; i < n; ++i)
218 p[i] = 0;
222 * True if the machine supports DAC addressing, and DEV can
223 * make use of it given MASK.
225 static int pci_dac_dma_supported(struct pci_dev *dev, u64 mask)
227 dma_addr_t dac_offset = alpha_mv.pci_dac_offset;
228 int ok = 1;
230 /* If this is not set, the machine doesn't support DAC at all. */
231 if (dac_offset == 0)
232 ok = 0;
234 /* The device has to be able to address our DAC bit. */
235 if ((dac_offset & dev->dma_mask) != dac_offset)
236 ok = 0;
238 /* If both conditions above are met, we are fine. */
239 DBGA("pci_dac_dma_supported %s from %pf\n",
240 ok ? "yes" : "no", __builtin_return_address(0));
242 return ok;
245 /* Map a single buffer of the indicated size for PCI DMA in streaming
246 mode. The 32-bit PCI bus mastering address to use is returned.
247 Once the device is given the dma address, the device owns this memory
248 until either pci_unmap_single or pci_dma_sync_single is performed. */
250 static dma_addr_t
251 pci_map_single_1(struct pci_dev *pdev, void *cpu_addr, size_t size,
252 int dac_allowed)
254 struct pci_controller *hose = pdev ? pdev->sysdata : pci_isa_hose;
255 dma_addr_t max_dma = pdev ? pdev->dma_mask : ISA_DMA_MASK;
256 struct pci_iommu_arena *arena;
257 long npages, dma_ofs, i;
258 unsigned long paddr;
259 dma_addr_t ret;
260 unsigned int align = 0;
261 struct device *dev = pdev ? &pdev->dev : NULL;
263 paddr = __pa(cpu_addr);
265 #if !DEBUG_NODIRECT
266 /* First check to see if we can use the direct map window. */
267 if (paddr + size + __direct_map_base - 1 <= max_dma
268 && paddr + size <= __direct_map_size) {
269 ret = paddr + __direct_map_base;
271 DBGA2("pci_map_single: [%p,%zx] -> direct %llx from %pf\n",
272 cpu_addr, size, ret, __builtin_return_address(0));
274 return ret;
276 #endif
278 /* Next, use DAC if selected earlier. */
279 if (dac_allowed) {
280 ret = paddr + alpha_mv.pci_dac_offset;
282 DBGA2("pci_map_single: [%p,%zx] -> DAC %llx from %pf\n",
283 cpu_addr, size, ret, __builtin_return_address(0));
285 return ret;
288 /* If the machine doesn't define a pci_tbi routine, we have to
289 assume it doesn't support sg mapping, and, since we tried to
290 use direct_map above, it now must be considered an error. */
291 if (! alpha_mv.mv_pci_tbi) {
292 printk_once(KERN_WARNING "pci_map_single: no HW sg\n");
293 return 0;
296 arena = hose->sg_pci;
297 if (!arena || arena->dma_base + arena->size - 1 > max_dma)
298 arena = hose->sg_isa;
300 npages = iommu_num_pages(paddr, size, PAGE_SIZE);
302 /* Force allocation to 64KB boundary for ISA bridges. */
303 if (pdev && pdev == isa_bridge)
304 align = 8;
305 dma_ofs = iommu_arena_alloc(dev, arena, npages, align);
306 if (dma_ofs < 0) {
307 printk(KERN_WARNING "pci_map_single failed: "
308 "could not allocate dma page tables\n");
309 return 0;
312 paddr &= PAGE_MASK;
313 for (i = 0; i < npages; ++i, paddr += PAGE_SIZE)
314 arena->ptes[i + dma_ofs] = mk_iommu_pte(paddr);
316 ret = arena->dma_base + dma_ofs * PAGE_SIZE;
317 ret += (unsigned long)cpu_addr & ~PAGE_MASK;
319 DBGA2("pci_map_single: [%p,%zx] np %ld -> sg %llx from %pf\n",
320 cpu_addr, size, npages, ret, __builtin_return_address(0));
322 return ret;
325 /* Helper for generic DMA-mapping functions. */
326 static struct pci_dev *alpha_gendev_to_pci(struct device *dev)
328 if (dev && dev_is_pci(dev))
329 return to_pci_dev(dev);
331 /* Assume that non-PCI devices asking for DMA are either ISA or EISA,
332 BUG() otherwise. */
333 BUG_ON(!isa_bridge);
335 /* Assume non-busmaster ISA DMA when dma_mask is not set (the ISA
336 bridge is bus master then). */
337 if (!dev || !dev->dma_mask || !*dev->dma_mask)
338 return isa_bridge;
340 /* For EISA bus masters, return isa_bridge (it might have smaller
341 dma_mask due to wiring limitations). */
342 if (*dev->dma_mask >= isa_bridge->dma_mask)
343 return isa_bridge;
345 /* This assumes ISA bus master with dma_mask 0xffffff. */
346 return NULL;
349 static dma_addr_t alpha_pci_map_page(struct device *dev, struct page *page,
350 unsigned long offset, size_t size,
351 enum dma_data_direction dir,
352 struct dma_attrs *attrs)
354 struct pci_dev *pdev = alpha_gendev_to_pci(dev);
355 int dac_allowed;
357 BUG_ON(dir == PCI_DMA_NONE);
359 dac_allowed = pdev ? pci_dac_dma_supported(pdev, pdev->dma_mask) : 0;
360 return pci_map_single_1(pdev, (char *)page_address(page) + offset,
361 size, dac_allowed);
364 /* Unmap a single streaming mode DMA translation. The DMA_ADDR and
365 SIZE must match what was provided for in a previous pci_map_single
366 call. All other usages are undefined. After this call, reads by
367 the cpu to the buffer are guaranteed to see whatever the device
368 wrote there. */
370 static void alpha_pci_unmap_page(struct device *dev, dma_addr_t dma_addr,
371 size_t size, enum dma_data_direction dir,
372 struct dma_attrs *attrs)
374 unsigned long flags;
375 struct pci_dev *pdev = alpha_gendev_to_pci(dev);
376 struct pci_controller *hose = pdev ? pdev->sysdata : pci_isa_hose;
377 struct pci_iommu_arena *arena;
378 long dma_ofs, npages;
380 BUG_ON(dir == PCI_DMA_NONE);
382 if (dma_addr >= __direct_map_base
383 && dma_addr < __direct_map_base + __direct_map_size) {
384 /* Nothing to do. */
386 DBGA2("pci_unmap_single: direct [%llx,%zx] from %pf\n",
387 dma_addr, size, __builtin_return_address(0));
389 return;
392 if (dma_addr > 0xffffffff) {
393 DBGA2("pci64_unmap_single: DAC [%llx,%zx] from %pf\n",
394 dma_addr, size, __builtin_return_address(0));
395 return;
398 arena = hose->sg_pci;
399 if (!arena || dma_addr < arena->dma_base)
400 arena = hose->sg_isa;
402 dma_ofs = (dma_addr - arena->dma_base) >> PAGE_SHIFT;
403 if (dma_ofs * PAGE_SIZE >= arena->size) {
404 printk(KERN_ERR "Bogus pci_unmap_single: dma_addr %llx "
405 " base %llx size %x\n",
406 dma_addr, arena->dma_base, arena->size);
407 return;
408 BUG();
411 npages = iommu_num_pages(dma_addr, size, PAGE_SIZE);
413 spin_lock_irqsave(&arena->lock, flags);
415 iommu_arena_free(arena, dma_ofs, npages);
417 /* If we're freeing ptes above the `next_entry' pointer (they
418 may have snuck back into the TLB since the last wrap flush),
419 we need to flush the TLB before reallocating the latter. */
420 if (dma_ofs >= arena->next_entry)
421 alpha_mv.mv_pci_tbi(hose, dma_addr, dma_addr + size - 1);
423 spin_unlock_irqrestore(&arena->lock, flags);
425 DBGA2("pci_unmap_single: sg [%llx,%zx] np %ld from %pf\n",
426 dma_addr, size, npages, __builtin_return_address(0));
429 /* Allocate and map kernel buffer using consistent mode DMA for PCI
430 device. Returns non-NULL cpu-view pointer to the buffer if
431 successful and sets *DMA_ADDRP to the pci side dma address as well,
432 else DMA_ADDRP is undefined. */
434 static void *alpha_pci_alloc_coherent(struct device *dev, size_t size,
435 dma_addr_t *dma_addrp, gfp_t gfp,
436 struct dma_attrs *attrs)
438 struct pci_dev *pdev = alpha_gendev_to_pci(dev);
439 void *cpu_addr;
440 long order = get_order(size);
442 gfp &= ~GFP_DMA;
444 try_again:
445 cpu_addr = (void *)__get_free_pages(gfp, order);
446 if (! cpu_addr) {
447 printk(KERN_INFO "pci_alloc_consistent: "
448 "get_free_pages failed from %pf\n",
449 __builtin_return_address(0));
450 /* ??? Really atomic allocation? Otherwise we could play
451 with vmalloc and sg if we can't find contiguous memory. */
452 return NULL;
454 memset(cpu_addr, 0, size);
456 *dma_addrp = pci_map_single_1(pdev, cpu_addr, size, 0);
457 if (*dma_addrp == 0) {
458 free_pages((unsigned long)cpu_addr, order);
459 if (alpha_mv.mv_pci_tbi || (gfp & GFP_DMA))
460 return NULL;
461 /* The address doesn't fit required mask and we
462 do not have iommu. Try again with GFP_DMA. */
463 gfp |= GFP_DMA;
464 goto try_again;
467 DBGA2("pci_alloc_consistent: %zx -> [%p,%llx] from %pf\n",
468 size, cpu_addr, *dma_addrp, __builtin_return_address(0));
470 return cpu_addr;
473 /* Free and unmap a consistent DMA buffer. CPU_ADDR and DMA_ADDR must
474 be values that were returned from pci_alloc_consistent. SIZE must
475 be the same as what as passed into pci_alloc_consistent.
476 References to the memory and mappings associated with CPU_ADDR or
477 DMA_ADDR past this call are illegal. */
479 static void alpha_pci_free_coherent(struct device *dev, size_t size,
480 void *cpu_addr, dma_addr_t dma_addr,
481 struct dma_attrs *attrs)
483 struct pci_dev *pdev = alpha_gendev_to_pci(dev);
484 pci_unmap_single(pdev, dma_addr, size, PCI_DMA_BIDIRECTIONAL);
485 free_pages((unsigned long)cpu_addr, get_order(size));
487 DBGA2("pci_free_consistent: [%llx,%zx] from %pf\n",
488 dma_addr, size, __builtin_return_address(0));
491 /* Classify the elements of the scatterlist. Write dma_address
492 of each element with:
493 0 : Followers all physically adjacent.
494 1 : Followers all virtually adjacent.
495 -1 : Not leader, physically adjacent to previous.
496 -2 : Not leader, virtually adjacent to previous.
497 Write dma_length of each leader with the combined lengths of
498 the mergable followers. */
500 #define SG_ENT_VIRT_ADDRESS(SG) (sg_virt((SG)))
501 #define SG_ENT_PHYS_ADDRESS(SG) __pa(SG_ENT_VIRT_ADDRESS(SG))
503 static void
504 sg_classify(struct device *dev, struct scatterlist *sg, struct scatterlist *end,
505 int virt_ok)
507 unsigned long next_paddr;
508 struct scatterlist *leader;
509 long leader_flag, leader_length;
510 unsigned int max_seg_size;
512 leader = sg;
513 leader_flag = 0;
514 leader_length = leader->length;
515 next_paddr = SG_ENT_PHYS_ADDRESS(leader) + leader_length;
517 /* we will not marge sg without device. */
518 max_seg_size = dev ? dma_get_max_seg_size(dev) : 0;
519 for (++sg; sg < end; ++sg) {
520 unsigned long addr, len;
521 addr = SG_ENT_PHYS_ADDRESS(sg);
522 len = sg->length;
524 if (leader_length + len > max_seg_size)
525 goto new_segment;
527 if (next_paddr == addr) {
528 sg->dma_address = -1;
529 leader_length += len;
530 } else if (((next_paddr | addr) & ~PAGE_MASK) == 0 && virt_ok) {
531 sg->dma_address = -2;
532 leader_flag = 1;
533 leader_length += len;
534 } else {
535 new_segment:
536 leader->dma_address = leader_flag;
537 leader->dma_length = leader_length;
538 leader = sg;
539 leader_flag = 0;
540 leader_length = len;
543 next_paddr = addr + len;
546 leader->dma_address = leader_flag;
547 leader->dma_length = leader_length;
550 /* Given a scatterlist leader, choose an allocation method and fill
551 in the blanks. */
553 static int
554 sg_fill(struct device *dev, struct scatterlist *leader, struct scatterlist *end,
555 struct scatterlist *out, struct pci_iommu_arena *arena,
556 dma_addr_t max_dma, int dac_allowed)
558 unsigned long paddr = SG_ENT_PHYS_ADDRESS(leader);
559 long size = leader->dma_length;
560 struct scatterlist *sg;
561 unsigned long *ptes;
562 long npages, dma_ofs, i;
564 #if !DEBUG_NODIRECT
565 /* If everything is physically contiguous, and the addresses
566 fall into the direct-map window, use it. */
567 if (leader->dma_address == 0
568 && paddr + size + __direct_map_base - 1 <= max_dma
569 && paddr + size <= __direct_map_size) {
570 out->dma_address = paddr + __direct_map_base;
571 out->dma_length = size;
573 DBGA(" sg_fill: [%p,%lx] -> direct %llx\n",
574 __va(paddr), size, out->dma_address);
576 return 0;
578 #endif
580 /* If physically contiguous and DAC is available, use it. */
581 if (leader->dma_address == 0 && dac_allowed) {
582 out->dma_address = paddr + alpha_mv.pci_dac_offset;
583 out->dma_length = size;
585 DBGA(" sg_fill: [%p,%lx] -> DAC %llx\n",
586 __va(paddr), size, out->dma_address);
588 return 0;
591 /* Otherwise, we'll use the iommu to make the pages virtually
592 contiguous. */
594 paddr &= ~PAGE_MASK;
595 npages = iommu_num_pages(paddr, size, PAGE_SIZE);
596 dma_ofs = iommu_arena_alloc(dev, arena, npages, 0);
597 if (dma_ofs < 0) {
598 /* If we attempted a direct map above but failed, die. */
599 if (leader->dma_address == 0)
600 return -1;
602 /* Otherwise, break up the remaining virtually contiguous
603 hunks into individual direct maps and retry. */
604 sg_classify(dev, leader, end, 0);
605 return sg_fill(dev, leader, end, out, arena, max_dma, dac_allowed);
608 out->dma_address = arena->dma_base + dma_ofs*PAGE_SIZE + paddr;
609 out->dma_length = size;
611 DBGA(" sg_fill: [%p,%lx] -> sg %llx np %ld\n",
612 __va(paddr), size, out->dma_address, npages);
614 /* All virtually contiguous. We need to find the length of each
615 physically contiguous subsegment to fill in the ptes. */
616 ptes = &arena->ptes[dma_ofs];
617 sg = leader;
618 do {
619 #if DEBUG_ALLOC > 0
620 struct scatterlist *last_sg = sg;
621 #endif
623 size = sg->length;
624 paddr = SG_ENT_PHYS_ADDRESS(sg);
626 while (sg+1 < end && (int) sg[1].dma_address == -1) {
627 size += sg[1].length;
628 sg++;
631 npages = iommu_num_pages(paddr, size, PAGE_SIZE);
633 paddr &= PAGE_MASK;
634 for (i = 0; i < npages; ++i, paddr += PAGE_SIZE)
635 *ptes++ = mk_iommu_pte(paddr);
637 #if DEBUG_ALLOC > 0
638 DBGA(" (%ld) [%p,%x] np %ld\n",
639 last_sg - leader, SG_ENT_VIRT_ADDRESS(last_sg),
640 last_sg->length, npages);
641 while (++last_sg <= sg) {
642 DBGA(" (%ld) [%p,%x] cont\n",
643 last_sg - leader, SG_ENT_VIRT_ADDRESS(last_sg),
644 last_sg->length);
646 #endif
647 } while (++sg < end && (int) sg->dma_address < 0);
649 return 1;
652 static int alpha_pci_map_sg(struct device *dev, struct scatterlist *sg,
653 int nents, enum dma_data_direction dir,
654 struct dma_attrs *attrs)
656 struct pci_dev *pdev = alpha_gendev_to_pci(dev);
657 struct scatterlist *start, *end, *out;
658 struct pci_controller *hose;
659 struct pci_iommu_arena *arena;
660 dma_addr_t max_dma;
661 int dac_allowed;
663 BUG_ON(dir == PCI_DMA_NONE);
665 dac_allowed = dev ? pci_dac_dma_supported(pdev, pdev->dma_mask) : 0;
667 /* Fast path single entry scatterlists. */
668 if (nents == 1) {
669 sg->dma_length = sg->length;
670 sg->dma_address
671 = pci_map_single_1(pdev, SG_ENT_VIRT_ADDRESS(sg),
672 sg->length, dac_allowed);
673 return sg->dma_address != 0;
676 start = sg;
677 end = sg + nents;
679 /* First, prepare information about the entries. */
680 sg_classify(dev, sg, end, alpha_mv.mv_pci_tbi != 0);
682 /* Second, figure out where we're going to map things. */
683 if (alpha_mv.mv_pci_tbi) {
684 hose = pdev ? pdev->sysdata : pci_isa_hose;
685 max_dma = pdev ? pdev->dma_mask : ISA_DMA_MASK;
686 arena = hose->sg_pci;
687 if (!arena || arena->dma_base + arena->size - 1 > max_dma)
688 arena = hose->sg_isa;
689 } else {
690 max_dma = -1;
691 arena = NULL;
692 hose = NULL;
695 /* Third, iterate over the scatterlist leaders and allocate
696 dma space as needed. */
697 for (out = sg; sg < end; ++sg) {
698 if ((int) sg->dma_address < 0)
699 continue;
700 if (sg_fill(dev, sg, end, out, arena, max_dma, dac_allowed) < 0)
701 goto error;
702 out++;
705 /* Mark the end of the list for pci_unmap_sg. */
706 if (out < end)
707 out->dma_length = 0;
709 if (out - start == 0)
710 printk(KERN_WARNING "pci_map_sg failed: no entries?\n");
711 DBGA("pci_map_sg: %ld entries\n", out - start);
713 return out - start;
715 error:
716 printk(KERN_WARNING "pci_map_sg failed: "
717 "could not allocate dma page tables\n");
719 /* Some allocation failed while mapping the scatterlist
720 entries. Unmap them now. */
721 if (out > start)
722 pci_unmap_sg(pdev, start, out - start, dir);
723 return 0;
726 /* Unmap a set of streaming mode DMA translations. Again, cpu read
727 rules concerning calls here are the same as for pci_unmap_single()
728 above. */
730 static void alpha_pci_unmap_sg(struct device *dev, struct scatterlist *sg,
731 int nents, enum dma_data_direction dir,
732 struct dma_attrs *attrs)
734 struct pci_dev *pdev = alpha_gendev_to_pci(dev);
735 unsigned long flags;
736 struct pci_controller *hose;
737 struct pci_iommu_arena *arena;
738 struct scatterlist *end;
739 dma_addr_t max_dma;
740 dma_addr_t fbeg, fend;
742 BUG_ON(dir == PCI_DMA_NONE);
744 if (! alpha_mv.mv_pci_tbi)
745 return;
747 hose = pdev ? pdev->sysdata : pci_isa_hose;
748 max_dma = pdev ? pdev->dma_mask : ISA_DMA_MASK;
749 arena = hose->sg_pci;
750 if (!arena || arena->dma_base + arena->size - 1 > max_dma)
751 arena = hose->sg_isa;
753 fbeg = -1, fend = 0;
755 spin_lock_irqsave(&arena->lock, flags);
757 for (end = sg + nents; sg < end; ++sg) {
758 dma_addr_t addr;
759 size_t size;
760 long npages, ofs;
761 dma_addr_t tend;
763 addr = sg->dma_address;
764 size = sg->dma_length;
765 if (!size)
766 break;
768 if (addr > 0xffffffff) {
769 /* It's a DAC address -- nothing to do. */
770 DBGA(" (%ld) DAC [%llx,%zx]\n",
771 sg - end + nents, addr, size);
772 continue;
775 if (addr >= __direct_map_base
776 && addr < __direct_map_base + __direct_map_size) {
777 /* Nothing to do. */
778 DBGA(" (%ld) direct [%llx,%zx]\n",
779 sg - end + nents, addr, size);
780 continue;
783 DBGA(" (%ld) sg [%llx,%zx]\n",
784 sg - end + nents, addr, size);
786 npages = iommu_num_pages(addr, size, PAGE_SIZE);
787 ofs = (addr - arena->dma_base) >> PAGE_SHIFT;
788 iommu_arena_free(arena, ofs, npages);
790 tend = addr + size - 1;
791 if (fbeg > addr) fbeg = addr;
792 if (fend < tend) fend = tend;
795 /* If we're freeing ptes above the `next_entry' pointer (they
796 may have snuck back into the TLB since the last wrap flush),
797 we need to flush the TLB before reallocating the latter. */
798 if ((fend - arena->dma_base) >> PAGE_SHIFT >= arena->next_entry)
799 alpha_mv.mv_pci_tbi(hose, fbeg, fend);
801 spin_unlock_irqrestore(&arena->lock, flags);
803 DBGA("pci_unmap_sg: %ld entries\n", nents - (end - sg));
806 /* Return whether the given PCI device DMA address mask can be
807 supported properly. */
809 static int alpha_pci_supported(struct device *dev, u64 mask)
811 struct pci_dev *pdev = alpha_gendev_to_pci(dev);
812 struct pci_controller *hose;
813 struct pci_iommu_arena *arena;
815 /* If there exists a direct map, and the mask fits either
816 the entire direct mapped space or the total system memory as
817 shifted by the map base */
818 if (__direct_map_size != 0
819 && (__direct_map_base + __direct_map_size - 1 <= mask ||
820 __direct_map_base + (max_low_pfn << PAGE_SHIFT) - 1 <= mask))
821 return 1;
823 /* Check that we have a scatter-gather arena that fits. */
824 hose = pdev ? pdev->sysdata : pci_isa_hose;
825 arena = hose->sg_isa;
826 if (arena && arena->dma_base + arena->size - 1 <= mask)
827 return 1;
828 arena = hose->sg_pci;
829 if (arena && arena->dma_base + arena->size - 1 <= mask)
830 return 1;
832 /* As last resort try ZONE_DMA. */
833 if (!__direct_map_base && MAX_DMA_ADDRESS - IDENT_ADDR - 1 <= mask)
834 return 1;
836 return 0;
841 * AGP GART extensions to the IOMMU
844 iommu_reserve(struct pci_iommu_arena *arena, long pg_count, long align_mask)
846 unsigned long flags;
847 unsigned long *ptes;
848 long i, p;
850 if (!arena) return -EINVAL;
852 spin_lock_irqsave(&arena->lock, flags);
854 /* Search for N empty ptes. */
855 ptes = arena->ptes;
856 p = iommu_arena_find_pages(NULL, arena, pg_count, align_mask);
857 if (p < 0) {
858 spin_unlock_irqrestore(&arena->lock, flags);
859 return -1;
862 /* Success. Mark them all reserved (ie not zero and invalid)
863 for the iommu tlb that could load them from under us.
864 They will be filled in with valid bits by _bind() */
865 for (i = 0; i < pg_count; ++i)
866 ptes[p+i] = IOMMU_RESERVED_PTE;
868 arena->next_entry = p + pg_count;
869 spin_unlock_irqrestore(&arena->lock, flags);
871 return p;
874 int
875 iommu_release(struct pci_iommu_arena *arena, long pg_start, long pg_count)
877 unsigned long *ptes;
878 long i;
880 if (!arena) return -EINVAL;
882 ptes = arena->ptes;
884 /* Make sure they're all reserved first... */
885 for(i = pg_start; i < pg_start + pg_count; i++)
886 if (ptes[i] != IOMMU_RESERVED_PTE)
887 return -EBUSY;
889 iommu_arena_free(arena, pg_start, pg_count);
890 return 0;
894 iommu_bind(struct pci_iommu_arena *arena, long pg_start, long pg_count,
895 struct page **pages)
897 unsigned long flags;
898 unsigned long *ptes;
899 long i, j;
901 if (!arena) return -EINVAL;
903 spin_lock_irqsave(&arena->lock, flags);
905 ptes = arena->ptes;
907 for(j = pg_start; j < pg_start + pg_count; j++) {
908 if (ptes[j] != IOMMU_RESERVED_PTE) {
909 spin_unlock_irqrestore(&arena->lock, flags);
910 return -EBUSY;
914 for(i = 0, j = pg_start; i < pg_count; i++, j++)
915 ptes[j] = mk_iommu_pte(page_to_phys(pages[i]));
917 spin_unlock_irqrestore(&arena->lock, flags);
919 return 0;
923 iommu_unbind(struct pci_iommu_arena *arena, long pg_start, long pg_count)
925 unsigned long *p;
926 long i;
928 if (!arena) return -EINVAL;
930 p = arena->ptes + pg_start;
931 for(i = 0; i < pg_count; i++)
932 p[i] = IOMMU_RESERVED_PTE;
934 return 0;
937 static int alpha_pci_mapping_error(struct device *dev, dma_addr_t dma_addr)
939 return dma_addr == 0;
942 struct dma_map_ops alpha_pci_ops = {
943 .alloc = alpha_pci_alloc_coherent,
944 .free = alpha_pci_free_coherent,
945 .map_page = alpha_pci_map_page,
946 .unmap_page = alpha_pci_unmap_page,
947 .map_sg = alpha_pci_map_sg,
948 .unmap_sg = alpha_pci_unmap_sg,
949 .mapping_error = alpha_pci_mapping_error,
950 .dma_supported = alpha_pci_supported,
953 struct dma_map_ops *dma_ops = &alpha_pci_ops;
954 EXPORT_SYMBOL(dma_ops);