sys/arch/powerpc/marvell/bus_dma.c

   1 /*      $NetBSD: bus_dma.c,v 1.19 2009/03/14 21:04:14 dsl Exp $ */
   2
   3 /*-
   4  * Copyright (c) 1997, 1998 The NetBSD Foundation, Inc.
   5  * All rights reserved.
   6  *
   7  * This code is derived from software contributed to The NetBSD Foundation
   8  * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
   9  * NASA Ames Research Center.
  10  *
  11  * Redistribution and use in source and binary forms, with or without
  12  * modification, are permitted provided that the following conditions
  13  * are met:
  14  * 1. Redistributions of source code must retain the above copyright
  15  *    notice, this list of conditions and the following disclaimer.
  16  * 2. Redistributions in binary form must reproduce the above copyright
  17  *    notice, this list of conditions and the following disclaimer in the
  18  *    documentation and/or other materials provided with the distribution.
  19  *
  20  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
  21  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  22  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  23  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  24  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30  * POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #include <sys/cdefs.h>
  34 __KERNEL_RCSID(0, "$NetBSD: bus_dma.c,v 1.19 2009/03/14 21:04:14 dsl Exp $");
  35
  36 #define DEBUG 1
  37
  38 #include <sys/param.h>
  39 #include <sys/systm.h>
  40 #include <sys/kernel.h>
  41 #include <sys/device.h>
  42 #include <sys/malloc.h>
  43 #include <sys/proc.h>
  44 #include <sys/mbuf.h>
  45
  46 #include <uvm/uvm.h>
  47 #include <uvm/uvm_extern.h>
  48
  49 #define _POWERPC_BUS_DMA_PRIVATE
  50 #include <machine/bus.h>
  51 #include <machine/intr.h>
  52 #include <machine/cpu.h>                /* for CACHELINESIZE */
  53
  54 #ifdef DEBUG
  55 int busdmadebug = 0;
  56 # define DPRINTF(x)     do { if (busdmadebug) printf x ; } while (0)
  57 #else
  58 # define DPRINTF(x)
  59 #endif
  60
  61 static inline void
  62 invaldcache(vaddr_t va, bus_size_t sz)
  63 {
  64         int off;
  65
  66         DPRINTF(("invaldcache: %#lx %ld\n", va, (long) sz));
  67         KASSERT(sz != 0);
  68
  69         __asm volatile("eieio;");
  70         off = (u_int)va & (curcpu()->ci_ci.dcache_line_size - 1);
  71         sz += off;
  72         va -= off;
  73         while ((int)sz > 0) {
  74                 __asm volatile("dcbi 0, %0;" :: "r"(va));
  75                 va += curcpu()->ci_ci.dcache_line_size;
  76                 sz -= curcpu()->ci_ci.dcache_line_size;
  77         }
  78         __asm volatile("sync;");
  79 }
  80
  81 static inline void
  82 flushdcache(vaddr_t va, bus_size_t sz)
  83 {
  84         int off;
  85
  86         DPRINTF(("flushdcache: %#lx %ld\n", va, (long) sz));
  87         KASSERT(sz != 0);
  88
  89         __asm volatile("eieio;");
  90         off = (u_int)va & (curcpu()->ci_ci.dcache_line_size - 1);
  91         sz += off;
  92         va -= off;
  93         while ((int)sz > curcpu()->ci_ci.dcache_line_size) {
  94                 __asm volatile("dcbf 0, %0;" :: "r"(va));
  95                 va += curcpu()->ci_ci.dcache_line_size;
  96                 sz -= curcpu()->ci_ci.dcache_line_size;
  97         }
  98
  99         /*
 100          * eieio ensures the last cache line flushed is ordered last
 101          * read-after-write ensures last cache line
 102          * (and therefore all cache lines) made it to  memory
 103          */
 104         __asm volatile("eieio; dcbf 0, %0;" :: "r"(va));
 105         __asm volatile("lwz %0,0(%0); sync;" : "+r"(va));
 106 }
 107
 108 static inline void
 109 storedcache(vaddr_t va, bus_size_t sz)
 110 {
 111         int off;
 112
 113         DPRINTF(("storedcache: %#lx %ld\n", va, (long) sz));
 114         KASSERT(sz != 0);
 115
 116         __asm volatile("eieio;");
 117         off = (u_int)va & (curcpu()->ci_ci.dcache_line_size - 1);
 118         sz += off;
 119         va -= off;
 120         while ((int)sz > 0) {
 121                 __asm volatile("dcbst 0, %0;" :: "r"(va));
 122                 va += curcpu()->ci_ci.dcache_line_size;
 123                 sz -= curcpu()->ci_ci.dcache_line_size;
 124         }
 125         __asm volatile("sync;");
 126 }
 127
 128 int     _bus_dmamap_load_buffer(bus_dma_tag_t, bus_dmamap_t, void *,
 129             bus_size_t, struct vmspace *, int, paddr_t *, int *, int);
 130
 131 /*
 132  * Common function for DMA map creation.  May be called by bus-specific
 133  * DMA map creation functions.
 134  */
 135 int
 136 _bus_dmamap_create(bus_dma_tag_t t, bus_size_t size, int nsegments, bus_size_t maxsegsz, bus_size_t boundary, int flags, bus_dmamap_t *dmamp)
 137 {
 138         struct powerpc_bus_dmamap *map;
 139         void *mapstore;
 140         size_t mapsize;
 141
 142         /*
 143          * Allocate and initialize the DMA map.  The end of the map
 144          * is a variable-sized array of segments, so we allocate enough
 145          * room for them in one shot.
 146          *
 147          * Note we don't preserve the WAITOK or NOWAIT flags.  Preservation
 148          * of ALLOCNOW notifies others that we've reserved these resources,
 149          * and they are not to be freed.
 150          *
 151          * The bus_dmamap_t includes one bus_dma_segment_t, hence
 152          * the (nsegments - 1).
 153          */
 154         mapsize = sizeof(struct powerpc_bus_dmamap) +
 155             (sizeof(bus_dma_segment_t) * (nsegments - 1));
 156         if ((mapstore = malloc(mapsize, M_DMAMAP,
 157             (flags & BUS_DMA_NOWAIT) ? M_NOWAIT : M_WAITOK)) == NULL)
 158                 return (ENOMEM);
 159
 160         memset(mapstore, 0, mapsize);
 161         map = (struct powerpc_bus_dmamap *)mapstore;
 162         map->_dm_size = size;
 163         map->_dm_segcnt = nsegments;
 164         map->_dm_maxmaxsegsz = maxsegsz;
 165         map->_dm_boundary = boundary;
 166         map->_dm_bounce_thresh = t->_bounce_thresh;
 167         map->_dm_flags = flags & ~(BUS_DMA_WAITOK|BUS_DMA_NOWAIT);
 168         map->dm_maxsegsz = maxsegsz;
 169         map->dm_mapsize = 0;            /* no valid mappings */
 170         map->dm_nsegs = 0;
 171
 172         *dmamp = map;
 173         return (0);
 174 }
 175
 176 /*
 177  * Common function for DMA map destruction.  May be called by bus-specific
 178  * DMA map destruction functions.
 179  */
 180 void
 181 _bus_dmamap_destroy(bus_dma_tag_t t, bus_dmamap_t map)
 182 {
 183
 184         free(map, M_DMAMAP);
 185 }
 186
 187 /*
 188  * Utility function to load a linear buffer.  lastaddrp holds state
 189  * between invocations (for multiple-buffer loads).  segp contains
 190  * the starting segment on entrance, and the ending segment on exit.
 191  * first indicates if this is the first invocation of this function.
 192  */
 193 int
 194 _bus_dmamap_load_buffer(bus_dma_tag_t t, bus_dmamap_t map, void *buf, bus_size_t buflen, struct vmspace *vm, int flags, paddr_t *lastaddrp, int *segp, int first)
 195 {
 196         bus_size_t sgsize;
 197         bus_addr_t curaddr, lastaddr, baddr, bmask;
 198         vaddr_t vaddr = (vaddr_t)buf;
 199         int seg;
 200
 201         lastaddr = *lastaddrp;
 202         bmask = ~(map->_dm_boundary - 1);
 203
 204         for (seg = *segp; buflen > 0 ; ) {
 205                 /*
 206                  * Get the physical address for this segment.
 207                  */
 208                 if (!VMSPACE_IS_KERNEL_P(vm))
 209                         (void) pmap_extract(vm_map_pmap(&vm->vm_map),
 210                             vaddr, (paddr_t *)&curaddr);
 211                 else
 212                         curaddr = vtophys(vaddr);
 213
 214                 /*
 215                  * If we're beyond the bounce threshold, notify
 216                  * the caller.
 217                  */
 218                 if (map->_dm_bounce_thresh != 0 &&
 219                     curaddr >= map->_dm_bounce_thresh)
 220                         return (EINVAL);
 221
 222                 /*
 223                  * Compute the segment size, and adjust counts.
 224                  */
 225                 sgsize = PAGE_SIZE - ((u_long)vaddr & PGOFSET);
 226                 if (buflen < sgsize)
 227                         sgsize = buflen;
 228
 229                 /*
 230                  * Make sure we don't cross any boundaries.
 231                  */
 232                 if (map->_dm_boundary > 0) {
 233                         baddr = (curaddr + map->_dm_boundary) & bmask;
 234                         if (sgsize > (baddr - curaddr))
 235                                 sgsize = (baddr - curaddr);
 236                 }
 237
 238                 /*
 239                  * Insert chunk into a segment, coalescing with
 240                  * the previous segment if possible.
 241                  */
 242                 if (first) {
 243                         map->dm_segs[seg].ds_addr = curaddr;
 244                         map->dm_segs[seg].ds_len = sgsize;
 245                         map->dm_segs[seg].ds_vaddr = vaddr;
 246                         first = 0;
 247                 } else {
 248                         if ((curaddr == lastaddr) &&
 249                            (vaddr == map->dm_segs[seg].ds_vaddr +
 250                                 (curaddr - map->dm_segs[seg].ds_addr)) &&
 251                             ((map->dm_segs[seg].ds_len + sgsize) <=
 252                              map->dm_maxsegsz) &&
 253                             ((map->_dm_boundary == 0) ||
 254                              ((map->dm_segs[seg].ds_addr & bmask) ==
 255                              (curaddr & bmask)))) {
 256                                 map->dm_segs[seg].ds_len += sgsize;
 257                         } else {
 258                                 if (++seg >= map->_dm_segcnt) {
 259 #ifdef DEBUG
 260                                         panic("_bus_dmamap_load_buffer: "
 261                                                 "seg %d >= _dm_segcnt %d\n",
 262                                                 seg, map->_dm_segcnt);
 263 #endif
 264                                         break;
 265                                 }
 266                                 map->dm_segs[seg].ds_addr = curaddr;
 267                                 map->dm_segs[seg].ds_len = sgsize;
 268                                 map->dm_segs[seg].ds_vaddr = vaddr;
 269                         }
 270                 }
 271
 272                 lastaddr = curaddr + sgsize;
 273                 vaddr += sgsize;
 274                 buflen -= sgsize;
 275         }
 276
 277         *segp = seg;
 278         *lastaddrp = lastaddr;
 279
 280         /*
 281          * Did we fit?
 282          */
 283         if (buflen != 0)
 284                 return (EFBIG);         /* XXX better return value here? */
 285
 286         return (0);
 287 }
 288
 289 /*
 290  * Common function for loading a DMA map with a linear buffer.  May
 291  * be called by bus-specific DMA map load functions.
 292  */
 293 int
 294 _bus_dmamap_load(bus_dma_tag_t t, bus_dmamap_t map, void *buf, bus_size_t buflen, struct proc *p, int flags)
 295 {
 296         paddr_t lastaddr;
 297         int seg, error;
 298         struct vmspace *vm;
 299
 300         /*
 301          * Make sure that on error condition we return "no valid mappings".
 302          */
 303         map->dm_mapsize = 0;
 304         map->dm_nsegs = 0;
 305         KASSERT(map->dm_maxsegsz <= map->_dm_maxmaxsegsz);
 306
 307         if (buflen > map->_dm_size)
 308                 return (EINVAL);
 309
 310         if (p != NULL) {
 311                 vm = p->p_vmspace;
 312         } else {
 313                 vm = vmspace_kernel();
 314         }
 315
 316         seg = 0;
 317         error = _bus_dmamap_load_buffer(t, map, buf, buflen, vm, flags,
 318                 &lastaddr, &seg, 1);
 319         if (error == 0) {
 320                 map->dm_mapsize = buflen;
 321                 map->dm_nsegs = seg + 1;
 322         }
 323         return (error);
 324 }
 325
 326 /*
 327  * Like _bus_dmamap_load(), but for mbufs.
 328  */
 329 int
 330 _bus_dmamap_load_mbuf(bus_dma_tag_t t, bus_dmamap_t map, struct mbuf *m0, int flags)
 331 {
 332         paddr_t lastaddr;
 333         int seg, error, first;
 334         struct mbuf *m;
 335
 336         /*
 337          * Make sure that on error condition we return "no valid mappings."
 338          */
 339         map->dm_mapsize = 0;
 340         map->dm_nsegs = 0;
 341         KASSERT(map->dm_maxsegsz <= map->_dm_maxmaxsegsz);
 342
 343 #ifdef DIAGNOSTIC
 344         if ((m0->m_flags & M_PKTHDR) == 0)
 345                 panic("_bus_dmamap_load_mbuf: no packet header");
 346 #endif
 347
 348         if (m0->m_pkthdr.len > map->_dm_size)
 349                 return (EINVAL);
 350
 351         first = 1;
 352         seg = 0;
 353         error = 0;
 354         for (m = m0; m != NULL && error == 0; m = m->m_next) {
 355                 if (m->m_len == 0)
 356                         continue;
 357                 error = _bus_dmamap_load_buffer(t, map, m->m_data, m->m_len,
 358                     vmspace_kernel(), flags, &lastaddr, &seg, first);
 359                 first = 0;
 360         }
 361         if (error == 0) {
 362                 map->dm_mapsize = m0->m_pkthdr.len;
 363                 map->dm_nsegs = seg + 1;
 364         }
 365         return (error);
 366 }
 367
 368 /*
 369  * Like _bus_dmamap_load(), but for uios.
 370  */
 371 int
 372 _bus_dmamap_load_uio(bus_dma_tag_t t, bus_dmamap_t map, struct uio *uio, int flags)
 373 {
 374         paddr_t lastaddr;
 375         int seg, i, error, first;
 376         bus_size_t minlen, resid;
 377         struct iovec *iov;
 378         void *addr;
 379
 380         /*
 381          * Make sure that on error condition we return "no valid mappings."
 382          */
 383         map->dm_mapsize = 0;
 384         map->dm_nsegs = 0;
 385         KASSERT(map->dm_maxsegsz <= map->_dm_maxmaxsegsz);
 386
 387         resid = uio->uio_resid;
 388         iov = uio->uio_iov;
 389
 390         first = 1;
 391         seg = 0;
 392         error = 0;
 393         for (i = 0; i < uio->uio_iovcnt && resid != 0 && error == 0; i++) {
 394                 /*
 395                  * Now at the first iovec to load.  Load each iovec
 396                  * until we have exhausted the residual count.
 397                  */
 398                 minlen = resid < iov[i].iov_len ? resid : iov[i].iov_len;
 399                 addr = (void *)iov[i].iov_base;
 400
 401                 error = _bus_dmamap_load_buffer(t, map, addr, minlen,
 402                     uio->uio_vmspace, flags, &lastaddr, &seg, first);
 403                 first = 0;
 404
 405                 resid -= minlen;
 406         }
 407         if (error == 0) {
 408                 map->dm_mapsize = uio->uio_resid;
 409                 map->dm_nsegs = seg + 1;
 410         }
 411         return (error);
 412 }
 413
 414 /*
 415  * Like _bus_dmamap_load(), but for raw memory.
 416  */
 417 int
 418 _bus_dmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map, bus_dma_segment_t *segs, int nsegs, bus_size_t size, int flags)
 419 {
 420
 421         panic("_bus_dmamap_load_raw: not implemented");
 422 }
 423
 424 /*
 425  * Common function for unloading a DMA map.  May be called by
 426  * chipset-specific DMA map unload functions.
 427  */
 428 void
 429 _bus_dmamap_unload(bus_dma_tag_t t, bus_dmamap_t map)
 430 {
 431         /*
 432          * No resources to free; just mark the mappings as
 433          * invalid.
 434          */
 435         map->dm_maxsegsz = map->_dm_maxmaxsegsz;
 436         map->dm_mapsize = 0;
 437         map->dm_nsegs = 0;
 438 }
 439
 440 /*
 441  * DMA map synchronization, provides software coherency
 442  */
 443 void
 444 _bus_dmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset, bus_size_t len, int ops)
 445 {
 446         int i;
 447         bus_size_t dslen;
 448         bus_size_t minlen;
 449
 450         DPRINTF(("_bus_dmamap_sync %p %p %#lx %ld %#x\n",
 451                 t, map, (unsigned long) offset, (unsigned long) len, ops));
 452
 453         /*
 454          * Mixing PRE and POST operations is not allowed.
 455          */
 456         if (((ops & (BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE)) != 0) &&
 457             ((ops & (BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE)) != 0))
 458                 panic("_bus_dmamap_sync: mix PRE and POST");
 459
 460 #ifdef DIAGNOSTIC
 461         if (offset >= map->dm_mapsize)
 462                 panic("_bus_dmamap_sync: bad offset %#lx (map size is %#lx)",
 463                       (unsigned long) offset, (unsigned long) map->dm_mapsize);
 464         if (len == 0 || (offset + len) > map->dm_mapsize)
 465                 panic("_bus_dmamap_sync: bad length");
 466 #endif
 467
 468         switch(ops) {
 469         case BUS_DMASYNC_PREWRITE:
 470                 for (i=0; i < map->dm_nsegs && len != 0; i++) {
 471                         /* find the beginning segment */
 472                         dslen = map->dm_segs[i].ds_len;
 473                         if (offset >= dslen) {
 474                                 offset -= dslen;
 475                                 continue;
 476                         }
 477                         dslen -= offset;
 478                         minlen = len;
 479                         if (len > dslen)
 480                                 minlen = dslen;
 481                         storedcache(map->dm_segs[i].ds_vaddr + offset, minlen);
 482                         len -= minlen;
 483                         offset = 0;
 484                 }
 485                 break;
 486         case BUS_DMASYNC_PREREAD:
 487                 for (i=0; i < map->dm_nsegs && len != 0; i++) {
 488                         vaddr_t va;
 489
 490                         /* find the beginning segment */
 491                         dslen = map->dm_segs[i].ds_len;
 492                         if (offset >= dslen) {
 493                                 offset -= dslen;
 494                                 continue;
 495                         }
 496                         dslen -= offset;
 497                         minlen = len;
 498                         if (len > dslen)
 499                                 minlen = dslen;
 500                         va = map->dm_segs[i].ds_vaddr + offset;
 501                         if (va & (curcpu()->ci_ci.dcache_line_size-1))
 502                                 storedcache(va, 1);
 503                         va += minlen;
 504                         if (va & (curcpu()->ci_ci.dcache_line_size-1))
 505                                 storedcache(va, 1);
 506                         invaldcache(map->dm_segs[i].ds_vaddr + offset, minlen);
 507                         len -= minlen;
 508                         offset = 0;
 509                 }
 510                 break;
 511         case BUS_DMASYNC_POSTREAD:
 512                 for (i=0; i < map->dm_nsegs && len != 0; i++) {
 513                         /* find the beginning segment */
 514                         dslen = map->dm_segs[i].ds_len;
 515                         if (offset >= dslen) {
 516                                 offset -= dslen;
 517                                 continue;
 518                         }
 519                         dslen -= offset;
 520                         minlen = len;
 521                         if (len > dslen)
 522                                 minlen = dslen;
 523                         invaldcache(map->dm_segs[i].ds_vaddr + offset, minlen);
 524                         len -= minlen;
 525                         offset = 0;
 526                 }
 527                 break;
 528         case BUS_DMASYNC_POSTWRITE:
 529                 break;
 530         case BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE:
 531                 for (i=0; i < map->dm_nsegs && len != 0; i++) {
 532                         /* find the beginning segment */
 533                         dslen = map->dm_segs[i].ds_len;
 534                         if (offset >= dslen) {
 535                                 offset -= dslen;
 536                                 continue;
 537                         }
 538                         dslen -= offset;
 539                         minlen = len;
 540                         if (len > dslen)
 541                                 minlen = dslen;
 542                         flushdcache(map->dm_segs[i].ds_vaddr + offset, minlen);
 543                         len -= minlen;
 544                         offset = 0;
 545                 }
 546                 break;
 547         case BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE:
 548                 for (i=0; i < map->dm_nsegs && len != 0; i++) {
 549                         /* find the beginning segment */
 550                         dslen = map->dm_segs[i].ds_len;
 551                         if (offset >= dslen) {
 552                                 offset -= dslen;
 553                                 continue;
 554                         }
 555                         dslen -= offset;
 556                         minlen = len;
 557                         if (len > dslen)
 558                                 minlen = dslen;
 559                         invaldcache(map->dm_segs[i].ds_vaddr + offset, minlen);
 560                         len -= minlen;
 561                         offset = 0;
 562                 }
 563                 break;
 564         }
 565 }
 566
 567 /*
 568  * Common function for DMA-safe memory allocation.  May be called
 569  * by bus-specific DMA memory allocation functions.
 570  */
 571 int
 572 _bus_dmamem_alloc(bus_dma_tag_t t, bus_size_t size, bus_size_t alignment, bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs, int flags)
 573 {
 574         paddr_t avail_start = 0xffffffff, avail_end = 0;
 575         paddr_t curaddr, lastaddr, high;
 576         struct vm_page *m;
 577         struct pglist mlist;
 578         int curseg, error, bank;
 579
 580         for (bank = 0; bank < vm_nphysseg; bank++) {
 581                 if (avail_start > vm_physmem[bank].avail_start << PGSHIFT)
 582                         avail_start = vm_physmem[bank].avail_start << PGSHIFT;
 583                 if (avail_end < vm_physmem[bank].avail_end << PGSHIFT)
 584                         avail_end = vm_physmem[bank].avail_end << PGSHIFT;
 585         }
 586
 587
 588         /* Always round the size. */
 589         size = round_page(size);
 590
 591         high = avail_end - PAGE_SIZE;
 592
 593         /*
 594          * Allocate pages from the VM system.
 595          */
 596         TAILQ_INIT(&mlist);
 597         error = uvm_pglistalloc(size, avail_start, high, alignment, boundary,
 598             &mlist, nsegs, (flags & BUS_DMA_NOWAIT) == 0);
 599         if (error)
 600                 return (error);
 601
 602         /*
 603          * Compute the location, size, and number of segments actually
 604          * returned by the VM code.
 605          */
 606         m = mlist.tqh_first;
 607         curseg = 0;
 608         lastaddr = segs[curseg].ds_addr = VM_PAGE_TO_PHYS(m);
 609         segs[curseg].ds_len = PAGE_SIZE;
 610         segs[curseg].ds_vaddr = (vaddr_t)0xdeadbeef;
 611         m = m->pageq.queue.tqe_next;
 612
 613         for (; m != NULL; m = m->pageq.queue.tqe_next) {
 614                 curaddr = VM_PAGE_TO_PHYS(m);
 615 #ifdef DIAGNOSTIC
 616                 if (curaddr < avail_start || curaddr >= high) {
 617                         printf("vm_page_alloc_memory returned non-sensical"
 618                             " address 0x%lx\n", curaddr);
 619                         panic("_bus_dmamem_alloc");
 620                 }
 621 #endif
 622                 if (curaddr == (lastaddr + PAGE_SIZE))
 623                         segs[curseg].ds_len += PAGE_SIZE;
 624                 else {
 625                         curseg++;
 626                         segs[curseg].ds_addr = curaddr;
 627                         segs[curseg].ds_len = PAGE_SIZE;
 628                         segs[curseg].ds_vaddr = (vaddr_t)0xdeadbeef;
 629                 }
 630                 lastaddr = curaddr;
 631         }
 632
 633         *rsegs = curseg + 1;
 634
 635         return (0);
 636 }
 637
 638 /*
 639  * Common function for freeing DMA-safe memory.  May be called by
 640  * bus-specific DMA memory free functions.
 641  */
 642 void
 643 _bus_dmamem_free(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs)
 644 {
 645         struct vm_page *m;
 646         bus_addr_t addr;
 647         struct pglist mlist;
 648         int curseg;
 649
 650         /*
 651          * Build a list of pages to free back to the VM system.
 652          */
 653         TAILQ_INIT(&mlist);
 654         for (curseg = 0; curseg < nsegs; curseg++) {
 655                 for (addr = segs[curseg].ds_addr;
 656                     addr < (segs[curseg].ds_addr + segs[curseg].ds_len);
 657                     addr += PAGE_SIZE) {
 658                         m = PHYS_TO_VM_PAGE(addr);
 659                         TAILQ_INSERT_TAIL(&mlist, m, pageq.queue);
 660                 }
 661         }
 662
 663         uvm_pglistfree(&mlist);
 664 }
 665
 666 /*
 667  * Common function for mapping DMA-safe memory.  May be called by
 668  * bus-specific DMA memory map functions.
 669  */
 670 int
 671 _bus_dmamem_map(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs, size_t size, void **kvap, int flags)
 672 {
 673         vaddr_t va;
 674         bus_addr_t addr;
 675         int curseg;
 676         const uvm_flag_t kmflags =
 677             (flags & BUS_DMA_NOWAIT) != 0 ? UVM_KMF_NOWAIT : 0;
 678
 679         size = round_page(size);
 680
 681         va = uvm_km_alloc(kernel_map, size, 0, UVM_KMF_VAONLY | kmflags);
 682
 683         if (va == 0)
 684                 return (ENOMEM);
 685
 686         *kvap = (void *)va;
 687
 688         for (curseg = 0; curseg < nsegs; curseg++) {
 689                 for (addr = segs[curseg].ds_addr;
 690                     addr < (segs[curseg].ds_addr + segs[curseg].ds_len);
 691                     addr += PAGE_SIZE, va += PAGE_SIZE, size -= PAGE_SIZE) {
 692                         if (size == 0)
 693                                 panic("_bus_dmamem_map: size botch");
 694                         pmap_enter(pmap_kernel(), va, addr,
 695                             VM_PROT_READ | VM_PROT_WRITE,
 696                             VM_PROT_READ | VM_PROT_WRITE | PMAP_WIRED|
 697                                     (flags & BUS_DMA_COHERENT ? PMAP_NC : 0));
 698                 }
 699         }
 700
 701         return (0);
 702 }
 703
 704 /*
 705  * Common function for unmapping DMA-safe memory.  May be called by
 706  * bus-specific DMA memory unmapping functions.
 707  */
 708 void
 709 _bus_dmamem_unmap(bus_dma_tag_t t, void *kva, size_t size)
 710 {
 711
 712 #ifdef DIAGNOSTIC
 713         if ((u_long)kva & PGOFSET)
 714                 panic("_bus_dmamem_unmap");
 715 #endif
 716
 717         size = round_page(size);
 718
 719         pmap_remove(pmap_kernel(), (vaddr_t)kva, (vaddr_t)kva + size);
 720         pmap_update(pmap_kernel());
 721         uvm_km_free(kernel_map, (vaddr_t)kva, size, UVM_KMF_VAONLY);
 722 }
 723
 724 /*
 725  * Common functin for mmap(2)'ing DMA-safe memory.  May be called by
 726  * bus-specific DMA mmap(2)'ing functions.
 727  */
 728 paddr_t
 729 _bus_dmamem_mmap(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs, off_t off, int prot, int flags)
 730 {
 731         int i;
 732
 733         for (i = 0; i < nsegs; i++) {
 734 #ifdef DIAGNOSTIC
 735                 if (off & PGOFSET)
 736                         panic("_bus_dmamem_mmap: offset unaligned");
 737                 if (segs[i].ds_addr & PGOFSET)
 738                         panic("_bus_dmamem_mmap: segment unaligned");
 739                 if (segs[i].ds_len & PGOFSET)
 740                         panic("_bus_dmamem_mmap: segment size not multiple"
 741                             " of page size");
 742 #endif
 743                 if (off >= segs[i].ds_len) {
 744                         off -= segs[i].ds_len;
 745                         continue;
 746                 }
 747
 748                 return (segs[i].ds_addr + off);
 749         }
 750
 751         /* Page not found. */
 752         return (-1);
 753 }