2 * drivers/gpu/drm/omapdrm/omap_gem.c
4 * Copyright (C) 2011 Texas Instruments
5 * Author: Rob Clark <rob.clark@linaro.org>
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 as published by
9 * the Free Software Foundation.
11 * This program is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16 * You should have received a copy of the GNU General Public License along with
17 * this program. If not, see <http://www.gnu.org/licenses/>.
20 #include <linux/shmem_fs.h>
21 #include <linux/spinlock.h>
23 #include <drm/drm_vma_manager.h>
26 #include "omap_dmm_tiler.h"
28 /* remove these once drm core helpers are merged */
29 struct page
**_drm_gem_get_pages(struct drm_gem_object
*obj
, gfp_t gfpmask
);
30 void _drm_gem_put_pages(struct drm_gem_object
*obj
, struct page
**pages
,
31 bool dirty
, bool accessed
);
32 int _drm_gem_create_mmap_offset_size(struct drm_gem_object
*obj
, size_t size
);
35 * GEM buffer object implementation.
38 #define to_omap_bo(x) container_of(x, struct omap_gem_object, base)
40 /* note: we use upper 8 bits of flags for driver-internal flags: */
41 #define OMAP_BO_DMA 0x01000000 /* actually is physically contiguous */
42 #define OMAP_BO_EXT_SYNC 0x02000000 /* externally allocated sync object */
43 #define OMAP_BO_EXT_MEM 0x04000000 /* externally allocated memory */
46 struct omap_gem_object
{
47 struct drm_gem_object base
;
49 struct list_head mm_list
;
53 /** width/height for tiled formats (rounded up to slot boundaries) */
54 uint16_t width
, height
;
56 /** roll applied when mapping to DMM */
60 * If buffer is allocated physically contiguous, the OMAP_BO_DMA flag
61 * is set and the paddr is valid. Also if the buffer is remapped in
62 * TILER and paddr_cnt > 0, then paddr is valid. But if you are using
63 * the physical address and OMAP_BO_DMA is not set, then you should
64 * be going thru omap_gem_{get,put}_paddr() to ensure the mapping is
65 * not removed from under your feet.
67 * Note that OMAP_BO_SCANOUT is a hint from userspace that DMA capable
68 * buffer is requested, but doesn't mean that it is. Use the
69 * OMAP_BO_DMA flag to determine if the buffer has a DMA capable
80 * tiler block used when buffer is remapped in DMM/TILER.
82 struct tiler_block
*block
;
85 * Array of backing pages, if allocated. Note that pages are never
86 * allocated for buffers originally allocated from contiguous memory
90 /** addresses corresponding to pages in above array */
94 * Virtual address, if mapped.
99 * sync-object allocated on demand (if needed)
101 * Per-buffer sync-object for tracking pending and completed hw/dma
102 * read and write operations. The layout in memory is dictated by
103 * the SGX firmware, which uses this information to stall the command
104 * stream if a surface is not ready yet.
106 * Note that when buffer is used by SGX, the sync-object needs to be
107 * allocated from a special heap of sync-objects. This way many sync
108 * objects can be packed in a page, and not waste GPU virtual address
109 * space. Because of this we have to have a omap_gem_set_sync_object()
110 * API to allow replacement of the syncobj after it has (potentially)
111 * already been allocated. A bit ugly but I haven't thought of a
112 * better alternative.
115 uint32_t write_pending
;
116 uint32_t write_complete
;
117 uint32_t read_pending
;
118 uint32_t read_complete
;
122 static int get_pages(struct drm_gem_object
*obj
, struct page
***pages
);
123 static uint64_t mmap_offset(struct drm_gem_object
*obj
);
125 /* To deal with userspace mmap'ings of 2d tiled buffers, which (a) are
126 * not necessarily pinned in TILER all the time, and (b) when they are
127 * they are not necessarily page aligned, we reserve one or more small
128 * regions in each of the 2d containers to use as a user-GART where we
129 * can create a second page-aligned mapping of parts of the buffer
130 * being accessed from userspace.
132 * Note that we could optimize slightly when we know that multiple
133 * tiler containers are backed by the same PAT.. but I'll leave that
136 #define NUM_USERGART_ENTRIES 2
137 struct usergart_entry
{
138 struct tiler_block
*block
; /* the reserved tiler block */
140 struct drm_gem_object
*obj
; /* the current pinned obj */
141 pgoff_t obj_pgoff
; /* page offset of obj currently
145 struct usergart_entry entry
[NUM_USERGART_ENTRIES
];
146 int height
; /* height in rows */
147 int height_shift
; /* ilog2(height in rows) */
148 int slot_shift
; /* ilog2(width per slot) */
149 int stride_pfn
; /* stride in pages */
150 int last
; /* index of last used entry */
153 static void evict_entry(struct drm_gem_object
*obj
,
154 enum tiler_fmt fmt
, struct usergart_entry
*entry
)
156 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
157 int n
= usergart
[fmt
].height
;
158 size_t size
= PAGE_SIZE
* n
;
159 loff_t off
= mmap_offset(obj
) +
160 (entry
->obj_pgoff
<< PAGE_SHIFT
);
161 const int m
= 1 + ((omap_obj
->width
<< fmt
) / PAGE_SIZE
);
165 /* if stride > than PAGE_SIZE then sparse mapping: */
166 for (i
= n
; i
> 0; i
--) {
167 unmap_mapping_range(obj
->dev
->anon_inode
->i_mapping
,
169 off
+= PAGE_SIZE
* m
;
172 unmap_mapping_range(obj
->dev
->anon_inode
->i_mapping
,
179 /* Evict a buffer from usergart, if it is mapped there */
180 static void evict(struct drm_gem_object
*obj
)
182 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
184 if (omap_obj
->flags
& OMAP_BO_TILED
) {
185 enum tiler_fmt fmt
= gem2fmt(omap_obj
->flags
);
191 for (i
= 0; i
< NUM_USERGART_ENTRIES
; i
++) {
192 struct usergart_entry
*entry
= &usergart
[fmt
].entry
[i
];
193 if (entry
->obj
== obj
)
194 evict_entry(obj
, fmt
, entry
);
199 /* GEM objects can either be allocated from contiguous memory (in which
200 * case obj->filp==NULL), or w/ shmem backing (obj->filp!=NULL). But non
201 * contiguous buffers can be remapped in TILER/DMM if they need to be
202 * contiguous... but we don't do this all the time to reduce pressure
203 * on TILER/DMM space when we know at allocation time that the buffer
204 * will need to be scanned out.
206 static inline bool is_shmem(struct drm_gem_object
*obj
)
208 return obj
->filp
!= NULL
;
212 * shmem buffers that are mapped cached can simulate coherency via using
213 * page faulting to keep track of dirty pages
215 static inline bool is_cached_coherent(struct drm_gem_object
*obj
)
217 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
218 return is_shmem(obj
) &&
219 ((omap_obj
->flags
& OMAP_BO_CACHE_MASK
) == OMAP_BO_CACHED
);
222 static DEFINE_SPINLOCK(sync_lock
);
224 /** ensure backing pages are allocated */
225 static int omap_gem_attach_pages(struct drm_gem_object
*obj
)
227 struct drm_device
*dev
= obj
->dev
;
228 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
230 int npages
= obj
->size
>> PAGE_SHIFT
;
234 WARN_ON(omap_obj
->pages
);
236 pages
= drm_gem_get_pages(obj
);
238 dev_err(obj
->dev
->dev
, "could not get pages: %ld\n", PTR_ERR(pages
));
239 return PTR_ERR(pages
);
242 /* for non-cached buffers, ensure the new pages are clean because
243 * DSS, GPU, etc. are not cache coherent:
245 if (omap_obj
->flags
& (OMAP_BO_WC
|OMAP_BO_UNCACHED
)) {
246 addrs
= kmalloc(npages
* sizeof(*addrs
), GFP_KERNEL
);
252 for (i
= 0; i
< npages
; i
++) {
253 addrs
[i
] = dma_map_page(dev
->dev
, pages
[i
],
254 0, PAGE_SIZE
, DMA_BIDIRECTIONAL
);
257 addrs
= kzalloc(npages
* sizeof(*addrs
), GFP_KERNEL
);
264 omap_obj
->addrs
= addrs
;
265 omap_obj
->pages
= pages
;
270 drm_gem_put_pages(obj
, pages
, true, false);
275 /** release backing pages */
276 static void omap_gem_detach_pages(struct drm_gem_object
*obj
)
278 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
280 /* for non-cached buffers, ensure the new pages are clean because
281 * DSS, GPU, etc. are not cache coherent:
283 if (omap_obj
->flags
& (OMAP_BO_WC
|OMAP_BO_UNCACHED
)) {
284 int i
, npages
= obj
->size
>> PAGE_SHIFT
;
285 for (i
= 0; i
< npages
; i
++) {
286 dma_unmap_page(obj
->dev
->dev
, omap_obj
->addrs
[i
],
287 PAGE_SIZE
, DMA_BIDIRECTIONAL
);
291 kfree(omap_obj
->addrs
);
292 omap_obj
->addrs
= NULL
;
294 drm_gem_put_pages(obj
, omap_obj
->pages
, true, false);
295 omap_obj
->pages
= NULL
;
298 /* get buffer flags */
299 uint32_t omap_gem_flags(struct drm_gem_object
*obj
)
301 return to_omap_bo(obj
)->flags
;
304 /** get mmap offset */
305 static uint64_t mmap_offset(struct drm_gem_object
*obj
)
307 struct drm_device
*dev
= obj
->dev
;
311 WARN_ON(!mutex_is_locked(&dev
->struct_mutex
));
313 /* Make it mmapable */
314 size
= omap_gem_mmap_size(obj
);
315 ret
= drm_gem_create_mmap_offset_size(obj
, size
);
317 dev_err(dev
->dev
, "could not allocate mmap offset\n");
321 return drm_vma_node_offset_addr(&obj
->vma_node
);
324 uint64_t omap_gem_mmap_offset(struct drm_gem_object
*obj
)
327 mutex_lock(&obj
->dev
->struct_mutex
);
328 offset
= mmap_offset(obj
);
329 mutex_unlock(&obj
->dev
->struct_mutex
);
334 size_t omap_gem_mmap_size(struct drm_gem_object
*obj
)
336 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
337 size_t size
= obj
->size
;
339 if (omap_obj
->flags
& OMAP_BO_TILED
) {
340 /* for tiled buffers, the virtual size has stride rounded up
341 * to 4kb.. (to hide the fact that row n+1 might start 16kb or
342 * 32kb later!). But we don't back the entire buffer with
343 * pages, only the valid picture part.. so need to adjust for
344 * this in the size used to mmap and generate mmap offset
346 size
= tiler_vsize(gem2fmt(omap_obj
->flags
),
347 omap_obj
->width
, omap_obj
->height
);
353 /* get tiled size, returns -EINVAL if not tiled buffer */
354 int omap_gem_tiled_size(struct drm_gem_object
*obj
, uint16_t *w
, uint16_t *h
)
356 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
357 if (omap_obj
->flags
& OMAP_BO_TILED
) {
358 *w
= omap_obj
->width
;
359 *h
= omap_obj
->height
;
365 /* Normal handling for the case of faulting in non-tiled buffers */
366 static int fault_1d(struct drm_gem_object
*obj
,
367 struct vm_area_struct
*vma
, struct vm_fault
*vmf
)
369 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
373 /* We don't use vmf->pgoff since that has the fake offset: */
374 pgoff
= ((unsigned long)vmf
->virtual_address
-
375 vma
->vm_start
) >> PAGE_SHIFT
;
377 if (omap_obj
->pages
) {
378 omap_gem_cpu_sync(obj
, pgoff
);
379 pfn
= page_to_pfn(omap_obj
->pages
[pgoff
]);
381 BUG_ON(!(omap_obj
->flags
& OMAP_BO_DMA
));
382 pfn
= (omap_obj
->paddr
>> PAGE_SHIFT
) + pgoff
;
385 VERB("Inserting %p pfn %lx, pa %lx", vmf
->virtual_address
,
386 pfn
, pfn
<< PAGE_SHIFT
);
388 return vm_insert_mixed(vma
, (unsigned long)vmf
->virtual_address
, pfn
);
391 /* Special handling for the case of faulting in 2d tiled buffers */
392 static int fault_2d(struct drm_gem_object
*obj
,
393 struct vm_area_struct
*vma
, struct vm_fault
*vmf
)
395 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
396 struct usergart_entry
*entry
;
397 enum tiler_fmt fmt
= gem2fmt(omap_obj
->flags
);
398 struct page
*pages
[64]; /* XXX is this too much to have on stack? */
400 pgoff_t pgoff
, base_pgoff
;
405 * Note the height of the slot is also equal to the number of pages
406 * that need to be mapped in to fill 4kb wide CPU page. If the slot
407 * height is 64, then 64 pages fill a 4kb wide by 64 row region.
409 const int n
= usergart
[fmt
].height
;
410 const int n_shift
= usergart
[fmt
].height_shift
;
413 * If buffer width in bytes > PAGE_SIZE then the virtual stride is
414 * rounded up to next multiple of PAGE_SIZE.. this need to be taken
415 * into account in some of the math, so figure out virtual stride
418 const int m
= 1 + ((omap_obj
->width
<< fmt
) / PAGE_SIZE
);
420 /* We don't use vmf->pgoff since that has the fake offset: */
421 pgoff
= ((unsigned long)vmf
->virtual_address
-
422 vma
->vm_start
) >> PAGE_SHIFT
;
425 * Actual address we start mapping at is rounded down to previous slot
426 * boundary in the y direction:
428 base_pgoff
= round_down(pgoff
, m
<< n_shift
);
430 /* figure out buffer width in slots */
431 slots
= omap_obj
->width
>> usergart
[fmt
].slot_shift
;
433 vaddr
= vmf
->virtual_address
- ((pgoff
- base_pgoff
) << PAGE_SHIFT
);
435 entry
= &usergart
[fmt
].entry
[usergart
[fmt
].last
];
437 /* evict previous buffer using this usergart entry, if any: */
439 evict_entry(entry
->obj
, fmt
, entry
);
442 entry
->obj_pgoff
= base_pgoff
;
444 /* now convert base_pgoff to phys offset from virt offset: */
445 base_pgoff
= (base_pgoff
>> n_shift
) * slots
;
447 /* for wider-than 4k.. figure out which part of the slot-row we want: */
450 entry
->obj_pgoff
+= off
;
452 slots
= min(slots
- (off
<< n_shift
), n
);
453 base_pgoff
+= off
<< n_shift
;
454 vaddr
+= off
<< PAGE_SHIFT
;
458 * Map in pages. Beyond the valid pixel part of the buffer, we set
459 * pages[i] to NULL to get a dummy page mapped in.. if someone
460 * reads/writes it they will get random/undefined content, but at
461 * least it won't be corrupting whatever other random page used to
462 * be mapped in, or other undefined behavior.
464 memcpy(pages
, &omap_obj
->pages
[base_pgoff
],
465 sizeof(struct page
*) * slots
);
466 memset(pages
+ slots
, 0,
467 sizeof(struct page
*) * (n
- slots
));
469 ret
= tiler_pin(entry
->block
, pages
, ARRAY_SIZE(pages
), 0, true);
471 dev_err(obj
->dev
->dev
, "failed to pin: %d\n", ret
);
475 pfn
= entry
->paddr
>> PAGE_SHIFT
;
477 VERB("Inserting %p pfn %lx, pa %lx", vmf
->virtual_address
,
478 pfn
, pfn
<< PAGE_SHIFT
);
480 for (i
= n
; i
> 0; i
--) {
481 vm_insert_mixed(vma
, (unsigned long)vaddr
, pfn
);
482 pfn
+= usergart
[fmt
].stride_pfn
;
483 vaddr
+= PAGE_SIZE
* m
;
486 /* simple round-robin: */
487 usergart
[fmt
].last
= (usergart
[fmt
].last
+ 1) % NUM_USERGART_ENTRIES
;
493 * omap_gem_fault - pagefault handler for GEM objects
494 * @vma: the VMA of the GEM object
497 * Invoked when a fault occurs on an mmap of a GEM managed area. GEM
498 * does most of the work for us including the actual map/unmap calls
499 * but we need to do the actual page work.
501 * The VMA was set up by GEM. In doing so it also ensured that the
502 * vma->vm_private_data points to the GEM object that is backing this
505 int omap_gem_fault(struct vm_area_struct
*vma
, struct vm_fault
*vmf
)
507 struct drm_gem_object
*obj
= vma
->vm_private_data
;
508 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
509 struct drm_device
*dev
= obj
->dev
;
513 /* Make sure we don't parallel update on a fault, nor move or remove
514 * something from beneath our feet
516 mutex_lock(&dev
->struct_mutex
);
518 /* if a shmem backed object, make sure we have pages attached now */
519 ret
= get_pages(obj
, &pages
);
523 /* where should we do corresponding put_pages().. we are mapping
524 * the original page, rather than thru a GART, so we can't rely
525 * on eviction to trigger this. But munmap() or all mappings should
526 * probably trigger put_pages()?
529 if (omap_obj
->flags
& OMAP_BO_TILED
)
530 ret
= fault_2d(obj
, vma
, vmf
);
532 ret
= fault_1d(obj
, vma
, vmf
);
536 mutex_unlock(&dev
->struct_mutex
);
541 return VM_FAULT_NOPAGE
;
545 return VM_FAULT_SIGBUS
;
549 /** We override mainly to fix up some of the vm mapping flags.. */
550 int omap_gem_mmap(struct file
*filp
, struct vm_area_struct
*vma
)
554 ret
= drm_gem_mmap(filp
, vma
);
556 DBG("mmap failed: %d", ret
);
560 return omap_gem_mmap_obj(vma
->vm_private_data
, vma
);
563 int omap_gem_mmap_obj(struct drm_gem_object
*obj
,
564 struct vm_area_struct
*vma
)
566 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
568 vma
->vm_flags
&= ~VM_PFNMAP
;
569 vma
->vm_flags
|= VM_MIXEDMAP
;
571 if (omap_obj
->flags
& OMAP_BO_WC
) {
572 vma
->vm_page_prot
= pgprot_writecombine(vm_get_page_prot(vma
->vm_flags
));
573 } else if (omap_obj
->flags
& OMAP_BO_UNCACHED
) {
574 vma
->vm_page_prot
= pgprot_noncached(vm_get_page_prot(vma
->vm_flags
));
577 * We do have some private objects, at least for scanout buffers
578 * on hardware without DMM/TILER. But these are allocated write-
581 if (WARN_ON(!obj
->filp
))
585 * Shunt off cached objs to shmem file so they have their own
586 * address_space (so unmap_mapping_range does what we want,
587 * in particular in the case of mmap'd dmabufs)
591 vma
->vm_file
= get_file(obj
->filp
);
593 vma
->vm_page_prot
= vm_get_page_prot(vma
->vm_flags
);
601 * omap_gem_dumb_create - create a dumb buffer
602 * @drm_file: our client file
604 * @args: the requested arguments copied from userspace
606 * Allocate a buffer suitable for use for a frame buffer of the
607 * form described by user space. Give userspace a handle by which
610 int omap_gem_dumb_create(struct drm_file
*file
, struct drm_device
*dev
,
611 struct drm_mode_create_dumb
*args
)
613 union omap_gem_size gsize
;
615 args
->pitch
= align_pitch(0, args
->width
, args
->bpp
);
616 args
->size
= PAGE_ALIGN(args
->pitch
* args
->height
);
618 gsize
= (union omap_gem_size
){
622 return omap_gem_new_handle(dev
, file
, gsize
,
623 OMAP_BO_SCANOUT
| OMAP_BO_WC
, &args
->handle
);
627 * omap_gem_dumb_map - buffer mapping for dumb interface
628 * @file: our drm client file
630 * @handle: GEM handle to the object (from dumb_create)
632 * Do the necessary setup to allow the mapping of the frame buffer
633 * into user memory. We don't have to do much here at the moment.
635 int omap_gem_dumb_map_offset(struct drm_file
*file
, struct drm_device
*dev
,
636 uint32_t handle
, uint64_t *offset
)
638 struct drm_gem_object
*obj
;
641 /* GEM does all our handle to object mapping */
642 obj
= drm_gem_object_lookup(dev
, file
, handle
);
648 *offset
= omap_gem_mmap_offset(obj
);
650 drm_gem_object_unreference_unlocked(obj
);
656 /* Set scrolling position. This allows us to implement fast scrolling
659 * Call only from non-atomic contexts.
661 int omap_gem_roll(struct drm_gem_object
*obj
, uint32_t roll
)
663 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
664 uint32_t npages
= obj
->size
>> PAGE_SHIFT
;
668 dev_err(obj
->dev
->dev
, "invalid roll: %d\n", roll
);
672 omap_obj
->roll
= roll
;
674 mutex_lock(&obj
->dev
->struct_mutex
);
676 /* if we aren't mapped yet, we don't need to do anything */
677 if (omap_obj
->block
) {
679 ret
= get_pages(obj
, &pages
);
682 ret
= tiler_pin(omap_obj
->block
, pages
, npages
, roll
, true);
684 dev_err(obj
->dev
->dev
, "could not repin: %d\n", ret
);
688 mutex_unlock(&obj
->dev
->struct_mutex
);
693 /* Sync the buffer for CPU access.. note pages should already be
694 * attached, ie. omap_gem_get_pages()
696 void omap_gem_cpu_sync(struct drm_gem_object
*obj
, int pgoff
)
698 struct drm_device
*dev
= obj
->dev
;
699 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
701 if (is_cached_coherent(obj
) && omap_obj
->addrs
[pgoff
]) {
702 dma_unmap_page(dev
->dev
, omap_obj
->addrs
[pgoff
],
703 PAGE_SIZE
, DMA_BIDIRECTIONAL
);
704 omap_obj
->addrs
[pgoff
] = 0;
708 /* sync the buffer for DMA access */
709 void omap_gem_dma_sync(struct drm_gem_object
*obj
,
710 enum dma_data_direction dir
)
712 struct drm_device
*dev
= obj
->dev
;
713 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
715 if (is_cached_coherent(obj
)) {
716 int i
, npages
= obj
->size
>> PAGE_SHIFT
;
717 struct page
**pages
= omap_obj
->pages
;
720 for (i
= 0; i
< npages
; i
++) {
721 if (!omap_obj
->addrs
[i
]) {
722 omap_obj
->addrs
[i
] = dma_map_page(dev
->dev
, pages
[i
], 0,
723 PAGE_SIZE
, DMA_BIDIRECTIONAL
);
729 unmap_mapping_range(obj
->filp
->f_mapping
, 0,
730 omap_gem_mmap_size(obj
), 1);
735 /* Get physical address for DMA.. if 'remap' is true, and the buffer is not
736 * already contiguous, remap it to pin in physically contiguous memory.. (ie.
739 int omap_gem_get_paddr(struct drm_gem_object
*obj
,
740 dma_addr_t
*paddr
, bool remap
)
742 struct omap_drm_private
*priv
= obj
->dev
->dev_private
;
743 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
746 mutex_lock(&obj
->dev
->struct_mutex
);
748 if (remap
&& is_shmem(obj
) && priv
->has_dmm
) {
749 if (omap_obj
->paddr_cnt
== 0) {
751 uint32_t npages
= obj
->size
>> PAGE_SHIFT
;
752 enum tiler_fmt fmt
= gem2fmt(omap_obj
->flags
);
753 struct tiler_block
*block
;
755 BUG_ON(omap_obj
->block
);
757 ret
= get_pages(obj
, &pages
);
761 if (omap_obj
->flags
& OMAP_BO_TILED
) {
762 block
= tiler_reserve_2d(fmt
,
764 omap_obj
->height
, 0);
766 block
= tiler_reserve_1d(obj
->size
);
770 ret
= PTR_ERR(block
);
771 dev_err(obj
->dev
->dev
,
772 "could not remap: %d (%d)\n", ret
, fmt
);
776 /* TODO: enable async refill.. */
777 ret
= tiler_pin(block
, pages
, npages
,
778 omap_obj
->roll
, true);
780 tiler_release(block
);
781 dev_err(obj
->dev
->dev
,
782 "could not pin: %d\n", ret
);
786 omap_obj
->paddr
= tiler_ssptr(block
);
787 omap_obj
->block
= block
;
789 DBG("got paddr: %pad", &omap_obj
->paddr
);
792 omap_obj
->paddr_cnt
++;
794 *paddr
= omap_obj
->paddr
;
795 } else if (omap_obj
->flags
& OMAP_BO_DMA
) {
796 *paddr
= omap_obj
->paddr
;
803 mutex_unlock(&obj
->dev
->struct_mutex
);
808 /* Release physical address, when DMA is no longer being performed.. this
809 * could potentially unpin and unmap buffers from TILER
811 void omap_gem_put_paddr(struct drm_gem_object
*obj
)
813 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
816 mutex_lock(&obj
->dev
->struct_mutex
);
817 if (omap_obj
->paddr_cnt
> 0) {
818 omap_obj
->paddr_cnt
--;
819 if (omap_obj
->paddr_cnt
== 0) {
820 ret
= tiler_unpin(omap_obj
->block
);
822 dev_err(obj
->dev
->dev
,
823 "could not unpin pages: %d\n", ret
);
825 ret
= tiler_release(omap_obj
->block
);
827 dev_err(obj
->dev
->dev
,
828 "could not release unmap: %d\n", ret
);
831 omap_obj
->block
= NULL
;
835 mutex_unlock(&obj
->dev
->struct_mutex
);
838 /* Get rotated scanout address (only valid if already pinned), at the
839 * specified orientation and x,y offset from top-left corner of buffer
840 * (only valid for tiled 2d buffers)
842 int omap_gem_rotated_paddr(struct drm_gem_object
*obj
, uint32_t orient
,
843 int x
, int y
, dma_addr_t
*paddr
)
845 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
848 mutex_lock(&obj
->dev
->struct_mutex
);
849 if ((omap_obj
->paddr_cnt
> 0) && omap_obj
->block
&&
850 (omap_obj
->flags
& OMAP_BO_TILED
)) {
851 *paddr
= tiler_tsptr(omap_obj
->block
, orient
, x
, y
);
854 mutex_unlock(&obj
->dev
->struct_mutex
);
858 /* Get tiler stride for the buffer (only valid for 2d tiled buffers) */
859 int omap_gem_tiled_stride(struct drm_gem_object
*obj
, uint32_t orient
)
861 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
863 if (omap_obj
->flags
& OMAP_BO_TILED
)
864 ret
= tiler_stride(gem2fmt(omap_obj
->flags
), orient
);
868 /* acquire pages when needed (for example, for DMA where physically
869 * contiguous buffer is not required
871 static int get_pages(struct drm_gem_object
*obj
, struct page
***pages
)
873 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
876 if (is_shmem(obj
) && !omap_obj
->pages
) {
877 ret
= omap_gem_attach_pages(obj
);
879 dev_err(obj
->dev
->dev
, "could not attach pages\n");
884 /* TODO: even phys-contig.. we should have a list of pages? */
885 *pages
= omap_obj
->pages
;
890 /* if !remap, and we don't have pages backing, then fail, rather than
891 * increasing the pin count (which we don't really do yet anyways,
892 * because we don't support swapping pages back out). And 'remap'
893 * might not be quite the right name, but I wanted to keep it working
894 * similarly to omap_gem_get_paddr(). Note though that mutex is not
895 * aquired if !remap (because this can be called in atomic ctxt),
896 * but probably omap_gem_get_paddr() should be changed to work in the
897 * same way. If !remap, a matching omap_gem_put_pages() call is not
898 * required (and should not be made).
900 int omap_gem_get_pages(struct drm_gem_object
*obj
, struct page
***pages
,
905 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
906 if (!omap_obj
->pages
)
908 *pages
= omap_obj
->pages
;
911 mutex_lock(&obj
->dev
->struct_mutex
);
912 ret
= get_pages(obj
, pages
);
913 mutex_unlock(&obj
->dev
->struct_mutex
);
917 /* release pages when DMA no longer being performed */
918 int omap_gem_put_pages(struct drm_gem_object
*obj
)
920 /* do something here if we dynamically attach/detach pages.. at
921 * least they would no longer need to be pinned if everyone has
922 * released the pages..
927 /* Get kernel virtual address for CPU access.. this more or less only
928 * exists for omap_fbdev. This should be called with struct_mutex
931 void *omap_gem_vaddr(struct drm_gem_object
*obj
)
933 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
934 WARN_ON(!mutex_is_locked(&obj
->dev
->struct_mutex
));
935 if (!omap_obj
->vaddr
) {
937 int ret
= get_pages(obj
, &pages
);
940 omap_obj
->vaddr
= vmap(pages
, obj
->size
>> PAGE_SHIFT
,
941 VM_MAP
, pgprot_writecombine(PAGE_KERNEL
));
943 return omap_obj
->vaddr
;
947 /* re-pin objects in DMM in resume path: */
948 int omap_gem_resume(struct device
*dev
)
950 struct drm_device
*drm_dev
= dev_get_drvdata(dev
);
951 struct omap_drm_private
*priv
= drm_dev
->dev_private
;
952 struct omap_gem_object
*omap_obj
;
955 list_for_each_entry(omap_obj
, &priv
->obj_list
, mm_list
) {
956 if (omap_obj
->block
) {
957 struct drm_gem_object
*obj
= &omap_obj
->base
;
958 uint32_t npages
= obj
->size
>> PAGE_SHIFT
;
959 WARN_ON(!omap_obj
->pages
); /* this can't happen */
960 ret
= tiler_pin(omap_obj
->block
,
961 omap_obj
->pages
, npages
,
962 omap_obj
->roll
, true);
964 dev_err(dev
, "could not repin: %d\n", ret
);
974 #ifdef CONFIG_DEBUG_FS
975 void omap_gem_describe(struct drm_gem_object
*obj
, struct seq_file
*m
)
977 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
980 off
= drm_vma_node_start(&obj
->vma_node
);
982 seq_printf(m
, "%08x: %2d (%2d) %08llx %pad (%2d) %p %4d",
983 omap_obj
->flags
, obj
->name
, obj
->refcount
.refcount
.counter
,
984 off
, &omap_obj
->paddr
, omap_obj
->paddr_cnt
,
985 omap_obj
->vaddr
, omap_obj
->roll
);
987 if (omap_obj
->flags
& OMAP_BO_TILED
) {
988 seq_printf(m
, " %dx%d", omap_obj
->width
, omap_obj
->height
);
989 if (omap_obj
->block
) {
990 struct tcm_area
*area
= &omap_obj
->block
->area
;
991 seq_printf(m
, " (%dx%d, %dx%d)",
992 area
->p0
.x
, area
->p0
.y
,
993 area
->p1
.x
, area
->p1
.y
);
996 seq_printf(m
, " %d", obj
->size
);
1002 void omap_gem_describe_objects(struct list_head
*list
, struct seq_file
*m
)
1004 struct omap_gem_object
*omap_obj
;
1008 list_for_each_entry(omap_obj
, list
, mm_list
) {
1009 struct drm_gem_object
*obj
= &omap_obj
->base
;
1011 omap_gem_describe(obj
, m
);
1016 seq_printf(m
, "Total %d objects, %zu bytes\n", count
, size
);
1020 /* Buffer Synchronization:
1023 struct omap_gem_sync_waiter
{
1024 struct list_head list
;
1025 struct omap_gem_object
*omap_obj
;
1026 enum omap_gem_op op
;
1027 uint32_t read_target
, write_target
;
1028 /* notify called w/ sync_lock held */
1029 void (*notify
)(void *arg
);
1033 /* list of omap_gem_sync_waiter.. the notify fxn gets called back when
1034 * the read and/or write target count is achieved which can call a user
1035 * callback (ex. to kick 3d and/or 2d), wakeup blocked task (prep for
1038 static LIST_HEAD(waiters
);
1040 static inline bool is_waiting(struct omap_gem_sync_waiter
*waiter
)
1042 struct omap_gem_object
*omap_obj
= waiter
->omap_obj
;
1043 if ((waiter
->op
& OMAP_GEM_READ
) &&
1044 (omap_obj
->sync
->write_complete
< waiter
->write_target
))
1046 if ((waiter
->op
& OMAP_GEM_WRITE
) &&
1047 (omap_obj
->sync
->read_complete
< waiter
->read_target
))
1052 /* macro for sync debug.. */
1054 #define SYNC(fmt, ...) do { if (SYNCDBG) \
1055 printk(KERN_ERR "%s:%d: "fmt"\n", \
1056 __func__, __LINE__, ##__VA_ARGS__); \
1060 static void sync_op_update(void)
1062 struct omap_gem_sync_waiter
*waiter
, *n
;
1063 list_for_each_entry_safe(waiter
, n
, &waiters
, list
) {
1064 if (!is_waiting(waiter
)) {
1065 list_del(&waiter
->list
);
1066 SYNC("notify: %p", waiter
);
1067 waiter
->notify(waiter
->arg
);
1073 static inline int sync_op(struct drm_gem_object
*obj
,
1074 enum omap_gem_op op
, bool start
)
1076 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
1079 spin_lock(&sync_lock
);
1081 if (!omap_obj
->sync
) {
1082 omap_obj
->sync
= kzalloc(sizeof(*omap_obj
->sync
), GFP_ATOMIC
);
1083 if (!omap_obj
->sync
) {
1090 if (op
& OMAP_GEM_READ
)
1091 omap_obj
->sync
->read_pending
++;
1092 if (op
& OMAP_GEM_WRITE
)
1093 omap_obj
->sync
->write_pending
++;
1095 if (op
& OMAP_GEM_READ
)
1096 omap_obj
->sync
->read_complete
++;
1097 if (op
& OMAP_GEM_WRITE
)
1098 omap_obj
->sync
->write_complete
++;
1103 spin_unlock(&sync_lock
);
1108 /* it is a bit lame to handle updates in this sort of polling way, but
1109 * in case of PVR, the GPU can directly update read/write complete
1110 * values, and not really tell us which ones it updated.. this also
1111 * means that sync_lock is not quite sufficient. So we'll need to
1112 * do something a bit better when it comes time to add support for
1115 void omap_gem_op_update(void)
1117 spin_lock(&sync_lock
);
1119 spin_unlock(&sync_lock
);
1122 /* mark the start of read and/or write operation */
1123 int omap_gem_op_start(struct drm_gem_object
*obj
, enum omap_gem_op op
)
1125 return sync_op(obj
, op
, true);
1128 int omap_gem_op_finish(struct drm_gem_object
*obj
, enum omap_gem_op op
)
1130 return sync_op(obj
, op
, false);
1133 static DECLARE_WAIT_QUEUE_HEAD(sync_event
);
1135 static void sync_notify(void *arg
)
1137 struct task_struct
**waiter_task
= arg
;
1138 *waiter_task
= NULL
;
1139 wake_up_all(&sync_event
);
1142 int omap_gem_op_sync(struct drm_gem_object
*obj
, enum omap_gem_op op
)
1144 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
1146 if (omap_obj
->sync
) {
1147 struct task_struct
*waiter_task
= current
;
1148 struct omap_gem_sync_waiter
*waiter
=
1149 kzalloc(sizeof(*waiter
), GFP_KERNEL
);
1154 waiter
->omap_obj
= omap_obj
;
1156 waiter
->read_target
= omap_obj
->sync
->read_pending
;
1157 waiter
->write_target
= omap_obj
->sync
->write_pending
;
1158 waiter
->notify
= sync_notify
;
1159 waiter
->arg
= &waiter_task
;
1161 spin_lock(&sync_lock
);
1162 if (is_waiting(waiter
)) {
1163 SYNC("waited: %p", waiter
);
1164 list_add_tail(&waiter
->list
, &waiters
);
1165 spin_unlock(&sync_lock
);
1166 ret
= wait_event_interruptible(sync_event
,
1167 (waiter_task
== NULL
));
1168 spin_lock(&sync_lock
);
1170 SYNC("interrupted: %p", waiter
);
1171 /* we were interrupted */
1172 list_del(&waiter
->list
);
1175 /* freed in sync_op_update() */
1179 spin_unlock(&sync_lock
);
1185 /* call fxn(arg), either synchronously or asynchronously if the op
1186 * is currently blocked.. fxn() can be called from any context
1188 * (TODO for now fxn is called back from whichever context calls
1189 * omap_gem_op_update().. but this could be better defined later
1192 * TODO more code in common w/ _sync()..
1194 int omap_gem_op_async(struct drm_gem_object
*obj
, enum omap_gem_op op
,
1195 void (*fxn
)(void *arg
), void *arg
)
1197 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
1198 if (omap_obj
->sync
) {
1199 struct omap_gem_sync_waiter
*waiter
=
1200 kzalloc(sizeof(*waiter
), GFP_ATOMIC
);
1205 waiter
->omap_obj
= omap_obj
;
1207 waiter
->read_target
= omap_obj
->sync
->read_pending
;
1208 waiter
->write_target
= omap_obj
->sync
->write_pending
;
1209 waiter
->notify
= fxn
;
1212 spin_lock(&sync_lock
);
1213 if (is_waiting(waiter
)) {
1214 SYNC("waited: %p", waiter
);
1215 list_add_tail(&waiter
->list
, &waiters
);
1216 spin_unlock(&sync_lock
);
1220 spin_unlock(&sync_lock
);
1231 /* special API so PVR can update the buffer to use a sync-object allocated
1232 * from it's sync-obj heap. Only used for a newly allocated (from PVR's
1233 * perspective) sync-object, so we overwrite the new syncobj w/ values
1234 * from the already allocated syncobj (if there is one)
1236 int omap_gem_set_sync_object(struct drm_gem_object
*obj
, void *syncobj
)
1238 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
1241 spin_lock(&sync_lock
);
1243 if ((omap_obj
->flags
& OMAP_BO_EXT_SYNC
) && !syncobj
) {
1244 /* clearing a previously set syncobj */
1245 syncobj
= kmemdup(omap_obj
->sync
, sizeof(*omap_obj
->sync
),
1251 omap_obj
->flags
&= ~OMAP_BO_EXT_SYNC
;
1252 omap_obj
->sync
= syncobj
;
1253 } else if (syncobj
&& !(omap_obj
->flags
& OMAP_BO_EXT_SYNC
)) {
1254 /* replacing an existing syncobj */
1255 if (omap_obj
->sync
) {
1256 memcpy(syncobj
, omap_obj
->sync
, sizeof(*omap_obj
->sync
));
1257 kfree(omap_obj
->sync
);
1259 omap_obj
->flags
|= OMAP_BO_EXT_SYNC
;
1260 omap_obj
->sync
= syncobj
;
1264 spin_unlock(&sync_lock
);
1268 /* don't call directly.. called from GEM core when it is time to actually
1271 void omap_gem_free_object(struct drm_gem_object
*obj
)
1273 struct drm_device
*dev
= obj
->dev
;
1274 struct omap_drm_private
*priv
= dev
->dev_private
;
1275 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
1279 WARN_ON(!mutex_is_locked(&dev
->struct_mutex
));
1281 spin_lock(&priv
->list_lock
);
1282 list_del(&omap_obj
->mm_list
);
1283 spin_unlock(&priv
->list_lock
);
1285 drm_gem_free_mmap_offset(obj
);
1287 /* this means the object is still pinned.. which really should
1288 * not happen. I think..
1290 WARN_ON(omap_obj
->paddr_cnt
> 0);
1292 /* don't free externally allocated backing memory */
1293 if (!(omap_obj
->flags
& OMAP_BO_EXT_MEM
)) {
1294 if (omap_obj
->pages
)
1295 omap_gem_detach_pages(obj
);
1297 if (!is_shmem(obj
)) {
1298 dma_free_writecombine(dev
->dev
, obj
->size
,
1299 omap_obj
->vaddr
, omap_obj
->paddr
);
1300 } else if (omap_obj
->vaddr
) {
1301 vunmap(omap_obj
->vaddr
);
1305 /* don't free externally allocated syncobj */
1306 if (!(omap_obj
->flags
& OMAP_BO_EXT_SYNC
))
1307 kfree(omap_obj
->sync
);
1309 drm_gem_object_release(obj
);
1314 /* convenience method to construct a GEM buffer object, and userspace handle */
1315 int omap_gem_new_handle(struct drm_device
*dev
, struct drm_file
*file
,
1316 union omap_gem_size gsize
, uint32_t flags
, uint32_t *handle
)
1318 struct drm_gem_object
*obj
;
1321 obj
= omap_gem_new(dev
, gsize
, flags
);
1325 ret
= drm_gem_handle_create(file
, obj
, handle
);
1327 drm_gem_object_release(obj
);
1328 kfree(obj
); /* TODO isn't there a dtor to call? just copying i915 */
1332 /* drop reference from allocate - handle holds it now */
1333 drm_gem_object_unreference_unlocked(obj
);
1338 /* GEM buffer object constructor */
1339 struct drm_gem_object
*omap_gem_new(struct drm_device
*dev
,
1340 union omap_gem_size gsize
, uint32_t flags
)
1342 struct omap_drm_private
*priv
= dev
->dev_private
;
1343 struct omap_gem_object
*omap_obj
;
1344 struct drm_gem_object
*obj
= NULL
;
1345 struct address_space
*mapping
;
1349 if (flags
& OMAP_BO_TILED
) {
1351 dev_err(dev
->dev
, "Tiled buffers require DMM\n");
1355 /* tiled buffers are always shmem paged backed.. when they are
1356 * scanned out, they are remapped into DMM/TILER
1358 flags
&= ~OMAP_BO_SCANOUT
;
1360 /* currently don't allow cached buffers.. there is some caching
1361 * stuff that needs to be handled better
1363 flags
&= ~(OMAP_BO_CACHED
|OMAP_BO_WC
|OMAP_BO_UNCACHED
);
1364 flags
|= tiler_get_cpu_cache_flags();
1366 /* align dimensions to slot boundaries... */
1367 tiler_align(gem2fmt(flags
),
1368 &gsize
.tiled
.width
, &gsize
.tiled
.height
);
1370 /* ...and calculate size based on aligned dimensions */
1371 size
= tiler_size(gem2fmt(flags
),
1372 gsize
.tiled
.width
, gsize
.tiled
.height
);
1374 size
= PAGE_ALIGN(gsize
.bytes
);
1377 omap_obj
= kzalloc(sizeof(*omap_obj
), GFP_KERNEL
);
1381 obj
= &omap_obj
->base
;
1383 if ((flags
& OMAP_BO_SCANOUT
) && !priv
->has_dmm
) {
1384 /* attempt to allocate contiguous memory if we don't
1385 * have DMM for remappign discontiguous buffers
1387 omap_obj
->vaddr
= dma_alloc_writecombine(dev
->dev
, size
,
1388 &omap_obj
->paddr
, GFP_KERNEL
);
1389 if (!omap_obj
->vaddr
) {
1395 flags
|= OMAP_BO_DMA
;
1398 spin_lock(&priv
->list_lock
);
1399 list_add(&omap_obj
->mm_list
, &priv
->obj_list
);
1400 spin_unlock(&priv
->list_lock
);
1402 omap_obj
->flags
= flags
;
1404 if (flags
& OMAP_BO_TILED
) {
1405 omap_obj
->width
= gsize
.tiled
.width
;
1406 omap_obj
->height
= gsize
.tiled
.height
;
1409 if (flags
& (OMAP_BO_DMA
|OMAP_BO_EXT_MEM
)) {
1410 drm_gem_private_object_init(dev
, obj
, size
);
1412 ret
= drm_gem_object_init(dev
, obj
, size
);
1416 mapping
= file_inode(obj
->filp
)->i_mapping
;
1417 mapping_set_gfp_mask(mapping
, GFP_USER
| __GFP_DMA32
);
1424 omap_gem_free_object(obj
);
1429 /* init/cleanup.. if DMM is used, we need to set some stuff up.. */
1430 void omap_gem_init(struct drm_device
*dev
)
1432 struct omap_drm_private
*priv
= dev
->dev_private
;
1433 const enum tiler_fmt fmts
[] = {
1434 TILFMT_8BIT
, TILFMT_16BIT
, TILFMT_32BIT
1438 if (!dmm_is_available()) {
1439 /* DMM only supported on OMAP4 and later, so this isn't fatal */
1440 dev_warn(dev
->dev
, "DMM not available, disable DMM support\n");
1444 usergart
= kcalloc(3, sizeof(*usergart
), GFP_KERNEL
);
1448 /* reserve 4k aligned/wide regions for userspace mappings: */
1449 for (i
= 0; i
< ARRAY_SIZE(fmts
); i
++) {
1450 uint16_t h
= 1, w
= PAGE_SIZE
>> i
;
1451 tiler_align(fmts
[i
], &w
, &h
);
1452 /* note: since each region is 1 4kb page wide, and minimum
1453 * number of rows, the height ends up being the same as the
1454 * # of pages in the region
1456 usergart
[i
].height
= h
;
1457 usergart
[i
].height_shift
= ilog2(h
);
1458 usergart
[i
].stride_pfn
= tiler_stride(fmts
[i
], 0) >> PAGE_SHIFT
;
1459 usergart
[i
].slot_shift
= ilog2((PAGE_SIZE
/ h
) >> i
);
1460 for (j
= 0; j
< NUM_USERGART_ENTRIES
; j
++) {
1461 struct usergart_entry
*entry
= &usergart
[i
].entry
[j
];
1462 struct tiler_block
*block
=
1463 tiler_reserve_2d(fmts
[i
], w
, h
,
1465 if (IS_ERR(block
)) {
1467 "reserve failed: %d, %d, %ld\n",
1468 i
, j
, PTR_ERR(block
));
1471 entry
->paddr
= tiler_ssptr(block
);
1472 entry
->block
= block
;
1474 DBG("%d:%d: %dx%d: paddr=%pad stride=%d", i
, j
, w
, h
,
1476 usergart
[i
].stride_pfn
<< PAGE_SHIFT
);
1480 priv
->has_dmm
= true;
1483 void omap_gem_deinit(struct drm_device
*dev
)
1485 /* I believe we can rely on there being no more outstanding GEM
1486 * objects which could depend on usergart/dmm at this point.