1 /**************************************************************************
3 * Copyright 2006 Tungsten Graphics, Inc., Bismarck, ND., USA.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
27 **************************************************************************/
30 * Generic simple memory manager implementation. Intended to be used as a base
31 * class implementation for more advanced memory managers.
33 * Note that the algorithm used is quite simple and there might be substantial
34 * performance gains if a smarter free list is implemented. Currently it is just an
35 * unordered stack of free regions. This could easily be improved if an RB-tree
36 * is used instead. At least if we expect heavy fragmentation.
38 * Aligned allocations can also see improvement.
41 * Thomas Hellström <thomas-at-tungstengraphics-dot-com>
46 #include <linux/slab.h>
47 #include <linux/seq_file.h>
49 #define MM_UNUSED_TARGET 4
51 static struct drm_mm_node
*drm_mm_kmalloc(struct drm_mm
*mm
, int atomic
)
53 struct drm_mm_node
*child
;
56 child
= kzalloc(sizeof(*child
), GFP_ATOMIC
);
58 child
= kzalloc(sizeof(*child
), GFP_KERNEL
);
60 if (unlikely(child
== NULL
)) {
61 spin_lock(&mm
->unused_lock
);
62 if (list_empty(&mm
->unused_nodes
))
66 list_entry(mm
->unused_nodes
.next
,
67 struct drm_mm_node
, node_list
);
68 list_del(&child
->node_list
);
71 spin_unlock(&mm
->unused_lock
);
76 /* drm_mm_pre_get() - pre allocate drm_mm_node structure
77 * drm_mm: memory manager struct we are pre-allocating for
79 * Returns 0 on success or -ENOMEM if allocation fails.
81 int drm_mm_pre_get(struct drm_mm
*mm
)
83 struct drm_mm_node
*node
;
85 spin_lock(&mm
->unused_lock
);
86 while (mm
->num_unused
< MM_UNUSED_TARGET
) {
87 spin_unlock(&mm
->unused_lock
);
88 node
= kzalloc(sizeof(*node
), GFP_KERNEL
);
89 spin_lock(&mm
->unused_lock
);
91 if (unlikely(node
== NULL
)) {
92 int ret
= (mm
->num_unused
< 2) ? -ENOMEM
: 0;
93 spin_unlock(&mm
->unused_lock
);
97 list_add_tail(&node
->node_list
, &mm
->unused_nodes
);
99 spin_unlock(&mm
->unused_lock
);
102 EXPORT_SYMBOL(drm_mm_pre_get
);
104 static inline unsigned long drm_mm_hole_node_start(struct drm_mm_node
*hole_node
)
106 return hole_node
->start
+ hole_node
->size
;
109 static inline unsigned long drm_mm_hole_node_end(struct drm_mm_node
*hole_node
)
111 struct drm_mm_node
*next_node
=
112 list_entry(hole_node
->node_list
.next
, struct drm_mm_node
,
115 return next_node
->start
;
118 static void drm_mm_insert_helper(struct drm_mm_node
*hole_node
,
119 struct drm_mm_node
*node
,
120 unsigned long size
, unsigned alignment
)
122 struct drm_mm
*mm
= hole_node
->mm
;
123 unsigned long tmp
= 0, wasted
= 0;
124 unsigned long hole_start
= drm_mm_hole_node_start(hole_node
);
125 unsigned long hole_end
= drm_mm_hole_node_end(hole_node
);
127 BUG_ON(!hole_node
->hole_follows
|| node
->allocated
);
130 tmp
= hole_start
% alignment
;
133 hole_node
->hole_follows
= 0;
134 list_del_init(&hole_node
->hole_stack
);
136 wasted
= alignment
- tmp
;
138 node
->start
= hole_start
+ wasted
;
143 INIT_LIST_HEAD(&node
->hole_stack
);
144 list_add(&node
->node_list
, &hole_node
->node_list
);
146 BUG_ON(node
->start
+ node
->size
> hole_end
);
148 if (node
->start
+ node
->size
< hole_end
) {
149 list_add(&node
->hole_stack
, &mm
->hole_stack
);
150 node
->hole_follows
= 1;
152 node
->hole_follows
= 0;
156 struct drm_mm_node
*drm_mm_get_block_generic(struct drm_mm_node
*hole_node
,
161 struct drm_mm_node
*node
;
163 node
= drm_mm_kmalloc(hole_node
->mm
, atomic
);
164 if (unlikely(node
== NULL
))
167 drm_mm_insert_helper(hole_node
, node
, size
, alignment
);
171 EXPORT_SYMBOL(drm_mm_get_block_generic
);
174 * Search for free space and insert a preallocated memory node. Returns
175 * -ENOSPC if no suitable free area is available. The preallocated memory node
178 int drm_mm_insert_node(struct drm_mm
*mm
, struct drm_mm_node
*node
,
179 unsigned long size
, unsigned alignment
)
181 struct drm_mm_node
*hole_node
;
183 hole_node
= drm_mm_search_free(mm
, size
, alignment
, 0);
187 drm_mm_insert_helper(hole_node
, node
, size
, alignment
);
191 EXPORT_SYMBOL(drm_mm_insert_node
);
193 static void drm_mm_insert_helper_range(struct drm_mm_node
*hole_node
,
194 struct drm_mm_node
*node
,
195 unsigned long size
, unsigned alignment
,
196 unsigned long start
, unsigned long end
)
198 struct drm_mm
*mm
= hole_node
->mm
;
199 unsigned long tmp
= 0, wasted
= 0;
200 unsigned long hole_start
= drm_mm_hole_node_start(hole_node
);
201 unsigned long hole_end
= drm_mm_hole_node_end(hole_node
);
203 BUG_ON(!hole_node
->hole_follows
|| node
->allocated
);
205 if (hole_start
< start
)
206 wasted
+= start
- hole_start
;
208 tmp
= (hole_start
+ wasted
) % alignment
;
211 wasted
+= alignment
- tmp
;
214 hole_node
->hole_follows
= 0;
215 list_del_init(&hole_node
->hole_stack
);
218 node
->start
= hole_start
+ wasted
;
223 INIT_LIST_HEAD(&node
->hole_stack
);
224 list_add(&node
->node_list
, &hole_node
->node_list
);
226 BUG_ON(node
->start
+ node
->size
> hole_end
);
227 BUG_ON(node
->start
+ node
->size
> end
);
229 if (node
->start
+ node
->size
< hole_end
) {
230 list_add(&node
->hole_stack
, &mm
->hole_stack
);
231 node
->hole_follows
= 1;
233 node
->hole_follows
= 0;
237 struct drm_mm_node
*drm_mm_get_block_range_generic(struct drm_mm_node
*hole_node
,
244 struct drm_mm_node
*node
;
246 node
= drm_mm_kmalloc(hole_node
->mm
, atomic
);
247 if (unlikely(node
== NULL
))
250 drm_mm_insert_helper_range(hole_node
, node
, size
, alignment
,
255 EXPORT_SYMBOL(drm_mm_get_block_range_generic
);
258 * Search for free space and insert a preallocated memory node. Returns
259 * -ENOSPC if no suitable free area is available. This is for range
260 * restricted allocations. The preallocated memory node must be cleared.
262 int drm_mm_insert_node_in_range(struct drm_mm
*mm
, struct drm_mm_node
*node
,
263 unsigned long size
, unsigned alignment
,
264 unsigned long start
, unsigned long end
)
266 struct drm_mm_node
*hole_node
;
268 hole_node
= drm_mm_search_free_in_range(mm
, size
, alignment
,
273 drm_mm_insert_helper_range(hole_node
, node
, size
, alignment
,
278 EXPORT_SYMBOL(drm_mm_insert_node_in_range
);
281 * Remove a memory node from the allocator.
283 void drm_mm_remove_node(struct drm_mm_node
*node
)
285 struct drm_mm
*mm
= node
->mm
;
286 struct drm_mm_node
*prev_node
;
288 BUG_ON(node
->scanned_block
|| node
->scanned_prev_free
289 || node
->scanned_next_free
);
292 list_entry(node
->node_list
.prev
, struct drm_mm_node
, node_list
);
294 if (node
->hole_follows
) {
295 BUG_ON(drm_mm_hole_node_start(node
)
296 == drm_mm_hole_node_end(node
));
297 list_del(&node
->hole_stack
);
299 BUG_ON(drm_mm_hole_node_start(node
)
300 != drm_mm_hole_node_end(node
));
302 if (!prev_node
->hole_follows
) {
303 prev_node
->hole_follows
= 1;
304 list_add(&prev_node
->hole_stack
, &mm
->hole_stack
);
306 list_move(&prev_node
->hole_stack
, &mm
->hole_stack
);
308 list_del(&node
->node_list
);
311 EXPORT_SYMBOL(drm_mm_remove_node
);
314 * Remove a memory node from the allocator and free the allocated struct
315 * drm_mm_node. Only to be used on a struct drm_mm_node obtained by one of the
316 * drm_mm_get_block functions.
318 void drm_mm_put_block(struct drm_mm_node
*node
)
321 struct drm_mm
*mm
= node
->mm
;
323 drm_mm_remove_node(node
);
325 spin_lock(&mm
->unused_lock
);
326 if (mm
->num_unused
< MM_UNUSED_TARGET
) {
327 list_add(&node
->node_list
, &mm
->unused_nodes
);
331 spin_unlock(&mm
->unused_lock
);
333 EXPORT_SYMBOL(drm_mm_put_block
);
335 static int check_free_hole(unsigned long start
, unsigned long end
,
336 unsigned long size
, unsigned alignment
)
340 if (end
- start
< size
)
344 unsigned tmp
= start
% alignment
;
346 wasted
= alignment
- tmp
;
349 if (end
>= start
+ size
+ wasted
) {
356 struct drm_mm_node
*drm_mm_search_free(const struct drm_mm
*mm
,
358 unsigned alignment
, int best_match
)
360 struct drm_mm_node
*entry
;
361 struct drm_mm_node
*best
;
362 unsigned long best_size
;
364 BUG_ON(mm
->scanned_blocks
);
369 list_for_each_entry(entry
, &mm
->hole_stack
, hole_stack
) {
370 BUG_ON(!entry
->hole_follows
);
371 if (!check_free_hole(drm_mm_hole_node_start(entry
),
372 drm_mm_hole_node_end(entry
),
379 if (entry
->size
< best_size
) {
381 best_size
= entry
->size
;
387 EXPORT_SYMBOL(drm_mm_search_free
);
389 struct drm_mm_node
*drm_mm_search_free_in_range(const struct drm_mm
*mm
,
396 struct drm_mm_node
*entry
;
397 struct drm_mm_node
*best
;
398 unsigned long best_size
;
400 BUG_ON(mm
->scanned_blocks
);
405 list_for_each_entry(entry
, &mm
->hole_stack
, hole_stack
) {
406 unsigned long adj_start
= drm_mm_hole_node_start(entry
) < start
?
407 start
: drm_mm_hole_node_start(entry
);
408 unsigned long adj_end
= drm_mm_hole_node_end(entry
) > end
?
409 end
: drm_mm_hole_node_end(entry
);
411 BUG_ON(!entry
->hole_follows
);
412 if (!check_free_hole(adj_start
, adj_end
, size
, alignment
))
418 if (entry
->size
< best_size
) {
420 best_size
= entry
->size
;
426 EXPORT_SYMBOL(drm_mm_search_free_in_range
);
429 * Moves an allocation. To be used with embedded struct drm_mm_node.
431 void drm_mm_replace_node(struct drm_mm_node
*old
, struct drm_mm_node
*new)
433 list_replace(&old
->node_list
, &new->node_list
);
434 list_replace(&old
->hole_stack
, &new->hole_stack
);
435 new->hole_follows
= old
->hole_follows
;
437 new->start
= old
->start
;
438 new->size
= old
->size
;
443 EXPORT_SYMBOL(drm_mm_replace_node
);
446 * Initializa lru scanning.
448 * This simply sets up the scanning routines with the parameters for the desired
451 * Warning: As long as the scan list is non-empty, no other operations than
452 * adding/removing nodes to/from the scan list are allowed.
454 void drm_mm_init_scan(struct drm_mm
*mm
, unsigned long size
,
457 mm
->scan_alignment
= alignment
;
458 mm
->scan_size
= size
;
459 mm
->scanned_blocks
= 0;
460 mm
->scan_hit_start
= 0;
461 mm
->scan_hit_size
= 0;
462 mm
->scan_check_range
= 0;
463 mm
->prev_scanned_node
= NULL
;
465 EXPORT_SYMBOL(drm_mm_init_scan
);
468 * Initializa lru scanning.
470 * This simply sets up the scanning routines with the parameters for the desired
471 * hole. This version is for range-restricted scans.
473 * Warning: As long as the scan list is non-empty, no other operations than
474 * adding/removing nodes to/from the scan list are allowed.
476 void drm_mm_init_scan_with_range(struct drm_mm
*mm
, unsigned long size
,
481 mm
->scan_alignment
= alignment
;
482 mm
->scan_size
= size
;
483 mm
->scanned_blocks
= 0;
484 mm
->scan_hit_start
= 0;
485 mm
->scan_hit_size
= 0;
486 mm
->scan_start
= start
;
488 mm
->scan_check_range
= 1;
489 mm
->prev_scanned_node
= NULL
;
491 EXPORT_SYMBOL(drm_mm_init_scan_with_range
);
494 * Add a node to the scan list that might be freed to make space for the desired
497 * Returns non-zero, if a hole has been found, zero otherwise.
499 int drm_mm_scan_add_block(struct drm_mm_node
*node
)
501 struct drm_mm
*mm
= node
->mm
;
502 struct drm_mm_node
*prev_node
;
503 unsigned long hole_start
, hole_end
;
504 unsigned long adj_start
;
505 unsigned long adj_end
;
507 mm
->scanned_blocks
++;
509 BUG_ON(node
->scanned_block
);
510 node
->scanned_block
= 1;
512 prev_node
= list_entry(node
->node_list
.prev
, struct drm_mm_node
,
515 node
->scanned_preceeds_hole
= prev_node
->hole_follows
;
516 prev_node
->hole_follows
= 1;
517 list_del(&node
->node_list
);
518 node
->node_list
.prev
= &prev_node
->node_list
;
519 node
->node_list
.next
= &mm
->prev_scanned_node
->node_list
;
520 mm
->prev_scanned_node
= node
;
522 hole_start
= drm_mm_hole_node_start(prev_node
);
523 hole_end
= drm_mm_hole_node_end(prev_node
);
524 if (mm
->scan_check_range
) {
525 adj_start
= hole_start
< mm
->scan_start
?
526 mm
->scan_start
: hole_start
;
527 adj_end
= hole_end
> mm
->scan_end
?
528 mm
->scan_end
: hole_end
;
530 adj_start
= hole_start
;
534 if (check_free_hole(adj_start
, adj_end
,
535 mm
->scan_size
, mm
->scan_alignment
)) {
536 mm
->scan_hit_start
= hole_start
;
537 mm
->scan_hit_size
= hole_end
;
544 EXPORT_SYMBOL(drm_mm_scan_add_block
);
547 * Remove a node from the scan list.
549 * Nodes _must_ be removed in the exact same order from the scan list as they
550 * have been added, otherwise the internal state of the memory manager will be
553 * When the scan list is empty, the selected memory nodes can be freed. An
554 * immediately following drm_mm_search_free with best_match = 0 will then return
555 * the just freed block (because its at the top of the free_stack list).
557 * Returns one if this block should be evicted, zero otherwise. Will always
558 * return zero when no hole has been found.
560 int drm_mm_scan_remove_block(struct drm_mm_node
*node
)
562 struct drm_mm
*mm
= node
->mm
;
563 struct drm_mm_node
*prev_node
;
565 mm
->scanned_blocks
--;
567 BUG_ON(!node
->scanned_block
);
568 node
->scanned_block
= 0;
570 prev_node
= list_entry(node
->node_list
.prev
, struct drm_mm_node
,
573 prev_node
->hole_follows
= node
->scanned_preceeds_hole
;
574 INIT_LIST_HEAD(&node
->node_list
);
575 list_add(&node
->node_list
, &prev_node
->node_list
);
577 /* Only need to check for containement because start&size for the
578 * complete resulting free block (not just the desired part) is
580 if (node
->start
>= mm
->scan_hit_start
&&
581 node
->start
+ node
->size
582 <= mm
->scan_hit_start
+ mm
->scan_hit_size
) {
588 EXPORT_SYMBOL(drm_mm_scan_remove_block
);
590 int drm_mm_clean(struct drm_mm
* mm
)
592 struct list_head
*head
= &mm
->head_node
.node_list
;
594 return (head
->next
->next
== head
);
596 EXPORT_SYMBOL(drm_mm_clean
);
598 int drm_mm_init(struct drm_mm
* mm
, unsigned long start
, unsigned long size
)
600 INIT_LIST_HEAD(&mm
->hole_stack
);
601 INIT_LIST_HEAD(&mm
->unused_nodes
);
603 mm
->scanned_blocks
= 0;
604 spin_lock_init(&mm
->unused_lock
);
606 /* Clever trick to avoid a special case in the free hole tracking. */
607 INIT_LIST_HEAD(&mm
->head_node
.node_list
);
608 INIT_LIST_HEAD(&mm
->head_node
.hole_stack
);
609 mm
->head_node
.hole_follows
= 1;
610 mm
->head_node
.scanned_block
= 0;
611 mm
->head_node
.scanned_prev_free
= 0;
612 mm
->head_node
.scanned_next_free
= 0;
613 mm
->head_node
.mm
= mm
;
614 mm
->head_node
.start
= start
+ size
;
615 mm
->head_node
.size
= start
- mm
->head_node
.start
;
616 list_add_tail(&mm
->head_node
.hole_stack
, &mm
->hole_stack
);
620 EXPORT_SYMBOL(drm_mm_init
);
622 void drm_mm_takedown(struct drm_mm
* mm
)
624 struct drm_mm_node
*entry
, *next
;
626 if (!list_empty(&mm
->head_node
.node_list
)) {
627 DRM_ERROR("Memory manager not clean. Delaying takedown\n");
631 spin_lock(&mm
->unused_lock
);
632 list_for_each_entry_safe(entry
, next
, &mm
->unused_nodes
, node_list
) {
633 list_del(&entry
->node_list
);
637 spin_unlock(&mm
->unused_lock
);
639 BUG_ON(mm
->num_unused
!= 0);
641 EXPORT_SYMBOL(drm_mm_takedown
);
643 void drm_mm_debug_table(struct drm_mm
*mm
, const char *prefix
)
645 struct drm_mm_node
*entry
;
646 unsigned long total_used
= 0, total_free
= 0, total
= 0;
647 unsigned long hole_start
, hole_end
, hole_size
;
649 hole_start
= drm_mm_hole_node_start(&mm
->head_node
);
650 hole_end
= drm_mm_hole_node_end(&mm
->head_node
);
651 hole_size
= hole_end
- hole_start
;
653 printk(KERN_DEBUG
"%s 0x%08lx-0x%08lx: %8lu: free\n",
654 prefix
, hole_start
, hole_end
,
656 total_free
+= hole_size
;
658 drm_mm_for_each_node(entry
, mm
) {
659 printk(KERN_DEBUG
"%s 0x%08lx-0x%08lx: %8lu: used\n",
660 prefix
, entry
->start
, entry
->start
+ entry
->size
,
662 total_used
+= entry
->size
;
664 if (entry
->hole_follows
) {
665 hole_start
= drm_mm_hole_node_start(entry
);
666 hole_end
= drm_mm_hole_node_end(entry
);
667 hole_size
= hole_end
- hole_start
;
668 printk(KERN_DEBUG
"%s 0x%08lx-0x%08lx: %8lu: free\n",
669 prefix
, hole_start
, hole_end
,
671 total_free
+= hole_size
;
674 total
= total_free
+ total_used
;
676 printk(KERN_DEBUG
"%s total: %lu, used %lu free %lu\n", prefix
, total
,
677 total_used
, total_free
);
679 EXPORT_SYMBOL(drm_mm_debug_table
);
681 #if defined(CONFIG_DEBUG_FS)
682 int drm_mm_dump_table(struct seq_file
*m
, struct drm_mm
*mm
)
684 struct drm_mm_node
*entry
;
685 unsigned long total_used
= 0, total_free
= 0, total
= 0;
686 unsigned long hole_start
, hole_end
, hole_size
;
688 hole_start
= drm_mm_hole_node_start(&mm
->head_node
);
689 hole_end
= drm_mm_hole_node_end(&mm
->head_node
);
690 hole_size
= hole_end
- hole_start
;
692 seq_printf(m
, "0x%08lx-0x%08lx: 0x%08lx: free\n",
693 hole_start
, hole_end
, hole_size
);
694 total_free
+= hole_size
;
696 drm_mm_for_each_node(entry
, mm
) {
697 seq_printf(m
, "0x%08lx-0x%08lx: 0x%08lx: used\n",
698 entry
->start
, entry
->start
+ entry
->size
,
700 total_used
+= entry
->size
;
701 if (entry
->hole_follows
) {
702 hole_start
= drm_mm_hole_node_start(entry
);
703 hole_end
= drm_mm_hole_node_end(entry
);
704 hole_size
= hole_end
- hole_start
;
705 seq_printf(m
, "0x%08lx-0x%08lx: 0x%08lx: free\n",
706 hole_start
, hole_end
, hole_size
);
707 total_free
+= hole_size
;
710 total
= total_free
+ total_used
;
712 seq_printf(m
, "total: %lu, used %lu free %lu\n", total
, total_used
, total_free
);
715 EXPORT_SYMBOL(drm_mm_dump_table
);