?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
include: replace linux/module.h with "struct module" wherever possible
[linux-2.6/next.git] / lib / dma-debug.c
blobb49f7b03c6da72f1ca706caa1f840a2d47d910a1
1 /*
2 * Copyright (C) 2008 Advanced Micro Devices, Inc.
3 *
4 * Author: Joerg Roedel <joerg.roedel@amd.com>
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 */
19
20 #include <linux/scatterlist.h>
21 #include <linux/dma-mapping.h>
22 #include <linux/stacktrace.h>
23 #include <linux/dma-debug.h>
24 #include <linux/spinlock.h>
25 #include <linux/debugfs.h>
26 #include <linux/uaccess.h>
27 #include <linux/export.h>
28 #include <linux/device.h>
29 #include <linux/types.h>
30 #include <linux/sched.h>
31 #include <linux/ctype.h>
32 #include <linux/list.h>
33 #include <linux/slab.h>
34
35 #include <asm/sections.h>
36
37 #define HASH_SIZE 1024ULL
38 #define HASH_FN_SHIFT 13
39 #define HASH_FN_MASK (HASH_SIZE - 1)
40
41 enum {
42 dma_debug_single,
43 dma_debug_page,
44 dma_debug_sg,
45 dma_debug_coherent,
46 };
47
48 #define DMA_DEBUG_STACKTRACE_ENTRIES 5
49
50 struct dma_debug_entry {
51 struct list_head list;
52 struct device *dev;
53 int type;
54 phys_addr_t paddr;
55 u64 dev_addr;
56 u64 size;
57 int direction;
58 int sg_call_ents;
59 int sg_mapped_ents;
60 #ifdef CONFIG_STACKTRACE
61 struct stack_trace stacktrace;
62 unsigned long st_entries[DMA_DEBUG_STACKTRACE_ENTRIES];
63 #endif
64 };
65
66 struct hash_bucket {
67 struct list_head list;
68 spinlock_t lock;
69 } ____cacheline_aligned_in_smp;
70
71 /* Hash list to save the allocated dma addresses */
72 static struct hash_bucket dma_entry_hash[HASH_SIZE];
73 /* List of pre-allocated dma_debug_entry's */
74 static LIST_HEAD(free_entries);
75 /* Lock for the list above */
76 static DEFINE_SPINLOCK(free_entries_lock);
77
78 /* Global disable flag - will be set in case of an error */
79 static bool global_disable __read_mostly;
80
81 /* Global error count */
82 static u32 error_count;
83
84 /* Global error show enable*/
85 static u32 show_all_errors __read_mostly;
86 /* Number of errors to show */
87 static u32 show_num_errors = 1;
88
89 static u32 num_free_entries;
90 static u32 min_free_entries;
91 static u32 nr_total_entries;
92
93 /* number of preallocated entries requested by kernel cmdline */
94 static u32 req_entries;
95
96 /* debugfs dentry's for the stuff above */
97 static struct dentry *dma_debug_dent __read_mostly;
98 static struct dentry *global_disable_dent __read_mostly;
99 static struct dentry *error_count_dent __read_mostly;
100 static struct dentry *show_all_errors_dent __read_mostly;
101 static struct dentry *show_num_errors_dent __read_mostly;
102 static struct dentry *num_free_entries_dent __read_mostly;
103 static struct dentry *min_free_entries_dent __read_mostly;
104 static struct dentry *filter_dent __read_mostly;
105
106 /* per-driver filter related state */
107
108 #define NAME_MAX_LEN 64
109
110 static char current_driver_name[NAME_MAX_LEN] __read_mostly;
111 static struct device_driver *current_driver __read_mostly;
112
113 static DEFINE_RWLOCK(driver_name_lock);
114
115 static const char *type2name[4] = { "single", "page",
116 "scather-gather", "coherent" };
117
118 static const char *dir2name[4] = { "DMA_BIDIRECTIONAL", "DMA_TO_DEVICE",
119 "DMA_FROM_DEVICE", "DMA_NONE" };
120
121 /* little merge helper - remove it after the merge window */
122 #ifndef BUS_NOTIFY_UNBOUND_DRIVER
123 #define BUS_NOTIFY_UNBOUND_DRIVER 0x0005
124 #endif
125
126 /*
127 * The access to some variables in this macro is racy. We can't use atomic_t
128 * here because all these variables are exported to debugfs. Some of them even
129 * writeable. This is also the reason why a lock won't help much. But anyway,
130 * the races are no big deal. Here is why:
131 *
132 * error_count: the addition is racy, but the worst thing that can happen is
133 * that we don't count some errors
134 * show_num_errors: the subtraction is racy. Also no big deal because in
135 * worst case this will result in one warning more in the
136 * system log than the user configured. This variable is
137 * writeable via debugfs.
138 */
139 static inline void dump_entry_trace(struct dma_debug_entry *entry)
140 {
141 #ifdef CONFIG_STACKTRACE
142 if (entry) {
143 pr_warning("Mapped at:\n");
144 print_stack_trace(&entry->stacktrace, 0);
145 }
146 #endif
147 }
148
149 static bool driver_filter(struct device *dev)
150 {
151 struct device_driver *drv;
152 unsigned long flags;
153 bool ret;
154
155 /* driver filter off */
156 if (likely(!current_driver_name[0]))
157 return true;
158
159 /* driver filter on and initialized */
160 if (current_driver && dev && dev->driver == current_driver)
161 return true;
162
163 /* driver filter on, but we can't filter on a NULL device... */
164 if (!dev)
165 return false;
166
167 if (current_driver || !current_driver_name[0])
168 return false;
169
170 /* driver filter on but not yet initialized */
171 drv = get_driver(dev->driver);
172 if (!drv)
173 return false;
174
175 /* lock to protect against change of current_driver_name */
176 read_lock_irqsave(&driver_name_lock, flags);
177
178 ret = false;
179 if (drv->name &&
180 strncmp(current_driver_name, drv->name, NAME_MAX_LEN - 1) == 0) {
181 current_driver = drv;
182 ret = true;
183 }
184
185 read_unlock_irqrestore(&driver_name_lock, flags);
186 put_driver(drv);
187
188 return ret;
189 }
190
191 #define err_printk(dev, entry, format, arg...) do { \
192 error_count += 1; \
193 if (driver_filter(dev) && \
194 (show_all_errors || show_num_errors > 0)) { \
195 WARN(1, "%s %s: " format, \
196 dev ? dev_driver_string(dev) : "NULL", \
197 dev ? dev_name(dev) : "NULL", ## arg); \
198 dump_entry_trace(entry); \
199 } \
200 if (!show_all_errors && show_num_errors > 0) \
201 show_num_errors -= 1; \
202 } while (0);
203
204 /*
205 * Hash related functions
206 *
207 * Every DMA-API request is saved into a struct dma_debug_entry. To
208 * have quick access to these structs they are stored into a hash.
209 */
210 static int hash_fn(struct dma_debug_entry *entry)
211 {
212 /*
213 * Hash function is based on the dma address.
214 * We use bits 20-27 here as the index into the hash
215 */
216 return (entry->dev_addr >> HASH_FN_SHIFT) & HASH_FN_MASK;
217 }
218
219 /*
220 * Request exclusive access to a hash bucket for a given dma_debug_entry.
221 */
222 static struct hash_bucket *get_hash_bucket(struct dma_debug_entry *entry,
223 unsigned long *flags)
224 {
225 int idx = hash_fn(entry);
226 unsigned long __flags;
227
228 spin_lock_irqsave(&dma_entry_hash[idx].lock, __flags);
229 *flags = __flags;
230 return &dma_entry_hash[idx];
231 }
232
233 /*
234 * Give up exclusive access to the hash bucket
235 */
236 static void put_hash_bucket(struct hash_bucket *bucket,
237 unsigned long *flags)
238 {
239 unsigned long __flags = *flags;
240
241 spin_unlock_irqrestore(&bucket->lock, __flags);
242 }
243
244 /*
245 * Search a given entry in the hash bucket list
246 */
247 static struct dma_debug_entry *hash_bucket_find(struct hash_bucket *bucket,
248 struct dma_debug_entry *ref)
249 {
250 struct dma_debug_entry *entry, *ret = NULL;
251 int matches = 0, match_lvl, last_lvl = 0;
252
253 list_for_each_entry(entry, &bucket->list, list) {
254 if ((entry->dev_addr != ref->dev_addr) ||
255 (entry->dev != ref->dev))
256 continue;
257
258 /*
259 * Some drivers map the same physical address multiple
260 * times. Without a hardware IOMMU this results in the
261 * same device addresses being put into the dma-debug
262 * hash multiple times too. This can result in false
263 * positives being reported. Therefore we implement a
264 * best-fit algorithm here which returns the entry from
265 * the hash which fits best to the reference value
266 * instead of the first-fit.
267 */
268 matches += 1;
269 match_lvl = 0;
270 entry->size == ref->size ? ++match_lvl : 0;
271 entry->type == ref->type ? ++match_lvl : 0;
272 entry->direction == ref->direction ? ++match_lvl : 0;
273 entry->sg_call_ents == ref->sg_call_ents ? ++match_lvl : 0;
274
275 if (match_lvl == 4) {
276 /* perfect-fit - return the result */
277 return entry;
278 } else if (match_lvl > last_lvl) {
279 /*
280 * We found an entry that fits better then the
281 * previous one
282 */
283 last_lvl = match_lvl;
284 ret = entry;
285 }
286 }
287
288 /*
289 * If we have multiple matches but no perfect-fit, just return
290 * NULL.
291 */
292 ret = (matches == 1) ? ret : NULL;
293
294 return ret;
295 }
296
297 /*
298 * Add an entry to a hash bucket
299 */
300 static void hash_bucket_add(struct hash_bucket *bucket,
301 struct dma_debug_entry *entry)
302 {
303 list_add_tail(&entry->list, &bucket->list);
304 }
305
306 /*
307 * Remove entry from a hash bucket list
308 */
309 static void hash_bucket_del(struct dma_debug_entry *entry)
310 {
311 list_del(&entry->list);
312 }
313
314 /*
315 * Dump mapping entries for debugging purposes
316 */
317 void debug_dma_dump_mappings(struct device *dev)
318 {
319 int idx;
320
321 for (idx = 0; idx < HASH_SIZE; idx++) {
322 struct hash_bucket *bucket = &dma_entry_hash[idx];
323 struct dma_debug_entry *entry;
324 unsigned long flags;
325
326 spin_lock_irqsave(&bucket->lock, flags);
327
328 list_for_each_entry(entry, &bucket->list, list) {
329 if (!dev || dev == entry->dev) {
330 dev_info(entry->dev,
331 "%s idx %d P=%Lx D=%Lx L=%Lx %s\n",
332 type2name[entry->type], idx,
333 (unsigned long long)entry->paddr,
334 entry->dev_addr, entry->size,
335 dir2name[entry->direction]);
336 }
337 }
338
339 spin_unlock_irqrestore(&bucket->lock, flags);
340 }
341 }
342 EXPORT_SYMBOL(debug_dma_dump_mappings);
343
344 /*
345 * Wrapper function for adding an entry to the hash.
346 * This function takes care of locking itself.
347 */
348 static void add_dma_entry(struct dma_debug_entry *entry)
349 {
350 struct hash_bucket *bucket;
351 unsigned long flags;
352
353 bucket = get_hash_bucket(entry, &flags);
354 hash_bucket_add(bucket, entry);
355 put_hash_bucket(bucket, &flags);
356 }
357
358 static struct dma_debug_entry *__dma_entry_alloc(void)
359 {
360 struct dma_debug_entry *entry;
361
362 entry = list_entry(free_entries.next, struct dma_debug_entry, list);
363 list_del(&entry->list);
364 memset(entry, 0, sizeof(*entry));
365
366 num_free_entries -= 1;
367 if (num_free_entries < min_free_entries)
368 min_free_entries = num_free_entries;
369
370 return entry;
371 }
372
373 /* struct dma_entry allocator
374 *
375 * The next two functions implement the allocator for
376 * struct dma_debug_entries.
377 */
378 static struct dma_debug_entry *dma_entry_alloc(void)
379 {
380 struct dma_debug_entry *entry = NULL;
381 unsigned long flags;
382
383 spin_lock_irqsave(&free_entries_lock, flags);
384
385 if (list_empty(&free_entries)) {
386 pr_err("DMA-API: debugging out of memory - disabling\n");
387 global_disable = true;
388 goto out;
389 }
390
391 entry = __dma_entry_alloc();
392
393 #ifdef CONFIG_STACKTRACE
394 entry->stacktrace.max_entries = DMA_DEBUG_STACKTRACE_ENTRIES;
395 entry->stacktrace.entries = entry->st_entries;
396 entry->stacktrace.skip = 2;
397 save_stack_trace(&entry->stacktrace);
398 #endif
399
400 out:
401 spin_unlock_irqrestore(&free_entries_lock, flags);
402
403 return entry;
404 }
405
406 static void dma_entry_free(struct dma_debug_entry *entry)
407 {
408 unsigned long flags;
409
410 /*
411 * add to beginning of the list - this way the entries are
412 * more likely cache hot when they are reallocated.
413 */
414 spin_lock_irqsave(&free_entries_lock, flags);
415 list_add(&entry->list, &free_entries);
416 num_free_entries += 1;
417 spin_unlock_irqrestore(&free_entries_lock, flags);
418 }
419
420 int dma_debug_resize_entries(u32 num_entries)
421 {
422 int i, delta, ret = 0;
423 unsigned long flags;
424 struct dma_debug_entry *entry;
425 LIST_HEAD(tmp);
426
427 spin_lock_irqsave(&free_entries_lock, flags);
428
429 if (nr_total_entries < num_entries) {
430 delta = num_entries - nr_total_entries;
431
432 spin_unlock_irqrestore(&free_entries_lock, flags);
433
434 for (i = 0; i < delta; i++) {
435 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
436 if (!entry)
437 break;
438
439 list_add_tail(&entry->list, &tmp);
440 }
441
442 spin_lock_irqsave(&free_entries_lock, flags);
443
444 list_splice(&tmp, &free_entries);
445 nr_total_entries += i;
446 num_free_entries += i;
447 } else {
448 delta = nr_total_entries - num_entries;
449
450 for (i = 0; i < delta && !list_empty(&free_entries); i++) {
451 entry = __dma_entry_alloc();
452 kfree(entry);
453 }
454
455 nr_total_entries -= i;
456 }
457
458 if (nr_total_entries != num_entries)
459 ret = 1;
460
461 spin_unlock_irqrestore(&free_entries_lock, flags);
462
463 return ret;
464 }
465 EXPORT_SYMBOL(dma_debug_resize_entries);
466
467 /*
468 * DMA-API debugging init code
469 *
470 * The init code does two things:
471 * 1. Initialize core data structures
472 * 2. Preallocate a given number of dma_debug_entry structs
473 */
474
475 static int prealloc_memory(u32 num_entries)
476 {
477 struct dma_debug_entry *entry, *next_entry;
478 int i;
479
480 for (i = 0; i < num_entries; ++i) {
481 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
482 if (!entry)
483 goto out_err;
484
485 list_add_tail(&entry->list, &free_entries);
486 }
487
488 num_free_entries = num_entries;
489 min_free_entries = num_entries;
490
491 pr_info("DMA-API: preallocated %d debug entries\n", num_entries);
492
493 return 0;
494
495 out_err:
496
497 list_for_each_entry_safe(entry, next_entry, &free_entries, list) {
498 list_del(&entry->list);
499 kfree(entry);
500 }
501
502 return -ENOMEM;
503 }
504
505 static ssize_t filter_read(struct file *file, char __user *user_buf,
506 size_t count, loff_t *ppos)
507 {
508 char buf[NAME_MAX_LEN + 1];
509 unsigned long flags;
510 int len;
511
512 if (!current_driver_name[0])
513 return 0;
514
515 /*
516 * We can't copy to userspace directly because current_driver_name can
517 * only be read under the driver_name_lock with irqs disabled. So
518 * create a temporary copy first.
519 */
520 read_lock_irqsave(&driver_name_lock, flags);
521 len = scnprintf(buf, NAME_MAX_LEN + 1, "%s\n", current_driver_name);
522 read_unlock_irqrestore(&driver_name_lock, flags);
523
524 return simple_read_from_buffer(user_buf, count, ppos, buf, len);
525 }
526
527 static ssize_t filter_write(struct file *file, const char __user *userbuf,
528 size_t count, loff_t *ppos)
529 {
530 char buf[NAME_MAX_LEN];
531 unsigned long flags;
532 size_t len;
533 int i;
534
535 /*
536 * We can't copy from userspace directly. Access to
537 * current_driver_name is protected with a write_lock with irqs
538 * disabled. Since copy_from_user can fault and may sleep we
539 * need to copy to temporary buffer first
540 */
541 len = min(count, (size_t)(NAME_MAX_LEN - 1));
542 if (copy_from_user(buf, userbuf, len))
543 return -EFAULT;
544
545 buf[len] = 0;
546
547 write_lock_irqsave(&driver_name_lock, flags);
548
549 /*
550 * Now handle the string we got from userspace very carefully.
551 * The rules are:
552 * - only use the first token we got
553 * - token delimiter is everything looking like a space
554 * character (' ', '\n', '\t' ...)
555 *
556 */
557 if (!isalnum(buf[0])) {
558 /*
559 * If the first character userspace gave us is not
560 * alphanumerical then assume the filter should be
561 * switched off.
562 */
563 if (current_driver_name[0])
564 pr_info("DMA-API: switching off dma-debug driver filter\n");
565 current_driver_name[0] = 0;
566 current_driver = NULL;
567 goto out_unlock;
568 }
569
570 /*
571 * Now parse out the first token and use it as the name for the
572 * driver to filter for.
573 */
574 for (i = 0; i < NAME_MAX_LEN - 1; ++i) {
575 current_driver_name[i] = buf[i];
576 if (isspace(buf[i]) || buf[i] == ' ' || buf[i] == 0)
577 break;
578 }
579 current_driver_name[i] = 0;
580 current_driver = NULL;
581
582 pr_info("DMA-API: enable driver filter for driver [%s]\n",
583 current_driver_name);
584
585 out_unlock:
586 write_unlock_irqrestore(&driver_name_lock, flags);
587
588 return count;
589 }
590
591 static const struct file_operations filter_fops = {
592 .read = filter_read,
593 .write = filter_write,
594 .llseek = default_llseek,
595 };
596
597 static int dma_debug_fs_init(void)
598 {
599 dma_debug_dent = debugfs_create_dir("dma-api", NULL);
600 if (!dma_debug_dent) {
601 pr_err("DMA-API: can not create debugfs directory\n");
602 return -ENOMEM;
603 }
604
605 global_disable_dent = debugfs_create_bool("disabled", 0444,
606 dma_debug_dent,
607 (u32 *)&global_disable);
608 if (!global_disable_dent)
609 goto out_err;
610
611 error_count_dent = debugfs_create_u32("error_count", 0444,
612 dma_debug_dent, &error_count);
613 if (!error_count_dent)
614 goto out_err;
615
616 show_all_errors_dent = debugfs_create_u32("all_errors", 0644,
617 dma_debug_dent,
618 &show_all_errors);
619 if (!show_all_errors_dent)
620 goto out_err;
621
622 show_num_errors_dent = debugfs_create_u32("num_errors", 0644,
623 dma_debug_dent,
624 &show_num_errors);
625 if (!show_num_errors_dent)
626 goto out_err;
627
628 num_free_entries_dent = debugfs_create_u32("num_free_entries", 0444,
629 dma_debug_dent,
630 &num_free_entries);
631 if (!num_free_entries_dent)
632 goto out_err;
633
634 min_free_entries_dent = debugfs_create_u32("min_free_entries", 0444,
635 dma_debug_dent,
636 &min_free_entries);
637 if (!min_free_entries_dent)
638 goto out_err;
639
640 filter_dent = debugfs_create_file("driver_filter", 0644,
641 dma_debug_dent, NULL, &filter_fops);
642 if (!filter_dent)
643 goto out_err;
644
645 return 0;
646
647 out_err:
648 debugfs_remove_recursive(dma_debug_dent);
649
650 return -ENOMEM;
651 }
652
653 static int device_dma_allocations(struct device *dev, struct dma_debug_entry **out_entry)
654 {
655 struct dma_debug_entry *entry;
656 unsigned long flags;
657 int count = 0, i;
658
659 local_irq_save(flags);
660
661 for (i = 0; i < HASH_SIZE; ++i) {
662 spin_lock(&dma_entry_hash[i].lock);
663 list_for_each_entry(entry, &dma_entry_hash[i].list, list) {
664 if (entry->dev == dev) {
665 count += 1;
666 *out_entry = entry;
667 }
668 }
669 spin_unlock(&dma_entry_hash[i].lock);
670 }
671
672 local_irq_restore(flags);
673
674 return count;
675 }
676
677 static int dma_debug_device_change(struct notifier_block *nb, unsigned long action, void *data)
678 {
679 struct device *dev = data;
680 struct dma_debug_entry *uninitialized_var(entry);
681 int count;
682
683 if (global_disable)
684 return 0;
685
686 switch (action) {
687 case BUS_NOTIFY_UNBOUND_DRIVER:
688 count = device_dma_allocations(dev, &entry);
689 if (count == 0)
690 break;
691 err_printk(dev, entry, "DMA-API: device driver has pending "
692 "DMA allocations while released from device "
693 "[count=%d]\n"
694 "One of leaked entries details: "
695 "[device address=0x%016llx] [size=%llu bytes] "
696 "[mapped with %s] [mapped as %s]\n",
697 count, entry->dev_addr, entry->size,
698 dir2name[entry->direction], type2name[entry->type]);
699 break;
700 default:
701 break;
702 }
703
704 return 0;
705 }
706
707 void dma_debug_add_bus(struct bus_type *bus)
708 {
709 struct notifier_block *nb;
710
711 if (global_disable)
712 return;
713
714 nb = kzalloc(sizeof(struct notifier_block), GFP_KERNEL);
715 if (nb == NULL) {
716 pr_err("dma_debug_add_bus: out of memory\n");
717 return;
718 }
719
720 nb->notifier_call = dma_debug_device_change;
721
722 bus_register_notifier(bus, nb);
723 }
724
725 /*
726 * Let the architectures decide how many entries should be preallocated.
727 */
728 void dma_debug_init(u32 num_entries)
729 {
730 int i;
731
732 if (global_disable)
733 return;
734
735 for (i = 0; i < HASH_SIZE; ++i) {
736 INIT_LIST_HEAD(&dma_entry_hash[i].list);
737 spin_lock_init(&dma_entry_hash[i].lock);
738 }
739
740 if (dma_debug_fs_init() != 0) {
741 pr_err("DMA-API: error creating debugfs entries - disabling\n");
742 global_disable = true;
743
744 return;
745 }
746
747 if (req_entries)
748 num_entries = req_entries;
749
750 if (prealloc_memory(num_entries) != 0) {
751 pr_err("DMA-API: debugging out of memory error - disabled\n");
752 global_disable = true;
753
754 return;
755 }
756
757 nr_total_entries = num_free_entries;
758
759 pr_info("DMA-API: debugging enabled by kernel config\n");
760 }
761
762 static __init int dma_debug_cmdline(char *str)
763 {
764 if (!str)
765 return -EINVAL;
766
767 if (strncmp(str, "off", 3) == 0) {
768 pr_info("DMA-API: debugging disabled on kernel command line\n");
769 global_disable = true;
770 }
771
772 return 0;
773 }
774
775 static __init int dma_debug_entries_cmdline(char *str)
776 {
777 int res;
778
779 if (!str)
780 return -EINVAL;
781
782 res = get_option(&str, &req_entries);
783
784 if (!res)
785 req_entries = 0;
786
787 return 0;
788 }
789
790 __setup("dma_debug=", dma_debug_cmdline);
791 __setup("dma_debug_entries=", dma_debug_entries_cmdline);
792
793 static void check_unmap(struct dma_debug_entry *ref)
794 {
795 struct dma_debug_entry *entry;
796 struct hash_bucket *bucket;
797 unsigned long flags;
798
799 if (dma_mapping_error(ref->dev, ref->dev_addr)) {
800 err_printk(ref->dev, NULL, "DMA-API: device driver tries "
801 "to free an invalid DMA memory address\n");
802 return;
803 }
804
805 bucket = get_hash_bucket(ref, &flags);
806 entry = hash_bucket_find(bucket, ref);
807
808 if (!entry) {
809 err_printk(ref->dev, NULL, "DMA-API: device driver tries "
810 "to free DMA memory it has not allocated "
811 "[device address=0x%016llx] [size=%llu bytes]\n",
812 ref->dev_addr, ref->size);
813 goto out;
814 }
815
816 if (ref->size != entry->size) {
817 err_printk(ref->dev, entry, "DMA-API: device driver frees "
818 "DMA memory with different size "
819 "[device address=0x%016llx] [map size=%llu bytes] "
820 "[unmap size=%llu bytes]\n",
821 ref->dev_addr, entry->size, ref->size);
822 }
823
824 if (ref->type != entry->type) {
825 err_printk(ref->dev, entry, "DMA-API: device driver frees "
826 "DMA memory with wrong function "
827 "[device address=0x%016llx] [size=%llu bytes] "
828 "[mapped as %s] [unmapped as %s]\n",
829 ref->dev_addr, ref->size,
830 type2name[entry->type], type2name[ref->type]);
831 } else if ((entry->type == dma_debug_coherent) &&
832 (ref->paddr != entry->paddr)) {
833 err_printk(ref->dev, entry, "DMA-API: device driver frees "
834 "DMA memory with different CPU address "
835 "[device address=0x%016llx] [size=%llu bytes] "
836 "[cpu alloc address=0x%016llx] "
837 "[cpu free address=0x%016llx]",
838 ref->dev_addr, ref->size,
839 (unsigned long long)entry->paddr,
840 (unsigned long long)ref->paddr);
841 }
842
843 if (ref->sg_call_ents && ref->type == dma_debug_sg &&
844 ref->sg_call_ents != entry->sg_call_ents) {
845 err_printk(ref->dev, entry, "DMA-API: device driver frees "
846 "DMA sg list with different entry count "
847 "[map count=%d] [unmap count=%d]\n",
848 entry->sg_call_ents, ref->sg_call_ents);
849 }
850
851 /*
852 * This may be no bug in reality - but most implementations of the
853 * DMA API don't handle this properly, so check for it here
854 */
855 if (ref->direction != entry->direction) {
856 err_printk(ref->dev, entry, "DMA-API: device driver frees "
857 "DMA memory with different direction "
858 "[device address=0x%016llx] [size=%llu bytes] "
859 "[mapped with %s] [unmapped with %s]\n",
860 ref->dev_addr, ref->size,
861 dir2name[entry->direction],
862 dir2name[ref->direction]);
863 }
864
865 hash_bucket_del(entry);
866 dma_entry_free(entry);
867
868 out:
869 put_hash_bucket(bucket, &flags);
870 }
871
872 static void check_for_stack(struct device *dev, void *addr)
873 {
874 if (object_is_on_stack(addr))
875 err_printk(dev, NULL, "DMA-API: device driver maps memory from"
876 "stack [addr=%p]\n", addr);
877 }
878
879 static inline bool overlap(void *addr, unsigned long len, void *start, void *end)
880 {
881 unsigned long a1 = (unsigned long)addr;
882 unsigned long b1 = a1 + len;
883 unsigned long a2 = (unsigned long)start;
884 unsigned long b2 = (unsigned long)end;
885
886 return !(b1 <= a2 || a1 >= b2);
887 }
888
889 static void check_for_illegal_area(struct device *dev, void *addr, unsigned long len)
890 {
891 if (overlap(addr, len, _text, _etext) ||
892 overlap(addr, len, __start_rodata, __end_rodata))
893 err_printk(dev, NULL, "DMA-API: device driver maps memory from kernel text or rodata [addr=%p] [len=%lu]\n", addr, len);
894 }
895
896 static void check_sync(struct device *dev,
897 struct dma_debug_entry *ref,
898 bool to_cpu)
899 {
900 struct dma_debug_entry *entry;
901 struct hash_bucket *bucket;
902 unsigned long flags;
903
904 bucket = get_hash_bucket(ref, &flags);
905
906 entry = hash_bucket_find(bucket, ref);
907
908 if (!entry) {
909 err_printk(dev, NULL, "DMA-API: device driver tries "
910 "to sync DMA memory it has not allocated "
911 "[device address=0x%016llx] [size=%llu bytes]\n",
912 (unsigned long long)ref->dev_addr, ref->size);
913 goto out;
914 }
915
916 if (ref->size > entry->size) {
917 err_printk(dev, entry, "DMA-API: device driver syncs"
918 " DMA memory outside allocated range "
919 "[device address=0x%016llx] "
920 "[allocation size=%llu bytes] "
921 "[sync offset+size=%llu]\n",
922 entry->dev_addr, entry->size,
923 ref->size);
924 }
925
926 if (entry->direction == DMA_BIDIRECTIONAL)
927 goto out;
928
929 if (ref->direction != entry->direction) {
930 err_printk(dev, entry, "DMA-API: device driver syncs "
931 "DMA memory with different direction "
932 "[device address=0x%016llx] [size=%llu bytes] "
933 "[mapped with %s] [synced with %s]\n",
934 (unsigned long long)ref->dev_addr, entry->size,
935 dir2name[entry->direction],
936 dir2name[ref->direction]);
937 }
938
939 if (to_cpu && !(entry->direction == DMA_FROM_DEVICE) &&
940 !(ref->direction == DMA_TO_DEVICE))
941 err_printk(dev, entry, "DMA-API: device driver syncs "
942 "device read-only DMA memory for cpu "
943 "[device address=0x%016llx] [size=%llu bytes] "
944 "[mapped with %s] [synced with %s]\n",
945 (unsigned long long)ref->dev_addr, entry->size,
946 dir2name[entry->direction],
947 dir2name[ref->direction]);
948
949 if (!to_cpu && !(entry->direction == DMA_TO_DEVICE) &&
950 !(ref->direction == DMA_FROM_DEVICE))
951 err_printk(dev, entry, "DMA-API: device driver syncs "
952 "device write-only DMA memory to device "
953 "[device address=0x%016llx] [size=%llu bytes] "
954 "[mapped with %s] [synced with %s]\n",
955 (unsigned long long)ref->dev_addr, entry->size,
956 dir2name[entry->direction],
957 dir2name[ref->direction]);
958
959 out:
960 put_hash_bucket(bucket, &flags);
961 }
962
963 void debug_dma_map_page(struct device *dev, struct page *page, size_t offset,
964 size_t size, int direction, dma_addr_t dma_addr,
965 bool map_single)
966 {
967 struct dma_debug_entry *entry;
968
969 if (unlikely(global_disable))
970 return;
971
972 if (unlikely(dma_mapping_error(dev, dma_addr)))
973 return;
974
975 entry = dma_entry_alloc();
976 if (!entry)
977 return;
978
979 entry->dev = dev;
980 entry->type = dma_debug_page;
981 entry->paddr = page_to_phys(page) + offset;
982 entry->dev_addr = dma_addr;
983 entry->size = size;
984 entry->direction = direction;
985
986 if (map_single)
987 entry->type = dma_debug_single;
988
989 if (!PageHighMem(page)) {
990 void *addr = page_address(page) + offset;
991
992 check_for_stack(dev, addr);
993 check_for_illegal_area(dev, addr, size);
994 }
995
996 add_dma_entry(entry);
997 }
998 EXPORT_SYMBOL(debug_dma_map_page);
999
1000 void debug_dma_unmap_page(struct device *dev, dma_addr_t addr,
1001 size_t size, int direction, bool map_single)
1002 {
1003 struct dma_debug_entry ref = {
1004 .type = dma_debug_page,
1005 .dev = dev,
1006 .dev_addr = addr,
1007 .size = size,
1008 .direction = direction,
1009 };
1010
1011 if (unlikely(global_disable))
1012 return;
1013
1014 if (map_single)
1015 ref.type = dma_debug_single;
1016
1017 check_unmap(&ref);
1018 }
1019 EXPORT_SYMBOL(debug_dma_unmap_page);
1020
1021 void debug_dma_map_sg(struct device *dev, struct scatterlist *sg,
1022 int nents, int mapped_ents, int direction)
1023 {
1024 struct dma_debug_entry *entry;
1025 struct scatterlist *s;
1026 int i;
1027
1028 if (unlikely(global_disable))
1029 return;
1030
1031 for_each_sg(sg, s, mapped_ents, i) {
1032 entry = dma_entry_alloc();
1033 if (!entry)
1034 return;
1035
1036 entry->type = dma_debug_sg;
1037 entry->dev = dev;
1038 entry->paddr = sg_phys(s);
1039 entry->size = sg_dma_len(s);
1040 entry->dev_addr = sg_dma_address(s);
1041 entry->direction = direction;
1042 entry->sg_call_ents = nents;
1043 entry->sg_mapped_ents = mapped_ents;
1044
1045 if (!PageHighMem(sg_page(s))) {
1046 check_for_stack(dev, sg_virt(s));
1047 check_for_illegal_area(dev, sg_virt(s), sg_dma_len(s));
1048 }
1049
1050 add_dma_entry(entry);
1051 }
1052 }
1053 EXPORT_SYMBOL(debug_dma_map_sg);
1054
1055 static int get_nr_mapped_entries(struct device *dev,
1056 struct dma_debug_entry *ref)
1057 {
1058 struct dma_debug_entry *entry;
1059 struct hash_bucket *bucket;
1060 unsigned long flags;
1061 int mapped_ents;
1062
1063 bucket = get_hash_bucket(ref, &flags);
1064 entry = hash_bucket_find(bucket, ref);
1065 mapped_ents = 0;
1066
1067 if (entry)
1068 mapped_ents = entry->sg_mapped_ents;
1069 put_hash_bucket(bucket, &flags);
1070
1071 return mapped_ents;
1072 }
1073
1074 void debug_dma_unmap_sg(struct device *dev, struct scatterlist *sglist,
1075 int nelems, int dir)
1076 {
1077 struct scatterlist *s;
1078 int mapped_ents = 0, i;
1079
1080 if (unlikely(global_disable))
1081 return;
1082
1083 for_each_sg(sglist, s, nelems, i) {
1084
1085 struct dma_debug_entry ref = {
1086 .type = dma_debug_sg,
1087 .dev = dev,
1088 .paddr = sg_phys(s),
1089 .dev_addr = sg_dma_address(s),
1090 .size = sg_dma_len(s),
1091 .direction = dir,
1092 .sg_call_ents = nelems,
1093 };
1094
1095 if (mapped_ents && i >= mapped_ents)
1096 break;
1097
1098 if (!i)
1099 mapped_ents = get_nr_mapped_entries(dev, &ref);
1100
1101 check_unmap(&ref);
1102 }
1103 }
1104 EXPORT_SYMBOL(debug_dma_unmap_sg);
1105
1106 void debug_dma_alloc_coherent(struct device *dev, size_t size,
1107 dma_addr_t dma_addr, void *virt)
1108 {
1109 struct dma_debug_entry *entry;
1110
1111 if (unlikely(global_disable))
1112 return;
1113
1114 if (unlikely(virt == NULL))
1115 return;
1116
1117 entry = dma_entry_alloc();
1118 if (!entry)
1119 return;
1120
1121 entry->type = dma_debug_coherent;
1122 entry->dev = dev;
1123 entry->paddr = virt_to_phys(virt);
1124 entry->size = size;
1125 entry->dev_addr = dma_addr;
1126 entry->direction = DMA_BIDIRECTIONAL;
1127
1128 add_dma_entry(entry);
1129 }
1130 EXPORT_SYMBOL(debug_dma_alloc_coherent);
1131
1132 void debug_dma_free_coherent(struct device *dev, size_t size,
1133 void *virt, dma_addr_t addr)
1134 {
1135 struct dma_debug_entry ref = {
1136 .type = dma_debug_coherent,
1137 .dev = dev,
1138 .paddr = virt_to_phys(virt),
1139 .dev_addr = addr,
1140 .size = size,
1141 .direction = DMA_BIDIRECTIONAL,
1142 };
1143
1144 if (unlikely(global_disable))
1145 return;
1146
1147 check_unmap(&ref);
1148 }
1149 EXPORT_SYMBOL(debug_dma_free_coherent);
1150
1151 void debug_dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle,
1152 size_t size, int direction)
1153 {
1154 struct dma_debug_entry ref;
1155
1156 if (unlikely(global_disable))
1157 return;
1158
1159 ref.type = dma_debug_single;
1160 ref.dev = dev;
1161 ref.dev_addr = dma_handle;
1162 ref.size = size;
1163 ref.direction = direction;
1164 ref.sg_call_ents = 0;
1165
1166 check_sync(dev, &ref, true);
1167 }
1168 EXPORT_SYMBOL(debug_dma_sync_single_for_cpu);
1169
1170 void debug_dma_sync_single_for_device(struct device *dev,
1171 dma_addr_t dma_handle, size_t size,
1172 int direction)
1173 {
1174 struct dma_debug_entry ref;
1175
1176 if (unlikely(global_disable))
1177 return;
1178
1179 ref.type = dma_debug_single;
1180 ref.dev = dev;
1181 ref.dev_addr = dma_handle;
1182 ref.size = size;
1183 ref.direction = direction;
1184 ref.sg_call_ents = 0;
1185
1186 check_sync(dev, &ref, false);
1187 }
1188 EXPORT_SYMBOL(debug_dma_sync_single_for_device);
1189
1190 void debug_dma_sync_single_range_for_cpu(struct device *dev,
1191 dma_addr_t dma_handle,
1192 unsigned long offset, size_t size,
1193 int direction)
1194 {
1195 struct dma_debug_entry ref;
1196
1197 if (unlikely(global_disable))
1198 return;
1199
1200 ref.type = dma_debug_single;
1201 ref.dev = dev;
1202 ref.dev_addr = dma_handle;
1203 ref.size = offset + size;
1204 ref.direction = direction;
1205 ref.sg_call_ents = 0;
1206
1207 check_sync(dev, &ref, true);
1208 }
1209 EXPORT_SYMBOL(debug_dma_sync_single_range_for_cpu);
1210
1211 void debug_dma_sync_single_range_for_device(struct device *dev,
1212 dma_addr_t dma_handle,
1213 unsigned long offset,
1214 size_t size, int direction)
1215 {
1216 struct dma_debug_entry ref;
1217
1218 if (unlikely(global_disable))
1219 return;
1220
1221 ref.type = dma_debug_single;
1222 ref.dev = dev;
1223 ref.dev_addr = dma_handle;
1224 ref.size = offset + size;
1225 ref.direction = direction;
1226 ref.sg_call_ents = 0;
1227
1228 check_sync(dev, &ref, false);
1229 }
1230 EXPORT_SYMBOL(debug_dma_sync_single_range_for_device);
1231
1232 void debug_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
1233 int nelems, int direction)
1234 {
1235 struct scatterlist *s;
1236 int mapped_ents = 0, i;
1237
1238 if (unlikely(global_disable))
1239 return;
1240
1241 for_each_sg(sg, s, nelems, i) {
1242
1243 struct dma_debug_entry ref = {
1244 .type = dma_debug_sg,
1245 .dev = dev,
1246 .paddr = sg_phys(s),
1247 .dev_addr = sg_dma_address(s),
1248 .size = sg_dma_len(s),
1249 .direction = direction,
1250 .sg_call_ents = nelems,
1251 };
1252
1253 if (!i)
1254 mapped_ents = get_nr_mapped_entries(dev, &ref);
1255
1256 if (i >= mapped_ents)
1257 break;
1258
1259 check_sync(dev, &ref, true);
1260 }
1261 }
1262 EXPORT_SYMBOL(debug_dma_sync_sg_for_cpu);
1263
1264 void debug_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
1265 int nelems, int direction)
1266 {
1267 struct scatterlist *s;
1268 int mapped_ents = 0, i;
1269
1270 if (unlikely(global_disable))
1271 return;
1272
1273 for_each_sg(sg, s, nelems, i) {
1274
1275 struct dma_debug_entry ref = {
1276 .type = dma_debug_sg,
1277 .dev = dev,
1278 .paddr = sg_phys(s),
1279 .dev_addr = sg_dma_address(s),
1280 .size = sg_dma_len(s),
1281 .direction = direction,
1282 .sg_call_ents = nelems,
1283 };
1284 if (!i)
1285 mapped_ents = get_nr_mapped_entries(dev, &ref);
1286
1287 if (i >= mapped_ents)
1288 break;
1289
1290 check_sync(dev, &ref, false);
1291 }
1292 }
1293 EXPORT_SYMBOL(debug_dma_sync_sg_for_device);
1294
1295 static int __init dma_debug_driver_setup(char *str)
1296 {
1297 int i;
1298
1299 for (i = 0; i < NAME_MAX_LEN - 1; ++i, ++str) {
1300 current_driver_name[i] = *str;
1301 if (*str == 0)
1302 break;
1303 }
1304
1305 if (current_driver_name[0])
1306 pr_info("DMA-API: enable driver filter for driver [%s]\n",
1307 current_driver_name);
1308
1309
1310 return 1;
1311 }
1312 __setup("dma_debug_driver=", dma_debug_driver_setup);
linux-next