2 * Asynchronous refcounty things
4 * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
5 * Copyright 2012 Google, Inc.
8 #include <linux/debugfs.h>
9 #include <linux/module.h>
10 #include <linux/seq_file.h>
14 static inline void closure_put_after_sub(struct closure
*cl
, int flags
)
16 int r
= flags
& CLOSURE_REMAINING_MASK
;
18 BUG_ON(flags
& CLOSURE_GUARD_MASK
);
19 BUG_ON(!r
&& (flags
& ~CLOSURE_DESTRUCTOR
));
21 /* Must deliver precisely one wakeup */
22 if (r
== 1 && (flags
& CLOSURE_SLEEPING
))
23 wake_up_process(cl
->task
);
26 if (cl
->fn
&& !(flags
& CLOSURE_DESTRUCTOR
)) {
27 atomic_set(&cl
->remaining
,
28 CLOSURE_REMAINING_INITIALIZER
);
31 struct closure
*parent
= cl
->parent
;
32 closure_fn
*destructor
= cl
->fn
;
34 closure_debug_destroy(cl
);
45 /* For clearing flags with the same atomic op as a put */
46 void closure_sub(struct closure
*cl
, int v
)
48 closure_put_after_sub(cl
, atomic_sub_return(v
, &cl
->remaining
));
50 EXPORT_SYMBOL(closure_sub
);
53 * closure_put - decrement a closure's refcount
55 void closure_put(struct closure
*cl
)
57 closure_put_after_sub(cl
, atomic_dec_return(&cl
->remaining
));
59 EXPORT_SYMBOL(closure_put
);
62 * closure_wake_up - wake up all closures on a wait list, without memory barrier
64 void __closure_wake_up(struct closure_waitlist
*wait_list
)
66 struct llist_node
*list
;
68 struct llist_node
*reverse
= NULL
;
70 list
= llist_del_all(&wait_list
->list
);
72 /* We first reverse the list to preserve FIFO ordering and fairness */
75 struct llist_node
*t
= list
;
76 list
= llist_next(list
);
82 /* Then do the wakeups */
85 cl
= container_of(reverse
, struct closure
, list
);
86 reverse
= llist_next(reverse
);
88 closure_set_waiting(cl
, 0);
89 closure_sub(cl
, CLOSURE_WAITING
+ 1);
92 EXPORT_SYMBOL(__closure_wake_up
);
95 * closure_wait - add a closure to a waitlist
97 * @waitlist will own a ref on @cl, which will be released when
98 * closure_wake_up() is called on @waitlist.
101 bool closure_wait(struct closure_waitlist
*waitlist
, struct closure
*cl
)
103 if (atomic_read(&cl
->remaining
) & CLOSURE_WAITING
)
106 closure_set_waiting(cl
, _RET_IP_
);
107 atomic_add(CLOSURE_WAITING
+ 1, &cl
->remaining
);
108 llist_add(&cl
->list
, &waitlist
->list
);
112 EXPORT_SYMBOL(closure_wait
);
115 * closure_sync - sleep until a closure a closure has nothing left to wait on
117 * Sleeps until the refcount hits 1 - the thread that's running the closure owns
120 void closure_sync(struct closure
*cl
)
123 __closure_start_sleep(cl
);
124 closure_set_ret_ip(cl
);
126 if ((atomic_read(&cl
->remaining
) &
127 CLOSURE_REMAINING_MASK
) == 1)
133 __closure_end_sleep(cl
);
135 EXPORT_SYMBOL(closure_sync
);
137 #ifdef CONFIG_BCACHE_CLOSURES_DEBUG
139 static LIST_HEAD(closure_list
);
140 static DEFINE_SPINLOCK(closure_list_lock
);
142 void closure_debug_create(struct closure
*cl
)
146 BUG_ON(cl
->magic
== CLOSURE_MAGIC_ALIVE
);
147 cl
->magic
= CLOSURE_MAGIC_ALIVE
;
149 spin_lock_irqsave(&closure_list_lock
, flags
);
150 list_add(&cl
->all
, &closure_list
);
151 spin_unlock_irqrestore(&closure_list_lock
, flags
);
153 EXPORT_SYMBOL(closure_debug_create
);
155 void closure_debug_destroy(struct closure
*cl
)
159 BUG_ON(cl
->magic
!= CLOSURE_MAGIC_ALIVE
);
160 cl
->magic
= CLOSURE_MAGIC_DEAD
;
162 spin_lock_irqsave(&closure_list_lock
, flags
);
164 spin_unlock_irqrestore(&closure_list_lock
, flags
);
166 EXPORT_SYMBOL(closure_debug_destroy
);
168 static struct dentry
*debug
;
170 #define work_data_bits(work) ((unsigned long *)(&(work)->data))
172 static int debug_seq_show(struct seq_file
*f
, void *data
)
175 spin_lock_irq(&closure_list_lock
);
177 list_for_each_entry(cl
, &closure_list
, all
) {
178 int r
= atomic_read(&cl
->remaining
);
180 seq_printf(f
, "%p: %pF -> %pf p %p r %i ",
181 cl
, (void *) cl
->ip
, cl
->fn
, cl
->parent
,
182 r
& CLOSURE_REMAINING_MASK
);
184 seq_printf(f
, "%s%s%s%s\n",
185 test_bit(WORK_STRUCT_PENDING
,
186 work_data_bits(&cl
->work
)) ? "Q" : "",
187 r
& CLOSURE_RUNNING
? "R" : "",
188 r
& CLOSURE_STACK
? "S" : "",
189 r
& CLOSURE_SLEEPING
? "Sl" : "");
191 if (r
& CLOSURE_WAITING
)
192 seq_printf(f
, " W %pF\n",
193 (void *) cl
->waiting_on
);
198 spin_unlock_irq(&closure_list_lock
);
202 static int debug_seq_open(struct inode
*inode
, struct file
*file
)
204 return single_open(file
, debug_seq_show
, NULL
);
207 static const struct file_operations debug_ops
= {
208 .owner
= THIS_MODULE
,
209 .open
= debug_seq_open
,
211 .release
= single_release
214 void __init
closure_debug_init(void)
216 debug
= debugfs_create_file("closures", 0400, NULL
, NULL
, &debug_ops
);
221 MODULE_AUTHOR("Kent Overstreet <koverstreet@google.com>");
222 MODULE_LICENSE("GPL");