drm/panthor: Don't add write fences to the shared BOs
[drm/drm-misc.git] / kernel / livepatch / shadow.c
blobc2e724d97ddf2da5dbbcbbca24ac573b3fc48c26
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * shadow.c - Shadow Variables
5 * Copyright (C) 2014 Josh Poimboeuf <jpoimboe@redhat.com>
6 * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com>
7 * Copyright (C) 2017 Joe Lawrence <joe.lawrence@redhat.com>
8 */
10 /**
11 * DOC: Shadow variable API concurrency notes:
13 * The shadow variable API provides a simple relationship between an
14 * <obj, id> pair and a pointer value. It is the responsibility of the
15 * caller to provide any mutual exclusion required of the shadow data.
17 * Once a shadow variable is attached to its parent object via the
18 * klp_shadow_*alloc() API calls, it is considered live: any subsequent
19 * call to klp_shadow_get() may then return the shadow variable's data
20 * pointer. Callers of klp_shadow_*alloc() should prepare shadow data
21 * accordingly.
23 * The klp_shadow_*alloc() API calls may allocate memory for new shadow
24 * variable structures. Their implementation does not call kmalloc
25 * inside any spinlocks, but API callers should pass GFP flags according
26 * to their specific needs.
28 * The klp_shadow_hash is an RCU-enabled hashtable and is safe against
29 * concurrent klp_shadow_free() and klp_shadow_get() operations.
32 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
34 #include <linux/hashtable.h>
35 #include <linux/slab.h>
36 #include <linux/livepatch.h>
38 static DEFINE_HASHTABLE(klp_shadow_hash, 12);
41 * klp_shadow_lock provides exclusive access to the klp_shadow_hash and
42 * the shadow variables it references.
44 static DEFINE_SPINLOCK(klp_shadow_lock);
46 /**
47 * struct klp_shadow - shadow variable structure
48 * @node: klp_shadow_hash hash table node
49 * @rcu_head: RCU is used to safely free this structure
50 * @obj: pointer to parent object
51 * @id: data identifier
52 * @data: data area
54 struct klp_shadow {
55 struct hlist_node node;
56 struct rcu_head rcu_head;
57 void *obj;
58 unsigned long id;
59 char data[];
62 /**
63 * klp_shadow_match() - verify a shadow variable matches given <obj, id>
64 * @shadow: shadow variable to match
65 * @obj: pointer to parent object
66 * @id: data identifier
68 * Return: true if the shadow variable matches.
70 static inline bool klp_shadow_match(struct klp_shadow *shadow, void *obj,
71 unsigned long id)
73 return shadow->obj == obj && shadow->id == id;
76 /**
77 * klp_shadow_get() - retrieve a shadow variable data pointer
78 * @obj: pointer to parent object
79 * @id: data identifier
81 * Return: the shadow variable data element, NULL on failure.
83 void *klp_shadow_get(void *obj, unsigned long id)
85 struct klp_shadow *shadow;
87 rcu_read_lock();
89 hash_for_each_possible_rcu(klp_shadow_hash, shadow, node,
90 (unsigned long)obj) {
92 if (klp_shadow_match(shadow, obj, id)) {
93 rcu_read_unlock();
94 return shadow->data;
98 rcu_read_unlock();
100 return NULL;
102 EXPORT_SYMBOL_GPL(klp_shadow_get);
104 static void *__klp_shadow_get_or_alloc(void *obj, unsigned long id,
105 size_t size, gfp_t gfp_flags,
106 klp_shadow_ctor_t ctor, void *ctor_data,
107 bool warn_on_exist)
109 struct klp_shadow *new_shadow;
110 void *shadow_data;
111 unsigned long flags;
113 /* Check if the shadow variable already exists */
114 shadow_data = klp_shadow_get(obj, id);
115 if (shadow_data)
116 goto exists;
119 * Allocate a new shadow variable. Fill it with zeroes by default.
120 * More complex setting can be done by @ctor function. But it is
121 * called only when the buffer is really used (under klp_shadow_lock).
123 new_shadow = kzalloc(size + sizeof(*new_shadow), gfp_flags);
124 if (!new_shadow)
125 return NULL;
127 /* Look for <obj, id> again under the lock */
128 spin_lock_irqsave(&klp_shadow_lock, flags);
129 shadow_data = klp_shadow_get(obj, id);
130 if (unlikely(shadow_data)) {
132 * Shadow variable was found, throw away speculative
133 * allocation.
135 spin_unlock_irqrestore(&klp_shadow_lock, flags);
136 kfree(new_shadow);
137 goto exists;
140 new_shadow->obj = obj;
141 new_shadow->id = id;
143 if (ctor) {
144 int err;
146 err = ctor(obj, new_shadow->data, ctor_data);
147 if (err) {
148 spin_unlock_irqrestore(&klp_shadow_lock, flags);
149 kfree(new_shadow);
150 pr_err("Failed to construct shadow variable <%p, %lx> (%d)\n",
151 obj, id, err);
152 return NULL;
156 /* No <obj, id> found, so attach the newly allocated one */
157 hash_add_rcu(klp_shadow_hash, &new_shadow->node,
158 (unsigned long)new_shadow->obj);
159 spin_unlock_irqrestore(&klp_shadow_lock, flags);
161 return new_shadow->data;
163 exists:
164 if (warn_on_exist) {
165 WARN(1, "Duplicate shadow variable <%p, %lx>\n", obj, id);
166 return NULL;
169 return shadow_data;
173 * klp_shadow_alloc() - allocate and add a new shadow variable
174 * @obj: pointer to parent object
175 * @id: data identifier
176 * @size: size of attached data
177 * @gfp_flags: GFP mask for allocation
178 * @ctor: custom constructor to initialize the shadow data (optional)
179 * @ctor_data: pointer to any data needed by @ctor (optional)
181 * Allocates @size bytes for new shadow variable data using @gfp_flags.
182 * The data are zeroed by default. They are further initialized by @ctor
183 * function if it is not NULL. The new shadow variable is then added
184 * to the global hashtable.
186 * If an existing <obj, id> shadow variable can be found, this routine will
187 * issue a WARN, exit early and return NULL.
189 * This function guarantees that the constructor function is called only when
190 * the variable did not exist before. The cost is that @ctor is called
191 * in atomic context under a spin lock.
193 * Return: the shadow variable data element, NULL on duplicate or
194 * failure.
196 void *klp_shadow_alloc(void *obj, unsigned long id,
197 size_t size, gfp_t gfp_flags,
198 klp_shadow_ctor_t ctor, void *ctor_data)
200 return __klp_shadow_get_or_alloc(obj, id, size, gfp_flags,
201 ctor, ctor_data, true);
203 EXPORT_SYMBOL_GPL(klp_shadow_alloc);
206 * klp_shadow_get_or_alloc() - get existing or allocate a new shadow variable
207 * @obj: pointer to parent object
208 * @id: data identifier
209 * @size: size of attached data
210 * @gfp_flags: GFP mask for allocation
211 * @ctor: custom constructor to initialize the shadow data (optional)
212 * @ctor_data: pointer to any data needed by @ctor (optional)
214 * Returns a pointer to existing shadow data if an <obj, id> shadow
215 * variable is already present. Otherwise, it creates a new shadow
216 * variable like klp_shadow_alloc().
218 * This function guarantees that only one shadow variable exists with the given
219 * @id for the given @obj. It also guarantees that the constructor function
220 * will be called only when the variable did not exist before. The cost is
221 * that @ctor is called in atomic context under a spin lock.
223 * Return: the shadow variable data element, NULL on failure.
225 void *klp_shadow_get_or_alloc(void *obj, unsigned long id,
226 size_t size, gfp_t gfp_flags,
227 klp_shadow_ctor_t ctor, void *ctor_data)
229 return __klp_shadow_get_or_alloc(obj, id, size, gfp_flags,
230 ctor, ctor_data, false);
232 EXPORT_SYMBOL_GPL(klp_shadow_get_or_alloc);
234 static void klp_shadow_free_struct(struct klp_shadow *shadow,
235 klp_shadow_dtor_t dtor)
237 hash_del_rcu(&shadow->node);
238 if (dtor)
239 dtor(shadow->obj, shadow->data);
240 kfree_rcu(shadow, rcu_head);
244 * klp_shadow_free() - detach and free a <obj, id> shadow variable
245 * @obj: pointer to parent object
246 * @id: data identifier
247 * @dtor: custom callback that can be used to unregister the variable
248 * and/or free data that the shadow variable points to (optional)
250 * This function releases the memory for this <obj, id> shadow variable
251 * instance, callers should stop referencing it accordingly.
253 void klp_shadow_free(void *obj, unsigned long id, klp_shadow_dtor_t dtor)
255 struct klp_shadow *shadow;
256 unsigned long flags;
258 spin_lock_irqsave(&klp_shadow_lock, flags);
260 /* Delete <obj, id> from hash */
261 hash_for_each_possible(klp_shadow_hash, shadow, node,
262 (unsigned long)obj) {
264 if (klp_shadow_match(shadow, obj, id)) {
265 klp_shadow_free_struct(shadow, dtor);
266 break;
270 spin_unlock_irqrestore(&klp_shadow_lock, flags);
272 EXPORT_SYMBOL_GPL(klp_shadow_free);
275 * klp_shadow_free_all() - detach and free all <_, id> shadow variables
276 * @id: data identifier
277 * @dtor: custom callback that can be used to unregister the variable
278 * and/or free data that the shadow variable points to (optional)
280 * This function releases the memory for all <_, id> shadow variable
281 * instances, callers should stop referencing them accordingly.
283 void klp_shadow_free_all(unsigned long id, klp_shadow_dtor_t dtor)
285 struct klp_shadow *shadow;
286 unsigned long flags;
287 int i;
289 spin_lock_irqsave(&klp_shadow_lock, flags);
291 /* Delete all <_, id> from hash */
292 hash_for_each(klp_shadow_hash, i, shadow, node) {
293 if (klp_shadow_match(shadow, shadow->obj, id))
294 klp_shadow_free_struct(shadow, dtor);
297 spin_unlock_irqrestore(&klp_shadow_lock, flags);
299 EXPORT_SYMBOL_GPL(klp_shadow_free_all);