1 // SPDX-License-Identifier: GPL-2.0 OR MIT
4 * Xen frontend/backend page directory based shared buffer
7 * Copyright (C) 2018 EPAM Systems Inc.
9 * Author: Oleksandr Andrushchenko <oleksandr_andrushchenko@epam.com>
12 #include <linux/module.h>
13 #include <linux/errno.h>
16 #include <asm/xen/hypervisor.h>
17 #include <xen/balloon.h>
19 #include <xen/xenbus.h>
20 #include <xen/interface/io/ring.h>
22 #include <xen/xen-front-pgdir-shbuf.h>
24 #ifndef GRANT_INVALID_REF
26 * FIXME: usage of grant reference 0 as invalid grant reference:
27 * grant reference 0 is valid, but never exposed to a PV driver,
28 * because of the fact it is already in use/reserved by the PV console.
30 #define GRANT_INVALID_REF 0
34 * This structure represents the structure of a shared page
35 * that contains grant references to the pages of the shared
36 * buffer. This structure is common to many Xen para-virtualized
37 * protocols at include/xen/interface/io/
39 struct xen_page_directory
{
40 grant_ref_t gref_dir_next_page
;
41 grant_ref_t gref
[1]; /* Variable length */
45 * Shared buffer ops which are differently implemented
46 * depending on the allocation mode, e.g. if the buffer
47 * is allocated by the corresponding backend or frontend.
48 * Some of the operations.
50 struct xen_front_pgdir_shbuf_ops
{
52 * Calculate number of grefs required to handle this buffer,
53 * e.g. if grefs are required for page directory only or the buffer
56 void (*calc_num_grefs
)(struct xen_front_pgdir_shbuf
*buf
);
58 /* Fill page directory according to para-virtual display protocol. */
59 void (*fill_page_dir
)(struct xen_front_pgdir_shbuf
*buf
);
61 /* Claim grant references for the pages of the buffer. */
62 int (*grant_refs_for_buffer
)(struct xen_front_pgdir_shbuf
*buf
,
63 grant_ref_t
*priv_gref_head
, int gref_idx
);
65 /* Map grant references of the buffer. */
66 int (*map
)(struct xen_front_pgdir_shbuf
*buf
);
68 /* Unmap grant references of the buffer. */
69 int (*unmap
)(struct xen_front_pgdir_shbuf
*buf
);
73 * Get granted reference to the very first page of the
74 * page directory. Usually this is passed to the backend,
75 * so it can find/fill the grant references to the buffer's
78 * \param buf shared buffer which page directory is of interest.
79 * \return granted reference to the very first page of the
83 xen_front_pgdir_shbuf_get_dir_start(struct xen_front_pgdir_shbuf
*buf
)
86 return GRANT_INVALID_REF
;
90 EXPORT_SYMBOL_GPL(xen_front_pgdir_shbuf_get_dir_start
);
93 * Map granted references of the shared buffer.
95 * Depending on the shared buffer mode of allocation
96 * (be_alloc flag) this can either do nothing (for buffers
97 * shared by the frontend itself) or map the provided granted
98 * references onto the backing storage (buf->pages).
100 * \param buf shared buffer which grants to be maped.
101 * \return zero on success or a negative number on failure.
103 int xen_front_pgdir_shbuf_map(struct xen_front_pgdir_shbuf
*buf
)
105 if (buf
->ops
&& buf
->ops
->map
)
106 return buf
->ops
->map(buf
);
108 /* No need to map own grant references. */
111 EXPORT_SYMBOL_GPL(xen_front_pgdir_shbuf_map
);
114 * Unmap granted references of the shared buffer.
116 * Depending on the shared buffer mode of allocation
117 * (be_alloc flag) this can either do nothing (for buffers
118 * shared by the frontend itself) or unmap the provided granted
121 * \param buf shared buffer which grants to be unmaped.
122 * \return zero on success or a negative number on failure.
124 int xen_front_pgdir_shbuf_unmap(struct xen_front_pgdir_shbuf
*buf
)
126 if (buf
->ops
&& buf
->ops
->unmap
)
127 return buf
->ops
->unmap(buf
);
129 /* No need to unmap own grant references. */
132 EXPORT_SYMBOL_GPL(xen_front_pgdir_shbuf_unmap
);
135 * Free all the resources of the shared buffer.
137 * \param buf shared buffer which resources to be freed.
139 void xen_front_pgdir_shbuf_free(struct xen_front_pgdir_shbuf
*buf
)
144 for (i
= 0; i
< buf
->num_grefs
; i
++)
145 if (buf
->grefs
[i
] != GRANT_INVALID_REF
)
146 gnttab_end_foreign_access(buf
->grefs
[i
],
150 kfree(buf
->directory
);
152 EXPORT_SYMBOL_GPL(xen_front_pgdir_shbuf_free
);
155 * Number of grefs a page can hold with respect to the
156 * struct xen_page_directory header.
158 #define XEN_NUM_GREFS_PER_PAGE ((PAGE_SIZE - \
159 offsetof(struct xen_page_directory, \
160 gref)) / sizeof(grant_ref_t))
163 * Get the number of pages the page directory consumes itself.
165 * \param buf shared buffer.
167 static int get_num_pages_dir(struct xen_front_pgdir_shbuf
*buf
)
169 return DIV_ROUND_UP(buf
->num_pages
, XEN_NUM_GREFS_PER_PAGE
);
173 * Calculate the number of grant references needed to share the buffer
174 * and its pages when backend allocates the buffer.
176 * \param buf shared buffer.
178 static void backend_calc_num_grefs(struct xen_front_pgdir_shbuf
*buf
)
180 /* Only for pages the page directory consumes itself. */
181 buf
->num_grefs
= get_num_pages_dir(buf
);
185 * Calculate the number of grant references needed to share the buffer
186 * and its pages when frontend allocates the buffer.
188 * \param buf shared buffer.
190 static void guest_calc_num_grefs(struct xen_front_pgdir_shbuf
*buf
)
193 * Number of pages the page directory consumes itself
194 * plus grefs for the buffer pages.
196 buf
->num_grefs
= get_num_pages_dir(buf
) + buf
->num_pages
;
199 #define xen_page_to_vaddr(page) \
200 ((uintptr_t)pfn_to_kaddr(page_to_xen_pfn(page)))
203 * Unmap the buffer previously mapped with grant references
204 * provided by the backend.
206 * \param buf shared buffer.
207 * \return zero on success or a negative number on failure.
209 static int backend_unmap(struct xen_front_pgdir_shbuf
*buf
)
211 struct gnttab_unmap_grant_ref
*unmap_ops
;
214 if (!buf
->pages
|| !buf
->backend_map_handles
|| !buf
->grefs
)
217 unmap_ops
= kcalloc(buf
->num_pages
, sizeof(*unmap_ops
),
222 for (i
= 0; i
< buf
->num_pages
; i
++) {
225 addr
= xen_page_to_vaddr(buf
->pages
[i
]);
226 gnttab_set_unmap_op(&unmap_ops
[i
], addr
, GNTMAP_host_map
,
227 buf
->backend_map_handles
[i
]);
230 ret
= gnttab_unmap_refs(unmap_ops
, NULL
, buf
->pages
,
233 for (i
= 0; i
< buf
->num_pages
; i
++) {
234 if (unlikely(unmap_ops
[i
].status
!= GNTST_okay
))
235 dev_err(&buf
->xb_dev
->dev
,
236 "Failed to unmap page %d: %d\n",
237 i
, unmap_ops
[i
].status
);
241 dev_err(&buf
->xb_dev
->dev
,
242 "Failed to unmap grant references, ret %d", ret
);
245 kfree(buf
->backend_map_handles
);
246 buf
->backend_map_handles
= NULL
;
251 * Map the buffer with grant references provided by the backend.
253 * \param buf shared buffer.
254 * \return zero on success or a negative number on failure.
256 static int backend_map(struct xen_front_pgdir_shbuf
*buf
)
258 struct gnttab_map_grant_ref
*map_ops
= NULL
;
260 int ret
, cur_gref
, cur_dir_page
, cur_page
, grefs_left
;
262 map_ops
= kcalloc(buf
->num_pages
, sizeof(*map_ops
), GFP_KERNEL
);
266 buf
->backend_map_handles
= kcalloc(buf
->num_pages
,
267 sizeof(*buf
->backend_map_handles
),
269 if (!buf
->backend_map_handles
) {
275 * Read page directory to get grefs from the backend: for external
276 * buffer we only allocate buf->grefs for the page directory,
277 * so buf->num_grefs has number of pages in the page directory itself.
279 ptr
= buf
->directory
;
280 grefs_left
= buf
->num_pages
;
282 for (cur_dir_page
= 0; cur_dir_page
< buf
->num_grefs
; cur_dir_page
++) {
283 struct xen_page_directory
*page_dir
=
284 (struct xen_page_directory
*)ptr
;
285 int to_copy
= XEN_NUM_GREFS_PER_PAGE
;
287 if (to_copy
> grefs_left
)
288 to_copy
= grefs_left
;
290 for (cur_gref
= 0; cur_gref
< to_copy
; cur_gref
++) {
293 addr
= xen_page_to_vaddr(buf
->pages
[cur_page
]);
294 gnttab_set_map_op(&map_ops
[cur_page
], addr
,
296 page_dir
->gref
[cur_gref
],
297 buf
->xb_dev
->otherend_id
);
301 grefs_left
-= to_copy
;
304 ret
= gnttab_map_refs(map_ops
, NULL
, buf
->pages
, buf
->num_pages
);
306 /* Save handles even if error, so we can unmap. */
307 for (cur_page
= 0; cur_page
< buf
->num_pages
; cur_page
++) {
308 buf
->backend_map_handles
[cur_page
] = map_ops
[cur_page
].handle
;
309 if (unlikely(map_ops
[cur_page
].status
!= GNTST_okay
))
310 dev_err(&buf
->xb_dev
->dev
,
311 "Failed to map page %d: %d\n",
312 cur_page
, map_ops
[cur_page
].status
);
316 dev_err(&buf
->xb_dev
->dev
,
317 "Failed to map grant references, ret %d", ret
);
326 * Fill page directory with grant references to the pages of the
327 * page directory itself.
329 * The grant references to the buffer pages are provided by the
330 * backend in this case.
332 * \param buf shared buffer.
334 static void backend_fill_page_dir(struct xen_front_pgdir_shbuf
*buf
)
336 struct xen_page_directory
*page_dir
;
338 int i
, num_pages_dir
;
340 ptr
= buf
->directory
;
341 num_pages_dir
= get_num_pages_dir(buf
);
343 /* Fill only grefs for the page directory itself. */
344 for (i
= 0; i
< num_pages_dir
- 1; i
++) {
345 page_dir
= (struct xen_page_directory
*)ptr
;
347 page_dir
->gref_dir_next_page
= buf
->grefs
[i
+ 1];
350 /* Last page must say there is no more pages. */
351 page_dir
= (struct xen_page_directory
*)ptr
;
352 page_dir
->gref_dir_next_page
= GRANT_INVALID_REF
;
356 * Fill page directory with grant references to the pages of the
357 * page directory and the buffer we share with the backend.
359 * \param buf shared buffer.
361 static void guest_fill_page_dir(struct xen_front_pgdir_shbuf
*buf
)
364 int cur_gref
, grefs_left
, to_copy
, i
, num_pages_dir
;
366 ptr
= buf
->directory
;
367 num_pages_dir
= get_num_pages_dir(buf
);
370 * While copying, skip grefs at start, they are for pages
371 * granted for the page directory itself.
373 cur_gref
= num_pages_dir
;
374 grefs_left
= buf
->num_pages
;
375 for (i
= 0; i
< num_pages_dir
; i
++) {
376 struct xen_page_directory
*page_dir
=
377 (struct xen_page_directory
*)ptr
;
379 if (grefs_left
<= XEN_NUM_GREFS_PER_PAGE
) {
380 to_copy
= grefs_left
;
381 page_dir
->gref_dir_next_page
= GRANT_INVALID_REF
;
383 to_copy
= XEN_NUM_GREFS_PER_PAGE
;
384 page_dir
->gref_dir_next_page
= buf
->grefs
[i
+ 1];
386 memcpy(&page_dir
->gref
, &buf
->grefs
[cur_gref
],
387 to_copy
* sizeof(grant_ref_t
));
389 grefs_left
-= to_copy
;
395 * Grant references to the frontend's buffer pages.
397 * These will be shared with the backend, so it can
398 * access the buffer's data.
400 * \param buf shared buffer.
401 * \return zero on success or a negative number on failure.
403 static int guest_grant_refs_for_buffer(struct xen_front_pgdir_shbuf
*buf
,
404 grant_ref_t
*priv_gref_head
,
407 int i
, cur_ref
, otherend_id
;
409 otherend_id
= buf
->xb_dev
->otherend_id
;
410 for (i
= 0; i
< buf
->num_pages
; i
++) {
411 cur_ref
= gnttab_claim_grant_reference(priv_gref_head
);
415 gnttab_grant_foreign_access_ref(cur_ref
, otherend_id
,
416 xen_page_to_gfn(buf
->pages
[i
]),
418 buf
->grefs
[gref_idx
++] = cur_ref
;
424 * Grant all the references needed to share the buffer.
426 * Grant references to the page directory pages and, if
427 * needed, also to the pages of the shared buffer data.
429 * \param buf shared buffer.
430 * \return zero on success or a negative number on failure.
432 static int grant_references(struct xen_front_pgdir_shbuf
*buf
)
434 grant_ref_t priv_gref_head
;
435 int ret
, i
, j
, cur_ref
;
436 int otherend_id
, num_pages_dir
;
438 ret
= gnttab_alloc_grant_references(buf
->num_grefs
, &priv_gref_head
);
440 dev_err(&buf
->xb_dev
->dev
,
441 "Cannot allocate grant references\n");
445 otherend_id
= buf
->xb_dev
->otherend_id
;
447 num_pages_dir
= get_num_pages_dir(buf
);
448 for (i
= 0; i
< num_pages_dir
; i
++) {
451 cur_ref
= gnttab_claim_grant_reference(&priv_gref_head
);
455 frame
= xen_page_to_gfn(virt_to_page(buf
->directory
+
457 gnttab_grant_foreign_access_ref(cur_ref
, otherend_id
, frame
, 0);
458 buf
->grefs
[j
++] = cur_ref
;
461 if (buf
->ops
->grant_refs_for_buffer
) {
462 ret
= buf
->ops
->grant_refs_for_buffer(buf
, &priv_gref_head
, j
);
467 gnttab_free_grant_references(priv_gref_head
);
472 * Allocate all required structures to mange shared buffer.
474 * \param buf shared buffer.
475 * \return zero on success or a negative number on failure.
477 static int alloc_storage(struct xen_front_pgdir_shbuf
*buf
)
479 buf
->grefs
= kcalloc(buf
->num_grefs
, sizeof(*buf
->grefs
), GFP_KERNEL
);
483 buf
->directory
= kcalloc(get_num_pages_dir(buf
), PAGE_SIZE
, GFP_KERNEL
);
491 * For backend allocated buffers we don't need grant_refs_for_buffer
492 * as those grant references are allocated at backend side.
494 static const struct xen_front_pgdir_shbuf_ops backend_ops
= {
495 .calc_num_grefs
= backend_calc_num_grefs
,
496 .fill_page_dir
= backend_fill_page_dir
,
498 .unmap
= backend_unmap
502 * For locally granted references we do not need to map/unmap
505 static const struct xen_front_pgdir_shbuf_ops local_ops
= {
506 .calc_num_grefs
= guest_calc_num_grefs
,
507 .fill_page_dir
= guest_fill_page_dir
,
508 .grant_refs_for_buffer
= guest_grant_refs_for_buffer
,
512 * Allocate a new instance of a shared buffer.
514 * \param cfg configuration to be used while allocating a new shared buffer.
515 * \return zero on success or a negative number on failure.
517 int xen_front_pgdir_shbuf_alloc(struct xen_front_pgdir_shbuf_cfg
*cfg
)
519 struct xen_front_pgdir_shbuf
*buf
= cfg
->pgdir
;
523 buf
->ops
= &backend_ops
;
525 buf
->ops
= &local_ops
;
526 buf
->xb_dev
= cfg
->xb_dev
;
527 buf
->num_pages
= cfg
->num_pages
;
528 buf
->pages
= cfg
->pages
;
530 buf
->ops
->calc_num_grefs(buf
);
532 ret
= alloc_storage(buf
);
536 ret
= grant_references(buf
);
540 buf
->ops
->fill_page_dir(buf
);
545 xen_front_pgdir_shbuf_free(buf
);
548 EXPORT_SYMBOL_GPL(xen_front_pgdir_shbuf_alloc
);
550 MODULE_DESCRIPTION("Xen frontend/backend page directory based "
551 "shared buffer handling");
552 MODULE_AUTHOR("Oleksandr Andrushchenko");
553 MODULE_LICENSE("GPL");