1 // SPDX-License-Identifier: GPL-2.0-only
3 * linux/net/sunrpc/xdr.c
7 * Copyright (C) 1995, 1996 Olaf Kirch <okir@monad.swb.de>
10 #include <linux/module.h>
11 #include <linux/slab.h>
12 #include <linux/types.h>
13 #include <linux/string.h>
14 #include <linux/kernel.h>
15 #include <linux/pagemap.h>
16 #include <linux/errno.h>
17 #include <linux/sunrpc/xdr.h>
18 #include <linux/sunrpc/msg_prot.h>
19 #include <linux/bvec.h>
20 #include <trace/events/sunrpc.h>
22 static void _copy_to_pages(struct page
**, size_t, const char *, size_t);
26 * XDR functions for basic NFS types
29 xdr_encode_netobj(__be32
*p
, const struct xdr_netobj
*obj
)
31 unsigned int quadlen
= XDR_QUADLEN(obj
->len
);
33 p
[quadlen
] = 0; /* zero trailing bytes */
34 *p
++ = cpu_to_be32(obj
->len
);
35 memcpy(p
, obj
->data
, obj
->len
);
36 return p
+ XDR_QUADLEN(obj
->len
);
38 EXPORT_SYMBOL_GPL(xdr_encode_netobj
);
41 xdr_decode_netobj(__be32
*p
, struct xdr_netobj
*obj
)
45 if ((len
= be32_to_cpu(*p
++)) > XDR_MAX_NETOBJ
)
49 return p
+ XDR_QUADLEN(len
);
51 EXPORT_SYMBOL_GPL(xdr_decode_netobj
);
54 * xdr_encode_opaque_fixed - Encode fixed length opaque data
55 * @p: pointer to current position in XDR buffer.
56 * @ptr: pointer to data to encode (or NULL)
57 * @nbytes: size of data.
59 * Copy the array of data of length nbytes at ptr to the XDR buffer
60 * at position p, then align to the next 32-bit boundary by padding
61 * with zero bytes (see RFC1832).
62 * Note: if ptr is NULL, only the padding is performed.
64 * Returns the updated current XDR buffer position
67 __be32
*xdr_encode_opaque_fixed(__be32
*p
, const void *ptr
, unsigned int nbytes
)
69 if (likely(nbytes
!= 0)) {
70 unsigned int quadlen
= XDR_QUADLEN(nbytes
);
71 unsigned int padding
= (quadlen
<< 2) - nbytes
;
74 memcpy(p
, ptr
, nbytes
);
76 memset((char *)p
+ nbytes
, 0, padding
);
81 EXPORT_SYMBOL_GPL(xdr_encode_opaque_fixed
);
84 * xdr_encode_opaque - Encode variable length opaque data
85 * @p: pointer to current position in XDR buffer.
86 * @ptr: pointer to data to encode (or NULL)
87 * @nbytes: size of data.
89 * Returns the updated current XDR buffer position
91 __be32
*xdr_encode_opaque(__be32
*p
, const void *ptr
, unsigned int nbytes
)
93 *p
++ = cpu_to_be32(nbytes
);
94 return xdr_encode_opaque_fixed(p
, ptr
, nbytes
);
96 EXPORT_SYMBOL_GPL(xdr_encode_opaque
);
99 xdr_encode_string(__be32
*p
, const char *string
)
101 return xdr_encode_array(p
, string
, strlen(string
));
103 EXPORT_SYMBOL_GPL(xdr_encode_string
);
106 xdr_decode_string_inplace(__be32
*p
, char **sp
,
107 unsigned int *lenp
, unsigned int maxlen
)
111 len
= be32_to_cpu(*p
++);
116 return p
+ XDR_QUADLEN(len
);
118 EXPORT_SYMBOL_GPL(xdr_decode_string_inplace
);
121 * xdr_terminate_string - '\0'-terminate a string residing in an xdr_buf
122 * @buf: XDR buffer where string resides
123 * @len: length of string, in bytes
126 void xdr_terminate_string(const struct xdr_buf
*buf
, const u32 len
)
130 kaddr
= kmap_atomic(buf
->pages
[0]);
131 kaddr
[buf
->page_base
+ len
] = '\0';
132 kunmap_atomic(kaddr
);
134 EXPORT_SYMBOL_GPL(xdr_terminate_string
);
136 size_t xdr_buf_pagecount(const struct xdr_buf
*buf
)
140 return (buf
->page_base
+ buf
->page_len
+ PAGE_SIZE
- 1) >> PAGE_SHIFT
;
144 xdr_alloc_bvec(struct xdr_buf
*buf
, gfp_t gfp
)
146 size_t i
, n
= xdr_buf_pagecount(buf
);
148 if (n
!= 0 && buf
->bvec
== NULL
) {
149 buf
->bvec
= kmalloc_array(n
, sizeof(buf
->bvec
[0]), gfp
);
152 for (i
= 0; i
< n
; i
++) {
153 buf
->bvec
[i
].bv_page
= buf
->pages
[i
];
154 buf
->bvec
[i
].bv_len
= PAGE_SIZE
;
155 buf
->bvec
[i
].bv_offset
= 0;
162 xdr_free_bvec(struct xdr_buf
*buf
)
169 * xdr_inline_pages - Prepare receive buffer for a large reply
170 * @xdr: xdr_buf into which reply will be placed
171 * @offset: expected offset where data payload will start, in bytes
172 * @pages: vector of struct page pointers
173 * @base: offset in first page where receive should start, in bytes
174 * @len: expected size of the upper layer data payload, in bytes
178 xdr_inline_pages(struct xdr_buf
*xdr
, unsigned int offset
,
179 struct page
**pages
, unsigned int base
, unsigned int len
)
181 struct kvec
*head
= xdr
->head
;
182 struct kvec
*tail
= xdr
->tail
;
183 char *buf
= (char *)head
->iov_base
;
184 unsigned int buflen
= head
->iov_len
;
186 head
->iov_len
= offset
;
189 xdr
->page_base
= base
;
192 tail
->iov_base
= buf
+ offset
;
193 tail
->iov_len
= buflen
- offset
;
196 EXPORT_SYMBOL_GPL(xdr_inline_pages
);
199 * Helper routines for doing 'memmove' like operations on a struct xdr_buf
203 * _shift_data_left_pages
204 * @pages: vector of pages containing both the source and dest memory area.
205 * @pgto_base: page vector address of destination
206 * @pgfrom_base: page vector address of source
207 * @len: number of bytes to copy
209 * Note: the addresses pgto_base and pgfrom_base are both calculated in
211 * if a memory area starts at byte 'base' in page 'pages[i]',
212 * then its address is given as (i << PAGE_CACHE_SHIFT) + base
213 * Alse note: pgto_base must be < pgfrom_base, but the memory areas
214 * they point to may overlap.
217 _shift_data_left_pages(struct page
**pages
, size_t pgto_base
,
218 size_t pgfrom_base
, size_t len
)
220 struct page
**pgfrom
, **pgto
;
224 BUG_ON(pgfrom_base
<= pgto_base
);
229 pgto
= pages
+ (pgto_base
>> PAGE_SHIFT
);
230 pgfrom
= pages
+ (pgfrom_base
>> PAGE_SHIFT
);
232 pgto_base
&= ~PAGE_MASK
;
233 pgfrom_base
&= ~PAGE_MASK
;
236 if (pgto_base
>= PAGE_SIZE
) {
240 if (pgfrom_base
>= PAGE_SIZE
){
246 if (copy
> (PAGE_SIZE
- pgto_base
))
247 copy
= PAGE_SIZE
- pgto_base
;
248 if (copy
> (PAGE_SIZE
- pgfrom_base
))
249 copy
= PAGE_SIZE
- pgfrom_base
;
251 vto
= kmap_atomic(*pgto
);
252 if (*pgto
!= *pgfrom
) {
253 vfrom
= kmap_atomic(*pgfrom
);
254 memcpy(vto
+ pgto_base
, vfrom
+ pgfrom_base
, copy
);
255 kunmap_atomic(vfrom
);
257 memmove(vto
+ pgto_base
, vto
+ pgfrom_base
, copy
);
258 flush_dcache_page(*pgto
);
264 } while ((len
-= copy
) != 0);
268 * _shift_data_right_pages
269 * @pages: vector of pages containing both the source and dest memory area.
270 * @pgto_base: page vector address of destination
271 * @pgfrom_base: page vector address of source
272 * @len: number of bytes to copy
274 * Note: the addresses pgto_base and pgfrom_base are both calculated in
276 * if a memory area starts at byte 'base' in page 'pages[i]',
277 * then its address is given as (i << PAGE_SHIFT) + base
278 * Also note: pgfrom_base must be < pgto_base, but the memory areas
279 * they point to may overlap.
282 _shift_data_right_pages(struct page
**pages
, size_t pgto_base
,
283 size_t pgfrom_base
, size_t len
)
285 struct page
**pgfrom
, **pgto
;
289 BUG_ON(pgto_base
<= pgfrom_base
);
297 pgto
= pages
+ (pgto_base
>> PAGE_SHIFT
);
298 pgfrom
= pages
+ (pgfrom_base
>> PAGE_SHIFT
);
300 pgto_base
&= ~PAGE_MASK
;
301 pgfrom_base
&= ~PAGE_MASK
;
304 /* Are any pointers crossing a page boundary? */
305 if (pgto_base
== 0) {
306 pgto_base
= PAGE_SIZE
;
309 if (pgfrom_base
== 0) {
310 pgfrom_base
= PAGE_SIZE
;
315 if (copy
> pgto_base
)
317 if (copy
> pgfrom_base
)
322 vto
= kmap_atomic(*pgto
);
323 if (*pgto
!= *pgfrom
) {
324 vfrom
= kmap_atomic(*pgfrom
);
325 memcpy(vto
+ pgto_base
, vfrom
+ pgfrom_base
, copy
);
326 kunmap_atomic(vfrom
);
328 memmove(vto
+ pgto_base
, vto
+ pgfrom_base
, copy
);
329 flush_dcache_page(*pgto
);
332 } while ((len
-= copy
) != 0);
337 * @pages: array of pages
338 * @pgbase: page vector address of destination
339 * @p: pointer to source data
342 * Copies data from an arbitrary memory location into an array of pages
343 * The copy is assumed to be non-overlapping.
346 _copy_to_pages(struct page
**pages
, size_t pgbase
, const char *p
, size_t len
)
355 pgto
= pages
+ (pgbase
>> PAGE_SHIFT
);
356 pgbase
&= ~PAGE_MASK
;
359 copy
= PAGE_SIZE
- pgbase
;
363 vto
= kmap_atomic(*pgto
);
364 memcpy(vto
+ pgbase
, p
, copy
);
372 if (pgbase
== PAGE_SIZE
) {
373 flush_dcache_page(*pgto
);
379 flush_dcache_page(*pgto
);
384 * @p: pointer to destination
385 * @pages: array of pages
386 * @pgbase: offset of source data
389 * Copies data into an arbitrary memory location from an array of pages
390 * The copy is assumed to be non-overlapping.
393 _copy_from_pages(char *p
, struct page
**pages
, size_t pgbase
, size_t len
)
395 struct page
**pgfrom
;
402 pgfrom
= pages
+ (pgbase
>> PAGE_SHIFT
);
403 pgbase
&= ~PAGE_MASK
;
406 copy
= PAGE_SIZE
- pgbase
;
410 vfrom
= kmap_atomic(*pgfrom
);
411 memcpy(p
, vfrom
+ pgbase
, copy
);
412 kunmap_atomic(vfrom
);
415 if (pgbase
== PAGE_SIZE
) {
421 } while ((len
-= copy
) != 0);
423 EXPORT_SYMBOL_GPL(_copy_from_pages
);
425 static void xdr_buf_iov_zero(const struct kvec
*iov
, unsigned int base
,
428 if (base
>= iov
->iov_len
)
430 if (len
> iov
->iov_len
- base
)
431 len
= iov
->iov_len
- base
;
432 memset(iov
->iov_base
+ base
, 0, len
);
438 * @pgbase: beginning offset
441 static void xdr_buf_pages_zero(const struct xdr_buf
*buf
, unsigned int pgbase
,
444 struct page
**pages
= buf
->pages
;
451 if (pgbase
>= buf
->page_len
) {
452 xdr_buf_iov_zero(buf
->tail
, pgbase
- buf
->page_len
, len
);
455 if (pgbase
+ len
> buf
->page_len
) {
456 xdr_buf_iov_zero(buf
->tail
, 0, pgbase
+ len
- buf
->page_len
);
457 len
= buf
->page_len
- pgbase
;
460 pgbase
+= buf
->page_base
;
462 page
= pages
+ (pgbase
>> PAGE_SHIFT
);
463 pgbase
&= ~PAGE_MASK
;
466 zero
= PAGE_SIZE
- pgbase
;
470 vpage
= kmap_atomic(*page
);
471 memset(vpage
+ pgbase
, 0, zero
);
472 kunmap_atomic(vpage
);
474 flush_dcache_page(*page
);
478 } while ((len
-= zero
) != 0);
481 static unsigned int xdr_buf_pages_fill_sparse(const struct xdr_buf
*buf
,
482 unsigned int buflen
, gfp_t gfp
)
484 unsigned int i
, npages
, pagelen
;
486 if (!(buf
->flags
& XDRBUF_SPARSE_PAGES
))
488 if (buflen
<= buf
->head
->iov_len
)
490 pagelen
= buflen
- buf
->head
->iov_len
;
491 if (pagelen
> buf
->page_len
)
492 pagelen
= buf
->page_len
;
493 npages
= (pagelen
+ buf
->page_base
+ PAGE_SIZE
- 1) >> PAGE_SHIFT
;
494 for (i
= 0; i
< npages
; i
++) {
497 buf
->pages
[i
] = alloc_page(gfp
);
498 if (likely(buf
->pages
[i
]))
501 pagelen
= i
<< PAGE_SHIFT
;
502 if (pagelen
> buf
->page_base
)
503 buflen
+= pagelen
- buf
->page_base
;
509 static void xdr_buf_try_expand(struct xdr_buf
*buf
, unsigned int len
)
511 struct kvec
*head
= buf
->head
;
512 struct kvec
*tail
= buf
->tail
;
513 unsigned int sum
= head
->iov_len
+ buf
->page_len
+ tail
->iov_len
;
514 unsigned int free_space
, newlen
;
516 if (sum
> buf
->len
) {
517 free_space
= min_t(unsigned int, sum
- buf
->len
, len
);
518 newlen
= xdr_buf_pages_fill_sparse(buf
, buf
->len
+ free_space
,
520 free_space
= newlen
- buf
->len
;
527 if (buf
->buflen
> sum
) {
528 /* Expand the tail buffer */
529 free_space
= min_t(unsigned int, buf
->buflen
- sum
, len
);
530 tail
->iov_len
+= free_space
;
531 buf
->len
+= free_space
;
535 static void xdr_buf_tail_copy_right(const struct xdr_buf
*buf
,
536 unsigned int base
, unsigned int len
,
539 const struct kvec
*tail
= buf
->tail
;
540 unsigned int to
= base
+ shift
;
542 if (to
>= tail
->iov_len
)
544 if (len
+ to
> tail
->iov_len
)
545 len
= tail
->iov_len
- to
;
546 memmove(tail
->iov_base
+ to
, tail
->iov_base
+ base
, len
);
549 static void xdr_buf_pages_copy_right(const struct xdr_buf
*buf
,
550 unsigned int base
, unsigned int len
,
553 const struct kvec
*tail
= buf
->tail
;
554 unsigned int to
= base
+ shift
;
555 unsigned int pglen
= 0;
556 unsigned int talen
= 0, tato
= 0;
558 if (base
>= buf
->page_len
)
560 if (len
> buf
->page_len
- base
)
561 len
= buf
->page_len
- base
;
562 if (to
>= buf
->page_len
) {
563 tato
= to
- buf
->page_len
;
564 if (tail
->iov_len
>= len
+ tato
)
566 else if (tail
->iov_len
> tato
)
567 talen
= tail
->iov_len
- tato
;
568 } else if (len
+ to
>= buf
->page_len
) {
569 pglen
= buf
->page_len
- to
;
571 if (talen
> tail
->iov_len
)
572 talen
= tail
->iov_len
;
576 _copy_from_pages(tail
->iov_base
+ tato
, buf
->pages
,
577 buf
->page_base
+ base
+ pglen
, talen
);
578 _shift_data_right_pages(buf
->pages
, buf
->page_base
+ to
,
579 buf
->page_base
+ base
, pglen
);
582 static void xdr_buf_head_copy_right(const struct xdr_buf
*buf
,
583 unsigned int base
, unsigned int len
,
586 const struct kvec
*head
= buf
->head
;
587 const struct kvec
*tail
= buf
->tail
;
588 unsigned int to
= base
+ shift
;
589 unsigned int pglen
= 0, pgto
= 0;
590 unsigned int talen
= 0, tato
= 0;
592 if (base
>= head
->iov_len
)
594 if (len
> head
->iov_len
- base
)
595 len
= head
->iov_len
- base
;
596 if (to
>= buf
->page_len
+ head
->iov_len
) {
597 tato
= to
- buf
->page_len
- head
->iov_len
;
599 } else if (to
>= head
->iov_len
) {
600 pgto
= to
- head
->iov_len
;
602 if (pgto
+ pglen
> buf
->page_len
) {
603 talen
= pgto
+ pglen
- buf
->page_len
;
608 if (pglen
> buf
->page_len
) {
609 talen
= pglen
- buf
->page_len
;
610 pglen
= buf
->page_len
;
616 if (talen
+ tato
> tail
->iov_len
)
617 talen
= tail
->iov_len
> tato
? tail
->iov_len
- tato
: 0;
618 memcpy(tail
->iov_base
+ tato
, head
->iov_base
+ base
, talen
);
622 _copy_to_pages(buf
->pages
, buf
->page_base
+ pgto
, head
->iov_base
+ base
,
626 memmove(head
->iov_base
+ to
, head
->iov_base
+ base
, len
);
629 static void xdr_buf_tail_shift_right(const struct xdr_buf
*buf
,
630 unsigned int base
, unsigned int len
,
633 const struct kvec
*tail
= buf
->tail
;
635 if (base
>= tail
->iov_len
|| !shift
|| !len
)
637 xdr_buf_tail_copy_right(buf
, base
, len
, shift
);
640 static void xdr_buf_pages_shift_right(const struct xdr_buf
*buf
,
641 unsigned int base
, unsigned int len
,
646 if (base
>= buf
->page_len
) {
647 xdr_buf_tail_shift_right(buf
, base
- buf
->page_len
, len
, shift
);
650 if (base
+ len
> buf
->page_len
)
651 xdr_buf_tail_shift_right(buf
, 0, base
+ len
- buf
->page_len
,
653 xdr_buf_pages_copy_right(buf
, base
, len
, shift
);
656 static void xdr_buf_head_shift_right(const struct xdr_buf
*buf
,
657 unsigned int base
, unsigned int len
,
660 const struct kvec
*head
= buf
->head
;
664 if (base
>= head
->iov_len
) {
665 xdr_buf_pages_shift_right(buf
, head
->iov_len
- base
, len
,
669 if (base
+ len
> head
->iov_len
)
670 xdr_buf_pages_shift_right(buf
, 0, base
+ len
- head
->iov_len
,
672 xdr_buf_head_copy_right(buf
, base
, len
, shift
);
675 static void xdr_buf_tail_copy_left(const struct xdr_buf
*buf
, unsigned int base
,
676 unsigned int len
, unsigned int shift
)
678 const struct kvec
*tail
= buf
->tail
;
680 if (base
>= tail
->iov_len
)
682 if (len
> tail
->iov_len
- base
)
683 len
= tail
->iov_len
- base
;
684 /* Shift data into head */
685 if (shift
> buf
->page_len
+ base
) {
686 const struct kvec
*head
= buf
->head
;
688 head
->iov_len
+ buf
->page_len
+ base
- shift
;
689 unsigned int hdlen
= len
;
691 if (WARN_ONCE(shift
> head
->iov_len
+ buf
->page_len
+ base
,
692 "SUNRPC: Misaligned data.\n"))
694 if (hdto
+ hdlen
> head
->iov_len
)
695 hdlen
= head
->iov_len
- hdto
;
696 memcpy(head
->iov_base
+ hdto
, tail
->iov_base
+ base
, hdlen
);
702 /* Shift data into pages */
704 unsigned int pgto
= buf
->page_len
+ base
- shift
;
705 unsigned int pglen
= len
;
707 if (pgto
+ pglen
> buf
->page_len
)
708 pglen
= buf
->page_len
- pgto
;
709 _copy_to_pages(buf
->pages
, buf
->page_base
+ pgto
,
710 tail
->iov_base
+ base
, pglen
);
716 memmove(tail
->iov_base
+ base
- shift
, tail
->iov_base
+ base
, len
);
719 static void xdr_buf_pages_copy_left(const struct xdr_buf
*buf
,
720 unsigned int base
, unsigned int len
,
725 if (base
>= buf
->page_len
)
727 if (len
> buf
->page_len
- base
)
728 len
= buf
->page_len
- base
;
729 /* Shift data into head */
731 const struct kvec
*head
= buf
->head
;
732 unsigned int hdto
= head
->iov_len
+ base
- shift
;
733 unsigned int hdlen
= len
;
735 if (WARN_ONCE(shift
> head
->iov_len
+ base
,
736 "SUNRPC: Misaligned data.\n"))
738 if (hdto
+ hdlen
> head
->iov_len
)
739 hdlen
= head
->iov_len
- hdto
;
740 _copy_from_pages(head
->iov_base
+ hdto
, buf
->pages
,
741 buf
->page_base
+ base
, hdlen
);
748 _shift_data_left_pages(buf
->pages
, buf
->page_base
+ pgto
,
749 buf
->page_base
+ base
, len
);
752 static void xdr_buf_tail_shift_left(const struct xdr_buf
*buf
,
753 unsigned int base
, unsigned int len
,
758 xdr_buf_tail_copy_left(buf
, base
, len
, shift
);
761 static void xdr_buf_pages_shift_left(const struct xdr_buf
*buf
,
762 unsigned int base
, unsigned int len
,
767 if (base
>= buf
->page_len
) {
768 xdr_buf_tail_shift_left(buf
, base
- buf
->page_len
, len
, shift
);
771 xdr_buf_pages_copy_left(buf
, base
, len
, shift
);
773 if (len
<= buf
->page_len
)
775 xdr_buf_tail_copy_left(buf
, 0, len
- buf
->page_len
, shift
);
781 * @len: new length of buf->head[0]
783 * Shrinks XDR buffer's header kvec buf->head[0], setting it to
784 * 'len' bytes. The extra data is not lost, but is instead
785 * moved into the inlined pages and/or the tail.
787 static unsigned int xdr_shrink_bufhead(struct xdr_buf
*buf
, unsigned int len
)
789 struct kvec
*head
= buf
->head
;
790 unsigned int shift
, buflen
= max(buf
->len
, len
);
792 WARN_ON_ONCE(len
> head
->iov_len
);
793 if (head
->iov_len
> buflen
) {
794 buf
->buflen
-= head
->iov_len
- buflen
;
795 head
->iov_len
= buflen
;
797 if (len
>= head
->iov_len
)
799 shift
= head
->iov_len
- len
;
800 xdr_buf_try_expand(buf
, shift
);
801 xdr_buf_head_shift_right(buf
, len
, buflen
- len
, shift
);
803 buf
->buflen
-= shift
;
809 * xdr_shrink_pagelen - shrinks buf->pages to @len bytes
811 * @len: new page buffer length
813 * The extra data is not lost, but is instead moved into buf->tail.
814 * Returns the actual number of bytes moved.
816 static unsigned int xdr_shrink_pagelen(struct xdr_buf
*buf
, unsigned int len
)
818 unsigned int shift
, buflen
= buf
->len
- buf
->head
->iov_len
;
820 WARN_ON_ONCE(len
> buf
->page_len
);
821 if (buf
->head
->iov_len
>= buf
->len
|| len
> buflen
)
823 if (buf
->page_len
> buflen
) {
824 buf
->buflen
-= buf
->page_len
- buflen
;
825 buf
->page_len
= buflen
;
827 if (len
>= buf
->page_len
)
829 shift
= buf
->page_len
- len
;
830 xdr_buf_try_expand(buf
, shift
);
831 xdr_buf_pages_shift_right(buf
, len
, buflen
- len
, shift
);
834 buf
->buflen
-= shift
;
839 xdr_shift_buf(struct xdr_buf
*buf
, size_t len
)
841 xdr_shrink_bufhead(buf
, buf
->head
->iov_len
- len
);
843 EXPORT_SYMBOL_GPL(xdr_shift_buf
);
846 * xdr_stream_pos - Return the current offset from the start of the xdr_stream
847 * @xdr: pointer to struct xdr_stream
849 unsigned int xdr_stream_pos(const struct xdr_stream
*xdr
)
851 return (unsigned int)(XDR_QUADLEN(xdr
->buf
->len
) - xdr
->nwords
) << 2;
853 EXPORT_SYMBOL_GPL(xdr_stream_pos
);
855 static void xdr_stream_set_pos(struct xdr_stream
*xdr
, unsigned int pos
)
857 unsigned int blen
= xdr
->buf
->len
;
859 xdr
->nwords
= blen
> pos
? XDR_QUADLEN(blen
) - XDR_QUADLEN(pos
) : 0;
862 static void xdr_stream_page_set_pos(struct xdr_stream
*xdr
, unsigned int pos
)
864 xdr_stream_set_pos(xdr
, pos
+ xdr
->buf
->head
[0].iov_len
);
868 * xdr_page_pos - Return the current offset from the start of the xdr pages
869 * @xdr: pointer to struct xdr_stream
871 unsigned int xdr_page_pos(const struct xdr_stream
*xdr
)
873 unsigned int pos
= xdr_stream_pos(xdr
);
875 WARN_ON(pos
< xdr
->buf
->head
[0].iov_len
);
876 return pos
- xdr
->buf
->head
[0].iov_len
;
878 EXPORT_SYMBOL_GPL(xdr_page_pos
);
881 * xdr_init_encode - Initialize a struct xdr_stream for sending data.
882 * @xdr: pointer to xdr_stream struct
883 * @buf: pointer to XDR buffer in which to encode data
884 * @p: current pointer inside XDR buffer
885 * @rqst: pointer to controlling rpc_rqst, for debugging
887 * Note: at the moment the RPC client only passes the length of our
888 * scratch buffer in the xdr_buf's header kvec. Previously this
889 * meant we needed to call xdr_adjust_iovec() after encoding the
890 * data. With the new scheme, the xdr_stream manages the details
891 * of the buffer length, and takes care of adjusting the kvec
894 void xdr_init_encode(struct xdr_stream
*xdr
, struct xdr_buf
*buf
, __be32
*p
,
895 struct rpc_rqst
*rqst
)
897 struct kvec
*iov
= buf
->head
;
898 int scratch_len
= buf
->buflen
- buf
->page_len
- buf
->tail
[0].iov_len
;
900 xdr_reset_scratch_buffer(xdr
);
901 BUG_ON(scratch_len
< 0);
904 xdr
->p
= (__be32
*)((char *)iov
->iov_base
+ iov
->iov_len
);
905 xdr
->end
= (__be32
*)((char *)iov
->iov_base
+ scratch_len
);
906 BUG_ON(iov
->iov_len
> scratch_len
);
908 if (p
!= xdr
->p
&& p
!= NULL
) {
911 BUG_ON(p
< xdr
->p
|| p
> xdr
->end
);
912 len
= (char *)p
- (char *)xdr
->p
;
919 EXPORT_SYMBOL_GPL(xdr_init_encode
);
922 * xdr_commit_encode - Ensure all data is written to buffer
923 * @xdr: pointer to xdr_stream
925 * We handle encoding across page boundaries by giving the caller a
926 * temporary location to write to, then later copying the data into
927 * place; xdr_commit_encode does that copying.
929 * Normally the caller doesn't need to call this directly, as the
930 * following xdr_reserve_space will do it. But an explicit call may be
931 * required at the end of encoding, or any other time when the xdr_buf
932 * data might be read.
934 inline void xdr_commit_encode(struct xdr_stream
*xdr
)
936 int shift
= xdr
->scratch
.iov_len
;
941 page
= page_address(*xdr
->page_ptr
);
942 memcpy(xdr
->scratch
.iov_base
, page
, shift
);
943 memmove(page
, page
+ shift
, (void *)xdr
->p
- page
);
944 xdr_reset_scratch_buffer(xdr
);
946 EXPORT_SYMBOL_GPL(xdr_commit_encode
);
948 static __be32
*xdr_get_next_encode_buffer(struct xdr_stream
*xdr
,
953 int frag1bytes
, frag2bytes
;
955 if (nbytes
> PAGE_SIZE
)
956 goto out_overflow
; /* Bigger buffers require special handling */
957 if (xdr
->buf
->len
+ nbytes
> xdr
->buf
->buflen
)
958 goto out_overflow
; /* Sorry, we're totally out of space */
959 frag1bytes
= (xdr
->end
- xdr
->p
) << 2;
960 frag2bytes
= nbytes
- frag1bytes
;
962 xdr
->iov
->iov_len
+= frag1bytes
;
964 xdr
->buf
->page_len
+= frag1bytes
;
968 * If the last encode didn't end exactly on a page boundary, the
969 * next one will straddle boundaries. Encode into the next
970 * page, then copy it back later in xdr_commit_encode. We use
971 * the "scratch" iov to track any temporarily unused fragment of
972 * space at the end of the previous buffer:
974 xdr_set_scratch_buffer(xdr
, xdr
->p
, frag1bytes
);
975 p
= page_address(*xdr
->page_ptr
);
977 * Note this is where the next encode will start after we've
978 * shifted this one back:
980 xdr
->p
= (void *)p
+ frag2bytes
;
981 space_left
= xdr
->buf
->buflen
- xdr
->buf
->len
;
982 xdr
->end
= (void *)p
+ min_t(int, space_left
, PAGE_SIZE
);
983 xdr
->buf
->page_len
+= frag2bytes
;
984 xdr
->buf
->len
+= nbytes
;
987 trace_rpc_xdr_overflow(xdr
, nbytes
);
992 * xdr_reserve_space - Reserve buffer space for sending
993 * @xdr: pointer to xdr_stream
994 * @nbytes: number of bytes to reserve
996 * Checks that we have enough buffer space to encode 'nbytes' more
997 * bytes of data. If so, update the total xdr_buf length, and
998 * adjust the length of the current kvec.
1000 __be32
* xdr_reserve_space(struct xdr_stream
*xdr
, size_t nbytes
)
1005 xdr_commit_encode(xdr
);
1006 /* align nbytes on the next 32-bit boundary */
1009 q
= p
+ (nbytes
>> 2);
1010 if (unlikely(q
> xdr
->end
|| q
< p
))
1011 return xdr_get_next_encode_buffer(xdr
, nbytes
);
1014 xdr
->iov
->iov_len
+= nbytes
;
1016 xdr
->buf
->page_len
+= nbytes
;
1017 xdr
->buf
->len
+= nbytes
;
1020 EXPORT_SYMBOL_GPL(xdr_reserve_space
);
1024 * xdr_reserve_space_vec - Reserves a large amount of buffer space for sending
1025 * @xdr: pointer to xdr_stream
1026 * @vec: pointer to a kvec array
1027 * @nbytes: number of bytes to reserve
1029 * Reserves enough buffer space to encode 'nbytes' of data and stores the
1030 * pointers in 'vec'. The size argument passed to xdr_reserve_space() is
1031 * determined based on the number of bytes remaining in the current page to
1032 * avoid invalidating iov_base pointers when xdr_commit_encode() is called.
1034 int xdr_reserve_space_vec(struct xdr_stream
*xdr
, struct kvec
*vec
, size_t nbytes
)
1041 * svcrdma requires every READ payload to start somewhere
1044 if (xdr
->iov
== xdr
->buf
->head
) {
1050 thislen
= xdr
->buf
->page_len
% PAGE_SIZE
;
1051 thislen
= min_t(size_t, nbytes
, PAGE_SIZE
- thislen
);
1053 p
= xdr_reserve_space(xdr
, thislen
);
1057 vec
[v
].iov_base
= p
;
1058 vec
[v
].iov_len
= thislen
;
1065 EXPORT_SYMBOL_GPL(xdr_reserve_space_vec
);
1068 * xdr_truncate_encode - truncate an encode buffer
1069 * @xdr: pointer to xdr_stream
1070 * @len: new length of buffer
1072 * Truncates the xdr stream, so that xdr->buf->len == len,
1073 * and xdr->p points at offset len from the start of the buffer, and
1074 * head, tail, and page lengths are adjusted to correspond.
1076 * If this means moving xdr->p to a different buffer, we assume that
1077 * the end pointer should be set to the end of the current page,
1078 * except in the case of the head buffer when we assume the head
1079 * buffer's current length represents the end of the available buffer.
1081 * This is *not* safe to use on a buffer that already has inlined page
1082 * cache pages (as in a zero-copy server read reply), except for the
1083 * simple case of truncating from one position in the tail to another.
1086 void xdr_truncate_encode(struct xdr_stream
*xdr
, size_t len
)
1088 struct xdr_buf
*buf
= xdr
->buf
;
1089 struct kvec
*head
= buf
->head
;
1090 struct kvec
*tail
= buf
->tail
;
1094 if (len
> buf
->len
) {
1098 xdr_commit_encode(xdr
);
1100 fraglen
= min_t(int, buf
->len
- len
, tail
->iov_len
);
1101 tail
->iov_len
-= fraglen
;
1102 buf
->len
-= fraglen
;
1103 if (tail
->iov_len
) {
1104 xdr
->p
= tail
->iov_base
+ tail
->iov_len
;
1105 WARN_ON_ONCE(!xdr
->end
);
1106 WARN_ON_ONCE(!xdr
->iov
);
1109 WARN_ON_ONCE(fraglen
);
1110 fraglen
= min_t(int, buf
->len
- len
, buf
->page_len
);
1111 buf
->page_len
-= fraglen
;
1112 buf
->len
-= fraglen
;
1114 new = buf
->page_base
+ buf
->page_len
;
1116 xdr
->page_ptr
= buf
->pages
+ (new >> PAGE_SHIFT
);
1118 if (buf
->page_len
) {
1119 xdr
->p
= page_address(*xdr
->page_ptr
);
1120 xdr
->end
= (void *)xdr
->p
+ PAGE_SIZE
;
1121 xdr
->p
= (void *)xdr
->p
+ (new % PAGE_SIZE
);
1122 WARN_ON_ONCE(xdr
->iov
);
1126 xdr
->end
= head
->iov_base
+ head
->iov_len
;
1127 /* (otherwise assume xdr->end is already set) */
1129 head
->iov_len
= len
;
1131 xdr
->p
= head
->iov_base
+ head
->iov_len
;
1132 xdr
->iov
= buf
->head
;
1134 EXPORT_SYMBOL(xdr_truncate_encode
);
1137 * xdr_restrict_buflen - decrease available buffer space
1138 * @xdr: pointer to xdr_stream
1139 * @newbuflen: new maximum number of bytes available
1141 * Adjust our idea of how much space is available in the buffer.
1142 * If we've already used too much space in the buffer, returns -1.
1143 * If the available space is already smaller than newbuflen, returns 0
1144 * and does nothing. Otherwise, adjusts xdr->buf->buflen to newbuflen
1145 * and ensures xdr->end is set at most offset newbuflen from the start
1148 int xdr_restrict_buflen(struct xdr_stream
*xdr
, int newbuflen
)
1150 struct xdr_buf
*buf
= xdr
->buf
;
1151 int left_in_this_buf
= (void *)xdr
->end
- (void *)xdr
->p
;
1152 int end_offset
= buf
->len
+ left_in_this_buf
;
1154 if (newbuflen
< 0 || newbuflen
< buf
->len
)
1156 if (newbuflen
> buf
->buflen
)
1158 if (newbuflen
< end_offset
)
1159 xdr
->end
= (void *)xdr
->end
+ newbuflen
- end_offset
;
1160 buf
->buflen
= newbuflen
;
1163 EXPORT_SYMBOL(xdr_restrict_buflen
);
1166 * xdr_write_pages - Insert a list of pages into an XDR buffer for sending
1167 * @xdr: pointer to xdr_stream
1168 * @pages: list of pages
1169 * @base: offset of first byte
1170 * @len: length of data in bytes
1173 void xdr_write_pages(struct xdr_stream
*xdr
, struct page
**pages
, unsigned int base
,
1176 struct xdr_buf
*buf
= xdr
->buf
;
1177 struct kvec
*iov
= buf
->tail
;
1179 buf
->page_base
= base
;
1180 buf
->page_len
= len
;
1182 iov
->iov_base
= (char *)xdr
->p
;
1187 unsigned int pad
= 4 - (len
& 3);
1189 BUG_ON(xdr
->p
>= xdr
->end
);
1190 iov
->iov_base
= (char *)xdr
->p
+ (len
& 3);
1191 iov
->iov_len
+= pad
;
1198 EXPORT_SYMBOL_GPL(xdr_write_pages
);
1200 static unsigned int xdr_set_iov(struct xdr_stream
*xdr
, struct kvec
*iov
,
1201 unsigned int base
, unsigned int len
)
1203 if (len
> iov
->iov_len
)
1205 if (unlikely(base
> len
))
1207 xdr
->p
= (__be32
*)(iov
->iov_base
+ base
);
1208 xdr
->end
= (__be32
*)(iov
->iov_base
+ len
);
1210 xdr
->page_ptr
= NULL
;
1214 static unsigned int xdr_set_tail_base(struct xdr_stream
*xdr
,
1215 unsigned int base
, unsigned int len
)
1217 struct xdr_buf
*buf
= xdr
->buf
;
1219 xdr_stream_set_pos(xdr
, base
+ buf
->page_len
+ buf
->head
->iov_len
);
1220 return xdr_set_iov(xdr
, buf
->tail
, base
, len
);
1223 static unsigned int xdr_set_page_base(struct xdr_stream
*xdr
,
1224 unsigned int base
, unsigned int len
)
1227 unsigned int maxlen
;
1232 maxlen
= xdr
->buf
->page_len
;
1233 if (base
>= maxlen
) {
1241 xdr_stream_page_set_pos(xdr
, base
);
1242 base
+= xdr
->buf
->page_base
;
1244 pgnr
= base
>> PAGE_SHIFT
;
1245 xdr
->page_ptr
= &xdr
->buf
->pages
[pgnr
];
1246 kaddr
= page_address(*xdr
->page_ptr
);
1248 pgoff
= base
& ~PAGE_MASK
;
1249 xdr
->p
= (__be32
*)(kaddr
+ pgoff
);
1251 pgend
= pgoff
+ len
;
1252 if (pgend
> PAGE_SIZE
)
1254 xdr
->end
= (__be32
*)(kaddr
+ pgend
);
1259 static void xdr_set_page(struct xdr_stream
*xdr
, unsigned int base
,
1262 if (xdr_set_page_base(xdr
, base
, len
) == 0) {
1263 base
-= xdr
->buf
->page_len
;
1264 xdr_set_tail_base(xdr
, base
, len
);
1268 static void xdr_set_next_page(struct xdr_stream
*xdr
)
1270 unsigned int newbase
;
1272 newbase
= (1 + xdr
->page_ptr
- xdr
->buf
->pages
) << PAGE_SHIFT
;
1273 newbase
-= xdr
->buf
->page_base
;
1274 if (newbase
< xdr
->buf
->page_len
)
1275 xdr_set_page_base(xdr
, newbase
, xdr_stream_remaining(xdr
));
1277 xdr_set_tail_base(xdr
, 0, xdr_stream_remaining(xdr
));
1280 static bool xdr_set_next_buffer(struct xdr_stream
*xdr
)
1282 if (xdr
->page_ptr
!= NULL
)
1283 xdr_set_next_page(xdr
);
1284 else if (xdr
->iov
== xdr
->buf
->head
)
1285 xdr_set_page(xdr
, 0, xdr_stream_remaining(xdr
));
1286 return xdr
->p
!= xdr
->end
;
1290 * xdr_init_decode - Initialize an xdr_stream for decoding data.
1291 * @xdr: pointer to xdr_stream struct
1292 * @buf: pointer to XDR buffer from which to decode data
1293 * @p: current pointer inside XDR buffer
1294 * @rqst: pointer to controlling rpc_rqst, for debugging
1296 void xdr_init_decode(struct xdr_stream
*xdr
, struct xdr_buf
*buf
, __be32
*p
,
1297 struct rpc_rqst
*rqst
)
1300 xdr_reset_scratch_buffer(xdr
);
1301 xdr
->nwords
= XDR_QUADLEN(buf
->len
);
1302 if (xdr_set_iov(xdr
, buf
->head
, 0, buf
->len
) == 0 &&
1303 xdr_set_page_base(xdr
, 0, buf
->len
) == 0)
1304 xdr_set_iov(xdr
, buf
->tail
, 0, buf
->len
);
1305 if (p
!= NULL
&& p
> xdr
->p
&& xdr
->end
>= p
) {
1306 xdr
->nwords
-= p
- xdr
->p
;
1311 EXPORT_SYMBOL_GPL(xdr_init_decode
);
1314 * xdr_init_decode_pages - Initialize an xdr_stream for decoding into pages
1315 * @xdr: pointer to xdr_stream struct
1316 * @buf: pointer to XDR buffer from which to decode data
1317 * @pages: list of pages to decode into
1318 * @len: length in bytes of buffer in pages
1320 void xdr_init_decode_pages(struct xdr_stream
*xdr
, struct xdr_buf
*buf
,
1321 struct page
**pages
, unsigned int len
)
1323 memset(buf
, 0, sizeof(*buf
));
1325 buf
->page_len
= len
;
1328 xdr_init_decode(xdr
, buf
, NULL
, NULL
);
1330 EXPORT_SYMBOL_GPL(xdr_init_decode_pages
);
1332 static __be32
* __xdr_inline_decode(struct xdr_stream
*xdr
, size_t nbytes
)
1334 unsigned int nwords
= XDR_QUADLEN(nbytes
);
1336 __be32
*q
= p
+ nwords
;
1338 if (unlikely(nwords
> xdr
->nwords
|| q
> xdr
->end
|| q
< p
))
1341 xdr
->nwords
-= nwords
;
1345 static __be32
*xdr_copy_to_scratch(struct xdr_stream
*xdr
, size_t nbytes
)
1348 char *cpdest
= xdr
->scratch
.iov_base
;
1349 size_t cplen
= (char *)xdr
->end
- (char *)xdr
->p
;
1351 if (nbytes
> xdr
->scratch
.iov_len
)
1353 p
= __xdr_inline_decode(xdr
, cplen
);
1356 memcpy(cpdest
, p
, cplen
);
1357 if (!xdr_set_next_buffer(xdr
))
1361 p
= __xdr_inline_decode(xdr
, nbytes
);
1364 memcpy(cpdest
, p
, nbytes
);
1365 return xdr
->scratch
.iov_base
;
1367 trace_rpc_xdr_overflow(xdr
, nbytes
);
1372 * xdr_inline_decode - Retrieve XDR data to decode
1373 * @xdr: pointer to xdr_stream struct
1374 * @nbytes: number of bytes of data to decode
1376 * Check if the input buffer is long enough to enable us to decode
1377 * 'nbytes' more bytes of data starting at the current position.
1378 * If so return the current pointer, then update the current
1381 __be32
* xdr_inline_decode(struct xdr_stream
*xdr
, size_t nbytes
)
1385 if (unlikely(nbytes
== 0))
1387 if (xdr
->p
== xdr
->end
&& !xdr_set_next_buffer(xdr
))
1389 p
= __xdr_inline_decode(xdr
, nbytes
);
1392 return xdr_copy_to_scratch(xdr
, nbytes
);
1394 trace_rpc_xdr_overflow(xdr
, nbytes
);
1397 EXPORT_SYMBOL_GPL(xdr_inline_decode
);
1399 static void xdr_realign_pages(struct xdr_stream
*xdr
)
1401 struct xdr_buf
*buf
= xdr
->buf
;
1402 struct kvec
*iov
= buf
->head
;
1403 unsigned int cur
= xdr_stream_pos(xdr
);
1404 unsigned int copied
;
1406 /* Realign pages to current pointer position */
1407 if (iov
->iov_len
> cur
) {
1408 copied
= xdr_shrink_bufhead(buf
, cur
);
1409 trace_rpc_xdr_alignment(xdr
, cur
, copied
);
1410 xdr_set_page(xdr
, 0, buf
->page_len
);
1414 static unsigned int xdr_align_pages(struct xdr_stream
*xdr
, unsigned int len
)
1416 struct xdr_buf
*buf
= xdr
->buf
;
1417 unsigned int nwords
= XDR_QUADLEN(len
);
1418 unsigned int copied
;
1420 if (xdr
->nwords
== 0)
1423 xdr_realign_pages(xdr
);
1424 if (nwords
> xdr
->nwords
) {
1425 nwords
= xdr
->nwords
;
1428 if (buf
->page_len
<= len
)
1429 len
= buf
->page_len
;
1430 else if (nwords
< xdr
->nwords
) {
1431 /* Truncate page data and move it into the tail */
1432 copied
= xdr_shrink_pagelen(buf
, len
);
1433 trace_rpc_xdr_alignment(xdr
, len
, copied
);
1439 * xdr_read_pages - align page-based XDR data to current pointer position
1440 * @xdr: pointer to xdr_stream struct
1441 * @len: number of bytes of page data
1443 * Moves data beyond the current pointer position from the XDR head[] buffer
1444 * into the page list. Any data that lies beyond current position + @len
1445 * bytes is moved into the XDR tail[]. The xdr_stream current position is
1446 * then advanced past that data to align to the next XDR object in the tail.
1448 * Returns the number of XDR encoded bytes now contained in the pages
1450 unsigned int xdr_read_pages(struct xdr_stream
*xdr
, unsigned int len
)
1452 unsigned int nwords
= XDR_QUADLEN(len
);
1453 unsigned int base
, end
, pglen
;
1455 pglen
= xdr_align_pages(xdr
, nwords
<< 2);
1459 base
= (nwords
<< 2) - pglen
;
1460 end
= xdr_stream_remaining(xdr
) - pglen
;
1462 xdr_set_tail_base(xdr
, base
, end
);
1463 return len
<= pglen
? len
: pglen
;
1465 EXPORT_SYMBOL_GPL(xdr_read_pages
);
1467 unsigned int xdr_align_data(struct xdr_stream
*xdr
, unsigned int offset
,
1468 unsigned int length
)
1470 struct xdr_buf
*buf
= xdr
->buf
;
1471 unsigned int from
, bytes
, len
;
1474 xdr_realign_pages(xdr
);
1475 from
= xdr_page_pos(xdr
);
1477 if (from
>= buf
->page_len
+ buf
->tail
->iov_len
)
1479 if (from
+ buf
->head
->iov_len
>= buf
->len
)
1482 len
= buf
->len
- buf
->head
->iov_len
;
1484 /* We only shift data left! */
1485 if (WARN_ONCE(from
< offset
, "SUNRPC: misaligned data src=%u dst=%u\n",
1488 if (WARN_ONCE(offset
> buf
->page_len
,
1489 "SUNRPC: buffer overflow. offset=%u, page_len=%u\n",
1490 offset
, buf
->page_len
))
1493 /* Move page data to the left */
1494 shift
= from
- offset
;
1495 xdr_buf_pages_shift_left(buf
, from
, len
, shift
);
1497 bytes
= xdr_stream_remaining(xdr
);
1502 xdr
->buf
->len
-= shift
;
1503 xdr_set_page(xdr
, offset
+ length
, bytes
);
1506 EXPORT_SYMBOL_GPL(xdr_align_data
);
1508 unsigned int xdr_expand_hole(struct xdr_stream
*xdr
, unsigned int offset
,
1509 unsigned int length
)
1511 struct xdr_buf
*buf
= xdr
->buf
;
1512 unsigned int from
, to
, shift
;
1514 xdr_realign_pages(xdr
);
1515 from
= xdr_page_pos(xdr
);
1516 to
= xdr_align_size(offset
+ length
);
1518 /* Could the hole be behind us? */
1520 unsigned int buflen
= buf
->len
- buf
->head
->iov_len
;
1522 xdr_buf_try_expand(buf
, shift
);
1523 xdr_buf_pages_shift_right(buf
, from
, buflen
, shift
);
1524 xdr_set_page(xdr
, to
, xdr_stream_remaining(xdr
));
1525 } else if (to
!= from
)
1526 xdr_align_data(xdr
, to
, 0);
1527 xdr_buf_pages_zero(buf
, offset
, length
);
1531 EXPORT_SYMBOL_GPL(xdr_expand_hole
);
1534 * xdr_enter_page - decode data from the XDR page
1535 * @xdr: pointer to xdr_stream struct
1536 * @len: number of bytes of page data
1538 * Moves data beyond the current pointer position from the XDR head[] buffer
1539 * into the page list. Any data that lies beyond current position + "len"
1540 * bytes is moved into the XDR tail[]. The current pointer is then
1541 * repositioned at the beginning of the first XDR page.
1543 void xdr_enter_page(struct xdr_stream
*xdr
, unsigned int len
)
1545 len
= xdr_align_pages(xdr
, len
);
1547 * Position current pointer at beginning of tail, and
1548 * set remaining message length.
1551 xdr_set_page_base(xdr
, 0, len
);
1553 EXPORT_SYMBOL_GPL(xdr_enter_page
);
1555 static const struct kvec empty_iov
= {.iov_base
= NULL
, .iov_len
= 0};
1557 void xdr_buf_from_iov(const struct kvec
*iov
, struct xdr_buf
*buf
)
1559 buf
->head
[0] = *iov
;
1560 buf
->tail
[0] = empty_iov
;
1562 buf
->buflen
= buf
->len
= iov
->iov_len
;
1564 EXPORT_SYMBOL_GPL(xdr_buf_from_iov
);
1567 * xdr_buf_subsegment - set subbuf to a portion of buf
1568 * @buf: an xdr buffer
1569 * @subbuf: the result buffer
1570 * @base: beginning of range in bytes
1571 * @len: length of range in bytes
1573 * sets @subbuf to an xdr buffer representing the portion of @buf of
1574 * length @len starting at offset @base.
1576 * @buf and @subbuf may be pointers to the same struct xdr_buf.
1578 * Returns -1 if base of length are out of bounds.
1580 int xdr_buf_subsegment(const struct xdr_buf
*buf
, struct xdr_buf
*subbuf
,
1581 unsigned int base
, unsigned int len
)
1583 subbuf
->buflen
= subbuf
->len
= len
;
1584 if (base
< buf
->head
[0].iov_len
) {
1585 subbuf
->head
[0].iov_base
= buf
->head
[0].iov_base
+ base
;
1586 subbuf
->head
[0].iov_len
= min_t(unsigned int, len
,
1587 buf
->head
[0].iov_len
- base
);
1588 len
-= subbuf
->head
[0].iov_len
;
1591 base
-= buf
->head
[0].iov_len
;
1592 subbuf
->head
[0].iov_base
= buf
->head
[0].iov_base
;
1593 subbuf
->head
[0].iov_len
= 0;
1596 if (base
< buf
->page_len
) {
1597 subbuf
->page_len
= min(buf
->page_len
- base
, len
);
1598 base
+= buf
->page_base
;
1599 subbuf
->page_base
= base
& ~PAGE_MASK
;
1600 subbuf
->pages
= &buf
->pages
[base
>> PAGE_SHIFT
];
1601 len
-= subbuf
->page_len
;
1604 base
-= buf
->page_len
;
1605 subbuf
->pages
= buf
->pages
;
1606 subbuf
->page_base
= 0;
1607 subbuf
->page_len
= 0;
1610 if (base
< buf
->tail
[0].iov_len
) {
1611 subbuf
->tail
[0].iov_base
= buf
->tail
[0].iov_base
+ base
;
1612 subbuf
->tail
[0].iov_len
= min_t(unsigned int, len
,
1613 buf
->tail
[0].iov_len
- base
);
1614 len
-= subbuf
->tail
[0].iov_len
;
1617 base
-= buf
->tail
[0].iov_len
;
1618 subbuf
->tail
[0].iov_base
= buf
->tail
[0].iov_base
;
1619 subbuf
->tail
[0].iov_len
= 0;
1626 EXPORT_SYMBOL_GPL(xdr_buf_subsegment
);
1629 * xdr_stream_subsegment - set @subbuf to a portion of @xdr
1630 * @xdr: an xdr_stream set up for decoding
1631 * @subbuf: the result buffer
1632 * @nbytes: length of @xdr to extract, in bytes
1634 * Sets up @subbuf to represent a portion of @xdr. The portion
1635 * starts at the current offset in @xdr, and extends for a length
1636 * of @nbytes. If this is successful, @xdr is advanced to the next
1637 * position following that portion.
1640 * %true: @subbuf has been initialized, and @xdr has been advanced.
1641 * %false: a bounds error has occurred
1643 bool xdr_stream_subsegment(struct xdr_stream
*xdr
, struct xdr_buf
*subbuf
,
1644 unsigned int nbytes
)
1646 unsigned int remaining
, offset
, len
;
1648 if (xdr_buf_subsegment(xdr
->buf
, subbuf
, xdr_stream_pos(xdr
), nbytes
))
1651 if (subbuf
->head
[0].iov_len
)
1652 if (!__xdr_inline_decode(xdr
, subbuf
->head
[0].iov_len
))
1655 remaining
= subbuf
->page_len
;
1656 offset
= subbuf
->page_base
;
1658 len
= min_t(unsigned int, remaining
, PAGE_SIZE
) - offset
;
1660 if (xdr
->p
== xdr
->end
&& !xdr_set_next_buffer(xdr
))
1662 if (!__xdr_inline_decode(xdr
, len
))
1671 EXPORT_SYMBOL_GPL(xdr_stream_subsegment
);
1674 * xdr_buf_trim - lop at most "len" bytes off the end of "buf"
1675 * @buf: buf to be trimmed
1676 * @len: number of bytes to reduce "buf" by
1678 * Trim an xdr_buf by the given number of bytes by fixing up the lengths. Note
1679 * that it's possible that we'll trim less than that amount if the xdr_buf is
1680 * too small, or if (for instance) it's all in the head and the parser has
1681 * already read too far into it.
1683 void xdr_buf_trim(struct xdr_buf
*buf
, unsigned int len
)
1686 unsigned int trim
= len
;
1688 if (buf
->tail
[0].iov_len
) {
1689 cur
= min_t(size_t, buf
->tail
[0].iov_len
, trim
);
1690 buf
->tail
[0].iov_len
-= cur
;
1696 if (buf
->page_len
) {
1697 cur
= min_t(unsigned int, buf
->page_len
, trim
);
1698 buf
->page_len
-= cur
;
1704 if (buf
->head
[0].iov_len
) {
1705 cur
= min_t(size_t, buf
->head
[0].iov_len
, trim
);
1706 buf
->head
[0].iov_len
-= cur
;
1710 buf
->len
-= (len
- trim
);
1712 EXPORT_SYMBOL_GPL(xdr_buf_trim
);
1714 static void __read_bytes_from_xdr_buf(const struct xdr_buf
*subbuf
,
1715 void *obj
, unsigned int len
)
1717 unsigned int this_len
;
1719 this_len
= min_t(unsigned int, len
, subbuf
->head
[0].iov_len
);
1720 memcpy(obj
, subbuf
->head
[0].iov_base
, this_len
);
1723 this_len
= min_t(unsigned int, len
, subbuf
->page_len
);
1724 _copy_from_pages(obj
, subbuf
->pages
, subbuf
->page_base
, this_len
);
1727 this_len
= min_t(unsigned int, len
, subbuf
->tail
[0].iov_len
);
1728 memcpy(obj
, subbuf
->tail
[0].iov_base
, this_len
);
1731 /* obj is assumed to point to allocated memory of size at least len: */
1732 int read_bytes_from_xdr_buf(const struct xdr_buf
*buf
, unsigned int base
,
1733 void *obj
, unsigned int len
)
1735 struct xdr_buf subbuf
;
1738 status
= xdr_buf_subsegment(buf
, &subbuf
, base
, len
);
1741 __read_bytes_from_xdr_buf(&subbuf
, obj
, len
);
1744 EXPORT_SYMBOL_GPL(read_bytes_from_xdr_buf
);
1746 static void __write_bytes_to_xdr_buf(const struct xdr_buf
*subbuf
,
1747 void *obj
, unsigned int len
)
1749 unsigned int this_len
;
1751 this_len
= min_t(unsigned int, len
, subbuf
->head
[0].iov_len
);
1752 memcpy(subbuf
->head
[0].iov_base
, obj
, this_len
);
1755 this_len
= min_t(unsigned int, len
, subbuf
->page_len
);
1756 _copy_to_pages(subbuf
->pages
, subbuf
->page_base
, obj
, this_len
);
1759 this_len
= min_t(unsigned int, len
, subbuf
->tail
[0].iov_len
);
1760 memcpy(subbuf
->tail
[0].iov_base
, obj
, this_len
);
1763 /* obj is assumed to point to allocated memory of size at least len: */
1764 int write_bytes_to_xdr_buf(const struct xdr_buf
*buf
, unsigned int base
,
1765 void *obj
, unsigned int len
)
1767 struct xdr_buf subbuf
;
1770 status
= xdr_buf_subsegment(buf
, &subbuf
, base
, len
);
1773 __write_bytes_to_xdr_buf(&subbuf
, obj
, len
);
1776 EXPORT_SYMBOL_GPL(write_bytes_to_xdr_buf
);
1778 int xdr_decode_word(const struct xdr_buf
*buf
, unsigned int base
, u32
*obj
)
1783 status
= read_bytes_from_xdr_buf(buf
, base
, &raw
, sizeof(*obj
));
1786 *obj
= be32_to_cpu(raw
);
1789 EXPORT_SYMBOL_GPL(xdr_decode_word
);
1791 int xdr_encode_word(const struct xdr_buf
*buf
, unsigned int base
, u32 obj
)
1793 __be32 raw
= cpu_to_be32(obj
);
1795 return write_bytes_to_xdr_buf(buf
, base
, &raw
, sizeof(obj
));
1797 EXPORT_SYMBOL_GPL(xdr_encode_word
);
1799 /* Returns 0 on success, or else a negative error code. */
1800 static int xdr_xcode_array2(const struct xdr_buf
*buf
, unsigned int base
,
1801 struct xdr_array2_desc
*desc
, int encode
)
1803 char *elem
= NULL
, *c
;
1804 unsigned int copied
= 0, todo
, avail_here
;
1805 struct page
**ppages
= NULL
;
1809 if (xdr_encode_word(buf
, base
, desc
->array_len
) != 0)
1812 if (xdr_decode_word(buf
, base
, &desc
->array_len
) != 0 ||
1813 desc
->array_len
> desc
->array_maxlen
||
1814 (unsigned long) base
+ 4 + desc
->array_len
*
1815 desc
->elem_size
> buf
->len
)
1823 todo
= desc
->array_len
* desc
->elem_size
;
1826 if (todo
&& base
< buf
->head
->iov_len
) {
1827 c
= buf
->head
->iov_base
+ base
;
1828 avail_here
= min_t(unsigned int, todo
,
1829 buf
->head
->iov_len
- base
);
1832 while (avail_here
>= desc
->elem_size
) {
1833 err
= desc
->xcode(desc
, c
);
1836 c
+= desc
->elem_size
;
1837 avail_here
-= desc
->elem_size
;
1841 elem
= kmalloc(desc
->elem_size
, GFP_KERNEL
);
1847 err
= desc
->xcode(desc
, elem
);
1850 memcpy(c
, elem
, avail_here
);
1852 memcpy(elem
, c
, avail_here
);
1853 copied
= avail_here
;
1855 base
= buf
->head
->iov_len
; /* align to start of pages */
1858 /* process pages array */
1859 base
-= buf
->head
->iov_len
;
1860 if (todo
&& base
< buf
->page_len
) {
1861 unsigned int avail_page
;
1863 avail_here
= min(todo
, buf
->page_len
- base
);
1866 base
+= buf
->page_base
;
1867 ppages
= buf
->pages
+ (base
>> PAGE_SHIFT
);
1869 avail_page
= min_t(unsigned int, PAGE_SIZE
- base
,
1871 c
= kmap(*ppages
) + base
;
1873 while (avail_here
) {
1874 avail_here
-= avail_page
;
1875 if (copied
|| avail_page
< desc
->elem_size
) {
1876 unsigned int l
= min(avail_page
,
1877 desc
->elem_size
- copied
);
1879 elem
= kmalloc(desc
->elem_size
,
1887 err
= desc
->xcode(desc
, elem
);
1891 memcpy(c
, elem
+ copied
, l
);
1893 if (copied
== desc
->elem_size
)
1896 memcpy(elem
+ copied
, c
, l
);
1898 if (copied
== desc
->elem_size
) {
1899 err
= desc
->xcode(desc
, elem
);
1908 while (avail_page
>= desc
->elem_size
) {
1909 err
= desc
->xcode(desc
, c
);
1912 c
+= desc
->elem_size
;
1913 avail_page
-= desc
->elem_size
;
1916 unsigned int l
= min(avail_page
,
1917 desc
->elem_size
- copied
);
1919 elem
= kmalloc(desc
->elem_size
,
1927 err
= desc
->xcode(desc
, elem
);
1931 memcpy(c
, elem
+ copied
, l
);
1933 if (copied
== desc
->elem_size
)
1936 memcpy(elem
+ copied
, c
, l
);
1938 if (copied
== desc
->elem_size
) {
1939 err
= desc
->xcode(desc
, elem
);
1952 avail_page
= min(avail_here
,
1953 (unsigned int) PAGE_SIZE
);
1955 base
= buf
->page_len
; /* align to start of tail */
1959 base
-= buf
->page_len
;
1961 c
= buf
->tail
->iov_base
+ base
;
1963 unsigned int l
= desc
->elem_size
- copied
;
1966 memcpy(c
, elem
+ copied
, l
);
1968 memcpy(elem
+ copied
, c
, l
);
1969 err
= desc
->xcode(desc
, elem
);
1977 err
= desc
->xcode(desc
, c
);
1980 c
+= desc
->elem_size
;
1981 todo
-= desc
->elem_size
;
1993 int xdr_decode_array2(const struct xdr_buf
*buf
, unsigned int base
,
1994 struct xdr_array2_desc
*desc
)
1996 if (base
>= buf
->len
)
1999 return xdr_xcode_array2(buf
, base
, desc
, 0);
2001 EXPORT_SYMBOL_GPL(xdr_decode_array2
);
2003 int xdr_encode_array2(const struct xdr_buf
*buf
, unsigned int base
,
2004 struct xdr_array2_desc
*desc
)
2006 if ((unsigned long) base
+ 4 + desc
->array_len
* desc
->elem_size
>
2007 buf
->head
->iov_len
+ buf
->page_len
+ buf
->tail
->iov_len
)
2010 return xdr_xcode_array2(buf
, base
, desc
, 1);
2012 EXPORT_SYMBOL_GPL(xdr_encode_array2
);
2014 int xdr_process_buf(const struct xdr_buf
*buf
, unsigned int offset
,
2016 int (*actor
)(struct scatterlist
*, void *), void *data
)
2019 unsigned int page_len
, thislen
, page_offset
;
2020 struct scatterlist sg
[1];
2022 sg_init_table(sg
, 1);
2024 if (offset
>= buf
->head
[0].iov_len
) {
2025 offset
-= buf
->head
[0].iov_len
;
2027 thislen
= buf
->head
[0].iov_len
- offset
;
2030 sg_set_buf(sg
, buf
->head
[0].iov_base
+ offset
, thislen
);
2031 ret
= actor(sg
, data
);
2040 if (offset
>= buf
->page_len
) {
2041 offset
-= buf
->page_len
;
2043 page_len
= buf
->page_len
- offset
;
2047 page_offset
= (offset
+ buf
->page_base
) & (PAGE_SIZE
- 1);
2048 i
= (offset
+ buf
->page_base
) >> PAGE_SHIFT
;
2049 thislen
= PAGE_SIZE
- page_offset
;
2051 if (thislen
> page_len
)
2053 sg_set_page(sg
, buf
->pages
[i
], thislen
, page_offset
);
2054 ret
= actor(sg
, data
);
2057 page_len
-= thislen
;
2060 thislen
= PAGE_SIZE
;
2061 } while (page_len
!= 0);
2066 if (offset
< buf
->tail
[0].iov_len
) {
2067 thislen
= buf
->tail
[0].iov_len
- offset
;
2070 sg_set_buf(sg
, buf
->tail
[0].iov_base
+ offset
, thislen
);
2071 ret
= actor(sg
, data
);
2079 EXPORT_SYMBOL_GPL(xdr_process_buf
);
2082 * xdr_stream_decode_opaque - Decode variable length opaque
2083 * @xdr: pointer to xdr_stream
2084 * @ptr: location to store opaque data
2085 * @size: size of storage buffer @ptr
2088 * On success, returns size of object stored in *@ptr
2089 * %-EBADMSG on XDR buffer overflow
2090 * %-EMSGSIZE on overflow of storage buffer @ptr
2092 ssize_t
xdr_stream_decode_opaque(struct xdr_stream
*xdr
, void *ptr
, size_t size
)
2097 ret
= xdr_stream_decode_opaque_inline(xdr
, &p
, size
);
2100 memcpy(ptr
, p
, ret
);
2103 EXPORT_SYMBOL_GPL(xdr_stream_decode_opaque
);
2106 * xdr_stream_decode_opaque_dup - Decode and duplicate variable length opaque
2107 * @xdr: pointer to xdr_stream
2108 * @ptr: location to store pointer to opaque data
2109 * @maxlen: maximum acceptable object size
2110 * @gfp_flags: GFP mask to use
2113 * On success, returns size of object stored in *@ptr
2114 * %-EBADMSG on XDR buffer overflow
2115 * %-EMSGSIZE if the size of the object would exceed @maxlen
2116 * %-ENOMEM on memory allocation failure
2118 ssize_t
xdr_stream_decode_opaque_dup(struct xdr_stream
*xdr
, void **ptr
,
2119 size_t maxlen
, gfp_t gfp_flags
)
2124 ret
= xdr_stream_decode_opaque_inline(xdr
, &p
, maxlen
);
2126 *ptr
= kmemdup(p
, ret
, gfp_flags
);
2134 EXPORT_SYMBOL_GPL(xdr_stream_decode_opaque_dup
);
2137 * xdr_stream_decode_string - Decode variable length string
2138 * @xdr: pointer to xdr_stream
2139 * @str: location to store string
2140 * @size: size of storage buffer @str
2143 * On success, returns length of NUL-terminated string stored in *@str
2144 * %-EBADMSG on XDR buffer overflow
2145 * %-EMSGSIZE on overflow of storage buffer @str
2147 ssize_t
xdr_stream_decode_string(struct xdr_stream
*xdr
, char *str
, size_t size
)
2152 ret
= xdr_stream_decode_opaque_inline(xdr
, &p
, size
);
2154 memcpy(str
, p
, ret
);
2161 EXPORT_SYMBOL_GPL(xdr_stream_decode_string
);
2164 * xdr_stream_decode_string_dup - Decode and duplicate variable length string
2165 * @xdr: pointer to xdr_stream
2166 * @str: location to store pointer to string
2167 * @maxlen: maximum acceptable string length
2168 * @gfp_flags: GFP mask to use
2171 * On success, returns length of NUL-terminated string stored in *@ptr
2172 * %-EBADMSG on XDR buffer overflow
2173 * %-EMSGSIZE if the size of the string would exceed @maxlen
2174 * %-ENOMEM on memory allocation failure
2176 ssize_t
xdr_stream_decode_string_dup(struct xdr_stream
*xdr
, char **str
,
2177 size_t maxlen
, gfp_t gfp_flags
)
2182 ret
= xdr_stream_decode_opaque_inline(xdr
, &p
, maxlen
);
2184 char *s
= kmemdup_nul(p
, ret
, gfp_flags
);
2194 EXPORT_SYMBOL_GPL(xdr_stream_decode_string_dup
);