1 /* RxRPC recvmsg() implementation
3 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
4 * Written by David Howells (dhowells@redhat.com)
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
12 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
14 #include <linux/net.h>
15 #include <linux/skbuff.h>
16 #include <linux/export.h>
17 #include <linux/sched/signal.h>
20 #include <net/af_rxrpc.h>
21 #include "ar-internal.h"
24 * Post a call for attention by the socket or kernel service. Further
25 * notifications are suppressed by putting recvmsg_link on a dummy queue.
27 void rxrpc_notify_socket(struct rxrpc_call
*call
)
29 struct rxrpc_sock
*rx
;
32 _enter("%d", call
->debug_id
);
34 if (!list_empty(&call
->recvmsg_link
))
39 rx
= rcu_dereference(call
->socket
);
41 if (rx
&& sk
->sk_state
< RXRPC_CLOSE
) {
42 if (call
->notify_rx
) {
43 call
->notify_rx(sk
, call
, call
->user_call_ID
);
45 write_lock_bh(&rx
->recvmsg_lock
);
46 if (list_empty(&call
->recvmsg_link
)) {
47 rxrpc_get_call(call
, rxrpc_call_got
);
48 list_add_tail(&call
->recvmsg_link
, &rx
->recvmsg_q
);
50 write_unlock_bh(&rx
->recvmsg_lock
);
52 if (!sock_flag(sk
, SOCK_DEAD
)) {
53 _debug("call %ps", sk
->sk_data_ready
);
54 sk
->sk_data_ready(sk
);
64 * Pass a call terminating message to userspace.
66 static int rxrpc_recvmsg_term(struct rxrpc_call
*call
, struct msghdr
*msg
)
71 switch (call
->completion
) {
72 case RXRPC_CALL_SUCCEEDED
:
74 if (rxrpc_is_service_call(call
))
75 ret
= put_cmsg(msg
, SOL_RXRPC
, RXRPC_ACK
, 0, &tmp
);
77 case RXRPC_CALL_REMOTELY_ABORTED
:
78 tmp
= call
->abort_code
;
79 ret
= put_cmsg(msg
, SOL_RXRPC
, RXRPC_ABORT
, 4, &tmp
);
81 case RXRPC_CALL_LOCALLY_ABORTED
:
82 tmp
= call
->abort_code
;
83 ret
= put_cmsg(msg
, SOL_RXRPC
, RXRPC_ABORT
, 4, &tmp
);
85 case RXRPC_CALL_NETWORK_ERROR
:
87 ret
= put_cmsg(msg
, SOL_RXRPC
, RXRPC_NET_ERROR
, 4, &tmp
);
89 case RXRPC_CALL_LOCAL_ERROR
:
91 ret
= put_cmsg(msg
, SOL_RXRPC
, RXRPC_LOCAL_ERROR
, 4, &tmp
);
94 pr_err("Invalid terminal call state %u\n", call
->state
);
99 trace_rxrpc_recvmsg(call
, rxrpc_recvmsg_terminal
, call
->rx_hard_ack
,
100 call
->rx_pkt_offset
, call
->rx_pkt_len
, ret
);
105 * Pass back notification of a new call. The call is added to the
106 * to-be-accepted list. This means that the next call to be accepted might not
107 * be the last call seen awaiting acceptance, but unless we leave this on the
108 * front of the queue and block all other messages until someone gives us a
109 * user_ID for it, there's not a lot we can do.
111 static int rxrpc_recvmsg_new_call(struct rxrpc_sock
*rx
,
112 struct rxrpc_call
*call
,
113 struct msghdr
*msg
, int flags
)
117 ret
= put_cmsg(msg
, SOL_RXRPC
, RXRPC_NEW_CALL
, 0, &tmp
);
119 if (ret
== 0 && !(flags
& MSG_PEEK
)) {
120 _debug("to be accepted");
121 write_lock_bh(&rx
->recvmsg_lock
);
122 list_del_init(&call
->recvmsg_link
);
123 write_unlock_bh(&rx
->recvmsg_lock
);
125 rxrpc_get_call(call
, rxrpc_call_got
);
126 write_lock(&rx
->call_lock
);
127 list_add_tail(&call
->accept_link
, &rx
->to_be_accepted
);
128 write_unlock(&rx
->call_lock
);
131 trace_rxrpc_recvmsg(call
, rxrpc_recvmsg_to_be_accepted
, 1, 0, 0, ret
);
136 * End the packet reception phase.
138 static void rxrpc_end_rx_phase(struct rxrpc_call
*call
, rxrpc_serial_t serial
)
140 _enter("%d,%s", call
->debug_id
, rxrpc_call_states
[call
->state
]);
142 trace_rxrpc_receive(call
, rxrpc_receive_end
, 0, call
->rx_top
);
143 ASSERTCMP(call
->rx_hard_ack
, ==, call
->rx_top
);
145 if (call
->state
== RXRPC_CALL_CLIENT_RECV_REPLY
) {
146 rxrpc_propose_ACK(call
, RXRPC_ACK_IDLE
, 0, serial
, true, false,
147 rxrpc_propose_ack_terminal_ack
);
148 rxrpc_send_ack_packet(call
, false);
151 write_lock_bh(&call
->state_lock
);
153 switch (call
->state
) {
154 case RXRPC_CALL_CLIENT_RECV_REPLY
:
155 __rxrpc_call_completed(call
);
156 write_unlock_bh(&call
->state_lock
);
159 case RXRPC_CALL_SERVER_RECV_REQUEST
:
160 call
->tx_phase
= true;
161 call
->state
= RXRPC_CALL_SERVER_ACK_REQUEST
;
162 call
->ack_at
= call
->expire_at
;
163 write_unlock_bh(&call
->state_lock
);
164 rxrpc_propose_ACK(call
, RXRPC_ACK_DELAY
, 0, serial
, false, true,
165 rxrpc_propose_ack_processing_op
);
168 write_unlock_bh(&call
->state_lock
);
174 * Discard a packet we've used up and advance the Rx window by one.
176 static void rxrpc_rotate_rx_window(struct rxrpc_call
*call
)
178 struct rxrpc_skb_priv
*sp
;
180 rxrpc_serial_t serial
;
181 rxrpc_seq_t hard_ack
, top
;
185 _enter("%d", call
->debug_id
);
187 hard_ack
= call
->rx_hard_ack
;
188 top
= smp_load_acquire(&call
->rx_top
);
189 ASSERT(before(hard_ack
, top
));
192 ix
= hard_ack
& RXRPC_RXTX_BUFF_MASK
;
193 skb
= call
->rxtx_buffer
[ix
];
194 rxrpc_see_skb(skb
, rxrpc_skb_rx_rotated
);
196 flags
= sp
->hdr
.flags
;
197 serial
= sp
->hdr
.serial
;
198 if (call
->rxtx_annotations
[ix
] & RXRPC_RX_ANNO_JUMBO
)
199 serial
+= (call
->rxtx_annotations
[ix
] & RXRPC_RX_ANNO_JUMBO
) - 1;
201 call
->rxtx_buffer
[ix
] = NULL
;
202 call
->rxtx_annotations
[ix
] = 0;
203 /* Barrier against rxrpc_input_data(). */
204 smp_store_release(&call
->rx_hard_ack
, hard_ack
);
206 rxrpc_free_skb(skb
, rxrpc_skb_rx_freed
);
208 _debug("%u,%u,%02x", hard_ack
, top
, flags
);
209 trace_rxrpc_receive(call
, rxrpc_receive_rotate
, serial
, hard_ack
);
210 if (flags
& RXRPC_LAST_PACKET
) {
211 rxrpc_end_rx_phase(call
, serial
);
213 /* Check to see if there's an ACK that needs sending. */
214 if (after_eq(hard_ack
, call
->ackr_consumed
+ 2) ||
215 after_eq(top
, call
->ackr_seen
+ 2) ||
216 (hard_ack
== top
&& after(hard_ack
, call
->ackr_consumed
)))
217 rxrpc_propose_ACK(call
, RXRPC_ACK_DELAY
, 0, serial
,
219 rxrpc_propose_ack_rotate_rx
);
220 if (call
->ackr_reason
)
221 rxrpc_send_ack_packet(call
, false);
226 * Decrypt and verify a (sub)packet. The packet's length may be changed due to
227 * padding, but if this is the case, the packet length will be resident in the
228 * socket buffer. Note that we can't modify the master skb info as the skb may
229 * be the home to multiple subpackets.
231 static int rxrpc_verify_packet(struct rxrpc_call
*call
, struct sk_buff
*skb
,
233 unsigned int offset
, unsigned int len
)
235 struct rxrpc_skb_priv
*sp
= rxrpc_skb(skb
);
236 rxrpc_seq_t seq
= sp
->hdr
.seq
;
237 u16 cksum
= sp
->hdr
.cksum
;
241 /* For all but the head jumbo subpacket, the security checksum is in a
242 * jumbo header immediately prior to the data.
244 if ((annotation
& RXRPC_RX_ANNO_JUMBO
) > 1) {
246 if (skb_copy_bits(skb
, offset
- 2, &tmp
, 2) < 0)
249 seq
+= (annotation
& RXRPC_RX_ANNO_JUMBO
) - 1;
252 return call
->conn
->security
->verify_packet(call
, skb
, offset
, len
,
257 * Locate the data within a packet. This is complicated by:
259 * (1) An skb may contain a jumbo packet - so we have to find the appropriate
262 * (2) The (sub)packets may be encrypted and, if so, the encrypted portion
263 * contains an extra header which includes the true length of the data,
264 * excluding any encrypted padding.
266 static int rxrpc_locate_data(struct rxrpc_call
*call
, struct sk_buff
*skb
,
268 unsigned int *_offset
, unsigned int *_len
)
270 unsigned int offset
= sizeof(struct rxrpc_wire_header
);
271 unsigned int len
= *_len
;
273 u8 annotation
= *_annotation
;
275 /* Locate the subpacket */
276 len
= skb
->len
- offset
;
277 if ((annotation
& RXRPC_RX_ANNO_JUMBO
) > 0) {
278 offset
+= (((annotation
& RXRPC_RX_ANNO_JUMBO
) - 1) *
279 RXRPC_JUMBO_SUBPKTLEN
);
280 len
= (annotation
& RXRPC_RX_ANNO_JLAST
) ?
281 skb
->len
- offset
: RXRPC_JUMBO_SUBPKTLEN
;
284 if (!(annotation
& RXRPC_RX_ANNO_VERIFIED
)) {
285 ret
= rxrpc_verify_packet(call
, skb
, annotation
, offset
, len
);
288 *_annotation
|= RXRPC_RX_ANNO_VERIFIED
;
293 call
->conn
->security
->locate_data(call
, skb
, _offset
, _len
);
298 * Deliver messages to a call. This keeps processing packets until the buffer
299 * is filled and we find either more DATA (returns 0) or the end of the DATA
300 * (returns 1). If more packets are required, it returns -EAGAIN.
302 static int rxrpc_recvmsg_data(struct socket
*sock
, struct rxrpc_call
*call
,
303 struct msghdr
*msg
, struct iov_iter
*iter
,
304 size_t len
, int flags
, size_t *_offset
)
306 struct rxrpc_skb_priv
*sp
;
308 rxrpc_seq_t hard_ack
, top
, seq
;
311 unsigned int rx_pkt_offset
, rx_pkt_len
;
312 int ix
, copy
, ret
= -EAGAIN
, ret2
;
314 rx_pkt_offset
= call
->rx_pkt_offset
;
315 rx_pkt_len
= call
->rx_pkt_len
;
317 if (call
->state
>= RXRPC_CALL_SERVER_ACK_REQUEST
) {
318 seq
= call
->rx_hard_ack
;
323 /* Barriers against rxrpc_input_data(). */
324 hard_ack
= call
->rx_hard_ack
;
326 while (top
= smp_load_acquire(&call
->rx_top
),
329 ix
= seq
& RXRPC_RXTX_BUFF_MASK
;
330 skb
= call
->rxtx_buffer
[ix
];
332 trace_rxrpc_recvmsg(call
, rxrpc_recvmsg_hole
, seq
,
333 rx_pkt_offset
, rx_pkt_len
, 0);
337 rxrpc_see_skb(skb
, rxrpc_skb_rx_seen
);
340 if (!(flags
& MSG_PEEK
))
341 trace_rxrpc_receive(call
, rxrpc_receive_front
,
342 sp
->hdr
.serial
, seq
);
345 sock_recv_timestamp(msg
, sock
->sk
, skb
);
347 if (rx_pkt_offset
== 0) {
348 ret2
= rxrpc_locate_data(call
, skb
,
349 &call
->rxtx_annotations
[ix
],
350 &rx_pkt_offset
, &rx_pkt_len
);
351 trace_rxrpc_recvmsg(call
, rxrpc_recvmsg_next
, seq
,
352 rx_pkt_offset
, rx_pkt_len
, ret2
);
358 trace_rxrpc_recvmsg(call
, rxrpc_recvmsg_cont
, seq
,
359 rx_pkt_offset
, rx_pkt_len
, 0);
362 /* We have to handle short, empty and used-up DATA packets. */
363 remain
= len
- *_offset
;
368 ret2
= skb_copy_datagram_iter(skb
, rx_pkt_offset
, iter
,
375 /* handle piecemeal consumption of data packets */
376 rx_pkt_offset
+= copy
;
381 if (rx_pkt_len
> 0) {
382 trace_rxrpc_recvmsg(call
, rxrpc_recvmsg_full
, seq
,
383 rx_pkt_offset
, rx_pkt_len
, 0);
384 ASSERTCMP(*_offset
, ==, len
);
389 /* The whole packet has been transferred. */
390 last
= sp
->hdr
.flags
& RXRPC_LAST_PACKET
;
391 if (!(flags
& MSG_PEEK
))
392 rxrpc_rotate_rx_window(call
);
397 ASSERTCMP(seq
, ==, READ_ONCE(call
->rx_top
));
406 if (!(flags
& MSG_PEEK
)) {
407 call
->rx_pkt_offset
= rx_pkt_offset
;
408 call
->rx_pkt_len
= rx_pkt_len
;
411 trace_rxrpc_recvmsg(call
, rxrpc_recvmsg_data_return
, seq
,
412 rx_pkt_offset
, rx_pkt_len
, ret
);
417 * Receive a message from an RxRPC socket
418 * - we need to be careful about two or more threads calling recvmsg
421 int rxrpc_recvmsg(struct socket
*sock
, struct msghdr
*msg
, size_t len
,
424 struct rxrpc_call
*call
;
425 struct rxrpc_sock
*rx
= rxrpc_sk(sock
->sk
);
433 trace_rxrpc_recvmsg(NULL
, rxrpc_recvmsg_enter
, 0, 0, 0, 0);
435 if (flags
& (MSG_OOB
| MSG_TRUNC
))
438 timeo
= sock_rcvtimeo(&rx
->sk
, flags
& MSG_DONTWAIT
);
443 /* Return immediately if a client socket has no outstanding calls */
444 if (RB_EMPTY_ROOT(&rx
->calls
) &&
445 list_empty(&rx
->recvmsg_q
) &&
446 rx
->sk
.sk_state
!= RXRPC_SERVER_LISTENING
) {
447 release_sock(&rx
->sk
);
451 if (list_empty(&rx
->recvmsg_q
)) {
458 release_sock(&rx
->sk
);
460 /* Wait for something to happen */
461 prepare_to_wait_exclusive(sk_sleep(&rx
->sk
), &wait
,
463 ret
= sock_error(&rx
->sk
);
467 if (list_empty(&rx
->recvmsg_q
)) {
468 if (signal_pending(current
))
469 goto wait_interrupted
;
470 trace_rxrpc_recvmsg(NULL
, rxrpc_recvmsg_wait
,
472 timeo
= schedule_timeout(timeo
);
474 finish_wait(sk_sleep(&rx
->sk
), &wait
);
478 /* Find the next call and dequeue it if we're not just peeking. If we
479 * do dequeue it, that comes with a ref that we will need to release.
481 write_lock_bh(&rx
->recvmsg_lock
);
482 l
= rx
->recvmsg_q
.next
;
483 call
= list_entry(l
, struct rxrpc_call
, recvmsg_link
);
484 if (!(flags
& MSG_PEEK
))
485 list_del_init(&call
->recvmsg_link
);
487 rxrpc_get_call(call
, rxrpc_call_got
);
488 write_unlock_bh(&rx
->recvmsg_lock
);
490 trace_rxrpc_recvmsg(call
, rxrpc_recvmsg_dequeue
, 0, 0, 0, 0);
492 /* We're going to drop the socket lock, so we need to lock the call
493 * against interference by sendmsg.
495 if (!mutex_trylock(&call
->user_mutex
)) {
497 if (flags
& MSG_DONTWAIT
)
498 goto error_requeue_call
;
500 if (mutex_lock_interruptible(&call
->user_mutex
) < 0)
501 goto error_requeue_call
;
504 release_sock(&rx
->sk
);
506 if (test_bit(RXRPC_CALL_RELEASED
, &call
->flags
))
509 if (test_bit(RXRPC_CALL_HAS_USERID
, &call
->flags
)) {
510 if (flags
& MSG_CMSG_COMPAT
) {
511 unsigned int id32
= call
->user_call_ID
;
513 ret
= put_cmsg(msg
, SOL_RXRPC
, RXRPC_USER_CALL_ID
,
514 sizeof(unsigned int), &id32
);
516 ret
= put_cmsg(msg
, SOL_RXRPC
, RXRPC_USER_CALL_ID
,
517 sizeof(unsigned long),
518 &call
->user_call_ID
);
521 goto error_unlock_call
;
525 struct sockaddr_rxrpc
*srx
= msg
->msg_name
;
526 size_t len
= sizeof(call
->peer
->srx
);
528 memcpy(msg
->msg_name
, &call
->peer
->srx
, len
);
529 srx
->srx_service
= call
->service_id
;
530 msg
->msg_namelen
= len
;
533 switch (READ_ONCE(call
->state
)) {
534 case RXRPC_CALL_SERVER_ACCEPTING
:
535 ret
= rxrpc_recvmsg_new_call(rx
, call
, msg
, flags
);
537 case RXRPC_CALL_CLIENT_RECV_REPLY
:
538 case RXRPC_CALL_SERVER_RECV_REQUEST
:
539 case RXRPC_CALL_SERVER_ACK_REQUEST
:
540 ret
= rxrpc_recvmsg_data(sock
, call
, msg
, &msg
->msg_iter
, len
,
545 if (after(call
->rx_top
, call
->rx_hard_ack
) &&
546 call
->rxtx_buffer
[(call
->rx_hard_ack
+ 1) & RXRPC_RXTX_BUFF_MASK
])
547 rxrpc_notify_socket(call
);
555 goto error_unlock_call
;
557 if (call
->state
== RXRPC_CALL_COMPLETE
) {
558 ret
= rxrpc_recvmsg_term(call
, msg
);
560 goto error_unlock_call
;
561 if (!(flags
& MSG_PEEK
))
562 rxrpc_release_call(rx
, call
);
563 msg
->msg_flags
|= MSG_EOR
;
568 msg
->msg_flags
|= MSG_MORE
;
570 msg
->msg_flags
&= ~MSG_MORE
;
574 mutex_unlock(&call
->user_mutex
);
575 rxrpc_put_call(call
, rxrpc_call_put
);
576 trace_rxrpc_recvmsg(call
, rxrpc_recvmsg_return
, 0, 0, 0, ret
);
580 if (!(flags
& MSG_PEEK
)) {
581 write_lock_bh(&rx
->recvmsg_lock
);
582 list_add(&call
->recvmsg_link
, &rx
->recvmsg_q
);
583 write_unlock_bh(&rx
->recvmsg_lock
);
584 trace_rxrpc_recvmsg(call
, rxrpc_recvmsg_requeue
, 0, 0, 0, 0);
586 rxrpc_put_call(call
, rxrpc_call_put
);
589 release_sock(&rx
->sk
);
590 trace_rxrpc_recvmsg(call
, rxrpc_recvmsg_return
, 0, 0, 0, ret
);
594 ret
= sock_intr_errno(timeo
);
596 finish_wait(sk_sleep(&rx
->sk
), &wait
);
602 * rxrpc_kernel_recv_data - Allow a kernel service to receive data/info
603 * @sock: The socket that the call exists on
604 * @call: The call to send data through
605 * @buf: The buffer to receive into
606 * @size: The size of the buffer, including data already read
607 * @_offset: The running offset into the buffer.
608 * @want_more: True if more data is expected to be read
609 * @_abort: Where the abort code is stored if -ECONNABORTED is returned
611 * Allow a kernel service to receive data and pick up information about the
612 * state of a call. Returns 0 if got what was asked for and there's more
613 * available, 1 if we got what was asked for and we're at the end of the data
614 * and -EAGAIN if we need more data.
616 * Note that we may return -EAGAIN to drain empty packets at the end of the
617 * data, even if we've already copied over the requested data.
619 * This function adds the amount it transfers to *_offset, so this should be
620 * precleared as appropriate. Note that the amount remaining in the buffer is
621 * taken to be size - *_offset.
623 * *_abort should also be initialised to 0.
625 int rxrpc_kernel_recv_data(struct socket
*sock
, struct rxrpc_call
*call
,
626 void *buf
, size_t size
, size_t *_offset
,
627 bool want_more
, u32
*_abort
)
629 struct iov_iter iter
;
633 _enter("{%d,%s},%zu/%zu,%d",
634 call
->debug_id
, rxrpc_call_states
[call
->state
],
635 *_offset
, size
, want_more
);
637 ASSERTCMP(*_offset
, <=, size
);
638 ASSERTCMP(call
->state
, !=, RXRPC_CALL_SERVER_ACCEPTING
);
640 iov
.iov_base
= buf
+ *_offset
;
641 iov
.iov_len
= size
- *_offset
;
642 iov_iter_kvec(&iter
, ITER_KVEC
| READ
, &iov
, 1, size
- *_offset
);
644 mutex_lock(&call
->user_mutex
);
646 switch (READ_ONCE(call
->state
)) {
647 case RXRPC_CALL_CLIENT_RECV_REPLY
:
648 case RXRPC_CALL_SERVER_RECV_REQUEST
:
649 case RXRPC_CALL_SERVER_ACK_REQUEST
:
650 ret
= rxrpc_recvmsg_data(sock
, call
, NULL
, &iter
, size
, 0,
655 /* We can only reach here with a partially full buffer if we
656 * have reached the end of the data. We must otherwise have a
657 * full buffer or have been given -EAGAIN.
663 goto read_phase_complete
;
672 case RXRPC_CALL_COMPLETE
:
683 mutex_unlock(&call
->user_mutex
);
684 _leave(" = %d [%zu,%d]", ret
, *_offset
, *_abort
);
688 trace_rxrpc_rx_eproto(call
, 0, tracepoint_string("short_data"));
692 trace_rxrpc_rx_eproto(call
, 0, tracepoint_string("excess_data"));
696 *_abort
= call
->abort_code
;
698 if (call
->completion
== RXRPC_CALL_SUCCEEDED
) {
705 EXPORT_SYMBOL(rxrpc_kernel_recv_data
);