?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Linux 5.11-rc1
[linux/fpc-iii.git] / net / ceph / messenger_v2.c
blobc1ebb2aa08b5cb604a085d06198767442ef6f28d
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Ceph msgr2 protocol implementation
4 *
5 * Copyright (C) 2020 Ilya Dryomov <idryomov@gmail.com>
6 */
7
8 #include <linux/ceph/ceph_debug.h>
9
10 #include <crypto/aead.h>
11 #include <crypto/algapi.h> /* for crypto_memneq() */
12 #include <crypto/hash.h>
13 #include <crypto/sha2.h>
14 #include <linux/bvec.h>
15 #include <linux/crc32c.h>
16 #include <linux/net.h>
17 #include <linux/scatterlist.h>
18 #include <linux/socket.h>
19 #include <linux/sched/mm.h>
20 #include <net/sock.h>
21 #include <net/tcp.h>
22
23 #include <linux/ceph/ceph_features.h>
24 #include <linux/ceph/decode.h>
25 #include <linux/ceph/libceph.h>
26 #include <linux/ceph/messenger.h>
27
28 #include "crypto.h" /* for CEPH_KEY_LEN and CEPH_MAX_CON_SECRET_LEN */
29
30 #define FRAME_TAG_HELLO 1
31 #define FRAME_TAG_AUTH_REQUEST 2
32 #define FRAME_TAG_AUTH_BAD_METHOD 3
33 #define FRAME_TAG_AUTH_REPLY_MORE 4
34 #define FRAME_TAG_AUTH_REQUEST_MORE 5
35 #define FRAME_TAG_AUTH_DONE 6
36 #define FRAME_TAG_AUTH_SIGNATURE 7
37 #define FRAME_TAG_CLIENT_IDENT 8
38 #define FRAME_TAG_SERVER_IDENT 9
39 #define FRAME_TAG_IDENT_MISSING_FEATURES 10
40 #define FRAME_TAG_SESSION_RECONNECT 11
41 #define FRAME_TAG_SESSION_RESET 12
42 #define FRAME_TAG_SESSION_RETRY 13
43 #define FRAME_TAG_SESSION_RETRY_GLOBAL 14
44 #define FRAME_TAG_SESSION_RECONNECT_OK 15
45 #define FRAME_TAG_WAIT 16
46 #define FRAME_TAG_MESSAGE 17
47 #define FRAME_TAG_KEEPALIVE2 18
48 #define FRAME_TAG_KEEPALIVE2_ACK 19
49 #define FRAME_TAG_ACK 20
50
51 #define FRAME_LATE_STATUS_ABORTED 0x1
52 #define FRAME_LATE_STATUS_COMPLETE 0xe
53 #define FRAME_LATE_STATUS_ABORTED_MASK 0xf
54
55 #define IN_S_HANDLE_PREAMBLE 1
56 #define IN_S_HANDLE_CONTROL 2
57 #define IN_S_HANDLE_CONTROL_REMAINDER 3
58 #define IN_S_PREPARE_READ_DATA 4
59 #define IN_S_PREPARE_READ_DATA_CONT 5
60 #define IN_S_HANDLE_EPILOGUE 6
61 #define IN_S_FINISH_SKIP 7
62
63 #define OUT_S_QUEUE_DATA 1
64 #define OUT_S_QUEUE_DATA_CONT 2
65 #define OUT_S_QUEUE_ENC_PAGE 3
66 #define OUT_S_QUEUE_ZEROS 4
67 #define OUT_S_FINISH_MESSAGE 5
68 #define OUT_S_GET_NEXT 6
69
70 #define CTRL_BODY(p) ((void *)(p) + CEPH_PREAMBLE_LEN)
71 #define FRONT_PAD(p) ((void *)(p) + CEPH_EPILOGUE_SECURE_LEN)
72 #define MIDDLE_PAD(p) (FRONT_PAD(p) + CEPH_GCM_BLOCK_LEN)
73 #define DATA_PAD(p) (MIDDLE_PAD(p) + CEPH_GCM_BLOCK_LEN)
74
75 #define CEPH_MSG_FLAGS (MSG_DONTWAIT | MSG_NOSIGNAL)
76
77 static int do_recvmsg(struct socket *sock, struct iov_iter *it)
78 {
79 struct msghdr msg = { .msg_flags = CEPH_MSG_FLAGS };
80 int ret;
81
82 msg.msg_iter = *it;
83 while (iov_iter_count(it)) {
84 ret = sock_recvmsg(sock, &msg, msg.msg_flags);
85 if (ret <= 0) {
86 if (ret == -EAGAIN)
87 ret = 0;
88 return ret;
89 }
90
91 iov_iter_advance(it, ret);
92 }
93
94 WARN_ON(msg_data_left(&msg));
95 return 1;
96 }
97
98 /*
99 * Read as much as possible.
100 *
101 * Return:
102 * 1 - done, nothing (else) to read
103 * 0 - socket is empty, need to wait
104 * <0 - error
105 */
106 static int ceph_tcp_recv(struct ceph_connection *con)
107 {
108 int ret;
109
110 dout("%s con %p %s %zu\n", __func__, con,
111 iov_iter_is_discard(&con->v2.in_iter) ? "discard" : "need",
112 iov_iter_count(&con->v2.in_iter));
113 ret = do_recvmsg(con->sock, &con->v2.in_iter);
114 dout("%s con %p ret %d left %zu\n", __func__, con, ret,
115 iov_iter_count(&con->v2.in_iter));
116 return ret;
117 }
118
119 static int do_sendmsg(struct socket *sock, struct iov_iter *it)
120 {
121 struct msghdr msg = { .msg_flags = CEPH_MSG_FLAGS };
122 int ret;
123
124 msg.msg_iter = *it;
125 while (iov_iter_count(it)) {
126 ret = sock_sendmsg(sock, &msg);
127 if (ret <= 0) {
128 if (ret == -EAGAIN)
129 ret = 0;
130 return ret;
131 }
132
133 iov_iter_advance(it, ret);
134 }
135
136 WARN_ON(msg_data_left(&msg));
137 return 1;
138 }
139
140 static int do_try_sendpage(struct socket *sock, struct iov_iter *it)
141 {
142 struct msghdr msg = { .msg_flags = CEPH_MSG_FLAGS };
143 struct bio_vec bv;
144 int ret;
145
146 if (WARN_ON(!iov_iter_is_bvec(it)))
147 return -EINVAL;
148
149 while (iov_iter_count(it)) {
150 /* iov_iter_iovec() for ITER_BVEC */
151 bv.bv_page = it->bvec->bv_page;
152 bv.bv_offset = it->bvec->bv_offset + it->iov_offset;
153 bv.bv_len = min(iov_iter_count(it),
154 it->bvec->bv_len - it->iov_offset);
155
156 /*
157 * sendpage cannot properly handle pages with
158 * page_count == 0, we need to fall back to sendmsg if
159 * that's the case.
160 *
161 * Same goes for slab pages: skb_can_coalesce() allows
162 * coalescing neighboring slab objects into a single frag
163 * which triggers one of hardened usercopy checks.
164 */
165 if (sendpage_ok(bv.bv_page)) {
166 ret = sock->ops->sendpage(sock, bv.bv_page,
167 bv.bv_offset, bv.bv_len,
168 CEPH_MSG_FLAGS);
169 } else {
170 iov_iter_bvec(&msg.msg_iter, WRITE, &bv, 1, bv.bv_len);
171 ret = sock_sendmsg(sock, &msg);
172 }
173 if (ret <= 0) {
174 if (ret == -EAGAIN)
175 ret = 0;
176 return ret;
177 }
178
179 iov_iter_advance(it, ret);
180 }
181
182 return 1;
183 }
184
185 /*
186 * Write as much as possible. The socket is expected to be corked,
187 * so we don't bother with MSG_MORE/MSG_SENDPAGE_NOTLAST here.
188 *
189 * Return:
190 * 1 - done, nothing (else) to write
191 * 0 - socket is full, need to wait
192 * <0 - error
193 */
194 static int ceph_tcp_send(struct ceph_connection *con)
195 {
196 int ret;
197
198 dout("%s con %p have %zu try_sendpage %d\n", __func__, con,
199 iov_iter_count(&con->v2.out_iter), con->v2.out_iter_sendpage);
200 if (con->v2.out_iter_sendpage)
201 ret = do_try_sendpage(con->sock, &con->v2.out_iter);
202 else
203 ret = do_sendmsg(con->sock, &con->v2.out_iter);
204 dout("%s con %p ret %d left %zu\n", __func__, con, ret,
205 iov_iter_count(&con->v2.out_iter));
206 return ret;
207 }
208
209 static void add_in_kvec(struct ceph_connection *con, void *buf, int len)
210 {
211 BUG_ON(con->v2.in_kvec_cnt >= ARRAY_SIZE(con->v2.in_kvecs));
212 WARN_ON(!iov_iter_is_kvec(&con->v2.in_iter));
213
214 con->v2.in_kvecs[con->v2.in_kvec_cnt].iov_base = buf;
215 con->v2.in_kvecs[con->v2.in_kvec_cnt].iov_len = len;
216 con->v2.in_kvec_cnt++;
217
218 con->v2.in_iter.nr_segs++;
219 con->v2.in_iter.count += len;
220 }
221
222 static void reset_in_kvecs(struct ceph_connection *con)
223 {
224 WARN_ON(iov_iter_count(&con->v2.in_iter));
225
226 con->v2.in_kvec_cnt = 0;
227 iov_iter_kvec(&con->v2.in_iter, READ, con->v2.in_kvecs, 0, 0);
228 }
229
230 static void set_in_bvec(struct ceph_connection *con, const struct bio_vec *bv)
231 {
232 WARN_ON(iov_iter_count(&con->v2.in_iter));
233
234 con->v2.in_bvec = *bv;
235 iov_iter_bvec(&con->v2.in_iter, READ, &con->v2.in_bvec, 1, bv->bv_len);
236 }
237
238 static void set_in_skip(struct ceph_connection *con, int len)
239 {
240 WARN_ON(iov_iter_count(&con->v2.in_iter));
241
242 dout("%s con %p len %d\n", __func__, con, len);
243 iov_iter_discard(&con->v2.in_iter, READ, len);
244 }
245
246 static void add_out_kvec(struct ceph_connection *con, void *buf, int len)
247 {
248 BUG_ON(con->v2.out_kvec_cnt >= ARRAY_SIZE(con->v2.out_kvecs));
249 WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter));
250 WARN_ON(con->v2.out_zero);
251
252 con->v2.out_kvecs[con->v2.out_kvec_cnt].iov_base = buf;
253 con->v2.out_kvecs[con->v2.out_kvec_cnt].iov_len = len;
254 con->v2.out_kvec_cnt++;
255
256 con->v2.out_iter.nr_segs++;
257 con->v2.out_iter.count += len;
258 }
259
260 static void reset_out_kvecs(struct ceph_connection *con)
261 {
262 WARN_ON(iov_iter_count(&con->v2.out_iter));
263 WARN_ON(con->v2.out_zero);
264
265 con->v2.out_kvec_cnt = 0;
266
267 iov_iter_kvec(&con->v2.out_iter, WRITE, con->v2.out_kvecs, 0, 0);
268 con->v2.out_iter_sendpage = false;
269 }
270
271 static void set_out_bvec(struct ceph_connection *con, const struct bio_vec *bv,
272 bool zerocopy)
273 {
274 WARN_ON(iov_iter_count(&con->v2.out_iter));
275 WARN_ON(con->v2.out_zero);
276
277 con->v2.out_bvec = *bv;
278 con->v2.out_iter_sendpage = zerocopy;
279 iov_iter_bvec(&con->v2.out_iter, WRITE, &con->v2.out_bvec, 1,
280 con->v2.out_bvec.bv_len);
281 }
282
283 static void set_out_bvec_zero(struct ceph_connection *con)
284 {
285 WARN_ON(iov_iter_count(&con->v2.out_iter));
286 WARN_ON(!con->v2.out_zero);
287
288 con->v2.out_bvec.bv_page = ceph_zero_page;
289 con->v2.out_bvec.bv_offset = 0;
290 con->v2.out_bvec.bv_len = min(con->v2.out_zero, (int)PAGE_SIZE);
291 con->v2.out_iter_sendpage = true;
292 iov_iter_bvec(&con->v2.out_iter, WRITE, &con->v2.out_bvec, 1,
293 con->v2.out_bvec.bv_len);
294 }
295
296 static void out_zero_add(struct ceph_connection *con, int len)
297 {
298 dout("%s con %p len %d\n", __func__, con, len);
299 con->v2.out_zero += len;
300 }
301
302 static void *alloc_conn_buf(struct ceph_connection *con, int len)
303 {
304 void *buf;
305
306 dout("%s con %p len %d\n", __func__, con, len);
307
308 if (WARN_ON(con->v2.conn_buf_cnt >= ARRAY_SIZE(con->v2.conn_bufs)))
309 return NULL;
310
311 buf = ceph_kvmalloc(len, GFP_NOIO);
312 if (!buf)
313 return NULL;
314
315 con->v2.conn_bufs[con->v2.conn_buf_cnt++] = buf;
316 return buf;
317 }
318
319 static void free_conn_bufs(struct ceph_connection *con)
320 {
321 while (con->v2.conn_buf_cnt)
322 kvfree(con->v2.conn_bufs[--con->v2.conn_buf_cnt]);
323 }
324
325 static void add_in_sign_kvec(struct ceph_connection *con, void *buf, int len)
326 {
327 BUG_ON(con->v2.in_sign_kvec_cnt >= ARRAY_SIZE(con->v2.in_sign_kvecs));
328
329 con->v2.in_sign_kvecs[con->v2.in_sign_kvec_cnt].iov_base = buf;
330 con->v2.in_sign_kvecs[con->v2.in_sign_kvec_cnt].iov_len = len;
331 con->v2.in_sign_kvec_cnt++;
332 }
333
334 static void clear_in_sign_kvecs(struct ceph_connection *con)
335 {
336 con->v2.in_sign_kvec_cnt = 0;
337 }
338
339 static void add_out_sign_kvec(struct ceph_connection *con, void *buf, int len)
340 {
341 BUG_ON(con->v2.out_sign_kvec_cnt >= ARRAY_SIZE(con->v2.out_sign_kvecs));
342
343 con->v2.out_sign_kvecs[con->v2.out_sign_kvec_cnt].iov_base = buf;
344 con->v2.out_sign_kvecs[con->v2.out_sign_kvec_cnt].iov_len = len;
345 con->v2.out_sign_kvec_cnt++;
346 }
347
348 static void clear_out_sign_kvecs(struct ceph_connection *con)
349 {
350 con->v2.out_sign_kvec_cnt = 0;
351 }
352
353 static bool con_secure(struct ceph_connection *con)
354 {
355 return con->v2.con_mode == CEPH_CON_MODE_SECURE;
356 }
357
358 static int front_len(const struct ceph_msg *msg)
359 {
360 return le32_to_cpu(msg->hdr.front_len);
361 }
362
363 static int middle_len(const struct ceph_msg *msg)
364 {
365 return le32_to_cpu(msg->hdr.middle_len);
366 }
367
368 static int data_len(const struct ceph_msg *msg)
369 {
370 return le32_to_cpu(msg->hdr.data_len);
371 }
372
373 static bool need_padding(int len)
374 {
375 return !IS_ALIGNED(len, CEPH_GCM_BLOCK_LEN);
376 }
377
378 static int padded_len(int len)
379 {
380 return ALIGN(len, CEPH_GCM_BLOCK_LEN);
381 }
382
383 static int padding_len(int len)
384 {
385 return padded_len(len) - len;
386 }
387
388 /* preamble + control segment */
389 static int head_onwire_len(int ctrl_len, bool secure)
390 {
391 int head_len;
392 int rem_len;
393
394 if (secure) {
395 head_len = CEPH_PREAMBLE_SECURE_LEN;
396 if (ctrl_len > CEPH_PREAMBLE_INLINE_LEN) {
397 rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN;
398 head_len += padded_len(rem_len) + CEPH_GCM_TAG_LEN;
399 }
400 } else {
401 head_len = CEPH_PREAMBLE_PLAIN_LEN;
402 if (ctrl_len)
403 head_len += ctrl_len + CEPH_CRC_LEN;
404 }
405 return head_len;
406 }
407
408 /* front, middle and data segments + epilogue */
409 static int __tail_onwire_len(int front_len, int middle_len, int data_len,
410 bool secure)
411 {
412 if (!front_len && !middle_len && !data_len)
413 return 0;
414
415 if (!secure)
416 return front_len + middle_len + data_len +
417 CEPH_EPILOGUE_PLAIN_LEN;
418
419 return padded_len(front_len) + padded_len(middle_len) +
420 padded_len(data_len) + CEPH_EPILOGUE_SECURE_LEN;
421 }
422
423 static int tail_onwire_len(const struct ceph_msg *msg, bool secure)
424 {
425 return __tail_onwire_len(front_len(msg), middle_len(msg),
426 data_len(msg), secure);
427 }
428
429 /* head_onwire_len(sizeof(struct ceph_msg_header2), false) */
430 #define MESSAGE_HEAD_PLAIN_LEN (CEPH_PREAMBLE_PLAIN_LEN + \
431 sizeof(struct ceph_msg_header2) + \
432 CEPH_CRC_LEN)
433
434 static const int frame_aligns[] = {
435 sizeof(void *),
436 sizeof(void *),
437 sizeof(void *),
438 PAGE_SIZE
439 };
440
441 /*
442 * Discards trailing empty segments, unless there is just one segment.
443 * A frame always has at least one (possibly empty) segment.
444 */
445 static int calc_segment_count(const int *lens, int len_cnt)
446 {
447 int i;
448
449 for (i = len_cnt - 1; i >= 0; i--) {
450 if (lens[i])
451 return i + 1;
452 }
453
454 return 1;
455 }
456
457 static void init_frame_desc(struct ceph_frame_desc *desc, int tag,
458 const int *lens, int len_cnt)
459 {
460 int i;
461
462 memset(desc, 0, sizeof(*desc));
463
464 desc->fd_tag = tag;
465 desc->fd_seg_cnt = calc_segment_count(lens, len_cnt);
466 BUG_ON(desc->fd_seg_cnt > CEPH_FRAME_MAX_SEGMENT_COUNT);
467 for (i = 0; i < desc->fd_seg_cnt; i++) {
468 desc->fd_lens[i] = lens[i];
469 desc->fd_aligns[i] = frame_aligns[i];
470 }
471 }
472
473 /*
474 * Preamble crc covers everything up to itself (28 bytes) and
475 * is calculated and verified irrespective of the connection mode
476 * (i.e. even if the frame is encrypted).
477 */
478 static void encode_preamble(const struct ceph_frame_desc *desc, void *p)
479 {
480 void *crcp = p + CEPH_PREAMBLE_LEN - CEPH_CRC_LEN;
481 void *start = p;
482 int i;
483
484 memset(p, 0, CEPH_PREAMBLE_LEN);
485
486 ceph_encode_8(&p, desc->fd_tag);
487 ceph_encode_8(&p, desc->fd_seg_cnt);
488 for (i = 0; i < desc->fd_seg_cnt; i++) {
489 ceph_encode_32(&p, desc->fd_lens[i]);
490 ceph_encode_16(&p, desc->fd_aligns[i]);
491 }
492
493 put_unaligned_le32(crc32c(0, start, crcp - start), crcp);
494 }
495
496 static int decode_preamble(void *p, struct ceph_frame_desc *desc)
497 {
498 void *crcp = p + CEPH_PREAMBLE_LEN - CEPH_CRC_LEN;
499 u32 crc, expected_crc;
500 int i;
501
502 crc = crc32c(0, p, crcp - p);
503 expected_crc = get_unaligned_le32(crcp);
504 if (crc != expected_crc) {
505 pr_err("bad preamble crc, calculated %u, expected %u\n",
506 crc, expected_crc);
507 return -EBADMSG;
508 }
509
510 memset(desc, 0, sizeof(*desc));
511
512 desc->fd_tag = ceph_decode_8(&p);
513 desc->fd_seg_cnt = ceph_decode_8(&p);
514 if (desc->fd_seg_cnt < 1 ||
515 desc->fd_seg_cnt > CEPH_FRAME_MAX_SEGMENT_COUNT) {
516 pr_err("bad segment count %d\n", desc->fd_seg_cnt);
517 return -EINVAL;
518 }
519 for (i = 0; i < desc->fd_seg_cnt; i++) {
520 desc->fd_lens[i] = ceph_decode_32(&p);
521 desc->fd_aligns[i] = ceph_decode_16(&p);
522 }
523
524 /*
525 * This would fire for FRAME_TAG_WAIT (it has one empty
526 * segment), but we should never get it as client.
527 */
528 if (!desc->fd_lens[desc->fd_seg_cnt - 1]) {
529 pr_err("last segment empty\n");
530 return -EINVAL;
531 }
532
533 if (desc->fd_lens[0] > CEPH_MSG_MAX_CONTROL_LEN) {
534 pr_err("control segment too big %d\n", desc->fd_lens[0]);
535 return -EINVAL;
536 }
537 if (desc->fd_lens[1] > CEPH_MSG_MAX_FRONT_LEN) {
538 pr_err("front segment too big %d\n", desc->fd_lens[1]);
539 return -EINVAL;
540 }
541 if (desc->fd_lens[2] > CEPH_MSG_MAX_MIDDLE_LEN) {
542 pr_err("middle segment too big %d\n", desc->fd_lens[2]);
543 return -EINVAL;
544 }
545 if (desc->fd_lens[3] > CEPH_MSG_MAX_DATA_LEN) {
546 pr_err("data segment too big %d\n", desc->fd_lens[3]);
547 return -EINVAL;
548 }
549
550 return 0;
551 }
552
553 static void encode_epilogue_plain(struct ceph_connection *con, bool aborted)
554 {
555 con->v2.out_epil.late_status = aborted ? FRAME_LATE_STATUS_ABORTED :
556 FRAME_LATE_STATUS_COMPLETE;
557 cpu_to_le32s(&con->v2.out_epil.front_crc);
558 cpu_to_le32s(&con->v2.out_epil.middle_crc);
559 cpu_to_le32s(&con->v2.out_epil.data_crc);
560 }
561
562 static void encode_epilogue_secure(struct ceph_connection *con, bool aborted)
563 {
564 memset(&con->v2.out_epil, 0, sizeof(con->v2.out_epil));
565 con->v2.out_epil.late_status = aborted ? FRAME_LATE_STATUS_ABORTED :
566 FRAME_LATE_STATUS_COMPLETE;
567 }
568
569 static int decode_epilogue(void *p, u32 *front_crc, u32 *middle_crc,
570 u32 *data_crc)
571 {
572 u8 late_status;
573
574 late_status = ceph_decode_8(&p);
575 if ((late_status & FRAME_LATE_STATUS_ABORTED_MASK) !=
576 FRAME_LATE_STATUS_COMPLETE) {
577 /* we should never get an aborted message as client */
578 pr_err("bad late_status 0x%x\n", late_status);
579 return -EINVAL;
580 }
581
582 if (front_crc && middle_crc && data_crc) {
583 *front_crc = ceph_decode_32(&p);
584 *middle_crc = ceph_decode_32(&p);
585 *data_crc = ceph_decode_32(&p);
586 }
587
588 return 0;
589 }
590
591 static void fill_header(struct ceph_msg_header *hdr,
592 const struct ceph_msg_header2 *hdr2,
593 int front_len, int middle_len, int data_len,
594 const struct ceph_entity_name *peer_name)
595 {
596 hdr->seq = hdr2->seq;
597 hdr->tid = hdr2->tid;
598 hdr->type = hdr2->type;
599 hdr->priority = hdr2->priority;
600 hdr->version = hdr2->version;
601 hdr->front_len = cpu_to_le32(front_len);
602 hdr->middle_len = cpu_to_le32(middle_len);
603 hdr->data_len = cpu_to_le32(data_len);
604 hdr->data_off = hdr2->data_off;
605 hdr->src = *peer_name;
606 hdr->compat_version = hdr2->compat_version;
607 hdr->reserved = 0;
608 hdr->crc = 0;
609 }
610
611 static void fill_header2(struct ceph_msg_header2 *hdr2,
612 const struct ceph_msg_header *hdr, u64 ack_seq)
613 {
614 hdr2->seq = hdr->seq;
615 hdr2->tid = hdr->tid;
616 hdr2->type = hdr->type;
617 hdr2->priority = hdr->priority;
618 hdr2->version = hdr->version;
619 hdr2->data_pre_padding_len = 0;
620 hdr2->data_off = hdr->data_off;
621 hdr2->ack_seq = cpu_to_le64(ack_seq);
622 hdr2->flags = 0;
623 hdr2->compat_version = hdr->compat_version;
624 hdr2->reserved = 0;
625 }
626
627 static int verify_control_crc(struct ceph_connection *con)
628 {
629 int ctrl_len = con->v2.in_desc.fd_lens[0];
630 u32 crc, expected_crc;
631
632 WARN_ON(con->v2.in_kvecs[0].iov_len != ctrl_len);
633 WARN_ON(con->v2.in_kvecs[1].iov_len != CEPH_CRC_LEN);
634
635 crc = crc32c(-1, con->v2.in_kvecs[0].iov_base, ctrl_len);
636 expected_crc = get_unaligned_le32(con->v2.in_kvecs[1].iov_base);
637 if (crc != expected_crc) {
638 pr_err("bad control crc, calculated %u, expected %u\n",
639 crc, expected_crc);
640 return -EBADMSG;
641 }
642
643 return 0;
644 }
645
646 static int verify_epilogue_crcs(struct ceph_connection *con, u32 front_crc,
647 u32 middle_crc, u32 data_crc)
648 {
649 if (front_len(con->in_msg)) {
650 con->in_front_crc = crc32c(-1, con->in_msg->front.iov_base,
651 front_len(con->in_msg));
652 } else {
653 WARN_ON(!middle_len(con->in_msg) && !data_len(con->in_msg));
654 con->in_front_crc = -1;
655 }
656
657 if (middle_len(con->in_msg))
658 con->in_middle_crc = crc32c(-1,
659 con->in_msg->middle->vec.iov_base,
660 middle_len(con->in_msg));
661 else if (data_len(con->in_msg))
662 con->in_middle_crc = -1;
663 else
664 con->in_middle_crc = 0;
665
666 if (!data_len(con->in_msg))
667 con->in_data_crc = 0;
668
669 dout("%s con %p msg %p crcs %u %u %u\n", __func__, con, con->in_msg,
670 con->in_front_crc, con->in_middle_crc, con->in_data_crc);
671
672 if (con->in_front_crc != front_crc) {
673 pr_err("bad front crc, calculated %u, expected %u\n",
674 con->in_front_crc, front_crc);
675 return -EBADMSG;
676 }
677 if (con->in_middle_crc != middle_crc) {
678 pr_err("bad middle crc, calculated %u, expected %u\n",
679 con->in_middle_crc, middle_crc);
680 return -EBADMSG;
681 }
682 if (con->in_data_crc != data_crc) {
683 pr_err("bad data crc, calculated %u, expected %u\n",
684 con->in_data_crc, data_crc);
685 return -EBADMSG;
686 }
687
688 return 0;
689 }
690
691 static int setup_crypto(struct ceph_connection *con,
692 u8 *session_key, int session_key_len,
693 u8 *con_secret, int con_secret_len)
694 {
695 unsigned int noio_flag;
696 void *p;
697 int ret;
698
699 dout("%s con %p con_mode %d session_key_len %d con_secret_len %d\n",
700 __func__, con, con->v2.con_mode, session_key_len, con_secret_len);
701 WARN_ON(con->v2.hmac_tfm || con->v2.gcm_tfm || con->v2.gcm_req);
702
703 if (con->v2.con_mode != CEPH_CON_MODE_CRC &&
704 con->v2.con_mode != CEPH_CON_MODE_SECURE) {
705 pr_err("bad con_mode %d\n", con->v2.con_mode);
706 return -EINVAL;
707 }
708
709 if (!session_key_len) {
710 WARN_ON(con->v2.con_mode != CEPH_CON_MODE_CRC);
711 WARN_ON(con_secret_len);
712 return 0; /* auth_none */
713 }
714
715 noio_flag = memalloc_noio_save();
716 con->v2.hmac_tfm = crypto_alloc_shash("hmac(sha256)", 0, 0);
717 memalloc_noio_restore(noio_flag);
718 if (IS_ERR(con->v2.hmac_tfm)) {
719 ret = PTR_ERR(con->v2.hmac_tfm);
720 con->v2.hmac_tfm = NULL;
721 pr_err("failed to allocate hmac tfm context: %d\n", ret);
722 return ret;
723 }
724
725 WARN_ON((unsigned long)session_key &
726 crypto_shash_alignmask(con->v2.hmac_tfm));
727 ret = crypto_shash_setkey(con->v2.hmac_tfm, session_key,
728 session_key_len);
729 if (ret) {
730 pr_err("failed to set hmac key: %d\n", ret);
731 return ret;
732 }
733
734 if (con->v2.con_mode == CEPH_CON_MODE_CRC) {
735 WARN_ON(con_secret_len);
736 return 0; /* auth_x, plain mode */
737 }
738
739 if (con_secret_len < CEPH_GCM_KEY_LEN + 2 * CEPH_GCM_IV_LEN) {
740 pr_err("con_secret too small %d\n", con_secret_len);
741 return -EINVAL;
742 }
743
744 noio_flag = memalloc_noio_save();
745 con->v2.gcm_tfm = crypto_alloc_aead("gcm(aes)", 0, 0);
746 memalloc_noio_restore(noio_flag);
747 if (IS_ERR(con->v2.gcm_tfm)) {
748 ret = PTR_ERR(con->v2.gcm_tfm);
749 con->v2.gcm_tfm = NULL;
750 pr_err("failed to allocate gcm tfm context: %d\n", ret);
751 return ret;
752 }
753
754 p = con_secret;
755 WARN_ON((unsigned long)p & crypto_aead_alignmask(con->v2.gcm_tfm));
756 ret = crypto_aead_setkey(con->v2.gcm_tfm, p, CEPH_GCM_KEY_LEN);
757 if (ret) {
758 pr_err("failed to set gcm key: %d\n", ret);
759 return ret;
760 }
761
762 p += CEPH_GCM_KEY_LEN;
763 WARN_ON(crypto_aead_ivsize(con->v2.gcm_tfm) != CEPH_GCM_IV_LEN);
764 ret = crypto_aead_setauthsize(con->v2.gcm_tfm, CEPH_GCM_TAG_LEN);
765 if (ret) {
766 pr_err("failed to set gcm tag size: %d\n", ret);
767 return ret;
768 }
769
770 con->v2.gcm_req = aead_request_alloc(con->v2.gcm_tfm, GFP_NOIO);
771 if (!con->v2.gcm_req) {
772 pr_err("failed to allocate gcm request\n");
773 return -ENOMEM;
774 }
775
776 crypto_init_wait(&con->v2.gcm_wait);
777 aead_request_set_callback(con->v2.gcm_req, CRYPTO_TFM_REQ_MAY_BACKLOG,
778 crypto_req_done, &con->v2.gcm_wait);
779
780 memcpy(&con->v2.in_gcm_nonce, p, CEPH_GCM_IV_LEN);
781 memcpy(&con->v2.out_gcm_nonce, p + CEPH_GCM_IV_LEN, CEPH_GCM_IV_LEN);
782 return 0; /* auth_x, secure mode */
783 }
784
785 static int hmac_sha256(struct ceph_connection *con, const struct kvec *kvecs,
786 int kvec_cnt, u8 *hmac)
787 {
788 SHASH_DESC_ON_STACK(desc, con->v2.hmac_tfm); /* tfm arg is ignored */
789 int ret;
790 int i;
791
792 dout("%s con %p hmac_tfm %p kvec_cnt %d\n", __func__, con,
793 con->v2.hmac_tfm, kvec_cnt);
794
795 if (!con->v2.hmac_tfm) {
796 memset(hmac, 0, SHA256_DIGEST_SIZE);
797 return 0; /* auth_none */
798 }
799
800 desc->tfm = con->v2.hmac_tfm;
801 ret = crypto_shash_init(desc);
802 if (ret)
803 return ret;
804
805 for (i = 0; i < kvec_cnt; i++) {
806 WARN_ON((unsigned long)kvecs[i].iov_base &
807 crypto_shash_alignmask(con->v2.hmac_tfm));
808 ret = crypto_shash_update(desc, kvecs[i].iov_base,
809 kvecs[i].iov_len);
810 if (ret)
811 return ret;
812 }
813
814 ret = crypto_shash_final(desc, hmac);
815 if (ret)
816 return ret;
817
818 shash_desc_zero(desc);
819 return 0; /* auth_x, both plain and secure modes */
820 }
821
822 static void gcm_inc_nonce(struct ceph_gcm_nonce *nonce)
823 {
824 u64 counter;
825
826 counter = le64_to_cpu(nonce->counter);
827 nonce->counter = cpu_to_le64(counter + 1);
828 }
829
830 static int gcm_crypt(struct ceph_connection *con, bool encrypt,
831 struct scatterlist *src, struct scatterlist *dst,
832 int src_len)
833 {
834 struct ceph_gcm_nonce *nonce;
835 int ret;
836
837 nonce = encrypt ? &con->v2.out_gcm_nonce : &con->v2.in_gcm_nonce;
838
839 aead_request_set_ad(con->v2.gcm_req, 0); /* no AAD */
840 aead_request_set_crypt(con->v2.gcm_req, src, dst, src_len, (u8 *)nonce);
841 ret = crypto_wait_req(encrypt ? crypto_aead_encrypt(con->v2.gcm_req) :
842 crypto_aead_decrypt(con->v2.gcm_req),
843 &con->v2.gcm_wait);
844 if (ret)
845 return ret;
846
847 gcm_inc_nonce(nonce);
848 return 0;
849 }
850
851 static void get_bvec_at(struct ceph_msg_data_cursor *cursor,
852 struct bio_vec *bv)
853 {
854 struct page *page;
855 size_t off, len;
856
857 WARN_ON(!cursor->total_resid);
858
859 /* skip zero-length data items */
860 while (!cursor->resid)
861 ceph_msg_data_advance(cursor, 0);
862
863 /* get a piece of data, cursor isn't advanced */
864 page = ceph_msg_data_next(cursor, &off, &len, NULL);
865
866 bv->bv_page = page;
867 bv->bv_offset = off;
868 bv->bv_len = len;
869 }
870
871 static int calc_sg_cnt(void *buf, int buf_len)
872 {
873 int sg_cnt;
874
875 if (!buf_len)
876 return 0;
877
878 sg_cnt = need_padding(buf_len) ? 1 : 0;
879 if (is_vmalloc_addr(buf)) {
880 WARN_ON(offset_in_page(buf));
881 sg_cnt += PAGE_ALIGN(buf_len) >> PAGE_SHIFT;
882 } else {
883 sg_cnt++;
884 }
885
886 return sg_cnt;
887 }
888
889 static int calc_sg_cnt_cursor(struct ceph_msg_data_cursor *cursor)
890 {
891 int data_len = cursor->total_resid;
892 struct bio_vec bv;
893 int sg_cnt;
894
895 if (!data_len)
896 return 0;
897
898 sg_cnt = need_padding(data_len) ? 1 : 0;
899 do {
900 get_bvec_at(cursor, &bv);
901 sg_cnt++;
902
903 ceph_msg_data_advance(cursor, bv.bv_len);
904 } while (cursor->total_resid);
905
906 return sg_cnt;
907 }
908
909 static void init_sgs(struct scatterlist **sg, void *buf, int buf_len, u8 *pad)
910 {
911 void *end = buf + buf_len;
912 struct page *page;
913 int len;
914 void *p;
915
916 if (!buf_len)
917 return;
918
919 if (is_vmalloc_addr(buf)) {
920 p = buf;
921 do {
922 page = vmalloc_to_page(p);
923 len = min_t(int, end - p, PAGE_SIZE);
924 WARN_ON(!page || !len || offset_in_page(p));
925 sg_set_page(*sg, page, len, 0);
926 *sg = sg_next(*sg);
927 p += len;
928 } while (p != end);
929 } else {
930 sg_set_buf(*sg, buf, buf_len);
931 *sg = sg_next(*sg);
932 }
933
934 if (need_padding(buf_len)) {
935 sg_set_buf(*sg, pad, padding_len(buf_len));
936 *sg = sg_next(*sg);
937 }
938 }
939
940 static void init_sgs_cursor(struct scatterlist **sg,
941 struct ceph_msg_data_cursor *cursor, u8 *pad)
942 {
943 int data_len = cursor->total_resid;
944 struct bio_vec bv;
945
946 if (!data_len)
947 return;
948
949 do {
950 get_bvec_at(cursor, &bv);
951 sg_set_page(*sg, bv.bv_page, bv.bv_len, bv.bv_offset);
952 *sg = sg_next(*sg);
953
954 ceph_msg_data_advance(cursor, bv.bv_len);
955 } while (cursor->total_resid);
956
957 if (need_padding(data_len)) {
958 sg_set_buf(*sg, pad, padding_len(data_len));
959 *sg = sg_next(*sg);
960 }
961 }
962
963 static int setup_message_sgs(struct sg_table *sgt, struct ceph_msg *msg,
964 u8 *front_pad, u8 *middle_pad, u8 *data_pad,
965 void *epilogue, bool add_tag)
966 {
967 struct ceph_msg_data_cursor cursor;
968 struct scatterlist *cur_sg;
969 int sg_cnt;
970 int ret;
971
972 if (!front_len(msg) && !middle_len(msg) && !data_len(msg))
973 return 0;
974
975 sg_cnt = 1; /* epilogue + [auth tag] */
976 if (front_len(msg))
977 sg_cnt += calc_sg_cnt(msg->front.iov_base,
978 front_len(msg));
979 if (middle_len(msg))
980 sg_cnt += calc_sg_cnt(msg->middle->vec.iov_base,
981 middle_len(msg));
982 if (data_len(msg)) {
983 ceph_msg_data_cursor_init(&cursor, msg, data_len(msg));
984 sg_cnt += calc_sg_cnt_cursor(&cursor);
985 }
986
987 ret = sg_alloc_table(sgt, sg_cnt, GFP_NOIO);
988 if (ret)
989 return ret;
990
991 cur_sg = sgt->sgl;
992 if (front_len(msg))
993 init_sgs(&cur_sg, msg->front.iov_base, front_len(msg),
994 front_pad);
995 if (middle_len(msg))
996 init_sgs(&cur_sg, msg->middle->vec.iov_base, middle_len(msg),
997 middle_pad);
998 if (data_len(msg)) {
999 ceph_msg_data_cursor_init(&cursor, msg, data_len(msg));
1000 init_sgs_cursor(&cur_sg, &cursor, data_pad);
1001 }
1002
1003 WARN_ON(!sg_is_last(cur_sg));
1004 sg_set_buf(cur_sg, epilogue,
1005 CEPH_GCM_BLOCK_LEN + (add_tag ? CEPH_GCM_TAG_LEN : 0));
1006 return 0;
1007 }
1008
1009 static int decrypt_preamble(struct ceph_connection *con)
1010 {
1011 struct scatterlist sg;
1012
1013 sg_init_one(&sg, con->v2.in_buf, CEPH_PREAMBLE_SECURE_LEN);
1014 return gcm_crypt(con, false, &sg, &sg, CEPH_PREAMBLE_SECURE_LEN);
1015 }
1016
1017 static int decrypt_control_remainder(struct ceph_connection *con)
1018 {
1019 int ctrl_len = con->v2.in_desc.fd_lens[0];
1020 int rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN;
1021 int pt_len = padding_len(rem_len) + CEPH_GCM_TAG_LEN;
1022 struct scatterlist sgs[2];
1023
1024 WARN_ON(con->v2.in_kvecs[0].iov_len != rem_len);
1025 WARN_ON(con->v2.in_kvecs[1].iov_len != pt_len);
1026
1027 sg_init_table(sgs, 2);
1028 sg_set_buf(&sgs[0], con->v2.in_kvecs[0].iov_base, rem_len);
1029 sg_set_buf(&sgs[1], con->v2.in_buf, pt_len);
1030
1031 return gcm_crypt(con, false, sgs, sgs,
1032 padded_len(rem_len) + CEPH_GCM_TAG_LEN);
1033 }
1034
1035 static int decrypt_message(struct ceph_connection *con)
1036 {
1037 struct sg_table sgt = {};
1038 int ret;
1039
1040 ret = setup_message_sgs(&sgt, con->in_msg, FRONT_PAD(con->v2.in_buf),
1041 MIDDLE_PAD(con->v2.in_buf), DATA_PAD(con->v2.in_buf),
1042 con->v2.in_buf, true);
1043 if (ret)
1044 goto out;
1045
1046 ret = gcm_crypt(con, false, sgt.sgl, sgt.sgl,
1047 tail_onwire_len(con->in_msg, true));
1048
1049 out:
1050 sg_free_table(&sgt);
1051 return ret;
1052 }
1053
1054 static int prepare_banner(struct ceph_connection *con)
1055 {
1056 int buf_len = CEPH_BANNER_V2_LEN + 2 + 8 + 8;
1057 void *buf, *p;
1058
1059 buf = alloc_conn_buf(con, buf_len);
1060 if (!buf)
1061 return -ENOMEM;
1062
1063 p = buf;
1064 ceph_encode_copy(&p, CEPH_BANNER_V2, CEPH_BANNER_V2_LEN);
1065 ceph_encode_16(&p, sizeof(u64) + sizeof(u64));
1066 ceph_encode_64(&p, CEPH_MSGR2_SUPPORTED_FEATURES);
1067 ceph_encode_64(&p, CEPH_MSGR2_REQUIRED_FEATURES);
1068 WARN_ON(p != buf + buf_len);
1069
1070 add_out_kvec(con, buf, buf_len);
1071 add_out_sign_kvec(con, buf, buf_len);
1072 ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING);
1073 return 0;
1074 }
1075
1076 /*
1077 * base:
1078 * preamble
1079 * control body (ctrl_len bytes)
1080 * space for control crc
1081 *
1082 * extdata (optional):
1083 * control body (extdata_len bytes)
1084 *
1085 * Compute control crc and gather base and extdata into:
1086 *
1087 * preamble
1088 * control body (ctrl_len + extdata_len bytes)
1089 * control crc
1090 *
1091 * Preamble should already be encoded at the start of base.
1092 */
1093 static void prepare_head_plain(struct ceph_connection *con, void *base,
1094 int ctrl_len, void *extdata, int extdata_len,
1095 bool to_be_signed)
1096 {
1097 int base_len = CEPH_PREAMBLE_LEN + ctrl_len + CEPH_CRC_LEN;
1098 void *crcp = base + base_len - CEPH_CRC_LEN;
1099 u32 crc;
1100
1101 crc = crc32c(-1, CTRL_BODY(base), ctrl_len);
1102 if (extdata_len)
1103 crc = crc32c(crc, extdata, extdata_len);
1104 put_unaligned_le32(crc, crcp);
1105
1106 if (!extdata_len) {
1107 add_out_kvec(con, base, base_len);
1108 if (to_be_signed)
1109 add_out_sign_kvec(con, base, base_len);
1110 return;
1111 }
1112
1113 add_out_kvec(con, base, crcp - base);
1114 add_out_kvec(con, extdata, extdata_len);
1115 add_out_kvec(con, crcp, CEPH_CRC_LEN);
1116 if (to_be_signed) {
1117 add_out_sign_kvec(con, base, crcp - base);
1118 add_out_sign_kvec(con, extdata, extdata_len);
1119 add_out_sign_kvec(con, crcp, CEPH_CRC_LEN);
1120 }
1121 }
1122
1123 static int prepare_head_secure_small(struct ceph_connection *con,
1124 void *base, int ctrl_len)
1125 {
1126 struct scatterlist sg;
1127 int ret;
1128
1129 /* inline buffer padding? */
1130 if (ctrl_len < CEPH_PREAMBLE_INLINE_LEN)
1131 memset(CTRL_BODY(base) + ctrl_len, 0,
1132 CEPH_PREAMBLE_INLINE_LEN - ctrl_len);
1133
1134 sg_init_one(&sg, base, CEPH_PREAMBLE_SECURE_LEN);
1135 ret = gcm_crypt(con, true, &sg, &sg,
1136 CEPH_PREAMBLE_SECURE_LEN - CEPH_GCM_TAG_LEN);
1137 if (ret)
1138 return ret;
1139
1140 add_out_kvec(con, base, CEPH_PREAMBLE_SECURE_LEN);
1141 return 0;
1142 }
1143
1144 /*
1145 * base:
1146 * preamble
1147 * control body (ctrl_len bytes)
1148 * space for padding, if needed
1149 * space for control remainder auth tag
1150 * space for preamble auth tag
1151 *
1152 * Encrypt preamble and the inline portion, then encrypt the remainder
1153 * and gather into:
1154 *
1155 * preamble
1156 * control body (48 bytes)
1157 * preamble auth tag
1158 * control body (ctrl_len - 48 bytes)
1159 * zero padding, if needed
1160 * control remainder auth tag
1161 *
1162 * Preamble should already be encoded at the start of base.
1163 */
1164 static int prepare_head_secure_big(struct ceph_connection *con,
1165 void *base, int ctrl_len)
1166 {
1167 int rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN;
1168 void *rem = CTRL_BODY(base) + CEPH_PREAMBLE_INLINE_LEN;
1169 void *rem_tag = rem + padded_len(rem_len);
1170 void *pmbl_tag = rem_tag + CEPH_GCM_TAG_LEN;
1171 struct scatterlist sgs[2];
1172 int ret;
1173
1174 sg_init_table(sgs, 2);
1175 sg_set_buf(&sgs[0], base, rem - base);
1176 sg_set_buf(&sgs[1], pmbl_tag, CEPH_GCM_TAG_LEN);
1177 ret = gcm_crypt(con, true, sgs, sgs, rem - base);
1178 if (ret)
1179 return ret;
1180
1181 /* control remainder padding? */
1182 if (need_padding(rem_len))
1183 memset(rem + rem_len, 0, padding_len(rem_len));
1184
1185 sg_init_one(&sgs[0], rem, pmbl_tag - rem);
1186 ret = gcm_crypt(con, true, sgs, sgs, rem_tag - rem);
1187 if (ret)
1188 return ret;
1189
1190 add_out_kvec(con, base, rem - base);
1191 add_out_kvec(con, pmbl_tag, CEPH_GCM_TAG_LEN);
1192 add_out_kvec(con, rem, pmbl_tag - rem);
1193 return 0;
1194 }
1195
1196 static int __prepare_control(struct ceph_connection *con, int tag,
1197 void *base, int ctrl_len, void *extdata,
1198 int extdata_len, bool to_be_signed)
1199 {
1200 int total_len = ctrl_len + extdata_len;
1201 struct ceph_frame_desc desc;
1202 int ret;
1203
1204 dout("%s con %p tag %d len %d (%d+%d)\n", __func__, con, tag,
1205 total_len, ctrl_len, extdata_len);
1206
1207 /* extdata may be vmalloc'ed but not base */
1208 if (WARN_ON(is_vmalloc_addr(base) || !ctrl_len))
1209 return -EINVAL;
1210
1211 init_frame_desc(&desc, tag, &total_len, 1);
1212 encode_preamble(&desc, base);
1213
1214 if (con_secure(con)) {
1215 if (WARN_ON(extdata_len || to_be_signed))
1216 return -EINVAL;
1217
1218 if (ctrl_len <= CEPH_PREAMBLE_INLINE_LEN)
1219 /* fully inlined, inline buffer may need padding */
1220 ret = prepare_head_secure_small(con, base, ctrl_len);
1221 else
1222 /* partially inlined, inline buffer is full */
1223 ret = prepare_head_secure_big(con, base, ctrl_len);
1224 if (ret)
1225 return ret;
1226 } else {
1227 prepare_head_plain(con, base, ctrl_len, extdata, extdata_len,
1228 to_be_signed);
1229 }
1230
1231 ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING);
1232 return 0;
1233 }
1234
1235 static int prepare_control(struct ceph_connection *con, int tag,
1236 void *base, int ctrl_len)
1237 {
1238 return __prepare_control(con, tag, base, ctrl_len, NULL, 0, false);
1239 }
1240
1241 static int prepare_hello(struct ceph_connection *con)
1242 {
1243 void *buf, *p;
1244 int ctrl_len;
1245
1246 ctrl_len = 1 + ceph_entity_addr_encoding_len(&con->peer_addr);
1247 buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, false));
1248 if (!buf)
1249 return -ENOMEM;
1250
1251 p = CTRL_BODY(buf);
1252 ceph_encode_8(&p, CEPH_ENTITY_TYPE_CLIENT);
1253 ceph_encode_entity_addr(&p, &con->peer_addr);
1254 WARN_ON(p != CTRL_BODY(buf) + ctrl_len);
1255
1256 return __prepare_control(con, FRAME_TAG_HELLO, buf, ctrl_len,
1257 NULL, 0, true);
1258 }
1259
1260 /* so that head_onwire_len(AUTH_BUF_LEN, false) is 512 */
1261 #define AUTH_BUF_LEN (512 - CEPH_CRC_LEN - CEPH_PREAMBLE_PLAIN_LEN)
1262
1263 static int prepare_auth_request(struct ceph_connection *con)
1264 {
1265 void *authorizer, *authorizer_copy;
1266 int ctrl_len, authorizer_len;
1267 void *buf;
1268 int ret;
1269
1270 ctrl_len = AUTH_BUF_LEN;
1271 buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, false));
1272 if (!buf)
1273 return -ENOMEM;
1274
1275 mutex_unlock(&con->mutex);
1276 ret = con->ops->get_auth_request(con, CTRL_BODY(buf), &ctrl_len,
1277 &authorizer, &authorizer_len);
1278 mutex_lock(&con->mutex);
1279 if (con->state != CEPH_CON_S_V2_HELLO) {
1280 dout("%s con %p state changed to %d\n", __func__, con,
1281 con->state);
1282 return -EAGAIN;
1283 }
1284
1285 dout("%s con %p get_auth_request ret %d\n", __func__, con, ret);
1286 if (ret)
1287 return ret;
1288
1289 authorizer_copy = alloc_conn_buf(con, authorizer_len);
1290 if (!authorizer_copy)
1291 return -ENOMEM;
1292
1293 memcpy(authorizer_copy, authorizer, authorizer_len);
1294
1295 return __prepare_control(con, FRAME_TAG_AUTH_REQUEST, buf, ctrl_len,
1296 authorizer_copy, authorizer_len, true);
1297 }
1298
1299 static int prepare_auth_request_more(struct ceph_connection *con,
1300 void *reply, int reply_len)
1301 {
1302 int ctrl_len, authorizer_len;
1303 void *authorizer;
1304 void *buf;
1305 int ret;
1306
1307 ctrl_len = AUTH_BUF_LEN;
1308 buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, false));
1309 if (!buf)
1310 return -ENOMEM;
1311
1312 mutex_unlock(&con->mutex);
1313 ret = con->ops->handle_auth_reply_more(con, reply, reply_len,
1314 CTRL_BODY(buf), &ctrl_len,
1315 &authorizer, &authorizer_len);
1316 mutex_lock(&con->mutex);
1317 if (con->state != CEPH_CON_S_V2_AUTH) {
1318 dout("%s con %p state changed to %d\n", __func__, con,
1319 con->state);
1320 return -EAGAIN;
1321 }
1322
1323 dout("%s con %p handle_auth_reply_more ret %d\n", __func__, con, ret);
1324 if (ret)
1325 return ret;
1326
1327 return __prepare_control(con, FRAME_TAG_AUTH_REQUEST_MORE, buf,
1328 ctrl_len, authorizer, authorizer_len, true);
1329 }
1330
1331 static int prepare_auth_signature(struct ceph_connection *con)
1332 {
1333 void *buf;
1334 int ret;
1335
1336 buf = alloc_conn_buf(con, head_onwire_len(SHA256_DIGEST_SIZE, false));
1337 if (!buf)
1338 return -ENOMEM;
1339
1340 ret = hmac_sha256(con, con->v2.in_sign_kvecs, con->v2.in_sign_kvec_cnt,
1341 CTRL_BODY(buf));
1342 if (ret)
1343 return ret;
1344
1345 return prepare_control(con, FRAME_TAG_AUTH_SIGNATURE, buf,
1346 SHA256_DIGEST_SIZE);
1347 }
1348
1349 static int prepare_client_ident(struct ceph_connection *con)
1350 {
1351 struct ceph_entity_addr *my_addr = &con->msgr->inst.addr;
1352 struct ceph_client *client = from_msgr(con->msgr);
1353 u64 global_id = ceph_client_gid(client);
1354 void *buf, *p;
1355 int ctrl_len;
1356
1357 WARN_ON(con->v2.server_cookie);
1358 WARN_ON(con->v2.connect_seq);
1359 WARN_ON(con->v2.peer_global_seq);
1360
1361 if (!con->v2.client_cookie) {
1362 do {
1363 get_random_bytes(&con->v2.client_cookie,
1364 sizeof(con->v2.client_cookie));
1365 } while (!con->v2.client_cookie);
1366 dout("%s con %p generated cookie 0x%llx\n", __func__, con,
1367 con->v2.client_cookie);
1368 } else {
1369 dout("%s con %p cookie already set 0x%llx\n", __func__, con,
1370 con->v2.client_cookie);
1371 }
1372
1373 dout("%s con %p my_addr %s/%u peer_addr %s/%u global_id %llu global_seq %llu features 0x%llx required_features 0x%llx cookie 0x%llx\n",
1374 __func__, con, ceph_pr_addr(my_addr), le32_to_cpu(my_addr->nonce),
1375 ceph_pr_addr(&con->peer_addr), le32_to_cpu(con->peer_addr.nonce),
1376 global_id, con->v2.global_seq, client->supported_features,
1377 client->required_features, con->v2.client_cookie);
1378
1379 ctrl_len = 1 + 4 + ceph_entity_addr_encoding_len(my_addr) +
1380 ceph_entity_addr_encoding_len(&con->peer_addr) + 6 * 8;
1381 buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, con_secure(con)));
1382 if (!buf)
1383 return -ENOMEM;
1384
1385 p = CTRL_BODY(buf);
1386 ceph_encode_8(&p, 2); /* addrvec marker */
1387 ceph_encode_32(&p, 1); /* addr_cnt */
1388 ceph_encode_entity_addr(&p, my_addr);
1389 ceph_encode_entity_addr(&p, &con->peer_addr);
1390 ceph_encode_64(&p, global_id);
1391 ceph_encode_64(&p, con->v2.global_seq);
1392 ceph_encode_64(&p, client->supported_features);
1393 ceph_encode_64(&p, client->required_features);
1394 ceph_encode_64(&p, 0); /* flags */
1395 ceph_encode_64(&p, con->v2.client_cookie);
1396 WARN_ON(p != CTRL_BODY(buf) + ctrl_len);
1397
1398 return prepare_control(con, FRAME_TAG_CLIENT_IDENT, buf, ctrl_len);
1399 }
1400
1401 static int prepare_session_reconnect(struct ceph_connection *con)
1402 {
1403 struct ceph_entity_addr *my_addr = &con->msgr->inst.addr;
1404 void *buf, *p;
1405 int ctrl_len;
1406
1407 WARN_ON(!con->v2.client_cookie);
1408 WARN_ON(!con->v2.server_cookie);
1409 WARN_ON(!con->v2.connect_seq);
1410 WARN_ON(!con->v2.peer_global_seq);
1411
1412 dout("%s con %p my_addr %s/%u client_cookie 0x%llx server_cookie 0x%llx global_seq %llu connect_seq %llu in_seq %llu\n",
1413 __func__, con, ceph_pr_addr(my_addr), le32_to_cpu(my_addr->nonce),
1414 con->v2.client_cookie, con->v2.server_cookie, con->v2.global_seq,
1415 con->v2.connect_seq, con->in_seq);
1416
1417 ctrl_len = 1 + 4 + ceph_entity_addr_encoding_len(my_addr) + 5 * 8;
1418 buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, con_secure(con)));
1419 if (!buf)
1420 return -ENOMEM;
1421
1422 p = CTRL_BODY(buf);
1423 ceph_encode_8(&p, 2); /* entity_addrvec_t marker */
1424 ceph_encode_32(&p, 1); /* my_addrs len */
1425 ceph_encode_entity_addr(&p, my_addr);
1426 ceph_encode_64(&p, con->v2.client_cookie);
1427 ceph_encode_64(&p, con->v2.server_cookie);
1428 ceph_encode_64(&p, con->v2.global_seq);
1429 ceph_encode_64(&p, con->v2.connect_seq);
1430 ceph_encode_64(&p, con->in_seq);
1431 WARN_ON(p != CTRL_BODY(buf) + ctrl_len);
1432
1433 return prepare_control(con, FRAME_TAG_SESSION_RECONNECT, buf, ctrl_len);
1434 }
1435
1436 static int prepare_keepalive2(struct ceph_connection *con)
1437 {
1438 struct ceph_timespec *ts = CTRL_BODY(con->v2.out_buf);
1439 struct timespec64 now;
1440
1441 ktime_get_real_ts64(&now);
1442 dout("%s con %p timestamp %lld.%09ld\n", __func__, con, now.tv_sec,
1443 now.tv_nsec);
1444
1445 ceph_encode_timespec64(ts, &now);
1446
1447 reset_out_kvecs(con);
1448 return prepare_control(con, FRAME_TAG_KEEPALIVE2, con->v2.out_buf,
1449 sizeof(struct ceph_timespec));
1450 }
1451
1452 static int prepare_ack(struct ceph_connection *con)
1453 {
1454 void *p;
1455
1456 dout("%s con %p in_seq_acked %llu -> %llu\n", __func__, con,
1457 con->in_seq_acked, con->in_seq);
1458 con->in_seq_acked = con->in_seq;
1459
1460 p = CTRL_BODY(con->v2.out_buf);
1461 ceph_encode_64(&p, con->in_seq_acked);
1462
1463 reset_out_kvecs(con);
1464 return prepare_control(con, FRAME_TAG_ACK, con->v2.out_buf, 8);
1465 }
1466
1467 static void prepare_epilogue_plain(struct ceph_connection *con, bool aborted)
1468 {
1469 dout("%s con %p msg %p aborted %d crcs %u %u %u\n", __func__, con,
1470 con->out_msg, aborted, con->v2.out_epil.front_crc,
1471 con->v2.out_epil.middle_crc, con->v2.out_epil.data_crc);
1472
1473 encode_epilogue_plain(con, aborted);
1474 add_out_kvec(con, &con->v2.out_epil, CEPH_EPILOGUE_PLAIN_LEN);
1475 }
1476
1477 /*
1478 * For "used" empty segments, crc is -1. For unused (trailing)
1479 * segments, crc is 0.
1480 */
1481 static void prepare_message_plain(struct ceph_connection *con)
1482 {
1483 struct ceph_msg *msg = con->out_msg;
1484
1485 prepare_head_plain(con, con->v2.out_buf,
1486 sizeof(struct ceph_msg_header2), NULL, 0, false);
1487
1488 if (!front_len(msg) && !middle_len(msg)) {
1489 if (!data_len(msg)) {
1490 /*
1491 * Empty message: once the head is written,
1492 * we are done -- there is no epilogue.
1493 */
1494 con->v2.out_state = OUT_S_FINISH_MESSAGE;
1495 return;
1496 }
1497
1498 con->v2.out_epil.front_crc = -1;
1499 con->v2.out_epil.middle_crc = -1;
1500 con->v2.out_state = OUT_S_QUEUE_DATA;
1501 return;
1502 }
1503
1504 if (front_len(msg)) {
1505 con->v2.out_epil.front_crc = crc32c(-1, msg->front.iov_base,
1506 front_len(msg));
1507 add_out_kvec(con, msg->front.iov_base, front_len(msg));
1508 } else {
1509 /* middle (at least) is there, checked above */
1510 con->v2.out_epil.front_crc = -1;
1511 }
1512
1513 if (middle_len(msg)) {
1514 con->v2.out_epil.middle_crc =
1515 crc32c(-1, msg->middle->vec.iov_base, middle_len(msg));
1516 add_out_kvec(con, msg->middle->vec.iov_base, middle_len(msg));
1517 } else {
1518 con->v2.out_epil.middle_crc = data_len(msg) ? -1 : 0;
1519 }
1520
1521 if (data_len(msg)) {
1522 con->v2.out_state = OUT_S_QUEUE_DATA;
1523 } else {
1524 con->v2.out_epil.data_crc = 0;
1525 prepare_epilogue_plain(con, false);
1526 con->v2.out_state = OUT_S_FINISH_MESSAGE;
1527 }
1528 }
1529
1530 /*
1531 * Unfortunately the kernel crypto API doesn't support streaming
1532 * (piecewise) operation for AEAD algorithms, so we can't get away
1533 * with a fixed size buffer and a couple sgs. Instead, we have to
1534 * allocate pages for the entire tail of the message (currently up
1535 * to ~32M) and two sgs arrays (up to ~256K each)...
1536 */
1537 static int prepare_message_secure(struct ceph_connection *con)
1538 {
1539 void *zerop = page_address(ceph_zero_page);
1540 struct sg_table enc_sgt = {};
1541 struct sg_table sgt = {};
1542 struct page **enc_pages;
1543 int enc_page_cnt;
1544 int tail_len;
1545 int ret;
1546
1547 ret = prepare_head_secure_small(con, con->v2.out_buf,
1548 sizeof(struct ceph_msg_header2));
1549 if (ret)
1550 return ret;
1551
1552 tail_len = tail_onwire_len(con->out_msg, true);
1553 if (!tail_len) {
1554 /*
1555 * Empty message: once the head is written,
1556 * we are done -- there is no epilogue.
1557 */
1558 con->v2.out_state = OUT_S_FINISH_MESSAGE;
1559 return 0;
1560 }
1561
1562 encode_epilogue_secure(con, false);
1563 ret = setup_message_sgs(&sgt, con->out_msg, zerop, zerop, zerop,
1564 &con->v2.out_epil, false);
1565 if (ret)
1566 goto out;
1567
1568 enc_page_cnt = calc_pages_for(0, tail_len);
1569 enc_pages = ceph_alloc_page_vector(enc_page_cnt, GFP_NOIO);
1570 if (IS_ERR(enc_pages)) {
1571 ret = PTR_ERR(enc_pages);
1572 goto out;
1573 }
1574
1575 WARN_ON(con->v2.out_enc_pages || con->v2.out_enc_page_cnt);
1576 con->v2.out_enc_pages = enc_pages;
1577 con->v2.out_enc_page_cnt = enc_page_cnt;
1578 con->v2.out_enc_resid = tail_len;
1579 con->v2.out_enc_i = 0;
1580
1581 ret = sg_alloc_table_from_pages(&enc_sgt, enc_pages, enc_page_cnt,
1582 0, tail_len, GFP_NOIO);
1583 if (ret)
1584 goto out;
1585
1586 ret = gcm_crypt(con, true, sgt.sgl, enc_sgt.sgl,
1587 tail_len - CEPH_GCM_TAG_LEN);
1588 if (ret)
1589 goto out;
1590
1591 dout("%s con %p msg %p sg_cnt %d enc_page_cnt %d\n", __func__, con,
1592 con->out_msg, sgt.orig_nents, enc_page_cnt);
1593 con->v2.out_state = OUT_S_QUEUE_ENC_PAGE;
1594
1595 out:
1596 sg_free_table(&sgt);
1597 sg_free_table(&enc_sgt);
1598 return ret;
1599 }
1600
1601 static int prepare_message(struct ceph_connection *con)
1602 {
1603 int lens[] = {
1604 sizeof(struct ceph_msg_header2),
1605 front_len(con->out_msg),
1606 middle_len(con->out_msg),
1607 data_len(con->out_msg)
1608 };
1609 struct ceph_frame_desc desc;
1610 int ret;
1611
1612 dout("%s con %p msg %p logical %d+%d+%d+%d\n", __func__, con,
1613 con->out_msg, lens[0], lens[1], lens[2], lens[3]);
1614
1615 if (con->in_seq > con->in_seq_acked) {
1616 dout("%s con %p in_seq_acked %llu -> %llu\n", __func__, con,
1617 con->in_seq_acked, con->in_seq);
1618 con->in_seq_acked = con->in_seq;
1619 }
1620
1621 reset_out_kvecs(con);
1622 init_frame_desc(&desc, FRAME_TAG_MESSAGE, lens, 4);
1623 encode_preamble(&desc, con->v2.out_buf);
1624 fill_header2(CTRL_BODY(con->v2.out_buf), &con->out_msg->hdr,
1625 con->in_seq_acked);
1626
1627 if (con_secure(con)) {
1628 ret = prepare_message_secure(con);
1629 if (ret)
1630 return ret;
1631 } else {
1632 prepare_message_plain(con);
1633 }
1634
1635 ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING);
1636 return 0;
1637 }
1638
1639 static int prepare_read_banner_prefix(struct ceph_connection *con)
1640 {
1641 void *buf;
1642
1643 buf = alloc_conn_buf(con, CEPH_BANNER_V2_PREFIX_LEN);
1644 if (!buf)
1645 return -ENOMEM;
1646
1647 reset_in_kvecs(con);
1648 add_in_kvec(con, buf, CEPH_BANNER_V2_PREFIX_LEN);
1649 add_in_sign_kvec(con, buf, CEPH_BANNER_V2_PREFIX_LEN);
1650 con->state = CEPH_CON_S_V2_BANNER_PREFIX;
1651 return 0;
1652 }
1653
1654 static int prepare_read_banner_payload(struct ceph_connection *con,
1655 int payload_len)
1656 {
1657 void *buf;
1658
1659 buf = alloc_conn_buf(con, payload_len);
1660 if (!buf)
1661 return -ENOMEM;
1662
1663 reset_in_kvecs(con);
1664 add_in_kvec(con, buf, payload_len);
1665 add_in_sign_kvec(con, buf, payload_len);
1666 con->state = CEPH_CON_S_V2_BANNER_PAYLOAD;
1667 return 0;
1668 }
1669
1670 static void prepare_read_preamble(struct ceph_connection *con)
1671 {
1672 reset_in_kvecs(con);
1673 add_in_kvec(con, con->v2.in_buf,
1674 con_secure(con) ? CEPH_PREAMBLE_SECURE_LEN :
1675 CEPH_PREAMBLE_PLAIN_LEN);
1676 con->v2.in_state = IN_S_HANDLE_PREAMBLE;
1677 }
1678
1679 static int prepare_read_control(struct ceph_connection *con)
1680 {
1681 int ctrl_len = con->v2.in_desc.fd_lens[0];
1682 int head_len;
1683 void *buf;
1684
1685 reset_in_kvecs(con);
1686 if (con->state == CEPH_CON_S_V2_HELLO ||
1687 con->state == CEPH_CON_S_V2_AUTH) {
1688 head_len = head_onwire_len(ctrl_len, false);
1689 buf = alloc_conn_buf(con, head_len);
1690 if (!buf)
1691 return -ENOMEM;
1692
1693 /* preserve preamble */
1694 memcpy(buf, con->v2.in_buf, CEPH_PREAMBLE_LEN);
1695
1696 add_in_kvec(con, CTRL_BODY(buf), ctrl_len);
1697 add_in_kvec(con, CTRL_BODY(buf) + ctrl_len, CEPH_CRC_LEN);
1698 add_in_sign_kvec(con, buf, head_len);
1699 } else {
1700 if (ctrl_len > CEPH_PREAMBLE_INLINE_LEN) {
1701 buf = alloc_conn_buf(con, ctrl_len);
1702 if (!buf)
1703 return -ENOMEM;
1704
1705 add_in_kvec(con, buf, ctrl_len);
1706 } else {
1707 add_in_kvec(con, CTRL_BODY(con->v2.in_buf), ctrl_len);
1708 }
1709 add_in_kvec(con, con->v2.in_buf, CEPH_CRC_LEN);
1710 }
1711 con->v2.in_state = IN_S_HANDLE_CONTROL;
1712 return 0;
1713 }
1714
1715 static int prepare_read_control_remainder(struct ceph_connection *con)
1716 {
1717 int ctrl_len = con->v2.in_desc.fd_lens[0];
1718 int rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN;
1719 void *buf;
1720
1721 buf = alloc_conn_buf(con, ctrl_len);
1722 if (!buf)
1723 return -ENOMEM;
1724
1725 memcpy(buf, CTRL_BODY(con->v2.in_buf), CEPH_PREAMBLE_INLINE_LEN);
1726
1727 reset_in_kvecs(con);
1728 add_in_kvec(con, buf + CEPH_PREAMBLE_INLINE_LEN, rem_len);
1729 add_in_kvec(con, con->v2.in_buf,
1730 padding_len(rem_len) + CEPH_GCM_TAG_LEN);
1731 con->v2.in_state = IN_S_HANDLE_CONTROL_REMAINDER;
1732 return 0;
1733 }
1734
1735 static void prepare_read_data(struct ceph_connection *con)
1736 {
1737 struct bio_vec bv;
1738
1739 if (!con_secure(con))
1740 con->in_data_crc = -1;
1741 ceph_msg_data_cursor_init(&con->v2.in_cursor, con->in_msg,
1742 data_len(con->in_msg));
1743
1744 get_bvec_at(&con->v2.in_cursor, &bv);
1745 set_in_bvec(con, &bv);
1746 con->v2.in_state = IN_S_PREPARE_READ_DATA_CONT;
1747 }
1748
1749 static void prepare_read_data_cont(struct ceph_connection *con)
1750 {
1751 struct bio_vec bv;
1752
1753 if (!con_secure(con))
1754 con->in_data_crc = ceph_crc32c_page(con->in_data_crc,
1755 con->v2.in_bvec.bv_page,
1756 con->v2.in_bvec.bv_offset,
1757 con->v2.in_bvec.bv_len);
1758
1759 ceph_msg_data_advance(&con->v2.in_cursor, con->v2.in_bvec.bv_len);
1760 if (con->v2.in_cursor.total_resid) {
1761 get_bvec_at(&con->v2.in_cursor, &bv);
1762 set_in_bvec(con, &bv);
1763 WARN_ON(con->v2.in_state != IN_S_PREPARE_READ_DATA_CONT);
1764 return;
1765 }
1766
1767 /*
1768 * We've read all data. Prepare to read data padding (if any)
1769 * and epilogue.
1770 */
1771 reset_in_kvecs(con);
1772 if (con_secure(con)) {
1773 if (need_padding(data_len(con->in_msg)))
1774 add_in_kvec(con, DATA_PAD(con->v2.in_buf),
1775 padding_len(data_len(con->in_msg)));
1776 add_in_kvec(con, con->v2.in_buf, CEPH_EPILOGUE_SECURE_LEN);
1777 } else {
1778 add_in_kvec(con, con->v2.in_buf, CEPH_EPILOGUE_PLAIN_LEN);
1779 }
1780 con->v2.in_state = IN_S_HANDLE_EPILOGUE;
1781 }
1782
1783 static void __finish_skip(struct ceph_connection *con)
1784 {
1785 con->in_seq++;
1786 prepare_read_preamble(con);
1787 }
1788
1789 static void prepare_skip_message(struct ceph_connection *con)
1790 {
1791 struct ceph_frame_desc *desc = &con->v2.in_desc;
1792 int tail_len;
1793
1794 dout("%s con %p %d+%d+%d\n", __func__, con, desc->fd_lens[1],
1795 desc->fd_lens[2], desc->fd_lens[3]);
1796
1797 tail_len = __tail_onwire_len(desc->fd_lens[1], desc->fd_lens[2],
1798 desc->fd_lens[3], con_secure(con));
1799 if (!tail_len) {
1800 __finish_skip(con);
1801 } else {
1802 set_in_skip(con, tail_len);
1803 con->v2.in_state = IN_S_FINISH_SKIP;
1804 }
1805 }
1806
1807 static int process_banner_prefix(struct ceph_connection *con)
1808 {
1809 int payload_len;
1810 void *p;
1811
1812 WARN_ON(con->v2.in_kvecs[0].iov_len != CEPH_BANNER_V2_PREFIX_LEN);
1813
1814 p = con->v2.in_kvecs[0].iov_base;
1815 if (memcmp(p, CEPH_BANNER_V2, CEPH_BANNER_V2_LEN)) {
1816 if (!memcmp(p, CEPH_BANNER, CEPH_BANNER_LEN))
1817 con->error_msg = "server is speaking msgr1 protocol";
1818 else
1819 con->error_msg = "protocol error, bad banner";
1820 return -EINVAL;
1821 }
1822
1823 p += CEPH_BANNER_V2_LEN;
1824 payload_len = ceph_decode_16(&p);
1825 dout("%s con %p payload_len %d\n", __func__, con, payload_len);
1826
1827 return prepare_read_banner_payload(con, payload_len);
1828 }
1829
1830 static int process_banner_payload(struct ceph_connection *con)
1831 {
1832 void *end = con->v2.in_kvecs[0].iov_base + con->v2.in_kvecs[0].iov_len;
1833 u64 feat = CEPH_MSGR2_SUPPORTED_FEATURES;
1834 u64 req_feat = CEPH_MSGR2_REQUIRED_FEATURES;
1835 u64 server_feat, server_req_feat;
1836 void *p;
1837 int ret;
1838
1839 p = con->v2.in_kvecs[0].iov_base;
1840 ceph_decode_64_safe(&p, end, server_feat, bad);
1841 ceph_decode_64_safe(&p, end, server_req_feat, bad);
1842
1843 dout("%s con %p server_feat 0x%llx server_req_feat 0x%llx\n",
1844 __func__, con, server_feat, server_req_feat);
1845
1846 if (req_feat & ~server_feat) {
1847 pr_err("msgr2 feature set mismatch: my required > server's supported 0x%llx, need 0x%llx\n",
1848 server_feat, req_feat & ~server_feat);
1849 con->error_msg = "missing required protocol features";
1850 return -EINVAL;
1851 }
1852 if (server_req_feat & ~feat) {
1853 pr_err("msgr2 feature set mismatch: server's required > my supported 0x%llx, missing 0x%llx\n",
1854 feat, server_req_feat & ~feat);
1855 con->error_msg = "missing required protocol features";
1856 return -EINVAL;
1857 }
1858
1859 /* no reset_out_kvecs() as our banner may still be pending */
1860 ret = prepare_hello(con);
1861 if (ret) {
1862 pr_err("prepare_hello failed: %d\n", ret);
1863 return ret;
1864 }
1865
1866 con->state = CEPH_CON_S_V2_HELLO;
1867 prepare_read_preamble(con);
1868 return 0;
1869
1870 bad:
1871 pr_err("failed to decode banner payload\n");
1872 return -EINVAL;
1873 }
1874
1875 static int process_hello(struct ceph_connection *con, void *p, void *end)
1876 {
1877 struct ceph_entity_addr *my_addr = &con->msgr->inst.addr;
1878 struct ceph_entity_addr addr_for_me;
1879 u8 entity_type;
1880 int ret;
1881
1882 if (con->state != CEPH_CON_S_V2_HELLO) {
1883 con->error_msg = "protocol error, unexpected hello";
1884 return -EINVAL;
1885 }
1886
1887 ceph_decode_8_safe(&p, end, entity_type, bad);
1888 ret = ceph_decode_entity_addr(&p, end, &addr_for_me);
1889 if (ret) {
1890 pr_err("failed to decode addr_for_me: %d\n", ret);
1891 return ret;
1892 }
1893
1894 dout("%s con %p entity_type %d addr_for_me %s\n", __func__, con,
1895 entity_type, ceph_pr_addr(&addr_for_me));
1896
1897 if (entity_type != con->peer_name.type) {
1898 pr_err("bad peer type, want %d, got %d\n",
1899 con->peer_name.type, entity_type);
1900 con->error_msg = "wrong peer at address";
1901 return -EINVAL;
1902 }
1903
1904 /*
1905 * Set our address to the address our first peer (i.e. monitor)
1906 * sees that we are connecting from. If we are behind some sort
1907 * of NAT and want to be identified by some private (not NATed)
1908 * address, ip option should be used.
1909 */
1910 if (ceph_addr_is_blank(my_addr)) {
1911 memcpy(&my_addr->in_addr, &addr_for_me.in_addr,
1912 sizeof(my_addr->in_addr));
1913 ceph_addr_set_port(my_addr, 0);
1914 dout("%s con %p set my addr %s, as seen by peer %s\n",
1915 __func__, con, ceph_pr_addr(my_addr),
1916 ceph_pr_addr(&con->peer_addr));
1917 } else {
1918 dout("%s con %p my addr already set %s\n",
1919 __func__, con, ceph_pr_addr(my_addr));
1920 }
1921
1922 WARN_ON(ceph_addr_is_blank(my_addr) || ceph_addr_port(my_addr));
1923 WARN_ON(my_addr->type != CEPH_ENTITY_ADDR_TYPE_ANY);
1924 WARN_ON(!my_addr->nonce);
1925
1926 /* no reset_out_kvecs() as our hello may still be pending */
1927 ret = prepare_auth_request(con);
1928 if (ret) {
1929 if (ret != -EAGAIN)
1930 pr_err("prepare_auth_request failed: %d\n", ret);
1931 return ret;
1932 }
1933
1934 con->state = CEPH_CON_S_V2_AUTH;
1935 return 0;
1936
1937 bad:
1938 pr_err("failed to decode hello\n");
1939 return -EINVAL;
1940 }
1941
1942 static int process_auth_bad_method(struct ceph_connection *con,
1943 void *p, void *end)
1944 {
1945 int allowed_protos[8], allowed_modes[8];
1946 int allowed_proto_cnt, allowed_mode_cnt;
1947 int used_proto, result;
1948 int ret;
1949 int i;
1950
1951 if (con->state != CEPH_CON_S_V2_AUTH) {
1952 con->error_msg = "protocol error, unexpected auth_bad_method";
1953 return -EINVAL;
1954 }
1955
1956 ceph_decode_32_safe(&p, end, used_proto, bad);
1957 ceph_decode_32_safe(&p, end, result, bad);
1958 dout("%s con %p used_proto %d result %d\n", __func__, con, used_proto,
1959 result);
1960
1961 ceph_decode_32_safe(&p, end, allowed_proto_cnt, bad);
1962 if (allowed_proto_cnt > ARRAY_SIZE(allowed_protos)) {
1963 pr_err("allowed_protos too big %d\n", allowed_proto_cnt);
1964 return -EINVAL;
1965 }
1966 for (i = 0; i < allowed_proto_cnt; i++) {
1967 ceph_decode_32_safe(&p, end, allowed_protos[i], bad);
1968 dout("%s con %p allowed_protos[%d] %d\n", __func__, con,
1969 i, allowed_protos[i]);
1970 }
1971
1972 ceph_decode_32_safe(&p, end, allowed_mode_cnt, bad);
1973 if (allowed_mode_cnt > ARRAY_SIZE(allowed_modes)) {
1974 pr_err("allowed_modes too big %d\n", allowed_mode_cnt);
1975 return -EINVAL;
1976 }
1977 for (i = 0; i < allowed_mode_cnt; i++) {
1978 ceph_decode_32_safe(&p, end, allowed_modes[i], bad);
1979 dout("%s con %p allowed_modes[%d] %d\n", __func__, con,
1980 i, allowed_modes[i]);
1981 }
1982
1983 mutex_unlock(&con->mutex);
1984 ret = con->ops->handle_auth_bad_method(con, used_proto, result,
1985 allowed_protos,
1986 allowed_proto_cnt,
1987 allowed_modes,
1988 allowed_mode_cnt);
1989 mutex_lock(&con->mutex);
1990 if (con->state != CEPH_CON_S_V2_AUTH) {
1991 dout("%s con %p state changed to %d\n", __func__, con,
1992 con->state);
1993 return -EAGAIN;
1994 }
1995
1996 dout("%s con %p handle_auth_bad_method ret %d\n", __func__, con, ret);
1997 return ret;
1998
1999 bad:
2000 pr_err("failed to decode auth_bad_method\n");
2001 return -EINVAL;
2002 }
2003
2004 static int process_auth_reply_more(struct ceph_connection *con,
2005 void *p, void *end)
2006 {
2007 int payload_len;
2008 int ret;
2009
2010 if (con->state != CEPH_CON_S_V2_AUTH) {
2011 con->error_msg = "protocol error, unexpected auth_reply_more";
2012 return -EINVAL;
2013 }
2014
2015 ceph_decode_32_safe(&p, end, payload_len, bad);
2016 ceph_decode_need(&p, end, payload_len, bad);
2017
2018 dout("%s con %p payload_len %d\n", __func__, con, payload_len);
2019
2020 reset_out_kvecs(con);
2021 ret = prepare_auth_request_more(con, p, payload_len);
2022 if (ret) {
2023 if (ret != -EAGAIN)
2024 pr_err("prepare_auth_request_more failed: %d\n", ret);
2025 return ret;
2026 }
2027
2028 return 0;
2029
2030 bad:
2031 pr_err("failed to decode auth_reply_more\n");
2032 return -EINVAL;
2033 }
2034
2035 static int process_auth_done(struct ceph_connection *con, void *p, void *end)
2036 {
2037 u8 session_key[CEPH_KEY_LEN];
2038 u8 con_secret[CEPH_MAX_CON_SECRET_LEN];
2039 int session_key_len, con_secret_len;
2040 int payload_len;
2041 u64 global_id;
2042 int ret;
2043
2044 if (con->state != CEPH_CON_S_V2_AUTH) {
2045 con->error_msg = "protocol error, unexpected auth_done";
2046 return -EINVAL;
2047 }
2048
2049 ceph_decode_64_safe(&p, end, global_id, bad);
2050 ceph_decode_32_safe(&p, end, con->v2.con_mode, bad);
2051 ceph_decode_32_safe(&p, end, payload_len, bad);
2052
2053 dout("%s con %p global_id %llu con_mode %d payload_len %d\n",
2054 __func__, con, global_id, con->v2.con_mode, payload_len);
2055
2056 mutex_unlock(&con->mutex);
2057 session_key_len = 0;
2058 con_secret_len = 0;
2059 ret = con->ops->handle_auth_done(con, global_id, p, payload_len,
2060 session_key, &session_key_len,
2061 con_secret, &con_secret_len);
2062 mutex_lock(&con->mutex);
2063 if (con->state != CEPH_CON_S_V2_AUTH) {
2064 dout("%s con %p state changed to %d\n", __func__, con,
2065 con->state);
2066 return -EAGAIN;
2067 }
2068
2069 dout("%s con %p handle_auth_done ret %d\n", __func__, con, ret);
2070 if (ret)
2071 return ret;
2072
2073 ret = setup_crypto(con, session_key, session_key_len, con_secret,
2074 con_secret_len);
2075 if (ret)
2076 return ret;
2077
2078 reset_out_kvecs(con);
2079 ret = prepare_auth_signature(con);
2080 if (ret) {
2081 pr_err("prepare_auth_signature failed: %d\n", ret);
2082 return ret;
2083 }
2084
2085 con->state = CEPH_CON_S_V2_AUTH_SIGNATURE;
2086 return 0;
2087
2088 bad:
2089 pr_err("failed to decode auth_done\n");
2090 return -EINVAL;
2091 }
2092
2093 static int process_auth_signature(struct ceph_connection *con,
2094 void *p, void *end)
2095 {
2096 u8 hmac[SHA256_DIGEST_SIZE];
2097 int ret;
2098
2099 if (con->state != CEPH_CON_S_V2_AUTH_SIGNATURE) {
2100 con->error_msg = "protocol error, unexpected auth_signature";
2101 return -EINVAL;
2102 }
2103
2104 ret = hmac_sha256(con, con->v2.out_sign_kvecs,
2105 con->v2.out_sign_kvec_cnt, hmac);
2106 if (ret)
2107 return ret;
2108
2109 ceph_decode_need(&p, end, SHA256_DIGEST_SIZE, bad);
2110 if (crypto_memneq(p, hmac, SHA256_DIGEST_SIZE)) {
2111 con->error_msg = "integrity error, bad auth signature";
2112 return -EBADMSG;
2113 }
2114
2115 dout("%s con %p auth signature ok\n", __func__, con);
2116
2117 /* no reset_out_kvecs() as our auth_signature may still be pending */
2118 if (!con->v2.server_cookie) {
2119 ret = prepare_client_ident(con);
2120 if (ret) {
2121 pr_err("prepare_client_ident failed: %d\n", ret);
2122 return ret;
2123 }
2124
2125 con->state = CEPH_CON_S_V2_SESSION_CONNECT;
2126 } else {
2127 ret = prepare_session_reconnect(con);
2128 if (ret) {
2129 pr_err("prepare_session_reconnect failed: %d\n", ret);
2130 return ret;
2131 }
2132
2133 con->state = CEPH_CON_S_V2_SESSION_RECONNECT;
2134 }
2135
2136 return 0;
2137
2138 bad:
2139 pr_err("failed to decode auth_signature\n");
2140 return -EINVAL;
2141 }
2142
2143 static int process_server_ident(struct ceph_connection *con,
2144 void *p, void *end)
2145 {
2146 struct ceph_client *client = from_msgr(con->msgr);
2147 u64 features, required_features;
2148 struct ceph_entity_addr addr;
2149 u64 global_seq;
2150 u64 global_id;
2151 u64 cookie;
2152 u64 flags;
2153 int ret;
2154
2155 if (con->state != CEPH_CON_S_V2_SESSION_CONNECT) {
2156 con->error_msg = "protocol error, unexpected server_ident";
2157 return -EINVAL;
2158 }
2159
2160 ret = ceph_decode_entity_addrvec(&p, end, true, &addr);
2161 if (ret) {
2162 pr_err("failed to decode server addrs: %d\n", ret);
2163 return ret;
2164 }
2165
2166 ceph_decode_64_safe(&p, end, global_id, bad);
2167 ceph_decode_64_safe(&p, end, global_seq, bad);
2168 ceph_decode_64_safe(&p, end, features, bad);
2169 ceph_decode_64_safe(&p, end, required_features, bad);
2170 ceph_decode_64_safe(&p, end, flags, bad);
2171 ceph_decode_64_safe(&p, end, cookie, bad);
2172
2173 dout("%s con %p addr %s/%u global_id %llu global_seq %llu features 0x%llx required_features 0x%llx flags 0x%llx cookie 0x%llx\n",
2174 __func__, con, ceph_pr_addr(&addr), le32_to_cpu(addr.nonce),
2175 global_id, global_seq, features, required_features, flags, cookie);
2176
2177 /* is this who we intended to talk to? */
2178 if (memcmp(&addr, &con->peer_addr, sizeof(con->peer_addr))) {
2179 pr_err("bad peer addr/nonce, want %s/%u, got %s/%u\n",
2180 ceph_pr_addr(&con->peer_addr),
2181 le32_to_cpu(con->peer_addr.nonce),
2182 ceph_pr_addr(&addr), le32_to_cpu(addr.nonce));
2183 con->error_msg = "wrong peer at address";
2184 return -EINVAL;
2185 }
2186
2187 if (client->required_features & ~features) {
2188 pr_err("RADOS feature set mismatch: my required > server's supported 0x%llx, need 0x%llx\n",
2189 features, client->required_features & ~features);
2190 con->error_msg = "missing required protocol features";
2191 return -EINVAL;
2192 }
2193
2194 /*
2195 * Both name->type and name->num are set in ceph_con_open() but
2196 * name->num may be bogus in the initial monmap. name->type is
2197 * verified in handle_hello().
2198 */
2199 WARN_ON(!con->peer_name.type);
2200 con->peer_name.num = cpu_to_le64(global_id);
2201 con->v2.peer_global_seq = global_seq;
2202 con->peer_features = features;
2203 WARN_ON(required_features & ~client->supported_features);
2204 con->v2.server_cookie = cookie;
2205
2206 if (flags & CEPH_MSG_CONNECT_LOSSY) {
2207 ceph_con_flag_set(con, CEPH_CON_F_LOSSYTX);
2208 WARN_ON(con->v2.server_cookie);
2209 } else {
2210 WARN_ON(!con->v2.server_cookie);
2211 }
2212
2213 clear_in_sign_kvecs(con);
2214 clear_out_sign_kvecs(con);
2215 free_conn_bufs(con);
2216 con->delay = 0; /* reset backoff memory */
2217
2218 con->state = CEPH_CON_S_OPEN;
2219 con->v2.out_state = OUT_S_GET_NEXT;
2220 return 0;
2221
2222 bad:
2223 pr_err("failed to decode server_ident\n");
2224 return -EINVAL;
2225 }
2226
2227 static int process_ident_missing_features(struct ceph_connection *con,
2228 void *p, void *end)
2229 {
2230 struct ceph_client *client = from_msgr(con->msgr);
2231 u64 missing_features;
2232
2233 if (con->state != CEPH_CON_S_V2_SESSION_CONNECT) {
2234 con->error_msg = "protocol error, unexpected ident_missing_features";
2235 return -EINVAL;
2236 }
2237
2238 ceph_decode_64_safe(&p, end, missing_features, bad);
2239 pr_err("RADOS feature set mismatch: server's required > my supported 0x%llx, missing 0x%llx\n",
2240 client->supported_features, missing_features);
2241 con->error_msg = "missing required protocol features";
2242 return -EINVAL;
2243
2244 bad:
2245 pr_err("failed to decode ident_missing_features\n");
2246 return -EINVAL;
2247 }
2248
2249 static int process_session_reconnect_ok(struct ceph_connection *con,
2250 void *p, void *end)
2251 {
2252 u64 seq;
2253
2254 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2255 con->error_msg = "protocol error, unexpected session_reconnect_ok";
2256 return -EINVAL;
2257 }
2258
2259 ceph_decode_64_safe(&p, end, seq, bad);
2260
2261 dout("%s con %p seq %llu\n", __func__, con, seq);
2262 ceph_con_discard_requeued(con, seq);
2263
2264 clear_in_sign_kvecs(con);
2265 clear_out_sign_kvecs(con);
2266 free_conn_bufs(con);
2267 con->delay = 0; /* reset backoff memory */
2268
2269 con->state = CEPH_CON_S_OPEN;
2270 con->v2.out_state = OUT_S_GET_NEXT;
2271 return 0;
2272
2273 bad:
2274 pr_err("failed to decode session_reconnect_ok\n");
2275 return -EINVAL;
2276 }
2277
2278 static int process_session_retry(struct ceph_connection *con,
2279 void *p, void *end)
2280 {
2281 u64 connect_seq;
2282 int ret;
2283
2284 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2285 con->error_msg = "protocol error, unexpected session_retry";
2286 return -EINVAL;
2287 }
2288
2289 ceph_decode_64_safe(&p, end, connect_seq, bad);
2290
2291 dout("%s con %p connect_seq %llu\n", __func__, con, connect_seq);
2292 WARN_ON(connect_seq <= con->v2.connect_seq);
2293 con->v2.connect_seq = connect_seq + 1;
2294
2295 free_conn_bufs(con);
2296
2297 reset_out_kvecs(con);
2298 ret = prepare_session_reconnect(con);
2299 if (ret) {
2300 pr_err("prepare_session_reconnect (cseq) failed: %d\n", ret);
2301 return ret;
2302 }
2303
2304 return 0;
2305
2306 bad:
2307 pr_err("failed to decode session_retry\n");
2308 return -EINVAL;
2309 }
2310
2311 static int process_session_retry_global(struct ceph_connection *con,
2312 void *p, void *end)
2313 {
2314 u64 global_seq;
2315 int ret;
2316
2317 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2318 con->error_msg = "protocol error, unexpected session_retry_global";
2319 return -EINVAL;
2320 }
2321
2322 ceph_decode_64_safe(&p, end, global_seq, bad);
2323
2324 dout("%s con %p global_seq %llu\n", __func__, con, global_seq);
2325 WARN_ON(global_seq <= con->v2.global_seq);
2326 con->v2.global_seq = ceph_get_global_seq(con->msgr, global_seq);
2327
2328 free_conn_bufs(con);
2329
2330 reset_out_kvecs(con);
2331 ret = prepare_session_reconnect(con);
2332 if (ret) {
2333 pr_err("prepare_session_reconnect (gseq) failed: %d\n", ret);
2334 return ret;
2335 }
2336
2337 return 0;
2338
2339 bad:
2340 pr_err("failed to decode session_retry_global\n");
2341 return -EINVAL;
2342 }
2343
2344 static int process_session_reset(struct ceph_connection *con,
2345 void *p, void *end)
2346 {
2347 bool full;
2348 int ret;
2349
2350 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2351 con->error_msg = "protocol error, unexpected session_reset";
2352 return -EINVAL;
2353 }
2354
2355 ceph_decode_8_safe(&p, end, full, bad);
2356 if (!full) {
2357 con->error_msg = "protocol error, bad session_reset";
2358 return -EINVAL;
2359 }
2360
2361 pr_info("%s%lld %s session reset\n", ENTITY_NAME(con->peer_name),
2362 ceph_pr_addr(&con->peer_addr));
2363 ceph_con_reset_session(con);
2364
2365 mutex_unlock(&con->mutex);
2366 if (con->ops->peer_reset)
2367 con->ops->peer_reset(con);
2368 mutex_lock(&con->mutex);
2369 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2370 dout("%s con %p state changed to %d\n", __func__, con,
2371 con->state);
2372 return -EAGAIN;
2373 }
2374
2375 free_conn_bufs(con);
2376
2377 reset_out_kvecs(con);
2378 ret = prepare_client_ident(con);
2379 if (ret) {
2380 pr_err("prepare_client_ident (rst) failed: %d\n", ret);
2381 return ret;
2382 }
2383
2384 con->state = CEPH_CON_S_V2_SESSION_CONNECT;
2385 return 0;
2386
2387 bad:
2388 pr_err("failed to decode session_reset\n");
2389 return -EINVAL;
2390 }
2391
2392 static int process_keepalive2_ack(struct ceph_connection *con,
2393 void *p, void *end)
2394 {
2395 if (con->state != CEPH_CON_S_OPEN) {
2396 con->error_msg = "protocol error, unexpected keepalive2_ack";
2397 return -EINVAL;
2398 }
2399
2400 ceph_decode_need(&p, end, sizeof(struct ceph_timespec), bad);
2401 ceph_decode_timespec64(&con->last_keepalive_ack, p);
2402
2403 dout("%s con %p timestamp %lld.%09ld\n", __func__, con,
2404 con->last_keepalive_ack.tv_sec, con->last_keepalive_ack.tv_nsec);
2405
2406 return 0;
2407
2408 bad:
2409 pr_err("failed to decode keepalive2_ack\n");
2410 return -EINVAL;
2411 }
2412
2413 static int process_ack(struct ceph_connection *con, void *p, void *end)
2414 {
2415 u64 seq;
2416
2417 if (con->state != CEPH_CON_S_OPEN) {
2418 con->error_msg = "protocol error, unexpected ack";
2419 return -EINVAL;
2420 }
2421
2422 ceph_decode_64_safe(&p, end, seq, bad);
2423
2424 dout("%s con %p seq %llu\n", __func__, con, seq);
2425 ceph_con_discard_sent(con, seq);
2426 return 0;
2427
2428 bad:
2429 pr_err("failed to decode ack\n");
2430 return -EINVAL;
2431 }
2432
2433 static int process_control(struct ceph_connection *con, void *p, void *end)
2434 {
2435 int tag = con->v2.in_desc.fd_tag;
2436 int ret;
2437
2438 dout("%s con %p tag %d len %d\n", __func__, con, tag, (int)(end - p));
2439
2440 switch (tag) {
2441 case FRAME_TAG_HELLO:
2442 ret = process_hello(con, p, end);
2443 break;
2444 case FRAME_TAG_AUTH_BAD_METHOD:
2445 ret = process_auth_bad_method(con, p, end);
2446 break;
2447 case FRAME_TAG_AUTH_REPLY_MORE:
2448 ret = process_auth_reply_more(con, p, end);
2449 break;
2450 case FRAME_TAG_AUTH_DONE:
2451 ret = process_auth_done(con, p, end);
2452 break;
2453 case FRAME_TAG_AUTH_SIGNATURE:
2454 ret = process_auth_signature(con, p, end);
2455 break;
2456 case FRAME_TAG_SERVER_IDENT:
2457 ret = process_server_ident(con, p, end);
2458 break;
2459 case FRAME_TAG_IDENT_MISSING_FEATURES:
2460 ret = process_ident_missing_features(con, p, end);
2461 break;
2462 case FRAME_TAG_SESSION_RECONNECT_OK:
2463 ret = process_session_reconnect_ok(con, p, end);
2464 break;
2465 case FRAME_TAG_SESSION_RETRY:
2466 ret = process_session_retry(con, p, end);
2467 break;
2468 case FRAME_TAG_SESSION_RETRY_GLOBAL:
2469 ret = process_session_retry_global(con, p, end);
2470 break;
2471 case FRAME_TAG_SESSION_RESET:
2472 ret = process_session_reset(con, p, end);
2473 break;
2474 case FRAME_TAG_KEEPALIVE2_ACK:
2475 ret = process_keepalive2_ack(con, p, end);
2476 break;
2477 case FRAME_TAG_ACK:
2478 ret = process_ack(con, p, end);
2479 break;
2480 default:
2481 pr_err("bad tag %d\n", tag);
2482 con->error_msg = "protocol error, bad tag";
2483 return -EINVAL;
2484 }
2485 if (ret) {
2486 dout("%s con %p error %d\n", __func__, con, ret);
2487 return ret;
2488 }
2489
2490 prepare_read_preamble(con);
2491 return 0;
2492 }
2493
2494 /*
2495 * Return:
2496 * 1 - con->in_msg set, read message
2497 * 0 - skip message
2498 * <0 - error
2499 */
2500 static int process_message_header(struct ceph_connection *con,
2501 void *p, void *end)
2502 {
2503 struct ceph_frame_desc *desc = &con->v2.in_desc;
2504 struct ceph_msg_header2 *hdr2 = p;
2505 struct ceph_msg_header hdr;
2506 int skip;
2507 int ret;
2508 u64 seq;
2509
2510 /* verify seq# */
2511 seq = le64_to_cpu(hdr2->seq);
2512 if ((s64)seq - (s64)con->in_seq < 1) {
2513 pr_info("%s%lld %s skipping old message: seq %llu, expected %llu\n",
2514 ENTITY_NAME(con->peer_name),
2515 ceph_pr_addr(&con->peer_addr),
2516 seq, con->in_seq + 1);
2517 return 0;
2518 }
2519 if ((s64)seq - (s64)con->in_seq > 1) {
2520 pr_err("bad seq %llu, expected %llu\n", seq, con->in_seq + 1);
2521 con->error_msg = "bad message sequence # for incoming message";
2522 return -EBADE;
2523 }
2524
2525 ceph_con_discard_sent(con, le64_to_cpu(hdr2->ack_seq));
2526
2527 fill_header(&hdr, hdr2, desc->fd_lens[1], desc->fd_lens[2],
2528 desc->fd_lens[3], &con->peer_name);
2529 ret = ceph_con_in_msg_alloc(con, &hdr, &skip);
2530 if (ret)
2531 return ret;
2532
2533 WARN_ON(!con->in_msg ^ skip);
2534 if (skip)
2535 return 0;
2536
2537 WARN_ON(!con->in_msg);
2538 WARN_ON(con->in_msg->con != con);
2539 return 1;
2540 }
2541
2542 static int process_message(struct ceph_connection *con)
2543 {
2544 ceph_con_process_message(con);
2545
2546 /*
2547 * We could have been closed by ceph_con_close() because
2548 * ceph_con_process_message() temporarily drops con->mutex.
2549 */
2550 if (con->state != CEPH_CON_S_OPEN) {
2551 dout("%s con %p state changed to %d\n", __func__, con,
2552 con->state);
2553 return -EAGAIN;
2554 }
2555
2556 prepare_read_preamble(con);
2557 return 0;
2558 }
2559
2560 static int __handle_control(struct ceph_connection *con, void *p)
2561 {
2562 void *end = p + con->v2.in_desc.fd_lens[0];
2563 struct ceph_msg *msg;
2564 int ret;
2565
2566 if (con->v2.in_desc.fd_tag != FRAME_TAG_MESSAGE)
2567 return process_control(con, p, end);
2568
2569 ret = process_message_header(con, p, end);
2570 if (ret < 0)
2571 return ret;
2572 if (ret == 0) {
2573 prepare_skip_message(con);
2574 return 0;
2575 }
2576
2577 msg = con->in_msg; /* set in process_message_header() */
2578 if (!front_len(msg) && !middle_len(msg)) {
2579 if (!data_len(msg))
2580 return process_message(con);
2581
2582 prepare_read_data(con);
2583 return 0;
2584 }
2585
2586 reset_in_kvecs(con);
2587 if (front_len(msg)) {
2588 WARN_ON(front_len(msg) > msg->front_alloc_len);
2589 add_in_kvec(con, msg->front.iov_base, front_len(msg));
2590 msg->front.iov_len = front_len(msg);
2591
2592 if (con_secure(con) && need_padding(front_len(msg)))
2593 add_in_kvec(con, FRONT_PAD(con->v2.in_buf),
2594 padding_len(front_len(msg)));
2595 } else {
2596 msg->front.iov_len = 0;
2597 }
2598 if (middle_len(msg)) {
2599 WARN_ON(middle_len(msg) > msg->middle->alloc_len);
2600 add_in_kvec(con, msg->middle->vec.iov_base, middle_len(msg));
2601 msg->middle->vec.iov_len = middle_len(msg);
2602
2603 if (con_secure(con) && need_padding(middle_len(msg)))
2604 add_in_kvec(con, MIDDLE_PAD(con->v2.in_buf),
2605 padding_len(middle_len(msg)));
2606 } else if (msg->middle) {
2607 msg->middle->vec.iov_len = 0;
2608 }
2609
2610 if (data_len(msg)) {
2611 con->v2.in_state = IN_S_PREPARE_READ_DATA;
2612 } else {
2613 add_in_kvec(con, con->v2.in_buf,
2614 con_secure(con) ? CEPH_EPILOGUE_SECURE_LEN :
2615 CEPH_EPILOGUE_PLAIN_LEN);
2616 con->v2.in_state = IN_S_HANDLE_EPILOGUE;
2617 }
2618 return 0;
2619 }
2620
2621 static int handle_preamble(struct ceph_connection *con)
2622 {
2623 struct ceph_frame_desc *desc = &con->v2.in_desc;
2624 int ret;
2625
2626 if (con_secure(con)) {
2627 ret = decrypt_preamble(con);
2628 if (ret) {
2629 if (ret == -EBADMSG)
2630 con->error_msg = "integrity error, bad preamble auth tag";
2631 return ret;
2632 }
2633 }
2634
2635 ret = decode_preamble(con->v2.in_buf, desc);
2636 if (ret) {
2637 if (ret == -EBADMSG)
2638 con->error_msg = "integrity error, bad crc";
2639 else
2640 con->error_msg = "protocol error, bad preamble";
2641 return ret;
2642 }
2643
2644 dout("%s con %p tag %d seg_cnt %d %d+%d+%d+%d\n", __func__,
2645 con, desc->fd_tag, desc->fd_seg_cnt, desc->fd_lens[0],
2646 desc->fd_lens[1], desc->fd_lens[2], desc->fd_lens[3]);
2647
2648 if (!con_secure(con))
2649 return prepare_read_control(con);
2650
2651 if (desc->fd_lens[0] > CEPH_PREAMBLE_INLINE_LEN)
2652 return prepare_read_control_remainder(con);
2653
2654 return __handle_control(con, CTRL_BODY(con->v2.in_buf));
2655 }
2656
2657 static int handle_control(struct ceph_connection *con)
2658 {
2659 int ctrl_len = con->v2.in_desc.fd_lens[0];
2660 void *buf;
2661 int ret;
2662
2663 WARN_ON(con_secure(con));
2664
2665 ret = verify_control_crc(con);
2666 if (ret) {
2667 con->error_msg = "integrity error, bad crc";
2668 return ret;
2669 }
2670
2671 if (con->state == CEPH_CON_S_V2_AUTH) {
2672 buf = alloc_conn_buf(con, ctrl_len);
2673 if (!buf)
2674 return -ENOMEM;
2675
2676 memcpy(buf, con->v2.in_kvecs[0].iov_base, ctrl_len);
2677 return __handle_control(con, buf);
2678 }
2679
2680 return __handle_control(con, con->v2.in_kvecs[0].iov_base);
2681 }
2682
2683 static int handle_control_remainder(struct ceph_connection *con)
2684 {
2685 int ret;
2686
2687 WARN_ON(!con_secure(con));
2688
2689 ret = decrypt_control_remainder(con);
2690 if (ret) {
2691 if (ret == -EBADMSG)
2692 con->error_msg = "integrity error, bad control remainder auth tag";
2693 return ret;
2694 }
2695
2696 return __handle_control(con, con->v2.in_kvecs[0].iov_base -
2697 CEPH_PREAMBLE_INLINE_LEN);
2698 }
2699
2700 static int handle_epilogue(struct ceph_connection *con)
2701 {
2702 u32 front_crc, middle_crc, data_crc;
2703 int ret;
2704
2705 if (con_secure(con)) {
2706 ret = decrypt_message(con);
2707 if (ret) {
2708 if (ret == -EBADMSG)
2709 con->error_msg = "integrity error, bad epilogue auth tag";
2710 return ret;
2711 }
2712
2713 /* just late_status */
2714 ret = decode_epilogue(con->v2.in_buf, NULL, NULL, NULL);
2715 if (ret) {
2716 con->error_msg = "protocol error, bad epilogue";
2717 return ret;
2718 }
2719 } else {
2720 ret = decode_epilogue(con->v2.in_buf, &front_crc,
2721 &middle_crc, &data_crc);
2722 if (ret) {
2723 con->error_msg = "protocol error, bad epilogue";
2724 return ret;
2725 }
2726
2727 ret = verify_epilogue_crcs(con, front_crc, middle_crc,
2728 data_crc);
2729 if (ret) {
2730 con->error_msg = "integrity error, bad crc";
2731 return ret;
2732 }
2733 }
2734
2735 return process_message(con);
2736 }
2737
2738 static void finish_skip(struct ceph_connection *con)
2739 {
2740 dout("%s con %p\n", __func__, con);
2741
2742 if (con_secure(con))
2743 gcm_inc_nonce(&con->v2.in_gcm_nonce);
2744
2745 __finish_skip(con);
2746 }
2747
2748 static int populate_in_iter(struct ceph_connection *con)
2749 {
2750 int ret;
2751
2752 dout("%s con %p state %d in_state %d\n", __func__, con, con->state,
2753 con->v2.in_state);
2754 WARN_ON(iov_iter_count(&con->v2.in_iter));
2755
2756 if (con->state == CEPH_CON_S_V2_BANNER_PREFIX) {
2757 ret = process_banner_prefix(con);
2758 } else if (con->state == CEPH_CON_S_V2_BANNER_PAYLOAD) {
2759 ret = process_banner_payload(con);
2760 } else if ((con->state >= CEPH_CON_S_V2_HELLO &&
2761 con->state <= CEPH_CON_S_V2_SESSION_RECONNECT) ||
2762 con->state == CEPH_CON_S_OPEN) {
2763 switch (con->v2.in_state) {
2764 case IN_S_HANDLE_PREAMBLE:
2765 ret = handle_preamble(con);
2766 break;
2767 case IN_S_HANDLE_CONTROL:
2768 ret = handle_control(con);
2769 break;
2770 case IN_S_HANDLE_CONTROL_REMAINDER:
2771 ret = handle_control_remainder(con);
2772 break;
2773 case IN_S_PREPARE_READ_DATA:
2774 prepare_read_data(con);
2775 ret = 0;
2776 break;
2777 case IN_S_PREPARE_READ_DATA_CONT:
2778 prepare_read_data_cont(con);
2779 ret = 0;
2780 break;
2781 case IN_S_HANDLE_EPILOGUE:
2782 ret = handle_epilogue(con);
2783 break;
2784 case IN_S_FINISH_SKIP:
2785 finish_skip(con);
2786 ret = 0;
2787 break;
2788 default:
2789 WARN(1, "bad in_state %d", con->v2.in_state);
2790 return -EINVAL;
2791 }
2792 } else {
2793 WARN(1, "bad state %d", con->state);
2794 return -EINVAL;
2795 }
2796 if (ret) {
2797 dout("%s con %p error %d\n", __func__, con, ret);
2798 return ret;
2799 }
2800
2801 if (WARN_ON(!iov_iter_count(&con->v2.in_iter)))
2802 return -ENODATA;
2803 dout("%s con %p populated %zu\n", __func__, con,
2804 iov_iter_count(&con->v2.in_iter));
2805 return 1;
2806 }
2807
2808 int ceph_con_v2_try_read(struct ceph_connection *con)
2809 {
2810 int ret;
2811
2812 dout("%s con %p state %d need %zu\n", __func__, con, con->state,
2813 iov_iter_count(&con->v2.in_iter));
2814
2815 if (con->state == CEPH_CON_S_PREOPEN)
2816 return 0;
2817
2818 /*
2819 * We should always have something pending here. If not,
2820 * avoid calling populate_in_iter() as if we read something
2821 * (ceph_tcp_recv() would immediately return 1).
2822 */
2823 if (WARN_ON(!iov_iter_count(&con->v2.in_iter)))
2824 return -ENODATA;
2825
2826 for (;;) {
2827 ret = ceph_tcp_recv(con);
2828 if (ret <= 0)
2829 return ret;
2830
2831 ret = populate_in_iter(con);
2832 if (ret <= 0) {
2833 if (ret && ret != -EAGAIN && !con->error_msg)
2834 con->error_msg = "read processing error";
2835 return ret;
2836 }
2837 }
2838 }
2839
2840 static void queue_data(struct ceph_connection *con)
2841 {
2842 struct bio_vec bv;
2843
2844 con->v2.out_epil.data_crc = -1;
2845 ceph_msg_data_cursor_init(&con->v2.out_cursor, con->out_msg,
2846 data_len(con->out_msg));
2847
2848 get_bvec_at(&con->v2.out_cursor, &bv);
2849 set_out_bvec(con, &bv, true);
2850 con->v2.out_state = OUT_S_QUEUE_DATA_CONT;
2851 }
2852
2853 static void queue_data_cont(struct ceph_connection *con)
2854 {
2855 struct bio_vec bv;
2856
2857 con->v2.out_epil.data_crc = ceph_crc32c_page(
2858 con->v2.out_epil.data_crc, con->v2.out_bvec.bv_page,
2859 con->v2.out_bvec.bv_offset, con->v2.out_bvec.bv_len);
2860
2861 ceph_msg_data_advance(&con->v2.out_cursor, con->v2.out_bvec.bv_len);
2862 if (con->v2.out_cursor.total_resid) {
2863 get_bvec_at(&con->v2.out_cursor, &bv);
2864 set_out_bvec(con, &bv, true);
2865 WARN_ON(con->v2.out_state != OUT_S_QUEUE_DATA_CONT);
2866 return;
2867 }
2868
2869 /*
2870 * We've written all data. Queue epilogue. Once it's written,
2871 * we are done.
2872 */
2873 reset_out_kvecs(con);
2874 prepare_epilogue_plain(con, false);
2875 con->v2.out_state = OUT_S_FINISH_MESSAGE;
2876 }
2877
2878 static void queue_enc_page(struct ceph_connection *con)
2879 {
2880 struct bio_vec bv;
2881
2882 dout("%s con %p i %d resid %d\n", __func__, con, con->v2.out_enc_i,
2883 con->v2.out_enc_resid);
2884 WARN_ON(!con->v2.out_enc_resid);
2885
2886 bv.bv_page = con->v2.out_enc_pages[con->v2.out_enc_i];
2887 bv.bv_offset = 0;
2888 bv.bv_len = min(con->v2.out_enc_resid, (int)PAGE_SIZE);
2889
2890 set_out_bvec(con, &bv, false);
2891 con->v2.out_enc_i++;
2892 con->v2.out_enc_resid -= bv.bv_len;
2893
2894 if (con->v2.out_enc_resid) {
2895 WARN_ON(con->v2.out_state != OUT_S_QUEUE_ENC_PAGE);
2896 return;
2897 }
2898
2899 /*
2900 * We've queued the last piece of ciphertext (ending with
2901 * epilogue) + auth tag. Once it's written, we are done.
2902 */
2903 WARN_ON(con->v2.out_enc_i != con->v2.out_enc_page_cnt);
2904 con->v2.out_state = OUT_S_FINISH_MESSAGE;
2905 }
2906
2907 static void queue_zeros(struct ceph_connection *con)
2908 {
2909 dout("%s con %p out_zero %d\n", __func__, con, con->v2.out_zero);
2910
2911 if (con->v2.out_zero) {
2912 set_out_bvec_zero(con);
2913 con->v2.out_zero -= con->v2.out_bvec.bv_len;
2914 con->v2.out_state = OUT_S_QUEUE_ZEROS;
2915 return;
2916 }
2917
2918 /*
2919 * We've zero-filled everything up to epilogue. Queue epilogue
2920 * with late_status set to ABORTED and crcs adjusted for zeros.
2921 * Once it's written, we are done patching up for the revoke.
2922 */
2923 reset_out_kvecs(con);
2924 prepare_epilogue_plain(con, true);
2925 con->v2.out_state = OUT_S_FINISH_MESSAGE;
2926 }
2927
2928 static void finish_message(struct ceph_connection *con)
2929 {
2930 dout("%s con %p msg %p\n", __func__, con, con->out_msg);
2931
2932 /* we end up here both plain and secure modes */
2933 if (con->v2.out_enc_pages) {
2934 WARN_ON(!con->v2.out_enc_page_cnt);
2935 ceph_release_page_vector(con->v2.out_enc_pages,
2936 con->v2.out_enc_page_cnt);
2937 con->v2.out_enc_pages = NULL;
2938 con->v2.out_enc_page_cnt = 0;
2939 }
2940 /* message may have been revoked */
2941 if (con->out_msg) {
2942 ceph_msg_put(con->out_msg);
2943 con->out_msg = NULL;
2944 }
2945
2946 con->v2.out_state = OUT_S_GET_NEXT;
2947 }
2948
2949 static int populate_out_iter(struct ceph_connection *con)
2950 {
2951 int ret;
2952
2953 dout("%s con %p state %d out_state %d\n", __func__, con, con->state,
2954 con->v2.out_state);
2955 WARN_ON(iov_iter_count(&con->v2.out_iter));
2956
2957 if (con->state != CEPH_CON_S_OPEN) {
2958 WARN_ON(con->state < CEPH_CON_S_V2_BANNER_PREFIX ||
2959 con->state > CEPH_CON_S_V2_SESSION_RECONNECT);
2960 goto nothing_pending;
2961 }
2962
2963 switch (con->v2.out_state) {
2964 case OUT_S_QUEUE_DATA:
2965 WARN_ON(!con->out_msg);
2966 queue_data(con);
2967 goto populated;
2968 case OUT_S_QUEUE_DATA_CONT:
2969 WARN_ON(!con->out_msg);
2970 queue_data_cont(con);
2971 goto populated;
2972 case OUT_S_QUEUE_ENC_PAGE:
2973 queue_enc_page(con);
2974 goto populated;
2975 case OUT_S_QUEUE_ZEROS:
2976 WARN_ON(con->out_msg); /* revoked */
2977 queue_zeros(con);
2978 goto populated;
2979 case OUT_S_FINISH_MESSAGE:
2980 finish_message(con);
2981 break;
2982 case OUT_S_GET_NEXT:
2983 break;
2984 default:
2985 WARN(1, "bad out_state %d", con->v2.out_state);
2986 return -EINVAL;
2987 }
2988
2989 WARN_ON(con->v2.out_state != OUT_S_GET_NEXT);
2990 if (ceph_con_flag_test_and_clear(con, CEPH_CON_F_KEEPALIVE_PENDING)) {
2991 ret = prepare_keepalive2(con);
2992 if (ret) {
2993 pr_err("prepare_keepalive2 failed: %d\n", ret);
2994 return ret;
2995 }
2996 } else if (!list_empty(&con->out_queue)) {
2997 ceph_con_get_out_msg(con);
2998 ret = prepare_message(con);
2999 if (ret) {
3000 pr_err("prepare_message failed: %d\n", ret);
3001 return ret;
3002 }
3003 } else if (con->in_seq > con->in_seq_acked) {
3004 ret = prepare_ack(con);
3005 if (ret) {
3006 pr_err("prepare_ack failed: %d\n", ret);
3007 return ret;
3008 }
3009 } else {
3010 goto nothing_pending;
3011 }
3012
3013 populated:
3014 if (WARN_ON(!iov_iter_count(&con->v2.out_iter)))
3015 return -ENODATA;
3016 dout("%s con %p populated %zu\n", __func__, con,
3017 iov_iter_count(&con->v2.out_iter));
3018 return 1;
3019
3020 nothing_pending:
3021 WARN_ON(iov_iter_count(&con->v2.out_iter));
3022 dout("%s con %p nothing pending\n", __func__, con);
3023 ceph_con_flag_clear(con, CEPH_CON_F_WRITE_PENDING);
3024 return 0;
3025 }
3026
3027 int ceph_con_v2_try_write(struct ceph_connection *con)
3028 {
3029 int ret;
3030
3031 dout("%s con %p state %d have %zu\n", __func__, con, con->state,
3032 iov_iter_count(&con->v2.out_iter));
3033
3034 /* open the socket first? */
3035 if (con->state == CEPH_CON_S_PREOPEN) {
3036 WARN_ON(con->peer_addr.type != CEPH_ENTITY_ADDR_TYPE_MSGR2);
3037
3038 /*
3039 * Always bump global_seq. Bump connect_seq only if
3040 * there is a session (i.e. we are reconnecting and will
3041 * send session_reconnect instead of client_ident).
3042 */
3043 con->v2.global_seq = ceph_get_global_seq(con->msgr, 0);
3044 if (con->v2.server_cookie)
3045 con->v2.connect_seq++;
3046
3047 ret = prepare_read_banner_prefix(con);
3048 if (ret) {
3049 pr_err("prepare_read_banner_prefix failed: %d\n", ret);
3050 con->error_msg = "connect error";
3051 return ret;
3052 }
3053
3054 reset_out_kvecs(con);
3055 ret = prepare_banner(con);
3056 if (ret) {
3057 pr_err("prepare_banner failed: %d\n", ret);
3058 con->error_msg = "connect error";
3059 return ret;
3060 }
3061
3062 ret = ceph_tcp_connect(con);
3063 if (ret) {
3064 pr_err("ceph_tcp_connect failed: %d\n", ret);
3065 con->error_msg = "connect error";
3066 return ret;
3067 }
3068 }
3069
3070 if (!iov_iter_count(&con->v2.out_iter)) {
3071 ret = populate_out_iter(con);
3072 if (ret <= 0) {
3073 if (ret && ret != -EAGAIN && !con->error_msg)
3074 con->error_msg = "write processing error";
3075 return ret;
3076 }
3077 }
3078
3079 tcp_sock_set_cork(con->sock->sk, true);
3080 for (;;) {
3081 ret = ceph_tcp_send(con);
3082 if (ret <= 0)
3083 break;
3084
3085 ret = populate_out_iter(con);
3086 if (ret <= 0) {
3087 if (ret && ret != -EAGAIN && !con->error_msg)
3088 con->error_msg = "write processing error";
3089 break;
3090 }
3091 }
3092
3093 tcp_sock_set_cork(con->sock->sk, false);
3094 return ret;
3095 }
3096
3097 static u32 crc32c_zeros(u32 crc, int zero_len)
3098 {
3099 int len;
3100
3101 while (zero_len) {
3102 len = min(zero_len, (int)PAGE_SIZE);
3103 crc = crc32c(crc, page_address(ceph_zero_page), len);
3104 zero_len -= len;
3105 }
3106
3107 return crc;
3108 }
3109
3110 static void prepare_zero_front(struct ceph_connection *con, int resid)
3111 {
3112 int sent;
3113
3114 WARN_ON(!resid || resid > front_len(con->out_msg));
3115 sent = front_len(con->out_msg) - resid;
3116 dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid);
3117
3118 if (sent) {
3119 con->v2.out_epil.front_crc =
3120 crc32c(-1, con->out_msg->front.iov_base, sent);
3121 con->v2.out_epil.front_crc =
3122 crc32c_zeros(con->v2.out_epil.front_crc, resid);
3123 } else {
3124 con->v2.out_epil.front_crc = crc32c_zeros(-1, resid);
3125 }
3126
3127 con->v2.out_iter.count -= resid;
3128 out_zero_add(con, resid);
3129 }
3130
3131 static void prepare_zero_middle(struct ceph_connection *con, int resid)
3132 {
3133 int sent;
3134
3135 WARN_ON(!resid || resid > middle_len(con->out_msg));
3136 sent = middle_len(con->out_msg) - resid;
3137 dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid);
3138
3139 if (sent) {
3140 con->v2.out_epil.middle_crc =
3141 crc32c(-1, con->out_msg->middle->vec.iov_base, sent);
3142 con->v2.out_epil.middle_crc =
3143 crc32c_zeros(con->v2.out_epil.middle_crc, resid);
3144 } else {
3145 con->v2.out_epil.middle_crc = crc32c_zeros(-1, resid);
3146 }
3147
3148 con->v2.out_iter.count -= resid;
3149 out_zero_add(con, resid);
3150 }
3151
3152 static void prepare_zero_data(struct ceph_connection *con)
3153 {
3154 dout("%s con %p\n", __func__, con);
3155 con->v2.out_epil.data_crc = crc32c_zeros(-1, data_len(con->out_msg));
3156 out_zero_add(con, data_len(con->out_msg));
3157 }
3158
3159 static void revoke_at_queue_data(struct ceph_connection *con)
3160 {
3161 int boundary;
3162 int resid;
3163
3164 WARN_ON(!data_len(con->out_msg));
3165 WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter));
3166 resid = iov_iter_count(&con->v2.out_iter);
3167
3168 boundary = front_len(con->out_msg) + middle_len(con->out_msg);
3169 if (resid > boundary) {
3170 resid -= boundary;
3171 WARN_ON(resid > MESSAGE_HEAD_PLAIN_LEN);
3172 dout("%s con %p was sending head\n", __func__, con);
3173 if (front_len(con->out_msg))
3174 prepare_zero_front(con, front_len(con->out_msg));
3175 if (middle_len(con->out_msg))
3176 prepare_zero_middle(con, middle_len(con->out_msg));
3177 prepare_zero_data(con);
3178 WARN_ON(iov_iter_count(&con->v2.out_iter) != resid);
3179 con->v2.out_state = OUT_S_QUEUE_ZEROS;
3180 return;
3181 }
3182
3183 boundary = middle_len(con->out_msg);
3184 if (resid > boundary) {
3185 resid -= boundary;
3186 dout("%s con %p was sending front\n", __func__, con);
3187 prepare_zero_front(con, resid);
3188 if (middle_len(con->out_msg))
3189 prepare_zero_middle(con, middle_len(con->out_msg));
3190 prepare_zero_data(con);
3191 queue_zeros(con);
3192 return;
3193 }
3194
3195 WARN_ON(!resid);
3196 dout("%s con %p was sending middle\n", __func__, con);
3197 prepare_zero_middle(con, resid);
3198 prepare_zero_data(con);
3199 queue_zeros(con);
3200 }
3201
3202 static void revoke_at_queue_data_cont(struct ceph_connection *con)
3203 {
3204 int sent, resid; /* current piece of data */
3205
3206 WARN_ON(!data_len(con->out_msg));
3207 WARN_ON(!iov_iter_is_bvec(&con->v2.out_iter));
3208 resid = iov_iter_count(&con->v2.out_iter);
3209 WARN_ON(!resid || resid > con->v2.out_bvec.bv_len);
3210 sent = con->v2.out_bvec.bv_len - resid;
3211 dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid);
3212
3213 if (sent) {
3214 con->v2.out_epil.data_crc = ceph_crc32c_page(
3215 con->v2.out_epil.data_crc, con->v2.out_bvec.bv_page,
3216 con->v2.out_bvec.bv_offset, sent);
3217 ceph_msg_data_advance(&con->v2.out_cursor, sent);
3218 }
3219 WARN_ON(resid > con->v2.out_cursor.total_resid);
3220 con->v2.out_epil.data_crc = crc32c_zeros(con->v2.out_epil.data_crc,
3221 con->v2.out_cursor.total_resid);
3222
3223 con->v2.out_iter.count -= resid;
3224 out_zero_add(con, con->v2.out_cursor.total_resid);
3225 queue_zeros(con);
3226 }
3227
3228 static void revoke_at_finish_message(struct ceph_connection *con)
3229 {
3230 int boundary;
3231 int resid;
3232
3233 WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter));
3234 resid = iov_iter_count(&con->v2.out_iter);
3235
3236 if (!front_len(con->out_msg) && !middle_len(con->out_msg) &&
3237 !data_len(con->out_msg)) {
3238 WARN_ON(!resid || resid > MESSAGE_HEAD_PLAIN_LEN);
3239 dout("%s con %p was sending head (empty message) - noop\n",
3240 __func__, con);
3241 return;
3242 }
3243
3244 boundary = front_len(con->out_msg) + middle_len(con->out_msg) +
3245 CEPH_EPILOGUE_PLAIN_LEN;
3246 if (resid > boundary) {
3247 resid -= boundary;
3248 WARN_ON(resid > MESSAGE_HEAD_PLAIN_LEN);
3249 dout("%s con %p was sending head\n", __func__, con);
3250 if (front_len(con->out_msg))
3251 prepare_zero_front(con, front_len(con->out_msg));
3252 if (middle_len(con->out_msg))
3253 prepare_zero_middle(con, middle_len(con->out_msg));
3254 con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN;
3255 WARN_ON(iov_iter_count(&con->v2.out_iter) != resid);
3256 con->v2.out_state = OUT_S_QUEUE_ZEROS;
3257 return;
3258 }
3259
3260 boundary = middle_len(con->out_msg) + CEPH_EPILOGUE_PLAIN_LEN;
3261 if (resid > boundary) {
3262 resid -= boundary;
3263 dout("%s con %p was sending front\n", __func__, con);
3264 prepare_zero_front(con, resid);
3265 if (middle_len(con->out_msg))
3266 prepare_zero_middle(con, middle_len(con->out_msg));
3267 con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN;
3268 queue_zeros(con);
3269 return;
3270 }
3271
3272 boundary = CEPH_EPILOGUE_PLAIN_LEN;
3273 if (resid > boundary) {
3274 resid -= boundary;
3275 dout("%s con %p was sending middle\n", __func__, con);
3276 prepare_zero_middle(con, resid);
3277 con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN;
3278 queue_zeros(con);
3279 return;
3280 }
3281
3282 WARN_ON(!resid);
3283 dout("%s con %p was sending epilogue - noop\n", __func__, con);
3284 }
3285
3286 void ceph_con_v2_revoke(struct ceph_connection *con)
3287 {
3288 WARN_ON(con->v2.out_zero);
3289
3290 if (con_secure(con)) {
3291 WARN_ON(con->v2.out_state != OUT_S_QUEUE_ENC_PAGE &&
3292 con->v2.out_state != OUT_S_FINISH_MESSAGE);
3293 dout("%s con %p secure - noop\n", __func__, con);
3294 return;
3295 }
3296
3297 switch (con->v2.out_state) {
3298 case OUT_S_QUEUE_DATA:
3299 revoke_at_queue_data(con);
3300 break;
3301 case OUT_S_QUEUE_DATA_CONT:
3302 revoke_at_queue_data_cont(con);
3303 break;
3304 case OUT_S_FINISH_MESSAGE:
3305 revoke_at_finish_message(con);
3306 break;
3307 default:
3308 WARN(1, "bad out_state %d", con->v2.out_state);
3309 break;
3310 }
3311 }
3312
3313 static void revoke_at_prepare_read_data(struct ceph_connection *con)
3314 {
3315 int remaining; /* data + [data padding] + epilogue */
3316 int resid;
3317
3318 WARN_ON(!data_len(con->in_msg));
3319 WARN_ON(!iov_iter_is_kvec(&con->v2.in_iter));
3320 resid = iov_iter_count(&con->v2.in_iter);
3321 WARN_ON(!resid);
3322
3323 if (con_secure(con))
3324 remaining = padded_len(data_len(con->in_msg)) +
3325 CEPH_EPILOGUE_SECURE_LEN;
3326 else
3327 remaining = data_len(con->in_msg) + CEPH_EPILOGUE_PLAIN_LEN;
3328
3329 dout("%s con %p resid %d remaining %d\n", __func__, con, resid,
3330 remaining);
3331 con->v2.in_iter.count -= resid;
3332 set_in_skip(con, resid + remaining);
3333 con->v2.in_state = IN_S_FINISH_SKIP;
3334 }
3335
3336 static void revoke_at_prepare_read_data_cont(struct ceph_connection *con)
3337 {
3338 int recved, resid; /* current piece of data */
3339 int remaining; /* [data padding] + epilogue */
3340
3341 WARN_ON(!data_len(con->in_msg));
3342 WARN_ON(!iov_iter_is_bvec(&con->v2.in_iter));
3343 resid = iov_iter_count(&con->v2.in_iter);
3344 WARN_ON(!resid || resid > con->v2.in_bvec.bv_len);
3345 recved = con->v2.in_bvec.bv_len - resid;
3346 dout("%s con %p recved %d resid %d\n", __func__, con, recved, resid);
3347
3348 if (recved)
3349 ceph_msg_data_advance(&con->v2.in_cursor, recved);
3350 WARN_ON(resid > con->v2.in_cursor.total_resid);
3351
3352 if (con_secure(con))
3353 remaining = padding_len(data_len(con->in_msg)) +
3354 CEPH_EPILOGUE_SECURE_LEN;
3355 else
3356 remaining = CEPH_EPILOGUE_PLAIN_LEN;
3357
3358 dout("%s con %p total_resid %zu remaining %d\n", __func__, con,
3359 con->v2.in_cursor.total_resid, remaining);
3360 con->v2.in_iter.count -= resid;
3361 set_in_skip(con, con->v2.in_cursor.total_resid + remaining);
3362 con->v2.in_state = IN_S_FINISH_SKIP;
3363 }
3364
3365 static void revoke_at_handle_epilogue(struct ceph_connection *con)
3366 {
3367 int resid;
3368
3369 WARN_ON(!iov_iter_is_kvec(&con->v2.in_iter));
3370 resid = iov_iter_count(&con->v2.in_iter);
3371 WARN_ON(!resid);
3372
3373 dout("%s con %p resid %d\n", __func__, con, resid);
3374 con->v2.in_iter.count -= resid;
3375 set_in_skip(con, resid);
3376 con->v2.in_state = IN_S_FINISH_SKIP;
3377 }
3378
3379 void ceph_con_v2_revoke_incoming(struct ceph_connection *con)
3380 {
3381 switch (con->v2.in_state) {
3382 case IN_S_PREPARE_READ_DATA:
3383 revoke_at_prepare_read_data(con);
3384 break;
3385 case IN_S_PREPARE_READ_DATA_CONT:
3386 revoke_at_prepare_read_data_cont(con);
3387 break;
3388 case IN_S_HANDLE_EPILOGUE:
3389 revoke_at_handle_epilogue(con);
3390 break;
3391 default:
3392 WARN(1, "bad in_state %d", con->v2.in_state);
3393 break;
3394 }
3395 }
3396
3397 bool ceph_con_v2_opened(struct ceph_connection *con)
3398 {
3399 return con->v2.peer_global_seq;
3400 }
3401
3402 void ceph_con_v2_reset_session(struct ceph_connection *con)
3403 {
3404 con->v2.client_cookie = 0;
3405 con->v2.server_cookie = 0;
3406 con->v2.global_seq = 0;
3407 con->v2.connect_seq = 0;
3408 con->v2.peer_global_seq = 0;
3409 }
3410
3411 void ceph_con_v2_reset_protocol(struct ceph_connection *con)
3412 {
3413 iov_iter_truncate(&con->v2.in_iter, 0);
3414 iov_iter_truncate(&con->v2.out_iter, 0);
3415 con->v2.out_zero = 0;
3416
3417 clear_in_sign_kvecs(con);
3418 clear_out_sign_kvecs(con);
3419 free_conn_bufs(con);
3420
3421 if (con->v2.out_enc_pages) {
3422 WARN_ON(!con->v2.out_enc_page_cnt);
3423 ceph_release_page_vector(con->v2.out_enc_pages,
3424 con->v2.out_enc_page_cnt);
3425 con->v2.out_enc_pages = NULL;
3426 con->v2.out_enc_page_cnt = 0;
3427 }
3428
3429 con->v2.con_mode = CEPH_CON_MODE_UNKNOWN;
3430
3431 if (con->v2.hmac_tfm) {
3432 crypto_free_shash(con->v2.hmac_tfm);
3433 con->v2.hmac_tfm = NULL;
3434 }
3435 if (con->v2.gcm_req) {
3436 aead_request_free(con->v2.gcm_req);
3437 con->v2.gcm_req = NULL;
3438 }
3439 if (con->v2.gcm_tfm) {
3440 crypto_free_aead(con->v2.gcm_tfm);
3441 con->v2.gcm_tfm = NULL;
3442 }
3443 }
Linux with added FPC-III support