tools/testing/selftests/net/tcp_mmap.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright 2018 Google Inc.
   4  * Author: Eric Dumazet (edumazet@google.com)
   5  *
   6  * Reference program demonstrating tcp mmap() usage,
   7  * and SO_RCVLOWAT hints for receiver.
   8  *
   9  * Note : NIC with header split is needed to use mmap() on TCP :
  10  * Each incoming frame must be a multiple of PAGE_SIZE bytes of TCP payload.
  11  *
  12  * How to use on loopback interface :
  13  *
  14  *  ifconfig lo mtu 61512  # 15*4096 + 40 (ipv6 header) + 32 (TCP with TS option header)
  15  *  tcp_mmap -s -z &
  16  *  tcp_mmap -H ::1 -z
  17  *
  18  *  Or leave default lo mtu, but use -M option to set TCP_MAXSEG option to (4096 + 12)
  19  *      (4096 : page size on x86, 12: TCP TS option length)
  20  *  tcp_mmap -s -z -M $((4096+12)) &
  21  *  tcp_mmap -H ::1 -z -M $((4096+12))
  22  *
  23  * Note: -z option on sender uses MSG_ZEROCOPY, which forces a copy when packets go through loopback interface.
  24  *       We might use sendfile() instead, but really this test program is about mmap(), for receivers ;)
  25  *
  26  * $ ./tcp_mmap -s &                                 # Without mmap()
  27  * $ for i in {1..4}; do ./tcp_mmap -H ::1 -z ; done
  28  * received 32768 MB (0 % mmap'ed) in 14.1157 s, 19.4732 Gbit
  29  *   cpu usage user:0.057 sys:7.815, 240.234 usec per MB, 65531 c-switches
  30  * received 32768 MB (0 % mmap'ed) in 14.6833 s, 18.7204 Gbit
  31  *  cpu usage user:0.043 sys:8.103, 248.596 usec per MB, 65524 c-switches
  32  * received 32768 MB (0 % mmap'ed) in 11.143 s, 24.6682 Gbit
  33  *   cpu usage user:0.044 sys:6.576, 202.026 usec per MB, 65519 c-switches
  34  * received 32768 MB (0 % mmap'ed) in 14.9056 s, 18.4413 Gbit
  35  *   cpu usage user:0.036 sys:8.193, 251.129 usec per MB, 65530 c-switches
  36  * $ kill %1   # kill tcp_mmap server
  37  *
  38  * $ ./tcp_mmap -s -z &                              # With mmap()
  39  * $ for i in {1..4}; do ./tcp_mmap -H ::1 -z ; done
  40  * received 32768 MB (99.9939 % mmap'ed) in 6.73792 s, 40.7956 Gbit
  41  *   cpu usage user:0.045 sys:2.827, 87.6465 usec per MB, 65532 c-switches
  42  * received 32768 MB (99.9939 % mmap'ed) in 7.26732 s, 37.8238 Gbit
  43  *   cpu usage user:0.037 sys:3.087, 95.3369 usec per MB, 65532 c-switches
  44  * received 32768 MB (99.9939 % mmap'ed) in 7.61661 s, 36.0893 Gbit
  45  *   cpu usage user:0.046 sys:3.559, 110.016 usec per MB, 65529 c-switches
  46  * received 32768 MB (99.9939 % mmap'ed) in 7.43764 s, 36.9577 Gbit
  47  *   cpu usage user:0.035 sys:3.467, 106.873 usec per MB, 65530 c-switches
  48  */
  49 #define _GNU_SOURCE
  50 #include <pthread.h>
  51 #include <sys/types.h>
  52 #include <fcntl.h>
  53 #include <error.h>
  54 #include <sys/socket.h>
  55 #include <sys/mman.h>
  56 #include <sys/resource.h>
  57 #include <unistd.h>
  58 #include <string.h>
  59 #include <stdlib.h>
  60 #include <stdio.h>
  61 #include <errno.h>
  62 #include <time.h>
  63 #include <sys/time.h>
  64 #include <netinet/in.h>
  65 #include <arpa/inet.h>
  66 #include <poll.h>
  67 #include <linux/tcp.h>
  68 #include <assert.h>
  69 #include <openssl/pem.h>
  70
  71 #ifndef MSG_ZEROCOPY
  72 #define MSG_ZEROCOPY    0x4000000
  73 #endif
  74
  75 #ifndef min
  76 #define min(a, b)  ((a) < (b) ? (a) : (b))
  77 #endif
  78
  79 #define FILE_SZ (1ULL << 35)
  80 static int cfg_family = AF_INET6;
  81 static socklen_t cfg_alen = sizeof(struct sockaddr_in6);
  82 static int cfg_port = 8787;
  83
  84 static int rcvbuf; /* Default: autotuning.  Can be set with -r <integer> option */
  85 static int sndbuf; /* Default: autotuning.  Can be set with -w <integer> option */
  86 static int zflg; /* zero copy option. (MSG_ZEROCOPY for sender, mmap() for receiver */
  87 static int xflg; /* hash received data (simple xor) (-h option) */
  88 static int keepflag; /* -k option: receiver shall keep all received file in memory (no munmap() calls) */
  89 static int integrity; /* -i option: sender and receiver compute sha256 over the data.*/
  90
  91 static size_t chunk_size  = 512*1024;
  92
  93 static size_t map_align;
  94
  95 unsigned long htotal;
  96 unsigned int digest_len;
  97
  98 static inline void prefetch(const void *x)
  99 {
 100 #if defined(__x86_64__)
 101         asm volatile("prefetcht0 %P0" : : "m" (*(const char *)x));
 102 #endif
 103 }
 104
 105 void hash_zone(void *zone, unsigned int length)
 106 {
 107         unsigned long temp = htotal;
 108
 109         while (length >= 8*sizeof(long)) {
 110                 prefetch(zone + 384);
 111                 temp ^= *(unsigned long *)zone;
 112                 temp ^= *(unsigned long *)(zone + sizeof(long));
 113                 temp ^= *(unsigned long *)(zone + 2*sizeof(long));
 114                 temp ^= *(unsigned long *)(zone + 3*sizeof(long));
 115                 temp ^= *(unsigned long *)(zone + 4*sizeof(long));
 116                 temp ^= *(unsigned long *)(zone + 5*sizeof(long));
 117                 temp ^= *(unsigned long *)(zone + 6*sizeof(long));
 118                 temp ^= *(unsigned long *)(zone + 7*sizeof(long));
 119                 zone += 8*sizeof(long);
 120                 length -= 8*sizeof(long);
 121         }
 122         while (length >= 1) {
 123                 temp ^= *(unsigned char *)zone;
 124                 zone += 1;
 125                 length--;
 126         }
 127         htotal = temp;
 128 }
 129
 130 #define ALIGN_UP(x, align_to)   (((x) + ((align_to)-1)) & ~((align_to)-1))
 131 #define ALIGN_PTR_UP(p, ptr_align_to)   ((typeof(p))ALIGN_UP((unsigned long)(p), ptr_align_to))
 132
 133
 134 static void *mmap_large_buffer(size_t need, size_t *allocated)
 135 {
 136         void *buffer;
 137         size_t sz;
 138
 139         /* Attempt to use huge pages if possible. */
 140         sz = ALIGN_UP(need, map_align);
 141         buffer = mmap(NULL, sz, PROT_READ | PROT_WRITE,
 142                       MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0);
 143
 144         if (buffer == (void *)-1) {
 145                 sz = need;
 146                 buffer = mmap(NULL, sz, PROT_READ | PROT_WRITE,
 147                               MAP_PRIVATE | MAP_ANONYMOUS | MAP_POPULATE,
 148                               -1, 0);
 149                 if (buffer != (void *)-1)
 150                         fprintf(stderr, "MAP_HUGETLB attempt failed, look at /sys/kernel/mm/hugepages for optimal performance\n");
 151         }
 152         *allocated = sz;
 153         return buffer;
 154 }
 155
 156 static uint32_t tcp_info_get_rcv_mss(int fd)
 157 {
 158         socklen_t sz = sizeof(struct tcp_info);
 159         struct tcp_info info;
 160
 161         if (getsockopt(fd, IPPROTO_TCP, TCP_INFO, &info, &sz)) {
 162                 fprintf(stderr, "Error fetching TCP_INFO\n");
 163                 return 0;
 164         }
 165
 166         return info.tcpi_rcv_mss;
 167 }
 168
 169 void *child_thread(void *arg)
 170 {
 171         unsigned char digest[SHA256_DIGEST_LENGTH];
 172         unsigned long total_mmap = 0, total = 0;
 173         struct tcp_zerocopy_receive zc;
 174         unsigned char *buffer = NULL;
 175         unsigned long delta_usec;
 176         EVP_MD_CTX *ctx = NULL;
 177         int flags = MAP_SHARED;
 178         struct timeval t0, t1;
 179         void *raddr = NULL;
 180         void *addr = NULL;
 181         double throughput;
 182         struct rusage ru;
 183         size_t buffer_sz;
 184         int lu, fd;
 185
 186         fd = (int)(unsigned long)arg;
 187
 188         gettimeofday(&t0, NULL);
 189
 190         fcntl(fd, F_SETFL, O_NDELAY);
 191         buffer = mmap_large_buffer(chunk_size, &buffer_sz);
 192         if (buffer == (void *)-1) {
 193                 perror("mmap");
 194                 goto error;
 195         }
 196         if (zflg) {
 197                 raddr = mmap(NULL, chunk_size + map_align, PROT_READ, flags, fd, 0);
 198                 if (raddr == (void *)-1) {
 199                         perror("mmap");
 200                         zflg = 0;
 201                 } else {
 202                         addr = ALIGN_PTR_UP(raddr, map_align);
 203                 }
 204         }
 205         if (integrity) {
 206                 ctx = EVP_MD_CTX_new();
 207                 if (!ctx) {
 208                         perror("cannot enable SHA computing");
 209                         goto error;
 210                 }
 211                 EVP_DigestInit_ex(ctx, EVP_sha256(), NULL);
 212         }
 213         while (1) {
 214                 struct pollfd pfd = { .fd = fd, .events = POLLIN, };
 215                 int sub;
 216
 217                 poll(&pfd, 1, 10000);
 218                 if (zflg) {
 219                         socklen_t zc_len = sizeof(zc);
 220                         int res;
 221
 222                         memset(&zc, 0, sizeof(zc));
 223                         zc.address = (__u64)((unsigned long)addr);
 224                         zc.length = min(chunk_size, FILE_SZ - total);
 225
 226                         res = getsockopt(fd, IPPROTO_TCP, TCP_ZEROCOPY_RECEIVE,
 227                                          &zc, &zc_len);
 228                         if (res == -1)
 229                                 break;
 230
 231                         if (zc.length) {
 232                                 assert(zc.length <= chunk_size);
 233                                 if (integrity)
 234                                         EVP_DigestUpdate(ctx, addr, zc.length);
 235                                 total_mmap += zc.length;
 236                                 if (xflg)
 237                                         hash_zone(addr, zc.length);
 238                                 /* It is more efficient to unmap the pages right now,
 239                                  * instead of doing this in next TCP_ZEROCOPY_RECEIVE.
 240                                  */
 241                                 madvise(addr, zc.length, MADV_DONTNEED);
 242                                 total += zc.length;
 243                         }
 244                         if (zc.recv_skip_hint) {
 245                                 assert(zc.recv_skip_hint <= chunk_size);
 246                                 lu = read(fd, buffer, min(zc.recv_skip_hint,
 247                                                           FILE_SZ - total));
 248                                 if (lu > 0) {
 249                                         if (integrity)
 250                                                 EVP_DigestUpdate(ctx, buffer, lu);
 251                                         if (xflg)
 252                                                 hash_zone(buffer, lu);
 253                                         total += lu;
 254                                 }
 255                                 if (lu == 0)
 256                                         goto end;
 257                         }
 258                         continue;
 259                 }
 260                 sub = 0;
 261                 while (sub < chunk_size) {
 262                         lu = read(fd, buffer + sub, min(chunk_size - sub,
 263                                                         FILE_SZ - total));
 264                         if (lu == 0)
 265                                 goto end;
 266                         if (lu < 0)
 267                                 break;
 268                         if (integrity)
 269                                 EVP_DigestUpdate(ctx, buffer + sub, lu);
 270                         if (xflg)
 271                                 hash_zone(buffer + sub, lu);
 272                         total += lu;
 273                         sub += lu;
 274                 }
 275         }
 276 end:
 277         gettimeofday(&t1, NULL);
 278         delta_usec = (t1.tv_sec - t0.tv_sec) * 1000000 + t1.tv_usec - t0.tv_usec;
 279
 280         if (integrity) {
 281                 fcntl(fd, F_SETFL, 0);
 282                 EVP_DigestFinal_ex(ctx, digest, &digest_len);
 283                 lu = read(fd, buffer, SHA256_DIGEST_LENGTH);
 284                 if (lu != SHA256_DIGEST_LENGTH)
 285                         perror("Error: Cannot read SHA256\n");
 286
 287                 if (memcmp(digest, buffer,
 288                            SHA256_DIGEST_LENGTH))
 289                         fprintf(stderr, "Error: SHA256 of the data is not right\n");
 290                 else
 291                         printf("\nSHA256 is correct\n");
 292         }
 293
 294         throughput = 0;
 295         if (delta_usec)
 296                 throughput = total * 8.0 / (double)delta_usec / 1000.0;
 297         getrusage(RUSAGE_THREAD, &ru);
 298         if (total > 1024*1024) {
 299                 unsigned long total_usec;
 300                 unsigned long mb = total >> 20;
 301                 total_usec = 1000000*ru.ru_utime.tv_sec + ru.ru_utime.tv_usec +
 302                              1000000*ru.ru_stime.tv_sec + ru.ru_stime.tv_usec;
 303                 printf("received %lg MB (%lg %% mmap'ed) in %lg s, %lg Gbit\n"
 304                        "  cpu usage user:%lg sys:%lg, %lg usec per MB, %lu c-switches, rcv_mss %u\n",
 305                                 total / (1024.0 * 1024.0),
 306                                 100.0*total_mmap/total,
 307                                 (double)delta_usec / 1000000.0,
 308                                 throughput,
 309                                 (double)ru.ru_utime.tv_sec + (double)ru.ru_utime.tv_usec / 1000000.0,
 310                                 (double)ru.ru_stime.tv_sec + (double)ru.ru_stime.tv_usec / 1000000.0,
 311                                 (double)total_usec/mb,
 312                                 ru.ru_nvcsw,
 313                                 tcp_info_get_rcv_mss(fd));
 314         }
 315 error:
 316         munmap(buffer, buffer_sz);
 317         close(fd);
 318         if (zflg)
 319                 munmap(raddr, chunk_size + map_align);
 320         pthread_exit(0);
 321 }
 322
 323 static void apply_rcvsnd_buf(int fd)
 324 {
 325         if (rcvbuf && setsockopt(fd, SOL_SOCKET,
 326                                  SO_RCVBUF, &rcvbuf, sizeof(rcvbuf)) == -1) {
 327                 perror("setsockopt SO_RCVBUF");
 328         }
 329
 330         if (sndbuf && setsockopt(fd, SOL_SOCKET,
 331                                  SO_SNDBUF, &sndbuf, sizeof(sndbuf)) == -1) {
 332                 perror("setsockopt SO_SNDBUF");
 333         }
 334 }
 335
 336
 337 static void setup_sockaddr(int domain, const char *str_addr,
 338                            struct sockaddr_storage *sockaddr)
 339 {
 340         struct sockaddr_in6 *addr6 = (void *) sockaddr;
 341         struct sockaddr_in *addr4 = (void *) sockaddr;
 342
 343         switch (domain) {
 344         case PF_INET:
 345                 memset(addr4, 0, sizeof(*addr4));
 346                 addr4->sin_family = AF_INET;
 347                 addr4->sin_port = htons(cfg_port);
 348                 if (str_addr &&
 349                     inet_pton(AF_INET, str_addr, &(addr4->sin_addr)) != 1)
 350                         error(1, 0, "ipv4 parse error: %s", str_addr);
 351                 break;
 352         case PF_INET6:
 353                 memset(addr6, 0, sizeof(*addr6));
 354                 addr6->sin6_family = AF_INET6;
 355                 addr6->sin6_port = htons(cfg_port);
 356                 if (str_addr &&
 357                     inet_pton(AF_INET6, str_addr, &(addr6->sin6_addr)) != 1)
 358                         error(1, 0, "ipv6 parse error: %s", str_addr);
 359                 break;
 360         default:
 361                 error(1, 0, "illegal domain");
 362         }
 363 }
 364
 365 static void do_accept(int fdlisten)
 366 {
 367         pthread_attr_t attr;
 368         int rcvlowat;
 369
 370         pthread_attr_init(&attr);
 371         pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
 372
 373         rcvlowat = chunk_size;
 374         if (setsockopt(fdlisten, SOL_SOCKET, SO_RCVLOWAT,
 375                        &rcvlowat, sizeof(rcvlowat)) == -1) {
 376                 perror("setsockopt SO_RCVLOWAT");
 377         }
 378
 379         apply_rcvsnd_buf(fdlisten);
 380
 381         while (1) {
 382                 struct sockaddr_in addr;
 383                 socklen_t addrlen = sizeof(addr);
 384                 pthread_t th;
 385                 int fd, res;
 386
 387                 fd = accept(fdlisten, (struct sockaddr *)&addr, &addrlen);
 388                 if (fd == -1) {
 389                         perror("accept");
 390                         continue;
 391                 }
 392                 res = pthread_create(&th, &attr, child_thread,
 393                                      (void *)(unsigned long)fd);
 394                 if (res) {
 395                         errno = res;
 396                         perror("pthread_create");
 397                         close(fd);
 398                 }
 399         }
 400 }
 401
 402 /* Each thread should reserve a big enough vma to avoid
 403  * spinlock collisions in ptl locks.
 404  * This size is 2MB on x86_64, and is exported in /proc/meminfo.
 405  */
 406 static unsigned long default_huge_page_size(void)
 407 {
 408         FILE *f = fopen("/proc/meminfo", "r");
 409         unsigned long hps = 0;
 410         size_t linelen = 0;
 411         char *line = NULL;
 412
 413         if (!f)
 414                 return 0;
 415         while (getline(&line, &linelen, f) > 0) {
 416                 if (sscanf(line, "Hugepagesize:       %lu kB", &hps) == 1) {
 417                         hps <<= 10;
 418                         break;
 419                 }
 420         }
 421         free(line);
 422         fclose(f);
 423         return hps;
 424 }
 425
 426 static void randomize(void *target, size_t count)
 427 {
 428         static int urandom = -1;
 429         ssize_t got;
 430
 431         urandom = open("/dev/urandom", O_RDONLY);
 432         if (urandom < 0) {
 433                 perror("open /dev/urandom");
 434                 exit(1);
 435         }
 436         got = read(urandom, target, count);
 437         if (got != count) {
 438                 perror("read /dev/urandom");
 439                 exit(1);
 440         }
 441 }
 442
 443 int main(int argc, char *argv[])
 444 {
 445         unsigned char digest[SHA256_DIGEST_LENGTH];
 446         struct sockaddr_storage listenaddr, addr;
 447         unsigned int max_pacing_rate = 0;
 448         EVP_MD_CTX *ctx = NULL;
 449         unsigned char *buffer;
 450         uint64_t total = 0;
 451         char *host = NULL;
 452         int fd, c, on = 1;
 453         size_t buffer_sz;
 454         int sflg = 0;
 455         int mss = 0;
 456
 457         while ((c = getopt(argc, argv, "46p:svr:w:H:zxkP:M:C:a:i")) != -1) {
 458                 switch (c) {
 459                 case '4':
 460                         cfg_family = PF_INET;
 461                         cfg_alen = sizeof(struct sockaddr_in);
 462                         break;
 463                 case '6':
 464                         cfg_family = PF_INET6;
 465                         cfg_alen = sizeof(struct sockaddr_in6);
 466                         break;
 467                 case 'p':
 468                         cfg_port = atoi(optarg);
 469                         break;
 470                 case 'H':
 471                         host = optarg;
 472                         break;
 473                 case 's': /* server : listen for incoming connections */
 474                         sflg++;
 475                         break;
 476                 case 'r':
 477                         rcvbuf = atoi(optarg);
 478                         break;
 479                 case 'w':
 480                         sndbuf = atoi(optarg);
 481                         break;
 482                 case 'z':
 483                         zflg = 1;
 484                         break;
 485                 case 'M':
 486                         mss = atoi(optarg);
 487                         break;
 488                 case 'x':
 489                         xflg = 1;
 490                         break;
 491                 case 'k':
 492                         keepflag = 1;
 493                         break;
 494                 case 'P':
 495                         max_pacing_rate = atoi(optarg) ;
 496                         break;
 497                 case 'C':
 498                         chunk_size = atol(optarg);
 499                         break;
 500                 case 'a':
 501                         map_align = atol(optarg);
 502                         break;
 503                 case 'i':
 504                         integrity = 1;
 505                         break;
 506                 default:
 507                         exit(1);
 508                 }
 509         }
 510         if (!map_align) {
 511                 map_align = default_huge_page_size();
 512                 /* if really /proc/meminfo is not helping,
 513                  * we use the default x86_64 hugepagesize.
 514                  */
 515                 if (!map_align)
 516                         map_align = 2*1024*1024;
 517         }
 518         if (sflg) {
 519                 int fdlisten = socket(cfg_family, SOCK_STREAM, 0);
 520
 521                 if (fdlisten == -1) {
 522                         perror("socket");
 523                         exit(1);
 524                 }
 525                 apply_rcvsnd_buf(fdlisten);
 526                 setsockopt(fdlisten, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
 527
 528                 setup_sockaddr(cfg_family, host, &listenaddr);
 529
 530                 if (mss &&
 531                     setsockopt(fdlisten, IPPROTO_TCP, TCP_MAXSEG,
 532                                &mss, sizeof(mss)) == -1) {
 533                         perror("setsockopt TCP_MAXSEG");
 534                         exit(1);
 535                 }
 536                 if (bind(fdlisten, (const struct sockaddr *)&listenaddr, cfg_alen) == -1) {
 537                         perror("bind");
 538                         exit(1);
 539                 }
 540                 if (listen(fdlisten, 128) == -1) {
 541                         perror("listen");
 542                         exit(1);
 543                 }
 544                 do_accept(fdlisten);
 545         }
 546
 547         buffer = mmap_large_buffer(chunk_size, &buffer_sz);
 548         if (buffer == (unsigned char *)-1) {
 549                 perror("mmap");
 550                 exit(1);
 551         }
 552
 553         fd = socket(cfg_family, SOCK_STREAM, 0);
 554         if (fd == -1) {
 555                 perror("socket");
 556                 exit(1);
 557         }
 558         apply_rcvsnd_buf(fd);
 559
 560         setup_sockaddr(cfg_family, host, &addr);
 561
 562         if (mss &&
 563             setsockopt(fd, IPPROTO_TCP, TCP_MAXSEG, &mss, sizeof(mss)) == -1) {
 564                 perror("setsockopt TCP_MAXSEG");
 565                 exit(1);
 566         }
 567         if (connect(fd, (const struct sockaddr *)&addr, cfg_alen) == -1) {
 568                 perror("connect");
 569                 exit(1);
 570         }
 571         if (max_pacing_rate &&
 572             setsockopt(fd, SOL_SOCKET, SO_MAX_PACING_RATE,
 573                        &max_pacing_rate, sizeof(max_pacing_rate)) == -1)
 574                 perror("setsockopt SO_MAX_PACING_RATE");
 575
 576         if (zflg && setsockopt(fd, SOL_SOCKET, SO_ZEROCOPY,
 577                                &on, sizeof(on)) == -1) {
 578                 perror("setsockopt SO_ZEROCOPY, (-z option disabled)");
 579                 zflg = 0;
 580         }
 581         if (integrity) {
 582                 randomize(buffer, buffer_sz);
 583                 ctx = EVP_MD_CTX_new();
 584                 if (!ctx) {
 585                         perror("cannot enable SHA computing");
 586                         exit(1);
 587                 }
 588                 EVP_DigestInit_ex(ctx, EVP_sha256(), NULL);
 589         }
 590         while (total < FILE_SZ) {
 591                 size_t offset = total % chunk_size;
 592                 int64_t wr = FILE_SZ - total;
 593
 594                 if (wr > chunk_size - offset)
 595                         wr = chunk_size - offset;
 596                 /* Note : we just want to fill the pipe with random bytes */
 597                 wr = send(fd, buffer + offset,
 598                           (size_t)wr, zflg ? MSG_ZEROCOPY : 0);
 599                 if (wr <= 0)
 600                         break;
 601                 if (integrity)
 602                         EVP_DigestUpdate(ctx, buffer + offset, wr);
 603                 total += wr;
 604         }
 605         if (integrity && total == FILE_SZ) {
 606                 EVP_DigestFinal_ex(ctx, digest, &digest_len);
 607                 send(fd, digest, (size_t)SHA256_DIGEST_LENGTH, 0);
 608         }
 609         close(fd);
 610         munmap(buffer, buffer_sz);
 611         return 0;
 612 }