Initial commit

[udp-dtls-proxy.git] / dtls-port-forwarder.c++

#include <climits>
#if defined(__OpenBSD__)
#include <errno.h>
#else
#include <asm-generic/errno.h>
#endif

#include <stdexcept>
#include <string>
#ifdef WIN32
#include <Ws2tcpip.h>
#include <winsock2.h>
#define in_port_t u_short
#define ssize_t int
#define my_tid_t DWORD
#else // UNIX/Linux
#include <arpa/inet.h>
#include <cstdlib>
#include <cstring>
#include <netdb.h>
#include <netinet/in.h>
#include <pthread.h>
#include <stdio.h>
#include <sys/socket.h>
#include <sys/time.h>
#include <sys/types.h>
#include <unistd.h>
#define my_tid_t pthread_t
#endif

#include <openssl/bio.h>
#include <openssl/err.h>
#include <openssl/opensslv.h>
#include <openssl/rand.h>
#include <openssl/ssl.h>

#include "3rdparty/popl.hpp"

using namespace std::string_literals;

static unsigned long id_thread(void) {
#ifdef WIN32
  return (unsigned long)GetCurrentThreadId();
#else
  return (unsigned long)pthread_self();
#endif
}

#define debug_print(x)                                                    \
  if (false)                                                              \
    fprintf(stderr,                                                       \
            "Thread %lx:%s:%lld:%d:%s\n",                                 \
            id_thread(),                                                  \
            __func__,                                                     \
            time(NULL),                                                   \
            __LINE__,                                                     \
            x);                                                           \
  ;                                                                       \
  ; // fflush(stderr);
//#define debug_print(x)

static auto closer = [](int * sock_ptr) {
#ifdef WIN32
  closesocket(*sock_ptr);
#else
  close(*sock_ptr);
#endif
  debug_print(("Closed socket="s + std::to_string(*sock_ptr)).c_str());
};

void my_explain_recvfrom_error(int arg) {
  switch (arg) {
  case EAGAIN:
    debug_print(
      "The socket is marked nonblocking and the receive operation would   "
      "  "
      "block, or a receive timeout had been set and the timeout expired   "
      "  "
      "before data was received. POSIX.1 allows either error to be     "
      "returned for this case, and does not require these constants to    "
      " "
      "have the same value, so a portable application should check for    "
      "both possibilities.");
    break;
  case EBADF:
    debug_print("The argument sockfd is an invalid file descriptor.");
    break;
  case ECONNREFUSED:
    debug_print(
      "A remote host refused to allow the network connection( "
      "typically because it is not running the requested service)    "
      ".");
    break;
  case EFAULT:
    debug_print(
      "The receive buffer pointer(s) point outside the process's address "
      "space.");
    break;
  case EINTR:
    debug_print(
      "The receive was interrupted by delivery of a signal before "
      "any data was available; see signal(7).");
    break;
  case EINVAL:
    debug_print("Invalid argument passed.");
    break;
  case ENOMEM:
    debug_print("Could not allocate memory for recvmsg().");
    break;
  case ENOTCONN:
    debug_print("The socket is associated with a connection-oriented "
                "protocol and has "
                "not been connected  (see  connect(2)  and accept(2)).");
    break;
  case ENOTSOCK:
    debug_print("The file descriptor sockfd does not refer to a socket.");
    break;
  default:
    debug_print(
      ("Cannot explain recvfrom error "s + std::to_string(errno)).c_str());
  }
}

typedef union {
  struct sockaddr_storage ss;
  struct sockaddr_in s4;
  struct sockaddr_in6 s6;
} my_ip46_address;

std::string my_ip46_address_to_string(my_ip46_address address_bin) {
  char addrbuf[INET6_ADDRSTRLEN];
  short port = -1;
  if (address_bin.ss.ss_family == AF_INET) {
    inet_ntop(
      AF_INET, &address_bin.s4.sin_addr, addrbuf, INET6_ADDRSTRLEN);
    port = ntohs(address_bin.s4.sin_port);

  } else {
    inet_ntop(
      AF_INET6, &address_bin.s6.sin6_addr, addrbuf, INET6_ADDRSTRLEN);
    port = ntohs(address_bin.s6.sin6_port);
  }

  return "["s + addrbuf + "]:" + std::to_string(port);
}

void my_ip46_address_init(my_ip46_address & address_bin,
                          char const * address_txt,
                          short const port) {
  // unp address resolution
  struct addrinfo hints;
  struct addrinfo * res;
  struct addrinfo * ressave;
  std::memset(&hints, 0, sizeof(struct addrinfo));
  hints.ai_flags = AI_PASSIVE;
  hints.ai_family = AF_UNSPEC;
  hints.ai_socktype = SOCK_DGRAM;
  int unp_n;
  // simple ip address code
  std::memset(&address_bin, 0, sizeof(struct sockaddr_storage));
  debug_print(
    ("Initing address "s + address_txt + ":" + std::to_string(port))
      .c_str());

#if !defined(__OpenBSD__)
  if (inet_pton(AF_INET6,
                ("::FFFF:"s + address_txt).c_str(),
                &address_bin.s6.sin6_addr) == 1) {
    address_bin.s6.sin6_family = AF_INET6;
    address_bin.s6.sin6_port = htons(port);
  } else /**/
#endif
    if (inet_pton(AF_INET, address_txt, &address_bin.s4.sin_addr) == 1) {
    address_bin.s4.sin_family = AF_INET;
    debug_print(("It is an ipv4 address "s).c_str());

#ifdef HAVE_SIN_LEN
    address_bin.s4.sin_len = sizeof(struct sockaddr_in);
#endif
    address_bin.s4.sin_port = htons(port);
  } else if (inet_pton(AF_INET6, address_txt, &address_bin.s6.sin6_addr) ==
             1) {
    address_bin.s6.sin6_family = AF_INET6;
    debug_print(("It is an ipv6 address "s).c_str());
#ifdef HAVE_SIN6_LEN
    address_bin.s6.sin6_len = sizeof(struct sockaddr_in6);
#endif
    address_bin.s6.sin6_port = htons(port);
  } else if ((unp_n = getaddrinfo(address_txt, NULL, &hints, &res)) == 0) {
    ressave = res;
    debug_print("It's a domain name!");
    address_bin.ss.ss_family = res->ai_family;
    if (res->ai_family == AF_INET) {
      debug_print(("ipv4 domain name"s).c_str());
      std::memcpy(&address_bin.s4.sin_addr,
                  &(((struct sockaddr_in *)res->ai_addr)->sin_addr),
                  sizeof(struct sockaddr_in));
      address_bin.s4.sin_port = htons(port);
#ifdef HAVE_SIN_LEN
      address_bin.s4.sin_len = sizeof(struct sockaddr_in);
#endif
    } else if (res->ai_family == AF_INET6) {
      debug_print(("ipv6 domain name"s).c_str());
      std::memcpy(&address_bin.s6.sin6_addr,
                  &(((struct sockaddr_in6 *)res->ai_addr)->sin6_addr),
                  sizeof(struct sockaddr_in6));
      address_bin.s6.sin6_port = htons(port);
#ifdef HAVE_SIN6_LEN
      address_bin.s6.sin6_len = sizeof(struct sockaddr_in6);
#endif
    } else {
      throw std::logic_error("Unknown address type for domain name.");
    }
    debug_print(
      ("Connecting to: "s + my_ip46_address_to_string(address_bin))
        .c_str());
    freeaddrinfo(res);
  } else {
    throw std::out_of_range("cannot set an address from address="s +
                            address_txt +
                            " and port=" + std::to_string(port));
  }
}
// line 1229 connect TODO

void my_ip46_connect(int socket, my_ip46_address remote_addr) {
  if (remote_addr.ss.ss_family == AF_INET) {
    if (connect(socket,
                (struct sockaddr *)&remote_addr,
                sizeof(struct sockaddr_in))) {
      perror("my_ip46_connect:connect4");
      throw std::runtime_error("connect ipv4 failed");
    }
  } else if (remote_addr.ss.ss_family == AF_INET6) {
    int const off = 0;
    setsockopt(
      socket, IPPROTO_IPV6, IPV6_V6ONLY, (char *)&off, sizeof(off));
    if (connect(socket,
                (struct sockaddr *)&remote_addr,
                sizeof(struct sockaddr_in6))) {
      perror("my_ip46_connect:connect6");
      throw std::runtime_error("connect ipv6 failed");
    }
  } else {
    debug_print("my_ip46_connect:wrong_family");
    throw std::logic_error("my_ip46_connect: remote_addr.ss.ss_family not "
                           "AF_INET and AF_INET6");
  }
}

void my_ip46_bind(int socket, my_ip46_address local_addr) {
  std::cout << "ss_family=" << local_addr.ss.ss_family << std::endl;
  if (local_addr.ss.ss_family == AF_INET) {
    if (bind(socket,
             (const struct sockaddr *)&local_addr,
             sizeof(struct sockaddr_in))) {
      perror("my_ip46_bind:ipv4");
      throw std::runtime_error("bind ipv4 failed");
    }
  } else if (local_addr.ss.ss_family == AF_INET6) {
    int const off = 0;
    setsockopt(
      socket, IPPROTO_IPV6, IPV6_V6ONLY, (char *)&off, sizeof(off));
    if (bind(socket,
             (const struct sockaddr *)&local_addr,
             sizeof(struct sockaddr_in6))) {
      perror("my_ip46_bind:ipv6");
      throw std::runtime_error("bind ipv6 failed");
    }
  } else {
    debug_print("my_ip46_bind:wrong_family");
    throw std::logic_error("my_ip46_bind: local_addr.ss.ss_family not "
                           "AF_INET and AF_INET6");
  }
}

typedef struct pass_info {
  my_ip46_address server_addr, client_addr;
  SSL * ssl;
  int socket_reverse_forward = -1;
  int socket_direct_forward = -1;
  my_ip46_address udp_ip_and_port;
  bool allow_udp_ip_port_from_inbound = true;
  unsigned int timeout_sec = 10;
  unsigned int ntimeouts = 3;
} pass_info_t;

void my_create_thread(
#ifdef WIN32
  (DWORD WINAPI)(*thread_function(LPVOID * info))
#else
  void(*thread_function(void * info))
#endif
    ,
  void * info,
  my_tid_t * tid) {
#ifdef WIN32
  if (CreateThread(
        NULL, 0, (LPTHREAD_START_ROUTINE)thread_function, info, 0, tid) ==
      NULL) {
    exit(-1);
  }
#else
  if (pthread_create(tid, NULL, thread_function, info) != 0) {
    perror("pthread_create");
    exit(-1);
  }
#endif
}

#ifdef WIN32
#define MY_EXIT_THREAD(retval) ExitThread(retval)
#else
#define MY_EXIT_THREAD(retval) pthread_exit((void *)retval)
#endif

#define BUFFER_SIZE (1 << 16)
#define COOKIE_SECRET_LENGTH 16

int verbose = 0;
int veryverbose = 0;
unsigned char cookie_secret[COOKIE_SECRET_LENGTH];
int cookie_initialized = 0;

#if WIN32
static HANDLE * mutex_buf = NULL;
#else
static pthread_mutex_t * mutex_buf = NULL;
#endif

static void locking_function(int mode, int n, const char * file, int line) {
  if (mode & CRYPTO_LOCK)
#ifdef WIN32
    WaitForSingleObject(mutex_buf[n], INFINITE);
  else
    ReleaseMutex(mutex_buf[n]);
#else
    pthread_mutex_lock(&mutex_buf[n]);
  else
    pthread_mutex_unlock(&mutex_buf[n]);
#endif
}

int THREAD_setup() {
  int i;

#ifdef WIN32
  mutex_buf = (HANDLE *)malloc(CRYPTO_num_locks() * sizeof(HANDLE));
#else
  mutex_buf = (pthread_mutex_t *)malloc(CRYPTO_num_locks() *
                                        sizeof(pthread_mutex_t));
#endif
  if (!mutex_buf)
    return 0;
  for (i = 0; i < CRYPTO_num_locks(); i++)
#ifdef WIN32
    mutex_buf[i] = CreateMutex(NULL, FALSE, NULL);
#else
    pthread_mutex_init(&mutex_buf[i], NULL);
#endif
  CRYPTO_set_id_callback(id_thread);
  CRYPTO_set_locking_callback(locking_function);
  return 1;
}

int THREAD_cleanup() {
  int i;

  if (!mutex_buf)
    return 0;

  CRYPTO_set_id_callback(NULL);
  CRYPTO_set_locking_callback(NULL);
  for (i = 0; i < CRYPTO_num_locks(); i++)
#ifdef WIN32
    CloseHandle(mutex_buf[i]);
#else
    pthread_mutex_destroy(&mutex_buf[i]);
#endif
  free(mutex_buf);
  mutex_buf = NULL;
  return 1;
}

int handle_socket_error() {
  switch (errno) {
  case EINTR:
    /* Interrupted system call.
     * Just ignore.
     */
    printf("Interrupted system call!\n");
    return 1;
  case EBADF:
    /* Invalid socket.
     * Must close connection.
     */
    printf("Invalid socket!\n");
    return 0;
    break;
#ifdef EHOSTDOWN
  case EHOSTDOWN:
    /* Host is down.
     * Just ignore, might be an attacker
     * sending fake ICMP messages.
     */
    printf("Host is down!\n");
    return 1;
#endif
#ifdef ECONNRESET
  case ECONNRESET:
    /* Connection reset by peer.
     * Just ignore, might be an attacker
     * sending fake ICMP messages.
     */
    printf("Connection reset by peer!\n");
    return 1;
#endif
  case ENOMEM:
    /* Out of memory.
     * Must close connection.
     */
    printf("Out of memory!\n");
    return 0;
    break;
  case EACCES:
    /* Permission denied.
     * Just ignore, we might be blocked
     * by some firewall policy. Try again
     * and hope for the best.
     */
    printf("Permission denied!\n");
    return 1;
    break;
  default:
    /* Something unexpected happened */
    printf("Unexpected error! (errno = %d)\n", errno);
    perror("Perror message");
    return 0;
    break;
  }
  return 0;
}

int generate_cookie(SSL * ssl,
                    unsigned char * cookie,
                    unsigned int * cookie_len) {
  unsigned char *buffer, result[EVP_MAX_MD_SIZE];
  unsigned int length = 0, resultlength;
  my_ip46_address peer;

  /* Initialize a random secret */
  if (!cookie_initialized) {
    if (!RAND_bytes(cookie_secret, COOKIE_SECRET_LENGTH)) {
      printf("error setting random cookie secret\n");
      return 0;
    }
    cookie_initialized = 1;
  }

  /* Read peer information */
  (void)BIO_dgram_get_peer(SSL_get_rbio(ssl), &peer);

  /* Create buffer with peer's address and port */
  length = 0;
  switch (peer.ss.ss_family) {
  case AF_INET:
    length += sizeof(struct in_addr);
    break;
  case AF_INET6:
    length += sizeof(struct in6_addr);
    break;
  default:
    OPENSSL_assert(0);
    break;
  }
  length += sizeof(in_port_t);
  buffer = (unsigned char *)OPENSSL_malloc(length);

  if (buffer == NULL) {
    printf("out of memory\n");
    return 0;
  }

  switch (peer.ss.ss_family) {
  case AF_INET:
    std::memcpy(buffer, &peer.s4.sin_port, sizeof(in_port_t));
    std::memcpy(buffer + sizeof(peer.s4.sin_port),
                &peer.s4.sin_addr,
                sizeof(struct in_addr));
    break;
  case AF_INET6:
    std::memcpy(buffer, &peer.s6.sin6_port, sizeof(in_port_t));
    std::memcpy(buffer + sizeof(in_port_t),
                &peer.s6.sin6_addr,
                sizeof(struct in6_addr));
    break;
  default:
    OPENSSL_assert(0);
    break;
  }

  /* Calculate HMAC of buffer using the secret */
  HMAC(EVP_sha1(),
       (const void *)cookie_secret,
       COOKIE_SECRET_LENGTH,
       (const unsigned char *)buffer,
       length,
       result,
       &resultlength);
  OPENSSL_free(buffer);

  std::memcpy(cookie, result, resultlength);
  *cookie_len = resultlength;

  return 1;
}

int verify_cookie(SSL * ssl,
                  const unsigned char * cookie,
                  unsigned int cookie_len) {
  unsigned char *buffer, result[EVP_MAX_MD_SIZE];
  unsigned int length = 0, resultlength;
  my_ip46_address peer;

  /* If secret isn't initialized yet, the cookie can't be valid */
  if (!cookie_initialized)
    return 0;

  /* Read peer information */
  (void)BIO_dgram_get_peer(SSL_get_rbio(ssl), &peer);

  /* Create buffer with peer's address and port */
  length = 0;
  switch (peer.ss.ss_family) {
  case AF_INET:
    length += sizeof(struct in_addr);
    break;
  case AF_INET6:
    length += sizeof(struct in6_addr);
    break;
  default:
    OPENSSL_assert(0);
    break;
  }
  length += sizeof(in_port_t);
  buffer = (unsigned char *)OPENSSL_malloc(length);

  if (buffer == NULL) {
    printf("out of memory\n");
    return 0;
  }

  switch (peer.ss.ss_family) {
  case AF_INET:
    std::memcpy(buffer, &peer.s4.sin_port, sizeof(in_port_t));
    std::memcpy(buffer + sizeof(in_port_t),
                &peer.s4.sin_addr,
                sizeof(struct in_addr));
    break;
  case AF_INET6:
    std::memcpy(buffer, &peer.s6.sin6_port, sizeof(in_port_t));
    std::memcpy(buffer + sizeof(in_port_t),
                &peer.s6.sin6_addr,
                sizeof(struct in6_addr));
    break;
  default:
    OPENSSL_assert(0);
    break;
  }

  /* Calculate HMAC of buffer using the secret */
  HMAC(EVP_sha1(),
       (const void *)cookie_secret,
       COOKIE_SECRET_LENGTH,
       (const unsigned char *)buffer,
       length,
       result,
       &resultlength);
  OPENSSL_free(buffer);

  if (cookie_len == resultlength &&
      memcmp(result, cookie, resultlength) == 0)
    return 1;

  return 0;
}

int dtls_verify_callback(int ok, X509_STORE_CTX * ctx) {
  /* This function should ask the user
   * if he trusts the received certificate.
   * Here we always trust.
   */
  return 1;
}

#define handle_error_en(en, msg)                                          \
  do {                                                                    \
    errno = en;                                                           \
    perror(msg);                                                          \
    exit(EXIT_FAILURE);                                                   \
  } while (0)

#ifdef WIN32
DWORD WINAPI thread_udp_to_dtls(LPVOID * info)
#else
void * thread_udp_to_dtls(void * info)
#endif
{
  debug_print("spawned");
  int s = pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL);
  debug_print("enabled cancel");
  if (s != 0)
    handle_error_en(s, "pthread_setcancelstate: write_dtls");
  debug_print("handled cancel error");
  pass_info_t * pinfo = static_cast<pass_info_t *>(info);
  SSL * ssl = pinfo->ssl;
  int udp_socket = pinfo->socket_reverse_forward;
  if (udp_socket < 0)
    throw std::logic_error("client_reverse_forward_socket is not init");
  struct sockaddr_storage useless_;
  struct sockaddr * recvfrom_udp_target =
    (pinfo->allow_udp_ip_port_from_inbound
       ? reinterpret_cast<struct sockaddr *>(&(pinfo->udp_ip_and_port.ss))
       : reinterpret_cast<struct sockaddr *>(&useless_));

  char dtls_send_buffer[BUFFER_SIZE] = "hello, world";
  int buffer_len = strlen(dtls_send_buffer);

  struct timeval t_begin {};
  gettimeofday(&t_begin, NULL);
  struct timeval t_end {};
  while (!(SSL_get_shutdown(ssl))) { // & SSL_RECEIVED_SHUTDOWN
    gettimeofday(&t_end, NULL);
    debug_print(("packet processing took "s +
                 std::to_string((t_end.tv_sec - t_begin.tv_sec) * 1000000 +
                                (t_end.tv_usec - t_begin.tv_usec)) +
                 " usec")
                  .c_str());
    t_begin = t_end;

    debug_print("-------------------------------------------------------");
    if (BUFFER_SIZE < 1600)
      throw std::logic_error("udp_to_dtls:buffer too small="s +
                             std::to_string(BUFFER_SIZE));
    socklen_t sa_len = sizeof(sockaddr_storage);

    ssize_t udp_read_len = recvfrom(udp_socket,
                                    dtls_send_buffer,
                                    BUFFER_SIZE,
                                    0,
                                    recvfrom_udp_target,
                                    &sa_len);
    int my_errno = errno;
    if (udp_read_len < 0) {
      if (errno == EWOULDBLOCK) {
        debug_print("udp_to_dtls:Reading from udp socket timed out."
                    " I need to send something like a keepalive?");
        continue;
      } else if (errno == ECONNREFUSED) {
        debug_print(
          "udp socket has ECONNREFUSED bogus error. (debug it!)");
        continue;
      } else if (errno == EINTR) {
        debug_print(
          "udp socket returned EINTR. dropping datagram. (Debug it!)");
        continue;
      } else {
        auto a =
          ("unexpected error when reading from udp socket on client, errno="s +
           std::to_string(errno));
        debug_print(a.c_str());
        my_explain_recvfrom_error(my_errno);
        throw std::runtime_error(a);
      }
    }
    // if (verbose) {
    //   fprintf(
    //     stderr,
    //     "\nThread %lx: got datagram from %s Size=%zd\n",
    //     id_thread(),
    //     my_ip46_address_to_string(pinfo->server_last_received_udp_addr)
    //       .c_str(),
    //     server_udp_read_len);
    // }
    socklen_t dtls_write_len =
      SSL_write(ssl, dtls_send_buffer, udp_read_len);
    debug_print("wrote one message");
    switch (SSL_get_error(ssl, dtls_write_len)) {
    case SSL_ERROR_NONE:
      if (verbose) {
        debug_print((""s + "wrote " + std::to_string(dtls_write_len) +
                     " bytes to dtls")
                      .c_str());
      }

      break;
    case SSL_ERROR_WANT_WRITE:
      // Can't write because of a renegotiation, so
      // we actually have to retry sending this message...
      debug_print("SSL_ERROR_WANT_WRITE");
      break;
    case SSL_ERROR_WANT_READ:
      debug_print("SSL_ERROR_WANT_READ");
      // continue with reading
      break;
    case SSL_ERROR_SYSCALL:
      debug_print("udp_to_dtls:SSL_ERROR_SYSCALL:Socket write error: ");
      if (!handle_socket_error()) {
        SSL_shutdown(ssl);
        MY_EXIT_THREAD(1);
      }
      break;
    case SSL_ERROR_SSL:
      fprintf(stderr, "udp_to_dtls:SSL write error: SSL_ERROR_SSL");
      fprintf(stderr,
              "%s (%d)\n",
              ERR_error_string(ERR_get_error(), dtls_send_buffer),
              SSL_get_error(ssl, dtls_write_len));
      MY_EXIT_THREAD(1);
      break;
    default:
      fprintf(stderr, "udp_to_dtls:Unexpected error while writing!\n");
      MY_EXIT_THREAD(1);
      break;
    }
  }
  MY_EXIT_THREAD(0);
}

#ifdef WIN32
DWORD WINAPI thread_dtls_to_udp(LPVOID * info)
#else
void * thread_dtls_to_udp(void * info)
#endif
{
  // This thread should be reading FROM DTLS, and putting the packets
  // into the udp socket of the proxied service.
  debug_print("spawned");
  int s = pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL);
  debug_print("enabled cancel");
  if (s != 0)
    handle_error_en(s, "pthread_setcancelstate: read_dtls");
  debug_print("handled cancel error");

  pass_info_t * pinfo = static_cast<pass_info_t *>(info);
  SSL * ssl = pinfo->ssl;
  //  std::unique_ptr<SSL, int (*)(SSL *)> ssl_guard(ssl, SSL_shutdown);
  int udp_socket = pinfo->socket_reverse_forward;
  if (udp_socket < 0)
    throw std::logic_error("dtls_to_udp:udp socket is not init");
  int num_timeouts = 0;
  int const max_timeouts = pinfo->ntimeouts;
  char dtls_to_udp_buffer[BUFFER_SIZE] = "hello world";

  struct timeval t_begin {};
  gettimeofday(&t_begin, NULL);
  struct timeval t_end {};

  while (!(SSL_get_shutdown(ssl))) { //  & SSL_RECEIVED_SHUTDOWN
    gettimeofday(&t_end, NULL);
    debug_print(("Iteration processing took "s +
                 std::to_string((t_end.tv_sec - t_begin.tv_sec) * 1000000 +
                                (t_end.tv_usec - t_begin.tv_usec)) +
                 " usec")
                  .c_str());
    t_begin = t_end;

    debug_print(("dtls_to_udp: already saw "s +
                 std::to_string(num_timeouts) + " timeouts")
                  .c_str());
    if (num_timeouts > max_timeouts) {
      debug_print(("No data received from dtls for too long: "s +
                   std::to_string(max_timeouts) +
                   " timeouts. Breaking dtls_to_udp loop.")
                    .c_str());
      // SSL_shutdown(ssl);
      break;
    }

    debug_print("-------------------------------------------------------");
    ssize_t dtls_read_len =
      SSL_read(ssl, dtls_to_udp_buffer, sizeof(dtls_to_udp_buffer));

    switch (SSL_get_error(ssl, dtls_read_len)) {
    case SSL_ERROR_NONE:
      if (verbose) {
        debug_print(
          ("dtls_to_udp read " + std::to_string(dtls_read_len) + " bytes")
            .c_str());
      }
      debug_print("Not printing received packet, not implemented.");
      break;
    case SSL_ERROR_WANT_READ:
      // Stop reading on socket timeout, otherwise try again
      debug_print("DTLS:SSL_ERROR_WANT_READ:on SSL_read().");
      if (BIO_ctrl(SSL_get_rbio(ssl),
                   BIO_CTRL_DGRAM_GET_RECV_TIMER_EXP,
                   0,
                   NULL)) {
        debug_print("Timeout! No packet received from dtls.\n");
        num_timeouts++;
      } else {
        debug_print(
          "SSL_ERROR_WANT_READ, but where does it come from? TODO");
      }
      // Just try again
      continue;

    case SSL_ERROR_ZERO_RETURN:
      debug_print("dtls_to_udp:SSL_ERROR_ZERO_RETURN");
      // Shall we break here? Wtf?
      continue;
    case SSL_ERROR_SYSCALL:
      if (errno == EAGAIN) {
        // Why does the socket timeout appear here, not in BIO_ctrl?
        debug_print(
          "dtls_to_udp:SSL_ERROR_SYSCALL:dtls read error:0: EAGAIN");
        debug_print(
          "Received nothing from the server for too long. (EAGAIN) "
          "Breaking connection. This method is bad, because we "
          "may be sending data, so the connection is still "
          "valid, but so far let us make a temporary solution.");
        num_timeouts++;
        continue;
      }
      fprintf(stderr, "dtls_to_udp:SSL_ERROR_SYSCALL:not EAGAIN\n");
      if (!handle_socket_error()) {
        debug_print(
          "dtls_to_udp:SSL_ERROR_SYSCALL:Unhandled syscall socket "
          "error on reading.");
        MY_EXIT_THREAD(1);
      }
      continue;
    case SSL_ERROR_SSL:
      debug_print(
        "dtls_to_udp:SSL_ERROR_SSL:exiting because of SSL read error: ");
      fprintf(stderr,
              "%s (%d)\n",
              ERR_error_string(ERR_get_error(), dtls_to_udp_buffer),
              SSL_get_error(ssl, dtls_read_len));
      MY_EXIT_THREAD(1);
    default:
      debug_print("dtls_to_udp:Unexpected error while reading!\n");
      MY_EXIT_THREAD(1);
    }
    debug_print("Received a message over dtls");
    // Processed all the reading crap, and ready to do something with the
    // data in "server_dtls_receive_buffer", of length
    // dtls_message_len
    // and we send it to pinfo->server_last_received_udp_addr
    // So, my datagram is in client_dtls_read_buffer, and the real
    // length of it is ssl_read_len. So, I need something like:
    // Sendto_flags(sockfd, dataBuffer, 0, pservaddr, servlen);
    // This will send that datagram to the 2ping-server.
    // This should be enough for simplicity.
    // client_reverse_socket is already bound, so it should
    // sendto_flagsORwrite(client_reverse_socket,client_dtls_read_buffer,ssl_read_len);
    // write(client_reverse_socket, client_dtls_read_buffer,
    //      ssl_read_len);
    //! Why exactly I chose "sendto", I am not very sure.
    //! I did not want to use write(), because it is not
    //! socket-specific. And sendmsg is a bit annoying with
    //! its scatter/gather machinery.

    if (pinfo->udp_ip_and_port.ss.ss_family == 0) {
      debug_print(
        "No return address received yet from client, or the address is "
        "not given when starting. Dropping datagram.");
      continue;
    }
    socklen_t sa_len = sizeof(struct sockaddr_storage);
    ssize_t dtls_to_udp_write_len;
    do {
      dtls_to_udp_write_len = sendto(
        udp_socket,
        dtls_to_udp_buffer,
        dtls_read_len,
        0,
        reinterpret_cast<struct sockaddr *>(&(pinfo->udp_ip_and_port)),
        sa_len);
    } while (dtls_to_udp_write_len < 0 && errno == EAGAIN);
    int my_errno = errno;
    debug_print(
      ("ssl_read_len="s + std::to_string(dtls_read_len)).c_str());
    debug_print(
      ("udp_write_len="s + std::to_string(dtls_to_udp_write_len)).c_str());

    if (dtls_to_udp_write_len >= 0 &&
        dtls_to_udp_write_len == dtls_read_len) {
      debug_print("Written packet to client socket successfully");
    }

    if (dtls_to_udp_write_len >= 0 &&
        dtls_to_udp_write_len != dtls_read_len) {
      throw std::logic_error("written bytes do not match requested");
    }

    if (dtls_to_udp_write_len < 0) {
      debug_print("Writing to client udp socket failed. Let's see why.");

      if (my_errno == EAGAIN) {
        debug_print("EAGAIN on writing to udp socket. Why? Dropping "
                    "packet. (this should not happen) TODO.");
        throw std::logic_error("EAGAIN on writing to udp socket");
      }

      if (my_errno == ECONNREFUSED) {
        debug_print("Udp socket has no service running, perhaps, "
                    "crashed, or whatever. Dropping packet.");
        continue;
      }

      if (dtls_read_len > SSIZE_MAX) {
        debug_print("dtls_to_udp:dtls_read_len > SSIZE_MAX. Logic error. "
                    "Terminating.");
        throw std::logic_error("dtls_read_len > SSIZE_MAX");
      }

      if (my_errno == EINVAL) {
        debug_print("UDP: EINVAL. Some argument to sendto is wrong");
        throw std::logic_error(
          "UDP: EINVAL. Some argument to sendto is wrong");
      }
      auto a =
        "unexpected error when writing to udp socket on client, errno="s +
        std::to_string(my_errno);
      debug_print(a.c_str());
      my_explain_recvfrom_error(my_errno);
      throw std::runtime_error(a);
    }
  }
  //  SSL_shutdown(ssl);
  MY_EXIT_THREAD(0);
}

#ifdef WIN32
DWORD WINAPI thread_server_handle_dtls_connection(LPVOID * info)
#else
void * thread_server_handle_dtls_connection(void * info)
#endif
{
  // Start making a new socket for this particular tls connection.
  auto myInfoGuard = std::unique_ptr<pass_info_t, void (*)(void *)>(
    reinterpret_cast<pass_info_t *>(info), free);
  pass_info_t * pinfo = reinterpret_cast<pass_info_t *>(info);
  SSL * ssl = pinfo->ssl;
  auto mySSLGuard = std::unique_ptr<SSL, void (*)(SSL *)>(ssl, SSL_free);

  int server_dtls_connection_socket;
  auto mySocketGuard = std::unique_ptr<int, void (*)(int *)>(
    &server_dtls_connection_socket, closer);
  if (sizeof(void *) != sizeof(int *))
    throw std::logic_error("Size of void* too small");

  //  SSL_free(ssl);
  ssize_t dtls_message_len;
  char buf[BUFFER_SIZE];
  char addrbuf[INET6_ADDRSTRLEN];

  int reading = 0;
  int ret;
  const int on = 1, off = 0;
  struct timeval timeout;
  int num_timeouts = 0;
  int const max_timeouts = pinfo->ntimeouts;
  void * pthread_join_retval = nullptr;

#ifndef WIN32
  pthread_detach(pthread_self());
#endif

  OPENSSL_assert(pinfo->client_addr.ss.ss_family ==
                 pinfo->server_addr.ss.ss_family);
  debug_print(("pinfo->client_addr.ss.ss_family="s +
               std::to_string(pinfo->client_addr.ss.ss_family))
                .c_str());
  debug_print(("pinfo->server_addr.ss.ss_family="s +
               std::to_string(pinfo->server_addr.ss.ss_family))
                .c_str());
  server_dtls_connection_socket =
    socket(pinfo->client_addr.ss.ss_family, SOCK_DGRAM, 0);
  if (server_dtls_connection_socket < 0) {
    perror("socket");
    // goto cleanup;
    MY_EXIT_THREAD(4);
  }

#ifdef WIN32
  setsockopt(server_dtls_connection_socket,
             SOL_SOCKET,
             SO_REUSEADDR,
             (const char *)&on,
             (socklen_t)sizeof(on));
#else
  setsockopt(server_dtls_connection_socket,
             SOL_SOCKET,
             SO_REUSEADDR,
             (const void *)&on,
             (socklen_t)sizeof(on));
#if defined(SO_REUSEPORT) && !defined(__linux__)
  setsockopt(server_dtls_connection_socket,
             SOL_SOCKET,
             SO_REUSEPORT,
             (const void *)&on,
             (socklen_t)sizeof(on));
#endif
#endif
  switch (pinfo->client_addr.ss.ss_family) {
  case AF_INET:
    debug_print("client_addr.ss.ss_family=AF_INET");
    break;
  case AF_INET6:
    debug_print("client_addr.ss.ss_family=AF_INET6");
    break;
  default:
    debug_print("wrong family");
    OPENSSL_assert(0);

    break;
  }
  try {
    my_ip46_bind(server_dtls_connection_socket, pinfo->server_addr);
    debug_print("Bind worked!");
    my_ip46_connect(server_dtls_connection_socket, pinfo->client_addr);
    debug_print("Connect worked!");
  } catch (std::runtime_error ex) {
    debug_print(("Exception!:"s + ex.what()).c_str());
    MY_EXIT_THREAD(4);
  } catch (...) {
    debug_print(("Big exception!"));
    exit(1);
  }

  /* Set new fd and set BIO to connected */
  BIO_set_fd(
    SSL_get_rbio(ssl), server_dtls_connection_socket, BIO_NOCLOSE);
  BIO_ctrl(SSL_get_rbio(ssl),
           BIO_CTRL_DGRAM_SET_CONNECTED,
           0,
           &pinfo->client_addr.ss);

  /* Finish handshake */
  do {
    ret = SSL_accept(ssl);
  } while (ret == 0);
  if (ret < 0) {
    perror("SSL_accept");
    fprintf(stderr, "%s\n", ERR_error_string(ERR_get_error(), buf));
    // goto cleanup;
    MY_EXIT_THREAD(2);
  }

  /* Set and activate timeouts */
  {
    timeout.tv_sec = pinfo->timeout_sec;
    timeout.tv_usec = 0;
    BIO_ctrl(
      SSL_get_rbio(ssl), BIO_CTRL_DGRAM_SET_RECV_TIMEOUT, 0, &timeout);
  }
#if defined(__OpenBSD__)
  BIO_set_buffer_size(SSL_get_rbio(ssl), 2000000);
  BIO_set_read_buffer_size(SSL_get_rbio(ssl), 2000000);
#else
  BIO_set_buffer_size(SSL_get_rbio(ssl), 20000000);
  BIO_set_read_buffer_size(SSL_get_rbio(ssl), 20000000);
#endif
  if (verbose) {
    if (pinfo->client_addr.ss.ss_family == AF_INET) {
      fprintf(stderr,
              "\nThread %lx: accepted connection from %s:%d\n",
              id_thread(),
              inet_ntop(AF_INET,
                        &pinfo->client_addr.s4.sin_addr,
                        addrbuf,
                        INET6_ADDRSTRLEN),
              ntohs(pinfo->client_addr.s4.sin_port));
    } else {
      fprintf(stderr,
              "\nThread %lx: accepted connection from %s:%d\n",
              id_thread(),
              inet_ntop(AF_INET6,
                        &pinfo->client_addr.s6.sin6_addr,
                        addrbuf,
                        INET6_ADDRSTRLEN),
              ntohs(pinfo->client_addr.s6.sin6_port));
    }
  }

  if (veryverbose && SSL_get_peer_certificate(ssl)) {
    fprintf(
      stderr,
      "------------------------------------------------------------\n");
    fflush(stderr);
    X509_NAME_print_ex_fp(
      stderr,
      X509_get_subject_name(SSL_get_peer_certificate(ssl)),
      1,
      XN_FLAG_MULTILINE);
    fflush(stderr);
    fprintf(stderr,
            "\n\n Cipher: %s",
            SSL_CIPHER_get_name(SSL_get_current_cipher(ssl)));
    fprintf(
      stderr,
      "\n------------------------------------------------------------"
      "\n\n");
    fflush(stderr);
  }
  // my: so here we have established a connection, and are using something
  // (let me check what) for reading and writing.
  // (1) ssl is used in SSL_get_shutdown,
  // (2) buf although buffers are cheap in C++
  // This loop will exist in both threads.
  debug_print("before two threads");
  my_tid_t tid_thread_server_dtls_to_udp;
  my_tid_t tid_thread_server_udp_to_dtls;
  my_create_thread(
    thread_dtls_to_udp, info, &tid_thread_server_dtls_to_udp);
  debug_print("created thread for server_dtls_to_udp");
  my_create_thread(
    thread_udp_to_dtls, info, &tid_thread_server_udp_to_dtls);
  debug_print("created thread for server_udp_to_dtls");

  pthread_join(tid_thread_server_dtls_to_udp, &pthread_join_retval);
  debug_print("joined thread_server_dtls_to_udp");
  SSL_shutdown(ssl);
  // if (pthread_join_retval != 0) {
  //   // pthread_cancel(tid_write_thread);
  //   goto cleanup;
  // }
  pthread_join(tid_thread_server_udp_to_dtls, &pthread_join_retval);
  debug_print("joined thread_server_udp_to_dtls");

  // if (pthread_join_retval != 0)
  //   goto cleanup;

  // Here I will have to wait for these threads to finish, or I will
  // never call this shutdown, so I cannot self-detach.
  // But what happens if this thread should die?
  // I guess, if my dispatcher thread receives a new incoming connection,
  // it should publish a tid in some std::set , and the two writing-reading
  // threads will have to check that std::set instance, and terminate
  SSL_shutdown(ssl);

cleanup:
  if (verbose)
    fprintf(stderr, "Thread %lx: done, connection closed.\n", id_thread());
  return 0;
}

std::pair<int, my_ip46_address>
setup_udp_socket(std::string forward_address,
                 short forward_port,
                 std::string local_address,
                 short local_port) {
  //  const int on = 1, off = 0;
  struct timeval socket_timeout {
    5, 0
  };
  my_ip46_address local_udp_addr;
  if (local_address == ""s) {
    local_udp_addr.s6.sin6_family = AF_INET6;
#ifdef HAVE_SIN6_LEN
    local_udp_addr.s6.sin6_len = sizeof(struct sockaddr_in6);
#endif
    local_udp_addr.s6.sin6_addr = in6addr_any;
    local_udp_addr.s6.sin6_port = htons(local_port);
  } else {
    my_ip46_address_init(
      local_udp_addr, local_address.c_str(), local_port);
  }
  my_ip46_address forward_addr{};
  if (!forward_address.empty()) {
    my_ip46_address_init(
      forward_addr, forward_address.c_str(), forward_port);
    debug_print(("making a socket for communicating with "s +
                 forward_address + ":" + std::to_string(forward_port))
                  .c_str());
  }
  debug_print(("Making udp socket. udp-local-address="s +
               my_ip46_address_to_string(local_udp_addr) +
               " forward_addr=" + my_ip46_address_to_string(forward_addr))
                .c_str());
  int udp_socket = socket(local_udp_addr.ss.ss_family, SOCK_DGRAM, 0);
  if (udp_socket < 0) {
    perror("server_udp_socket:socket()_fail");
    exit(-1);
  }
  int const off = 0, on = 1;
#if !defined(__OpenBSD__)
  if (setsockopt(
        udp_socket, IPPROTO_IPV6, IPV6_V6ONLY, (char *)&off, sizeof(off)) <
      0)
    throw std::logic_error(
      "setup_server_udp_socket failed:setsockopt IPV6_ONLY=off");
#endif
#if defined(__OpenBSD__)
  int size = 2000000;
#else
  int size = 20000000;
#endif
  if (-1 ==
      setsockopt(udp_socket, SOL_SOCKET, SO_SNDBUF, &size, sizeof(size)))
    throw std::logic_error("cannot set UDP SO_SNDBUF");
  if (-1 ==
      setsockopt(udp_socket, SOL_SOCKET, SO_RCVBUF, &size, sizeof(size)))
    throw std::logic_error("cannot set UDP SO_RCVBUF");
  my_ip46_bind(udp_socket, local_udp_addr);
  if (setsockopt(udp_socket,
                 SOL_SOCKET,
                 SO_RCVTIMEO,
                 &socket_timeout,
                 sizeof(socket_timeout)) < 0) {
    debug_print("setup_server_udp_socket failed:setsockopt");
    throw std::logic_error("setup_server_udp_socket failed:setsockopt");
  }
  debug_print("server udp socket succeeded");
  return std::make_pair(udp_socket, forward_addr);
}

void start_server_dtls(int port,
                       char const * dtls_local_address,
                       int udp_socket,
                       my_ip46_address forward_address,
                       bool allow_update_forward_address,
                       std::string cert_path,
                       std::string key_path,
                       std::string ca_path,
                       unsigned int dtls_timeout,
                       unsigned int dtls_ntimeouts) {
  int dtls_dispatcher_socket;
  my_ip46_address server_addr, client_addr;
  my_tid_t tid;
  SSL_CTX * ctx;
  SSL * ssl;
  BIO * bio;
  debug_print(("Starting dtls server on "s + dtls_local_address + ":" +
               std::to_string(port))
                .c_str());
  const int on = 1, off = 0;

  std::memset(&server_addr, 0, sizeof(struct sockaddr_storage));
  if (std::strlen(dtls_local_address) == 0) {
    server_addr.s6.sin6_family = AF_INET6;
#ifdef HAVE_SIN6_LEN
    server_addr.s6.sin6_len = sizeof(struct sockaddr_in6);
#endif
    server_addr.s6.sin6_addr = in6addr_any;
    server_addr.s6.sin6_port = htons(port);
  } else {
    try {
      my_ip46_address_init(server_addr, dtls_local_address, port);
    } catch (...) {
      return;
    }
  }

  THREAD_setup();
  OpenSSL_add_ssl_algorithms();
  SSL_load_error_strings();
  ctx = SSL_CTX_new(DTLS_server_method());
  /* We accept all ciphers, including NULL.
   * Not recommended beyond testing and debugging
   */
  SSL_CTX_set_security_level(ctx, 0);
  SSL_CTX_set_cipher_list(ctx, "CHACHA20");
  SSL_CTX_set_session_cache_mode(ctx, SSL_SESS_CACHE_OFF);

  if (SSL_CTX_load_verify_locations(ctx, ca_path.c_str(), "certs") == 0)
    throw std::logic_error("CA directory or file not found.");

  if (!SSL_CTX_use_certificate_file(
        ctx, cert_path.c_str(), SSL_FILETYPE_PEM))
    printf("\nERROR: no certificate found!");

  if (!SSL_CTX_use_PrivateKey_file(
        ctx, key_path.c_str(), SSL_FILETYPE_PEM))
    printf("\nERROR: no private key found!");

  if (!SSL_CTX_check_private_key(ctx))
    printf("\nERROR: invalid private key!");

  /* Client has to authenticate */
  SSL_CTX_set_verify(
    ctx, SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, NULL);
  //                     dtls_verify_callback);

  SSL_CTX_set_read_ahead(ctx, 1);
  SSL_CTX_set_cookie_generate_cb(ctx, generate_cookie);
  SSL_CTX_set_cookie_verify_cb(ctx, &verify_cookie);

  dtls_dispatcher_socket = socket(server_addr.ss.ss_family, SOCK_DGRAM, 0);
  if (dtls_dispatcher_socket < 0) {
    perror("dtls_dispatch_socket");
    exit(-1);
  }
#if defined(__OpenBSD__)
  int size = 2000000;
#else
  int size = 20000000;
#endif
  if (-1 ==
      setsockopt(udp_socket, SOL_SOCKET, SO_SNDBUF, &size, sizeof(size)))
    throw std::logic_error("cannot set UDP SO_SNDBUF");
  if (-1 ==
      setsockopt(udp_socket, SOL_SOCKET, SO_RCVBUF, &size, sizeof(size)))
    throw std::logic_error("cannot set UDP SO_RCVBUF");

#ifdef WIN32
  setsockopt(dtls_dispatcher_socket,
             SOL_SOCKET,
             SO_REUSEADDR,
             (const char *)&on,
             (socklen_t)sizeof(on));
#else
  setsockopt(dtls_dispatcher_socket,
             SOL_SOCKET,
             SO_REUSEADDR,
             (const void *)&on,
             (socklen_t)sizeof(on));
#if defined(SO_REUSEPORT) && !defined(__linux__)
  setsockopt(dtls_dispatcher_socket,
             SOL_SOCKET,
             SO_REUSEPORT,
             (const void *)&on,
             (socklen_t)sizeof(on));
#endif
#endif
  my_ip46_bind(dtls_dispatcher_socket, server_addr);

  while (1) {
    debug_print("=============================================");

    std::memset(&client_addr, 0, sizeof(struct sockaddr_storage));

    /* Create BIO */
    bio = BIO_new_dgram(dtls_dispatcher_socket, BIO_NOCLOSE);

    /* Set and activate timeouts */
    struct timeval timeout;
    timeout.tv_sec = dtls_timeout;
    timeout.tv_usec = 0;
    BIO_ctrl(bio, BIO_CTRL_DGRAM_SET_RECV_TIMEOUT, 0, &timeout);
    BIO_set_buffer_size(bio, 2000000);
    BIO_set_read_buffer_size(bio, 2000000);
    ssl = SSL_new(ctx);

    SSL_set_bio(ssl, bio, bio);
    SSL_set_options(ssl, SSL_OP_COOKIE_EXCHANGE);

    while (DTLSv1_listen(ssl, (BIO_ADDR *)&client_addr) <= 0)
      ;
    pass_info_t * dtls_info;
    dtls_info =
      reinterpret_cast<pass_info_t *>(malloc(sizeof(pass_info_t)));
    std::memcpy(&dtls_info->server_addr,
                &server_addr,
                sizeof(struct sockaddr_storage));
    std::memcpy(&dtls_info->client_addr,
                &client_addr,
                sizeof(struct sockaddr_storage));
    dtls_info->ssl = ssl;
    dtls_info->socket_reverse_forward = udp_socket;
    dtls_info->udp_ip_and_port = forward_address;
    dtls_info->allow_udp_ip_port_from_inbound =
      allow_update_forward_address;
    dtls_info->timeout_sec = dtls_timeout;
    dtls_info->ntimeouts = dtls_ntimeouts;

    my_create_thread(
      thread_server_handle_dtls_connection, dtls_info, &tid);
  }
  THREAD_cleanup();
}

void start_client_dtls(char const * remote_address,
                       char const * local_address,
                       int port,
                       int client_udp_socket,
                       my_ip46_address forward_address,
                       bool allow_update_forward_address,
                       std::string cert_path,
                       std::string key_path,
                       std::string ca_path,
                       unsigned int dtls_timeout,
                       unsigned int dtls_ntimeouts) {
  int client_dtls_socket;
  int ssl_connect_retval;
  void * pthread_join_retval = nullptr;
  my_ip46_address remote_addr;
  my_ip46_address local_addr;
  // I will have to somehow prepare a this buffer for writing,
  // so that it contains a header with my protocol's metadata.
  // The metadata is probably simple:
  // {
  //   bool my_message_or_not_mine;
  //   float loss;
  //   float keepalive timeout;
  // };
  // or something like that.
  //  char message_buf[BUFFER_SIZE];
  // but at the moment I have no header, I just reconnect.
  char addrbuf[INET6_ADDRSTRLEN];
  socklen_t len;
  SSL_CTX * ctx;
  SSL * ssl;
  BIO * bio;
  int reading = 0;

  std::memset((void *)&remote_addr, 0, sizeof(struct sockaddr_storage));
  std::memset((void *)&local_addr, 0, sizeof(struct sockaddr_storage));

  try {
    my_ip46_address_init(remote_addr, remote_address, port);
  } catch (...) {
    return;
  }

  client_dtls_socket = socket(remote_addr.ss.ss_family, SOCK_DGRAM, 0);
  if (client_dtls_socket < 0) {
    perror("client socket");
    exit(-1);
  }
  auto mySocketGuard =
    std::unique_ptr<int, void (*)(int *)>(&client_dtls_socket, closer);

  if (strlen(local_address) > 0) {
    debug_print("DTLS:Before local bind. ");
    try {
      my_ip46_address_init(local_addr, local_address, port);
    } catch (...) {
      return;
    }

    OPENSSL_assert(remote_addr.ss.ss_family == local_addr.ss.ss_family);
    my_ip46_bind(client_dtls_socket, local_addr); // throws
    debug_print("DTLS:Local bind succeeded.");
  }

  OpenSSL_add_ssl_algorithms();
  SSL_load_error_strings();
  ctx = SSL_CTX_new(DTLS_client_method());
  // SSL_CTX_set_cipher_list(ctx, "eNULL:!MD5");
  SSL_CTX_set_security_level(ctx, 0);
  SSL_CTX_set_cipher_list(ctx, "CHACHA20");
  if (SSL_CTX_load_verify_locations(ctx, ca_path.c_str(), "certs") == 0)
    throw std::logic_error("CA directory not found.");

  if (!SSL_CTX_use_certificate_file(
        ctx, cert_path.c_str(), SSL_FILETYPE_PEM))
    printf("\nERROR: no certificate found!");

  if (!SSL_CTX_use_PrivateKey_file(
        ctx, key_path.c_str(), SSL_FILETYPE_PEM))
    printf("\nERROR: no private key found!");

  if (!SSL_CTX_check_private_key(ctx))
    printf("\nERROR: invalid private key!");

  // client also verifies server
  SSL_CTX_set_verify(
    ctx, SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, NULL);

  SSL_CTX_set_verify_depth(ctx, 2);
  SSL_CTX_set_read_ahead(ctx, 1);

  ssl = SSL_new(ctx);
  std::unique_ptr<SSL, void (*)(SSL *)> ssl_guard(ssl, SSL_free);

  /* Create BIO, connect and set to already connected */
  bio = BIO_new_dgram(client_dtls_socket, BIO_CLOSE);
  debug_print("DTLS: before connect.");
  try {
    my_ip46_connect(client_dtls_socket, remote_addr);
  } catch (std::runtime_error & ex) {
    debug_print(("Exception:dtls connect failed: "s + ex.what() +
                 "ending this attempt.")
                  .c_str());
    return;
  } catch (...) {
    debug_print("Unknown exception. Dying.");
  }
  debug_print("DTLS: connect succeeded.");

  BIO_ctrl(bio, BIO_CTRL_DGRAM_SET_CONNECTED, 0, &remote_addr.ss);

  SSL_set_bio(ssl, bio, bio);
  debug_print("DTLS: before SSL_connect.");
  // Set and activate timeouts
  // Maybe we need to
  {
    struct timeval timeout;
    timeout.tv_sec = dtls_timeout;
    timeout.tv_usec = 0;
    BIO_ctrl(bio, BIO_CTRL_DGRAM_SET_RECV_TIMEOUT, 0, &timeout);
    BIO_set_buffer_size(bio, 2000000);
    BIO_set_read_buffer_size(bio, 2000000);
  }
  ssl_connect_retval =
    SSL_connect(ssl); // TODO: for some reason this doesn't timeout when I
                      // expect it to.
  debug_print("DTLS: after SSL_connect.");
  if (ssl_connect_retval <= 0) {
    switch (SSL_get_error(ssl, ssl_connect_retval)) {
    case SSL_ERROR_ZERO_RETURN:
      fprintf(stderr, "SSL_connect failed with SSL_ERROR_ZERO_RETURN\n");
      break;
    case SSL_ERROR_WANT_READ:
      fprintf(stderr, "SSL_connect failed with SSL_ERROR_WANT_READ\n");
      break;
    case SSL_ERROR_WANT_WRITE:
      fprintf(stderr, "SSL_connect failed with SSL_ERROR_WANT_WRITE\n");
      break;
    case SSL_ERROR_WANT_CONNECT:
      fprintf(stderr, "SSL_connect failed with SSL_ERROR_WANT_CONNECT\n");
      break;
    case SSL_ERROR_WANT_ACCEPT:
      fprintf(stderr, "SSL_connect failed with SSL_ERROR_WANT_ACCEPT\n");
      break;
    case SSL_ERROR_WANT_X509_LOOKUP:
      fprintf(stderr,
              "SSL_connect failed with SSL_ERROR_WANT_X509_LOOKUP\n");
      break;
    case SSL_ERROR_SYSCALL:
      debug_print("SSL_connect failed with SSL_ERROR_SYSCALL\n");
      debug_print("TODO: process this syscall error.");
      // sleep(1);
      break;
    case SSL_ERROR_SSL:
      fprintf(stderr, "SSL_connect failed with SSL_ERROR_SSL\n");
      break;
    default:
      fprintf(stderr, "SSL_connect failed with unknown error\n");
      break;
    }
    //    goto cleanup_client;
    // exit(EXIT_FAILURE);
    debug_print("sleeping 5 seconds");
    sleep(5);
    debug_print("SSL_connect failed, terminating attempt.");
    return;
  }

  if (verbose) {
    if (remote_addr.ss.ss_family == AF_INET) {
      printf(
        "\nConnected to %s\n",
        inet_ntop(
          AF_INET, &remote_addr.s4.sin_addr, addrbuf, INET6_ADDRSTRLEN));
    } else {
      printf(
        "\nConnected to %s\n",
        inet_ntop(
          AF_INET6, &remote_addr.s6.sin6_addr, addrbuf, INET6_ADDRSTRLEN));
    }
  }

  if (veryverbose && SSL_get_peer_certificate(ssl)) {
    printf(
      "------------------------------------------------------------\n");
    X509_NAME_print_ex_fp(
      stdout,
      X509_get_subject_name(SSL_get_peer_certificate(ssl)),
      1,
      XN_FLAG_MULTILINE);
    printf("\n\n Cipher: %s",
           SSL_CIPHER_get_name(SSL_get_current_cipher(ssl)));
    printf("\n------------------------------------------------------------"
           "\n\n");
  }
  // Okay, here we seem to have obtained all the stuff for reading and
  // writing to/from the socket.
  // So this while loop should be going on in the two threads.

  pass_info_t dtls_info;
  std::memcpy(
    &dtls_info.server_addr, &remote_addr, sizeof(struct sockaddr_storage));
  std::memcpy(
    &dtls_info.client_addr, &local_addr, sizeof(struct sockaddr_storage));
  dtls_info.ssl = ssl;
  dtls_info.socket_reverse_forward = client_udp_socket;
  dtls_info.udp_ip_and_port = forward_address;
  dtls_info.allow_udp_ip_port_from_inbound = allow_update_forward_address;
  dtls_info.timeout_sec = dtls_timeout;
  dtls_info.ntimeouts = dtls_ntimeouts;

  debug_print("before two threads");
  my_tid_t tid_client_dtls_to_udp;
  my_tid_t tid_client_udp_to_dtls;
  my_create_thread(
    thread_dtls_to_udp, &dtls_info, &tid_client_dtls_to_udp);
  debug_print("created reading thread");
  my_create_thread(
    thread_udp_to_dtls, &dtls_info, &tid_client_udp_to_dtls);
  debug_print("created writing thread");
  pthread_join(tid_client_dtls_to_udp, &pthread_join_retval);
  debug_print("joined client_dtls_to_udp thread");
  SSL_shutdown(ssl);
  pthread_join(tid_client_udp_to_dtls, &pthread_join_retval);
  debug_print("joined client_udp_to_dtls thread");
}

int main(int argc, char ** argv) {
  if (EAGAIN != EWOULDBLOCK)
    throw std::logic_error("EAGAIN != EWOULDBLOCK, which is an error.");
  bool arg_verbose;
  //    char local_addr[INET6_ADDRSTRLEN+1];
  // std::memset(local_addr, 0, INET6_ADDRSTRLEN+1);

  popl::OptionParser op("Allowed options");
  auto help_option =
    op.add<popl::Switch>("h", "help", "produce help message");
  auto groff_option = op.add<popl::Switch>(
    "", "groff", "produce groff formatted help message");
  auto bash_option =
    op.add<popl::Switch>("", "bash", "produce bash completion script");
  auto verbose_option = op.add<popl::Switch, popl::Attribute::optional>(
    "v", "verbose", "be verbose", &arg_verbose);
  auto dtls_local_address_option = op.add<popl::Implicit<std::string>>(
    "L",
    "local-address",
    "local dtls address ",
    std::string('\0', INET6_ADDRSTRLEN + 1));

  auto remote_address_option = op.add<popl::Value<std::string>>(
    "R", "remote-address", "remote dtls address");
  auto dtls_port_option =
    op.add<popl::Implicit<int>>("p",
                                "port",
                                "port (default 23232) (TODO: remove or "
                                "rename. only for compatibility)",
                                23232);

  auto server_or_client_option =
    op.add<popl::Value<std::string>>("", "mode", "server or client");

  auto udp_forward_address_option = op.add<popl::Implicit<std::string>>(
    "",
    "forward-address",
    "forward address for udp socket. Default means that value will be "
    "taken from the last client.",
    ""s);

  auto udp_local_address_option = op.add<popl::Implicit<std::string>>(
    "", "udp-local-address", "local bind address for udp socket.", ""s);

  auto udp_forward_port_option = op.add<popl::Implicit<unsigned short>>(
    "",
    "forward-port",
    "UDP packets received over dtls will be forwarded to this address. "
    "Default is 0, which means 'get from the last client'",
    0);

  auto udp_local_port_option = op.add<popl::Implicit<unsigned short>>(
    "",
    "udp-local-port",
    "UDP packets sent to this port will be forwarded to the dtls",
    0);

  auto udp_get_addr_from_inbound_packets_option =
    op.add<popl::Implicit<bool>>(
      "",
      "udp-get-addr-from-inbound-packets",
      "UDP target address-port will be updated from incoming packets. "
      "Setting this option to false disables auto-update of address and "
      "port.",
      true);

  auto dtls_public_cert_path_option = op.add<popl::Value<std::string>>(
    "", "dtls-public-cert-path", "path to a public cert file");

  auto dtls_private_key_path_option = op.add<popl::Value<std::string>>(
    "", "dtls-private-key-path", "path to a private key file");

  auto dtls_ca_path_option = op.add<popl::Value<std::string>>(
    "", "dtls-ca-cert-path", "path to a CA file");

  auto dtls_timeout_option = op.add<popl::Implicit<unsigned int>>(
    "",
    "dtls-timeout",
    "How long to wait before timing out. Too short, and your tunnel will "
    "not connect on a lossy channel. Too long, and switching between IPs "
    "will be a pain. Recommended value 5-10, but you have to experiment.",
    30);

  auto dtls_ntimeouts_option = op.add<popl::Implicit<unsigned int>>(
    "", "dtls-ntimeouts", "break connection after this many timeouts", 3);

  try {
    op.parse(argc, argv);
    if (help_option->count() == 1) {
      std::cout << op << "\n";
      exit(EXIT_SUCCESS);
    }

    if (groff_option->is_set()) {
      popl::GroffOptionPrinter option_printer(&op);
      std::cout << option_printer.print();
      exit(EXIT_SUCCESS);
    }

    if (bash_option->is_set()) {
      popl::BashCompletionOptionPrinter option_printer(&op,
                                                       "popl_example");
      std::cout << option_printer.print();
      exit(EXIT_SUCCESS);
    }
    if (!(server_or_client_option->is_set())) {
      std::cerr << "--mode= option must be server or client" << std::endl;
      return 1;
    }

    if (verbose_option->is_set()) {
      verbose = 1;
      veryverbose = 1;
    }
  } catch (const popl::invalid_option & e) {
    std::cerr << "Invalid Option Exception: " << e.what() << "\n";
    std::cerr << "error:  ";
    if (e.error() == popl::invalid_option::Error::missing_argument)
      std::cerr << "missing_argument\n";
    else if (e.error() == popl::invalid_option::Error::invalid_argument)
      std::cerr << "invalid_argument\n";
    else if (e.error() == popl::invalid_option::Error::too_many_arguments)
      std::cerr << "too_many_arguments\n";
    else if (e.error() == popl::invalid_option::Error::missing_option)
      std::cerr << "missing_option\n";

    if (e.error() == popl::invalid_option::Error::missing_option) {
      std::string option_name(
        e.option()->name(popl::OptionName::short_name, true));
      if (option_name.empty())
        option_name = e.option()->name(popl::OptionName::long_name, true);
      std::cerr << "option: " << option_name << "\n";
    } else {
      std::cerr << "option: " << e.option()->name(e.what_name()) << "\n";
      std::cerr << "value:  " << e.value() << "\n";
    }
    std::cout << op << "\n";
    return EXIT_FAILURE;
  } catch (const std::exception & e) {
    std::cerr << "Exception: " << e.what() << "\n";
    return EXIT_FAILURE;
  }

  if (OpenSSL_version_num() != OPENSSL_VERSION_NUMBER) {
    printf("Warning: OpenSSL version mismatch!\n");
    printf("Compiled against %s\n", OPENSSL_VERSION_TEXT);
    printf("Linked against   %s\n", OpenSSL_version(OPENSSL_VERSION));

    if (OpenSSL_version_num() >> 20 != OPENSSL_VERSION_NUMBER >> 20) {
      printf(
        "Error: Major and minor version numbers must match, exiting.\n");
      exit(EXIT_FAILURE);
    }
  } else if (verbose) {
    printf("Using %s\n", OpenSSL_version(OPENSSL_VERSION));
  }

  if (OPENSSL_VERSION_NUMBER < 0x1010102fL) {
    printf(
      "Error: %s is unsupported, use OpenSSL Version 1.1.1a or higher\n",
      OpenSSL_version(OPENSSL_VERSION));
    exit(EXIT_FAILURE);
  }
#if WIN32
  WSADATA wsaData;
#endif
#ifdef WIN32
  WSAStartup(MAKEWORD(2, 2), &wsaData);
#endif

  auto [udp_socket, forward_addr] = setup_udp_socket
    /**/ (udp_forward_address_option->value().c_str(),
          udp_forward_port_option->value(),
          udp_local_address_option->value(),
          udp_local_port_option->value());

  debug_print(("udp_socket="s + std::to_string(udp_socket)).c_str());

  if (server_or_client_option->value() == "client") {
    while (true) {
      debug_print(
        "=======================================================");
      start_client_dtls(remote_address_option->value().c_str(),
                        dtls_local_address_option->value().c_str(),
                        dtls_port_option->value(),
                        udp_socket,
                        forward_addr,
                        udp_get_addr_from_inbound_packets_option->value(),
                        dtls_public_cert_path_option->value(),
                        dtls_private_key_path_option->value(),
                        dtls_ca_path_option->value(),
                        dtls_timeout_option->value(),
                        dtls_ntimeouts_option->value());
      debug_print("Client dtls connection died. Sleeping and restarting.");
      fprintf(stderr, "\n\n");
      fflush(stderr);
      sleep(1);
    }
  } else if (server_or_client_option->value() == "server") {
    start_server_dtls(dtls_port_option->value(),
                      dtls_local_address_option->value().c_str(),
                      udp_socket,
                      forward_addr,
                      udp_get_addr_from_inbound_packets_option->value(),
                      dtls_public_cert_path_option->value(),
                      dtls_private_key_path_option->value(),
                      dtls_ca_path_option->value(),
                      dtls_timeout_option->value(),
                      dtls_ntimeouts_option->value());
  }
#ifdef WIN32
  WSACleanup();
#endif
  return 0;
}