Remove building with NOCRYPTO option

[minix.git] / minix / net / lwip / tcpsock.c

/* LWIP service - tcpsock.c - TCP sockets */
/*
 * This module implements support for TCP sockets based on lwIP's core TCP PCB
 * module, which is largely but not fully cooperative with exactly what we want
 * to achieve, with as a result that this module is rather complicated.
 *
 * Each socket has a send queue and a receive queue.  Both are using lwIP's own
 * (pbuf) buffers, which largely come out of the main 512-byte buffer pool.
 * The buffers on the send queue are allocated and freed by us--the latter only
 * once they are no longer in use by lwIP as well.  A bit counterintuitively,
 * we deliberately use a smaller lwIP per-PCB TCP send buffer limit
 * (TCP_SND_BUF) in the lwIP send configuration (lwipopts.h) in order to more
 * easily trigger conditions where we cannot enqueue data (or the final FIN)
 * right away.  This way, we get to test the internal logic of this module a
 * lot more easily.  The small lwIP send queue size should not have any impact
 * on performance, as our own per-socket send queues can be much larger and we
 * enqueue more of that on the lwIP PCB as soon as we can in all cases.
 *
 * The receive queue consists of whatever buffers were given to us by lwIP, but
 * since those may be many buffers with small amounts of data each, we perform
 * fairly aggressive merging of consecutive buffers.  The intended result is
 * that we waste no more than 50% of memory within the receive queue.  Merging
 * requires memory copies, which makes it expensive, but we do not configure
 * lwIP with enough buffers to make running out of buffers a non-issue, so this
 * trade-off is necessary.  Practical experience and measurements of the merge
 * policy will have to show whether and how the current policy may be improved.
 *
 * As can be expected, the connection close semantics are by far the most
 * complicated part of this module.  We attempt to get rid of the lwIP PCB as
 * soon as we can, letting lwIP take care of the TIME_WAIT state for example.
 * However, there are various conditions that have to be met before we can
 * forget about the PCB here--most importantly, that none of our sent data
 * blocks are still referenced by lwIP because they have not yet been sent or
 * acknowledged.  We can only free the data blocks once lwIP is done with them.
 *
 * We do consider the TCP state of lwIP's PCB, in order to avoid duplicating
 * full state tracking here.  However, we do not look at a socket's TCP state
 * while in a lwIP-generated event for that socket, because the state may not
 * necessarily reflect the (correct or new) TCP state of the connection, nor
 * may the PCB be available--this is the case for error events.  For these
 * reasons we use a few internal TCPF_ flags to perform partial state tracking.
 *
 * More generally, we tend to access lwIP PCB fields directly only when lwIP's
 * own BSD API implementation does that too and there is no better alternative.
 * One example of this is the check to see if our FIN was acknowledged, for
 * SO_LINGER support.  In terms of maintenance, our hope is that if lwIP's API
 * changes later, we can change our code to imitate whatever lwIP's BSD API
 * implementation does at that point.
 */

#include <sys/socketvar.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <netinet/ip_var.h>
#include <netinet/tcp_timer.h>
#include <netinet/tcp_var.h>
#include <netinet/tcp_fsm.h>

/*
 * Unfortunately, NetBSD and lwIP have different definitions of a few relevant
 * preprocessor variables.  Make sure we do not attempt to use the NetBSD one
 * where it matters.  We do need one of the NetBSD definitions though.
 */
static const unsigned int NETBSD_TF_NODELAY = TF_NODELAY;
#undef TF_NODELAY
#undef TCP_MSS

#include "lwip.h"
#include "tcpisn.h"

#include "lwip/tcp.h"
#include "lwip/priv/tcp_priv.h" /* for tcp_pcb_lists */

/*
 * The number of TCP sockets (NR_TCPSOCK) is defined in the lwIP configuration.
 */

/*
 * We fully control the send buffer, so we can let its size be set to whatever
 * we want.  The receive buffer is different: if it is smaller than the window
 * size, we may have to refuse data that lwIP hands us, at which point more
 * incoming data will cause lwIP to abort the TCP connection--even aside from
 * performance issues.  Therefore, we must make sure the receive buffer is
 * larger than the TCP window at all times.
 */
#define TCP_SNDBUF_MIN  1               /* minimum TCP send buffer size */
#define TCP_SNDBUF_DEF  32768           /* default TCP send buffer size */
#define TCP_SNDBUF_MAX  131072          /* maximum TCP send buffer size */
#define TCP_RCVBUF_MIN  TCP_WND         /* minimum TCP receive buffer size */
#define TCP_RCVBUF_DEF  MAX(TCP_WND, 32768) /* default TCP recv buffer size */
#define TCP_RCVBUF_MAX  MAX(TCP_WND, 131072) /* maximum TCP recv buffer size */

/*
 * The total number of buffers that may in use for TCP socket send queues.  The
 * goal is to allow at least some progress to be made on receiving from TCP
 * sockets and on differently-typed sockets, at least as long as the LWIP
 * service can manage to allocate the memory it wants.  For the case that it
 * does not, we can only reactively kill off TCP sockets and/or free enqueued
 * ethernet packets, neither of which is currently implemented (TODO).
 */
#define TCP_MAX_SENDBUFS        (mempool_max_buffers() * 3 / 4)

/* Polling intervals, in 500-millsecond units. */
#define TCP_POLL_REG_INTERVAL   10      /* interval for reattempting sends */
#define TCP_POLL_CLOSE_INTERVAL 1       /* interval while closing connection */

static struct tcpsock {
        struct ipsock tcp_ipsock;               /* IP socket, MUST be first */
        struct tcp_pcb *tcp_pcb;                /* lwIP TCP control block */
        union pxfer_tcp_queue {                 /* free/accept queue */
                TAILQ_ENTRY(tcpsock) tq_next;   /* next in queue */
                TAILQ_HEAD(, tcpsock) tq_head;  /* head of queue */
        } tcp_queue;
        struct tcpsock *tcp_listener;           /* listener if on accept q. */
        struct {                                /* send queue */
                struct pbuf *ts_head;           /* first pbuf w/unacked data */
                struct pbuf *ts_unsent;         /* first pbuf w/unsent data */
                struct pbuf *ts_tail;           /* most recently added data */
                size_t ts_len;                  /* total sent + unsent */
                unsigned short ts_head_off;     /* offset into head pbuf */
                unsigned short ts_unsent_off;   /* offset into unsent pbuf */
        } tcp_snd;
        struct {                                /* receive queue */
                struct pbuf *tr_head;           /* first pbuf w/unrecvd data */
                struct pbuf **tr_pre_tailp;     /* ptr-ptr to newest pbuf */
                size_t tr_len;                  /* bytes on receive queue */
                unsigned short tr_head_off;     /* offset into head pbuf */
                unsigned short tr_unacked;      /* current window reduction */
        } tcp_rcv;
} tcp_array[NR_TCPSOCK];

static TAILQ_HEAD(, tcpsock) tcp_freelist;      /* list of free TCP sockets */

static const struct sockevent_ops tcpsock_ops;

static unsigned int tcpsock_sendbufs;           /* # send buffers in use */
static unsigned int tcpsock_recvbufs;           /* # receive buffers in use */

/* A bunch of macros that are just for convenience. */
#define tcpsock_get_id(tcp)     (SOCKID_TCP | (sockid_t)((tcp) - tcp_array))
#define tcpsock_get_ipsock(tcp) (&(tcp)->tcp_ipsock)
#define tcpsock_get_sock(tcp)   (ipsock_get_sock(tcpsock_get_ipsock(tcp)))
#define tcpsock_get_sndbuf(tcp) (ipsock_get_sndbuf(tcpsock_get_ipsock(tcp)))
#define tcpsock_get_rcvbuf(tcp) (ipsock_get_rcvbuf(tcpsock_get_ipsock(tcp)))
#define tcpsock_is_ipv6(tcp)    (ipsock_is_ipv6(tcpsock_get_ipsock(tcp)))
#define tcpsock_is_shutdown(tcp,fl) \
        (sockevent_is_shutdown(tcpsock_get_sock(tcp), fl))
#define tcpsock_is_listening(tcp) \
        (sockevent_is_listening(tcpsock_get_sock(tcp)))
#define tcpsock_get_flags(tcp)  (ipsock_get_flags(tcpsock_get_ipsock(tcp)))
#define tcpsock_set_flag(tcp,fl) \
        (ipsock_set_flag(tcpsock_get_ipsock(tcp), fl))
#define tcpsock_clear_flag(tcp,fl) \
        (ipsock_clear_flag(tcpsock_get_ipsock(tcp), fl))

static ssize_t tcpsock_pcblist(struct rmib_call *, struct rmib_node *,
        struct rmib_oldp *, struct rmib_newp *);

/* The CTL_NET {PF_INET,PF_INET6} IPPROTO_TCP subtree. */
/* TODO: add many more and make some of them writable.. */
static struct rmib_node net_inet_tcp_table[] = {
/* 2*/  [TCPCTL_SENDSPACE]      = RMIB_INT(RMIB_RO, TCP_SNDBUF_DEF,
                                    "sendspace",
                                    "Default TCP send buffer size"),
/* 3*/  [TCPCTL_RECVSPACE]      = RMIB_INT(RMIB_RO, TCP_RCVBUF_DEF,
                                    "recvspace",
                                    "Default TCP receive buffer size"),
/*29*/  [TCPCTL_LOOPBACKCKSUM]  = RMIB_FUNC(RMIB_RW | CTLTYPE_INT, sizeof(int),
                                    loopif_cksum, "do_loopback_cksum",
                                    "Perform TCP checksum on loopback"),
/*+0*/  [TCPCTL_MAXID]          = RMIB_FUNC(RMIB_RO | CTLTYPE_NODE, 0,
                                    tcpsock_pcblist, "pcblist",
                                    "TCP protocol control block list"),
/*+1*/  [TCPCTL_MAXID + 1]      = RMIB_FUNC(RMIB_RW | CTLFLAG_PRIVATE |
                                    CTLFLAG_HIDDEN | CTLTYPE_STRING,
                                    TCPISN_SECRET_HEX_LENGTH, tcpisn_secret,
                                    "isn_secret",
                                    "TCP ISN secret (MINIX 3 specific)")
};

static struct rmib_node net_inet_tcp_node =
    RMIB_NODE(RMIB_RO, net_inet_tcp_table, "tcp", "TCP related settings");
static struct rmib_node net_inet6_tcp6_node =
    RMIB_NODE(RMIB_RO, net_inet_tcp_table, "tcp6", "TCP related settings");

/*
 * Initialize the TCP sockets module.
 */
void
tcpsock_init(void)
{
        unsigned int slot;

        /* Initialize the list of free TCP sockets. */
        TAILQ_INIT(&tcp_freelist);

        for (slot = 0; slot < __arraycount(tcp_array); slot++)
                TAILQ_INSERT_TAIL(&tcp_freelist, &tcp_array[slot],
                    tcp_queue.tq_next);

        /* Initialize other variables. */
        tcpsock_sendbufs = 0;

        /* Register the net.inet.tcp and net.inet6.tcp6 RMIB subtrees. */
        mibtree_register_inet(PF_INET, IPPROTO_TCP, &net_inet_tcp_node);
        mibtree_register_inet(PF_INET6, IPPROTO_TCP, &net_inet6_tcp6_node);
}

/*
 * Initialize the state of a TCP socket's send queue.
 */
static void
tcpsock_reset_send(struct tcpsock * tcp)
{

        tcp->tcp_snd.ts_tail = NULL;
        tcp->tcp_snd.ts_unsent = NULL;
        tcp->tcp_snd.ts_head = NULL;
        tcp->tcp_snd.ts_len = 0;
        tcp->tcp_snd.ts_unsent_off = 0;
        tcp->tcp_snd.ts_head_off = 0;
}

/*
 * Initialize the state of a TCP socket's receive queue.
 */
static void
tcpsock_reset_recv(struct tcpsock * tcp)
{

        tcp->tcp_rcv.tr_pre_tailp = NULL;
        tcp->tcp_rcv.tr_head = NULL;
        tcp->tcp_rcv.tr_len = 0;
        tcp->tcp_rcv.tr_head_off = 0;
        tcp->tcp_rcv.tr_unacked = 0;
}

/*
 * Create a TCP socket.
 */
sockid_t
tcpsock_socket(int domain, int protocol, struct sock ** sockp,
        const struct sockevent_ops ** ops)
{
        struct tcpsock *tcp;
        uint8_t ip_type;

        switch (protocol) {
        case 0:
        case IPPROTO_TCP:
                break;

        default:
                return EPROTONOSUPPORT;
        }

        if (TAILQ_EMPTY(&tcp_freelist))
                return ENOBUFS;

        tcp = TAILQ_FIRST(&tcp_freelist);

        /*
         * Initialize the structure.  Do not memset it to zero, as it is still
         * part of the linked free list.  Initialization may still fail.  When
         * adding new fields, make sure to change tcpsock_clone() accordingly.
         */

        ip_type = ipsock_socket(tcpsock_get_ipsock(tcp), domain,
            TCP_SNDBUF_DEF, TCP_RCVBUF_DEF, sockp);

        if ((tcp->tcp_pcb = tcp_new_ip_type(ip_type)) == NULL)
                return ENOBUFS;
        tcp_arg(tcp->tcp_pcb, tcp);

        tcp->tcp_listener = NULL;

        tcpsock_reset_send(tcp);
        tcpsock_reset_recv(tcp);

        TAILQ_REMOVE(&tcp_freelist, tcp, tcp_queue.tq_next);

        *ops = &tcpsock_ops;
        return tcpsock_get_id(tcp);
}

/*
 * Create a TCP socket for the TCP PCB 'pcb' which identifies a new connection
 * incoming on listening socket 'listener'.  The new socket is essentially a
 * "clone" of the listening TCP socket, in that it should inherit any settings
 * from the listening socket.  The socket has not yet been accepted by userland
 * so add it to the queue of connetions pending for the listening socket.  On
 * success, return OK.  On failure, return a negative error code.
 */
static int
tcpsock_clone(struct tcpsock * listener, struct tcp_pcb * pcb)
{
        struct tcpsock *tcp;

        if (TAILQ_EMPTY(&tcp_freelist))
                return ENOBUFS;

        tcp = TAILQ_FIRST(&tcp_freelist);

        /*
         * Initialize the structure.  Do not memset it to zero, as it is still
         * part of the linked free list.  Initialization may still fail.  Most
         * settings should be inherited from the listening socket here, rather
         * than being initialized to their default state.
         */

        ipsock_clone(tcpsock_get_ipsock(listener), tcpsock_get_ipsock(tcp),
            tcpsock_get_id(tcp));

        tcp->tcp_pcb = pcb;
        tcp_arg(pcb, tcp);

        tcpsock_reset_send(tcp);
        tcpsock_reset_recv(tcp);

        /*
         * Remove the new socket from the free list, and add it to the queue of
         * the listening socket--in this order, because the same next pointer
         * is used for both.
         */
        TAILQ_REMOVE(&tcp_freelist, tcp, tcp_queue.tq_next);

        TAILQ_INSERT_TAIL(&listener->tcp_queue.tq_head, tcp,
            tcp_queue.tq_next);
        tcp->tcp_listener = listener;

        return OK;
}

/*
 * Allocate a buffer from the pool, using the standard pool size.  The returned
 * buffer is a single element--never a chain.
 */
static struct pbuf *
tcpsock_alloc_buf(void)
{
        struct pbuf *pbuf;

        pbuf = pbuf_alloc(PBUF_RAW, MEMPOOL_BUFSIZE, PBUF_RAM);

        assert(pbuf == NULL || pbuf->len == pbuf->tot_len);

        return pbuf;
}

/*
 * Free the given buffer.  Ensure that pbuf_free() will not attempt to free the
 * next buffer(s) in the chain as well.  This may be called for pbufs other
 * than those allocated with tcpsock_alloc_buf().
 */
static void
tcpsock_free_buf(struct pbuf * pbuf)
{

        /*
         * Resetting the length is currently not necessary, but better safe
         * than sorry..
         */
        pbuf->len = pbuf->tot_len;
        pbuf->next = NULL;

        pbuf_free(pbuf);
}

/*
 * Clear the send queue of a TCP socket.  The caller must ensure that lwIP will
 * no longer access any of data on the send queue.
 */
static void
tcpsock_clear_send(struct tcpsock * tcp)
{
        struct pbuf *phead;

        assert(tcp->tcp_pcb == NULL);

        while ((phead = tcp->tcp_snd.ts_head) != NULL) {
                tcp->tcp_snd.ts_head = phead->next;

                assert(tcpsock_sendbufs > 0);
                tcpsock_sendbufs--;

                tcpsock_free_buf(phead);
        }

        tcpsock_reset_send(tcp);
}

/*
 * Clear the receive queue of a TCP socket.  If 'ack_data' is set, also
 * acknowledge the previous contents of the receive queue to lwIP.
 */
static size_t
tcpsock_clear_recv(struct tcpsock * tcp, int ack_data)
{
        struct pbuf *phead;
        size_t rlen;

        rlen = tcp->tcp_rcv.tr_len;

        while ((phead = tcp->tcp_rcv.tr_head) != NULL) {
                tcp->tcp_rcv.tr_head = phead->next;

                assert(tcpsock_recvbufs > 0);
                tcpsock_recvbufs--;

                tcpsock_free_buf(phead);
        }

        /*
         * From now on, we will basically be discarding incoming data as fast
         * as possible, to keep the full window open at all times.
         */
        if (ack_data && tcp->tcp_pcb != NULL && tcp->tcp_rcv.tr_unacked > 0)
                tcp_recved(tcp->tcp_pcb, tcp->tcp_rcv.tr_unacked);

        tcpsock_reset_recv(tcp);

        return rlen;
}

/*
 * The TCP socket's PCB has been detached from the socket, typically because
 * the connection was aborted, either by us or by lwIP.  Either way, any TCP
 * connection is gone.  Clear the socket's send queue, remove the socket from
 * a listening socket's queue, and if the socket itself is ready and allowed to
 * be freed, free it now.  The socket is ready to be freed if it was either on
 * a listening queue or being closed already.  The socket is allowed to be
 * freed only if 'may_free' is TRUE.  If the socket is not freed, its receive
 * queue is left as is, as it may still have data to be received by userland.
 */
static int
tcpsock_cleanup(struct tcpsock * tcp, int may_free)
{
        int destroy;

        assert(tcp->tcp_pcb == NULL);

        /*
         * Free any data on the send queue.  This is safe to do right now,
         * because the PCB has been aborted (or was already gone).  We must be
         * very careful about clearing the send queue in all other situations.
         */
        tcpsock_clear_send(tcp);

        /*
         * If this was a socket pending acceptance, remove it from the
         * corresponding listener socket's queue, and free it.  Otherwise, free
         * the socket only if it suspended a graceful close operation.
         */
        if (tcp->tcp_listener != NULL) {
                TAILQ_REMOVE(&tcp->tcp_listener->tcp_queue.tq_head, tcp,
                    tcp_queue.tq_next);
                tcp->tcp_listener = NULL;

                /*
                 * The listener socket's backlog count should be adjusted by
                 * lwIP whenever the PCB is freed up, so we need (and must) not
                 * attempt to do that here.
                 */

                destroy = TRUE;
        } else
                destroy = sockevent_is_closing(tcpsock_get_sock(tcp));

        /*
         * Do not free the socket if 'may_free' is FALSE.  That flag may be set
         * if we are currently in the second tcpsock_close() call on the
         * socket, in which case sockevent_is_closing() is TRUE but we must
         * still not free the socket now: doing so would derail libsockevent.
         */
        if (destroy && may_free) {
                (void)tcpsock_clear_recv(tcp, FALSE /*ack_data*/);

                sockevent_raise(tcpsock_get_sock(tcp), SEV_CLOSE);
        }

        return destroy;
}

/*
 * Abort the lwIP PCB for the given socket, using tcp_abort().  If the PCB is
 * connected, this will cause the connection to be reset.  The PCB, which must
 * have still been present before the call, will be gone after the call.
 */
static void
tcpsock_pcb_abort(struct tcpsock * tcp)
{

        assert(tcp->tcp_pcb != NULL);
        assert(!tcpsock_is_listening(tcp));

        tcp_recv(tcp->tcp_pcb, NULL);
        tcp_sent(tcp->tcp_pcb, NULL);
        tcp_err(tcp->tcp_pcb, NULL);
        tcp_poll(tcp->tcp_pcb, NULL, TCP_POLL_REG_INTERVAL);

        tcp_arg(tcp->tcp_pcb, NULL);

        tcp_abort(tcp->tcp_pcb);

        tcp->tcp_pcb = NULL;
}

/*
 * Close the lwIP PCB for the given socket, using tcp_close().  If the PCB is
 * connected, its graceful close will be finished by lwIP in the background.
 * The PCB, which must have still been present before the call, will be gone
 * after the call.
 */
static void
tcpsock_pcb_close(struct tcpsock * tcp)
{
        err_t err;

        assert(tcp->tcp_pcb != NULL);
        assert(tcp->tcp_snd.ts_len == 0);

        if (!tcpsock_is_listening(tcp)) {
                tcp_recv(tcp->tcp_pcb, NULL);
                tcp_sent(tcp->tcp_pcb, NULL);
                tcp_err(tcp->tcp_pcb, NULL);
                tcp_poll(tcp->tcp_pcb, NULL, TCP_POLL_REG_INTERVAL);
        }

        tcp_arg(tcp->tcp_pcb, NULL);

        if ((err = tcp_close(tcp->tcp_pcb)) != ERR_OK)
                panic("unexpected TCP close failure: %d", err);

        tcp->tcp_pcb = NULL;
}

/*
 * Return TRUE if all conditions are met for closing the TCP socket's PCB, or
 * FALSE if they are not.  Upon calling this function, the socket's PCB must
 * still be around.
 */
static int
tcpsock_may_close(struct tcpsock * tcp)
{

        assert(tcp->tcp_pcb != NULL);

        /*
         * Regular closing of the PCB requires three conditions to be met:
         *
         * 1. all our data has been transmitted AND acknowledged, so that we do
         *    not risk corruption in case there are still unsent or unack'ed
         *    data buffers that may otherwise be recycled too soon;
         * 2. we have sent our FIN to the peer; and,
         * 3. we have received a FIN from the peer.
         */
        return ((tcpsock_get_flags(tcp) & (TCPF_SENT_FIN | TCPF_RCVD_FIN)) ==
            (TCPF_SENT_FIN | TCPF_RCVD_FIN) && tcp->tcp_snd.ts_len == 0);
}

/*
 * The given socket is ready to be closed as per the tcpsock_may_close() rules.
 * This implies that its send queue is already empty.  Gracefully close the
 * PCB.  In addition, if the socket is being closed gracefully, meaning we
 * suspended an earlier tcpsock_close() call (and as such already emptied the
 * receive queue as well), then tell libsockevent that the close is finished,
 * freeing the socket.  Return TRUE if the socket has indeed been freed this
 * way, or FALSE if the socket is still around.
 */
static int
tcpsock_finish_close(struct tcpsock * tcp)
{

        assert(tcp->tcp_snd.ts_len == 0);
        assert(tcp->tcp_listener == NULL);

        /*
         * If we get here, we have already shut down the sending side of the
         * PCB.  Technically, we are interested only in shutting down the
         * receiving side of the PCB here, so that lwIP may decide to recycle
         * the socket later etcetera.  We call tcp_close() because we do not
         * want to rely on tcp_shutdown(RX) doing the exact same thing.
         * However, we do rely on the fact that the PCB is not immediately
         * destroyed by the tcp_close() call: otherwise we may have to return
         * ERR_ABRT if this function is called from a lwIP-generated event.
         */
        tcpsock_pcb_close(tcp);

        /*
         * If we suspended an earlier tcpsock_close() call, we have to tell
         * libsockevent that the close operation is now complete.
         */
        if (sockevent_is_closing(tcpsock_get_sock(tcp))) {
                assert(tcp->tcp_rcv.tr_len == 0);

                sockevent_raise(tcpsock_get_sock(tcp), SEV_CLOSE);

                return TRUE;
        } else
                return FALSE;
}

/*
 * Attempt to start or resume enqueuing data and/or a FIN to send on the given
 * TCP socket.  Return TRUE if anything at all could be newly enqueued on the
 * lwIP PCB, even if less than desired.  In that case, the caller should try to
 * send whatever was enqueued, and if applicable, check if the socket may now
 * be closed (due to the FIN being enqueued).  In particular, in any situation
 * where the socket may be in the process of being closed, the caller must use
 * tcpsock_may_close() if TRUE is returned.  Return FALSE if nothing new could
 * be enqueued, in which case no send attempt need to be made either.
 */
static int
tcpsock_pcb_enqueue(struct tcpsock * tcp)
{
        struct pbuf *punsent;
        size_t space, chunk;
        unsigned int flags;
        err_t err;
        int enqueued;

        assert(tcp->tcp_pcb != NULL);

        if (tcpsock_get_flags(tcp) & TCPF_FULL)
                return FALSE;

        /*
         * Attempt to enqueue more unsent data, if any, on the PCB's send
         * queue.
         */
        enqueued = FALSE;

        while (tcp->tcp_snd.ts_unsent != NULL) {
                if ((space = tcp_sndbuf(tcp->tcp_pcb)) == 0)
                        break;

                /*
                 * We may maintain a non-NULL unsent pointer even when there is
                 * nothing more to send right now, because the tail buffer may
                 * be filled up further later on.
                 */
                punsent = tcp->tcp_snd.ts_unsent;

                assert(punsent->len >= tcp->tcp_snd.ts_unsent_off);

                chunk = (size_t)punsent->len - tcp->tcp_snd.ts_unsent_off;
                if (chunk == 0)
                        break;

                if (chunk > space)
                        chunk = space;

                /* Try to enqueue more data for sending. */
                if (chunk < punsent->len || punsent->next != NULL)
                        flags = TCP_WRITE_FLAG_MORE;
                else
                        flags = 0;

                err = tcp_write(tcp->tcp_pcb, (char *)punsent->payload +
                    tcp->tcp_snd.ts_unsent_off, chunk, flags);

                /*
                 * Since tcp_write() enqueues data only, it should only return
                 * out-of-memory errors; no fatal ones.  In any case, stop.
                 */
                if (err != ERR_OK) {
                        assert(err == ERR_MEM);

                        break;
                }

                /* We have successfully enqueued data. */
                enqueued = TRUE;

                tcp->tcp_snd.ts_unsent_off += chunk;

                if (tcp->tcp_snd.ts_unsent_off < punsent->tot_len) {
                        assert(tcp->tcp_snd.ts_unsent_off < punsent->len ||
                            punsent->next == NULL);

                        break;
                }

                tcp->tcp_snd.ts_unsent = punsent->next;
                tcp->tcp_snd.ts_unsent_off = 0;
        }

        /*
         * If all pending data has been enqueued for sending, and we should
         * shut down the sending end of the socket, try that now.
         */
        if ((tcp->tcp_snd.ts_unsent == NULL ||
            tcp->tcp_snd.ts_unsent_off == tcp->tcp_snd.ts_unsent->len) &&
            tcpsock_is_shutdown(tcp, SFL_SHUT_WR) &&
            !(tcpsock_get_flags(tcp) & TCPF_SENT_FIN)) {
                err = tcp_shutdown(tcp->tcp_pcb, 0 /*shut_rx*/, 1 /*shut_tx*/);

                if (err == ERR_OK) {
                        /*
                         * We have successfully enqueued a FIN.  The caller is
                         * now responsible for checking whether the PCB and
                         * possibly even the socket object can now be freed.
                         */
                        tcpsock_set_flag(tcp, TCPF_SENT_FIN);

                        enqueued = TRUE;
                } else {
                        assert(err == ERR_MEM);

                        /*
                         * FIXME: the resolution for lwIP bug #47485 has taken
                         * away even more control over the closing process from
                         * us, making tracking sockets especially for SO_LINGER
                         * even harder.  For now, we simply effectively undo
                         * the patch by clearing TF_CLOSEPEND if tcp_shutdown()
                         * returns ERR_MEM.  This will not be sustainable in
                         * the long term, though.
                         */
                        tcp->tcp_pcb->flags &= ~TF_CLOSEPEND;

                        tcpsock_set_flag(tcp, TCPF_FULL);
                }
        }

        return enqueued;
}

/*
 * Request lwIP to start sending any enqueued data and/or FIN on the TCP
 * socket's lwIP PCB.  On success, return OK.  On failure, return a negative
 * error code, after cleaning up the socket, freeing the PCB.  If the socket
 * was already being closed, also free the socket object in that case; the
 * caller must then not touch the socket object anymore upon return.  If the
 * socket object is not freed, and if 'raise_error' is TRUE, raise the error
 * on the socket object.
 */
static int
tcpsock_pcb_send(struct tcpsock * tcp, int raise_error)
{
        err_t err;
        int r;

        assert(tcp->tcp_pcb != NULL);

        /*
         * If we have enqueued something, ask lwIP to send TCP packets now.
         * This may result in a fatal error, in which case we clean up the
         * socket and return the error to the caller.  Since cleaning up the
         * socket may free the socket object, and the caller cannot tell
         * whether that will happen or has happened, also possibly raise the
         * error on the socket object if it is not gone.  As such, callers that
         * set 'raise_error' to FALSE must know for sure that the socket was
         * not being closed, for example because the caller is processing a
         * (send) call from userland.
         */
        err = tcp_output(tcp->tcp_pcb);

        if (err != ERR_OK && err != ERR_MEM) {
                tcpsock_pcb_abort(tcp);

                r = util_convert_err(err);

                if (!tcpsock_cleanup(tcp, TRUE /*may_free*/)) {
                        if (raise_error)
                                sockevent_set_error(tcpsock_get_sock(tcp), r);
                }
                /* Otherwise, do not touch the socket object anymore! */

                return r;
        } else
                return OK;
}

/*
 * Callback from lwIP.  The given number of data bytes have been acknowledged
 * as received by the remote end.  Dequeue and free data from the TCP socket's
 * send queue as appropriate.
 */
static err_t
tcpsock_event_sent(void * arg, struct tcp_pcb * pcb __unused, uint16_t len)
{
        struct tcpsock *tcp = (struct tcpsock *)arg;
        struct pbuf *phead;
        size_t left;

        assert(tcp != NULL);
        assert(pcb == tcp->tcp_pcb);
        assert(len > 0);

        assert(tcp->tcp_snd.ts_len >= len);
        assert(tcp->tcp_snd.ts_head != NULL);

        left = len;

        /*
         * First see if we can free up whole buffers.  Check against the head
         * buffer's 'len' rather than 'tot_len', or we may end up leaving an
         * empty buffer on the chain.
         */
        while ((phead = tcp->tcp_snd.ts_head) != NULL &&
            left >= (size_t)phead->len - tcp->tcp_snd.ts_head_off) {
                left -= (size_t)phead->len - tcp->tcp_snd.ts_head_off;

                tcp->tcp_snd.ts_head = phead->next;
                tcp->tcp_snd.ts_head_off = 0;

                if (phead == tcp->tcp_snd.ts_unsent) {
                        assert(tcp->tcp_snd.ts_unsent_off == phead->len);

                        tcp->tcp_snd.ts_unsent = phead->next;
                        tcp->tcp_snd.ts_unsent_off = 0;
                }

                assert(tcpsock_sendbufs > 0);
                tcpsock_sendbufs--;

                tcpsock_free_buf(phead);
        }

        /*
         * The rest of the given length is for less than the current head
         * buffer.
         */
        if (left > 0) {
                assert(tcp->tcp_snd.ts_head != NULL);
                assert((size_t)tcp->tcp_snd.ts_head->len -
                    tcp->tcp_snd.ts_head_off > left);

                tcp->tcp_snd.ts_head_off += left;
        }

        tcp->tcp_snd.ts_len -= (size_t)len;

        if (tcp->tcp_snd.ts_head == NULL) {
                assert(tcp->tcp_snd.ts_len == 0);
                assert(tcp->tcp_snd.ts_unsent == NULL);
                tcp->tcp_snd.ts_tail = NULL;
        } else
                assert(tcp->tcp_snd.ts_len > 0);

        /*
         * If we emptied the send queue, and we already managed to send a FIN
         * earlier, we may now have met all requirements to close the socket's
         * PCB.  Otherwise, we may also be able to send more now, so try to
         * resume sending.  Since we are invoked from the "sent" event,
         * tcp_output() will not actually process anything, and so we do not
         * call it either.  If we did, we would have to deal with errors here.
         */
        if (tcpsock_may_close(tcp)) {
                if (tcpsock_finish_close(tcp))
                        return ERR_OK;
        } else {
                tcpsock_clear_flag(tcp, TCPF_FULL);

                /*
                 * If we now manage to enqueue a FIN, we may be ready to close
                 * the PCB after all.
                 */
                if (tcpsock_pcb_enqueue(tcp)) {
                        if (tcpsock_may_close(tcp) &&
                            tcpsock_finish_close(tcp))
                                return ERR_OK;
                }
        }

        /* The user may also be able to send more now. */
        sockevent_raise(tcpsock_get_sock(tcp), SEV_SEND);

        return ERR_OK;
}

/*
 * Check whether any (additional) data previously received on a TCP socket
 * should be acknowledged, possibly allowing the remote end to send additional
 * data as a result.
 */
static void
tcpsock_ack_recv(struct tcpsock * tcp)
{
        size_t rcvbuf, left, delta, ack;

        assert(tcp->tcp_pcb != NULL);

        /*
         * We must make sure that at all times, we can still add an entire
         * window's worth of data to the receive queue.  If the amount of free
         * space drops below that threshold, we stop acknowledging received
         * data.  The user may change the receive buffer size at all times; we
         * update the window size lazily as appropriate.
         */
        rcvbuf = tcpsock_get_rcvbuf(tcp);

        if (rcvbuf > tcp->tcp_rcv.tr_len && tcp->tcp_rcv.tr_unacked > 0) {
                /*
                 * The number of bytes that lwIP can still give us at any time
                 * is represented as 'left'.  The number of bytes that we still
                 * allow to be stored in the receive queue is represented as
                 * 'delta'.  We must make sure that 'left' does not ever exceed
                 * 'delta' while acknowledging as many bytes as possible under
                 * that rule.
                 */
                left = TCP_WND - tcp->tcp_rcv.tr_unacked;
                delta = rcvbuf - tcp->tcp_rcv.tr_len;

                if (left < delta) {
                        ack = delta - left;

                        if (ack > tcp->tcp_rcv.tr_unacked)
                                ack = tcp->tcp_rcv.tr_unacked;

                        tcp_recved(tcp->tcp_pcb, ack);

                        tcp->tcp_rcv.tr_unacked -= ack;

                        assert(tcp->tcp_rcv.tr_len + TCP_WND -
                            tcp->tcp_rcv.tr_unacked <= rcvbuf);
                }
        }
}

/*
 * Attempt to merge two consecutive underfilled buffers in the receive queue of
 * a TCP socket, freeing up one of the two buffers as a result.  The first
 * (oldest) buffer is 'ptail', and the pointer to this buffer is stored at
 * 'pnext'.  The second (new) buffer is 'pbuf', which is already attached to
 * the first buffer.  The second buffer may be followed by additional buffers
 * with even more new data.  Return TRUE if buffers have been merged, in which
 * case the pointer at 'pnext' may have changed, and no assumptions should be
 * made about whether 'ptail' and 'pbuf' still exist in any form.  Return FALSE
 * if no merging was necessary or if no new buffer could be allocated.
 */
static int
tcpsock_try_merge(struct pbuf **pnext, struct pbuf * ptail, struct pbuf * pbuf)
{
        struct pbuf *pnew;

        assert(*pnext == ptail);
        assert(ptail->next == pbuf);

        /*
         * Unfortunately, we cannot figure out what kind of pbuf we were given
         * by the lower layers, so we cannot merge two buffers without first
         * allocating a third.  Once we have done that, though, we can easily
         * merge more into that new buffer.  For now we use the following
         * policies:
         *
         * 1. if two consecutive lwIP-provided buffers are both used less than
         *    half the size of a full buffer, try to allocate a new buffer and
         *    copy both lwIP-provided buffers into that new buffer, freeing up
         *    the pair afterwards;
         * 2. if the tail buffer on the chain is allocated by us and not yet
         *    full, and the next buffer's contents can be added to the tail
         *    buffer in their entirety, do just that.
         *
         * Obviously there is a trade-off between the performance overhead of
         * copying and the resource overhead of keeping less-than-full buffers
         * on the receive queue, but this policy should both keep actual memory
         * usage to no more than twice the receive queue length and prevent
         * excessive copying.  The policy deliberately performs more aggressive
         * merging into a buffer that we allocated ourselves.
         */
        if (ptail->tot_len <= MEMPOOL_BUFSIZE / 2 &&
            pbuf->len <= MEMPOOL_BUFSIZE / 2) {
                /*
                 * Case #1.
                 */
                assert(ptail->tot_len == ptail->len);
                assert(pbuf->tot_len == pbuf->len);

                pnew = tcpsock_alloc_buf();
                if (pnew == NULL)
                        return FALSE;

                memcpy(pnew->payload, ptail->payload, ptail->len);
                memcpy((char *)pnew->payload + ptail->len, pbuf->payload,
                    pbuf->len);
                pnew->len = ptail->len + pbuf->len;
                assert(pnew->len <= pnew->tot_len);

                pnew->next = pbuf->next;
                /* For now, we need not inherit any flags from either pbuf. */

                *pnext = pnew;

                /* One allocated, two about to be deallocated. */
                assert(tcpsock_recvbufs > 0);
                tcpsock_recvbufs--;

                tcpsock_free_buf(ptail);
                tcpsock_free_buf(pbuf);

                return TRUE;
        } else if (ptail->tot_len - ptail->len >= pbuf->len) {
                /*
                 * Case #2.
                 */
                memcpy((char *)ptail->payload + ptail->len, pbuf->payload,
                    pbuf->len);

                ptail->len += pbuf->len;

                ptail->next = pbuf->next;

                assert(tcpsock_recvbufs > 0);
                tcpsock_recvbufs--;

                tcpsock_free_buf(pbuf);

                return TRUE;
        } else
                return FALSE;
}

/*
 * Callback from lwIP.  New data or flags have been received on a TCP socket.
 */
static err_t
tcpsock_event_recv(void * arg, struct tcp_pcb * pcb __unused,
        struct pbuf * pbuf, err_t err)
{
        struct tcpsock *tcp = (struct tcpsock *)arg;
        struct pbuf *ptail, **pprevp;
        size_t len;

        assert(tcp != NULL);
        assert(pcb == tcp->tcp_pcb);

        /*
         * lwIP should never provide anything other than ERR_OK in 'err', and
         * it is not clear what we should do if it would.  If lwIP ever changes
         * in this regard, we will likely have to change this code accordingly.
         */
        if (err != ERR_OK)
                panic("TCP receive event with error: %d", err);

        /* If the given buffer is NULL, we have received a FIN. */
        if (pbuf == NULL) {
                tcpsock_set_flag(tcp, TCPF_RCVD_FIN);

                /* Userland may now receive EOF. */
                if (!tcpsock_is_shutdown(tcp, SFL_SHUT_RD))
                        sockevent_raise(tcpsock_get_sock(tcp), SEV_RECV);

                /*
                 * If we were in the process of closing the socket, and we
                 * receive a FIN before our FIN got acknowledged, we close the
                 * socket anyway, as described in tcpsock_close().  However, if
                 * there is still unacknowledged outgoing data or we did not
                 * even manage to send our FIN yet, hold off closing the socket
                 * for now.
                 */
                if (tcpsock_may_close(tcp))
                        (void)tcpsock_finish_close(tcp);

                return ERR_OK;
        }

        /*
         * If the socket is being closed, receiving new data should cause a
         * reset.
         */
        if (sockevent_is_closing(tcpsock_get_sock(tcp))) {
                tcpsock_pcb_abort(tcp);

                (void)tcpsock_cleanup(tcp, TRUE /*may_free*/);
                /* Do not touch the socket object anymore! */

                pbuf_free(pbuf);

                return ERR_ABRT;
        }

        /*
         * If the socket has already been shut down for reading, discard the
         * incoming data and do nothing else.
         */
        if (tcpsock_is_shutdown(tcp, SFL_SHUT_RD)) {
                tcp_recved(tcp->tcp_pcb, pbuf->tot_len);

                pbuf_free(pbuf);

                return ERR_OK;
        }

        /*
         * We deliberately ignore the PBUF_FLAG_PUSH flag.  This flag would
         * enable the receive functionality to delay delivering "un-pushed"
         * data to applications.  The implementation of this scheme could track
         * the amount of data up to and including the last-pushed segment using
         * a "tr_push_len" field or so.  Deciding when to deliver "un-pushed"
         * data after all is a bit tricker though.  As far as I can tell, the
         * BSDs do not implement anything like that.  Windows does, and this
         * results in interaction problems with even more lightweight TCP/IP
         * stacks that do not send the TCP PSH flag.  Currently, there is no
         * obvious benefit for us to support delaying data delivery like that.
         * In addition, testing its implementation reliably would be difficult.
         */

        len = (size_t)pbuf->tot_len;

        /*
         * Count the number of buffers that are now owned by us.  The new total
         * of buffers owned by us must not exceed the size of the memory pool.
         * Any more would indicate an accounting error.  Note that
         * tcpsock_recvbufs is currently used for debugging only!
         */
        tcpsock_recvbufs += pbuf_clen(pbuf);
        assert(tcpsock_recvbufs < mempool_cur_buffers());

        /*
         * The pre-tail pointer points to whatever is pointing to the tail
         * buffer.  The latter pointer may be the 'tr_head' field in our
         * tcpsock structure, or the 'next' field in the penultimate buffer,
         * or NULL if there are currently no buffers on the receive queue.
         */
        if ((pprevp = tcp->tcp_rcv.tr_pre_tailp) != NULL) {
                ptail = *pprevp;

                assert(ptail != NULL);
                assert(ptail->next == NULL);
                assert(tcp->tcp_rcv.tr_head != NULL);

                ptail->next = pbuf;
                pbuf->tot_len = pbuf->len;      /* to help freeing on merges */

                if (tcpsock_try_merge(pprevp, ptail, pbuf)) {
                        ptail = *pprevp;
                        pbuf = ptail->next;
                }

                if (pbuf != NULL)
                        pprevp = &ptail->next;
        } else {
                assert(tcp->tcp_rcv.tr_head == NULL);
                assert(tcp->tcp_rcv.tr_head_off == 0);

                tcp->tcp_rcv.tr_head = pbuf;

                pprevp = &tcp->tcp_rcv.tr_head;
        }

        /*
         * Chop up the chain into individual buffers.  This is necessary as we
         * overload 'tot_len' to mean "space available in the buffer", as we
         * want for buffers allocated by us as part of buffer merges.  Also get
         * a pointer to the pointer to the new penultimate tail buffer.  Due to
         * merging, the chain may already be empty by now, though.
         */
        if (pbuf != NULL) {
                for (; pbuf->next != NULL; pbuf = pbuf->next) {
                        pbuf->tot_len = pbuf->len;

                        pprevp = &pbuf->next;
                }
                assert(pbuf->len == pbuf->tot_len);
        }

        assert(*pprevp != NULL);
        assert((*pprevp)->next == NULL);
        tcp->tcp_rcv.tr_pre_tailp = pprevp;

        tcp->tcp_rcv.tr_len += len;
        tcp->tcp_rcv.tr_unacked += len;

        assert(tcp->tcp_rcv.tr_unacked <= TCP_WND);

        /*
         * Note that tr_len may now exceed the receive buffer size in the
         * highly exceptional case that the user is adjusting the latter after
         * the socket had already received data.
         */

        /* See if we can immediately acknowledge some or all of the data. */
        tcpsock_ack_recv(tcp);

        /* Also wake up any receivers now. */
        sockevent_raise(tcpsock_get_sock(tcp), SEV_RECV);

        return ERR_OK;
}

/*
 * Callback from lwIP.  The PCB corresponding to the socket identified by 'arg'
 * has been closed by lwIP, with the reason specified in 'err': either the
 * connection has been aborted locally (ERR_ABRT), it has been reset by the
 * remote end (ERR_RST), or it is closed due to state transitions (ERR_CLSD).
 */
static void
tcpsock_event_err(void * arg, err_t err)
{
        struct tcpsock *tcp = (struct tcpsock *)arg;
        int r;

        assert(tcp != NULL);
        assert(tcp->tcp_pcb != NULL);
        assert(err != ERR_OK);

        /* The original PCB is now gone, or will be shortly. */
        tcp->tcp_pcb = NULL;

        /*
         * Clean up the socket.  As a result it may be freed, in which case we
         * must not touch it anymore.  No need to return ERR_ABRT from here, as
         * the PCB has been aborted already.
         */
        if (tcpsock_cleanup(tcp, TRUE /*may_free*/))
                return;

        if (err == ERR_CLSD) {
                /*
                 * We may get here if the socket is shut down for writing and
                 * we already received a FIN from the remote side, thus putting
                 * the socket in LAST_ACK state, and we receive that last
                 * acknowledgment.  There is nothing more we need to do.
                 *
                 * We will never get here in the other case that ERR_CLSD is
                 * raised, which is when the socket is reset because of
                 * unacknowledged data while closing: we handle the
                 * reset-on-ACK case ourselves in tcpsock_close(), and the
                 * socket is in closing state after that.
                 */
                assert(tcpsock_is_shutdown(tcp, SFL_SHUT_WR));
                assert(tcpsock_get_flags(tcp) & TCPF_RCVD_FIN);
        } else {
                /*
                 * Anything else should be an error directly from lwIP;
                 * currently either ERR_ABRT and ERR_RST.  Covert it to a
                 * regular error and set it on the socket.  Doing so will also
                 * raise the appropriate events.
                 */
                /*
                 * Unfortunately, lwIP is not throwing accurate errors even
                 * when it can.  We convert some errors to reflect more
                 * accurately the most likely cause.
                 *
                 * TODO: fix lwIP in this regard..
                 */
                r = util_convert_err(err);

                if (tcpsock_get_flags(tcp) & TCPF_CONNECTING) {
                        switch (err) {
                        case ERR_ABRT:  r = ETIMEDOUT;          break;
                        case ERR_RST:   r = ECONNREFUSED;       break;
                        }
                }

                sockevent_set_error(tcpsock_get_sock(tcp), r);
        }
}

/*
 * Callback from lwIP.  Perform regular checks on a TCP socket.  This function
 * is called one per five seconds on connected sockets, and twice per second on
 * closing sockets.
 */
static err_t
tcpsock_event_poll(void * arg, struct tcp_pcb * pcb __unused)
{
        struct tcpsock *tcp = (struct tcpsock *)arg;
        err_t err;
        int r;

        assert(tcp != NULL);
        assert(pcb == tcp->tcp_pcb);

        /*
         * If we ended up running out of buffers earlier, try resuming any send
         * requests now, both for enqueuing TCP data with lwIP and for user
         * requests.
         */
        if (tcpsock_get_flags(tcp) & (TCPF_FULL | TCPF_OOM)) {
                tcpsock_clear_flag(tcp, TCPF_FULL);
                tcpsock_clear_flag(tcp, TCPF_OOM);

                /* See if we can enqueue more data with lwIP. */
                if (tcpsock_pcb_enqueue(tcp)) {
                        /* In some cases, we can now close the PCB. */
                        if (tcpsock_may_close(tcp)) {
                                (void)tcpsock_finish_close(tcp);
                                /*
                                 * The PCB is definitely gone here, and the
                                 * entire socket object may be gone now too.
                                 * Do not touch either anymore!
                                 */

                                return ERR_OK;
                        }

                        /*
                         * If actually sending the data fails, the PCB will be
                         * gone, and the socket object may be gone as well.  Do
                         * not touch either anymore in that case!
                         */
                        if (tcpsock_pcb_send(tcp, TRUE /*raise_error*/) != OK)
                                return ERR_ABRT;
                }

                /*
                 * If we ran out of buffers earlier, it may be possible to take
                 * in more data from a user process now, even if we did not
                 * manage to enqueue any more pending data with lwIP.
                 */
                sockevent_raise(tcpsock_get_sock(tcp), SEV_SEND);

                assert(tcp->tcp_pcb != NULL);
        } else if (tcp->tcp_snd.ts_unsent != NULL &&
            tcp->tcp_snd.ts_unsent_off < tcp->tcp_snd.ts_unsent->len) {
                /*
                 * If the send buffer is full, we will no longer call
                 * tcp_output(), which means we may also miss out on fatal
                 * errors that would otherwise kill the connection (e.g., no
                 * route).  As a result, the connection may erroneously
                 * continue to exist for a long time.  To avoid this, we call
                 * tcp_output() every once in a while when there are still
                 * unsent data.
                 */
                err = tcp_output(tcp->tcp_pcb);

                if (err != ERR_OK && err != ERR_MEM) {
                        tcpsock_pcb_abort(tcp);

                        if (!tcpsock_cleanup(tcp, TRUE /*may_free*/)) {
                                r = util_convert_err(err);

                                sockevent_set_error(tcpsock_get_sock(tcp), r);
                        }
                        /* Otherwise do not touch the socket object anymore! */

                        return ERR_ABRT;
                }
        }

        /*
         * If we are closing the socket, and we sent a FIN, see if the FIN got
         * acknowledged.  If so, finish closing the socket.  Unfortunately, we
         * can perform this check by polling only.  TODO: change lwIP..
         */
        if (sockevent_is_closing(tcpsock_get_sock(tcp)) &&
            (tcpsock_get_flags(tcp) & TCPF_SENT_FIN) &&
            tcp->tcp_pcb->unsent == NULL && tcp->tcp_pcb->unacked == NULL) {
                assert(tcp->tcp_snd.ts_len == 0);

                tcpsock_finish_close(tcp);
        }

        return ERR_OK;
}

/*
 * Bind a TCP socket to a local address.
 */
static int
tcpsock_bind(struct sock * sock, const struct sockaddr * addr,
        socklen_t addr_len, endpoint_t user_endpt)
{
        struct tcpsock *tcp = (struct tcpsock *)sock;
        ip_addr_t ipaddr;
        uint16_t port;
        err_t err;
        int r;

        if (tcp->tcp_pcb == NULL || tcp->tcp_pcb->state != CLOSED)
                return EINVAL;

        if ((r = ipsock_get_src_addr(tcpsock_get_ipsock(tcp), addr, addr_len,
            user_endpt, &tcp->tcp_pcb->local_ip, tcp->tcp_pcb->local_port,
            FALSE /*allow_mcast*/, &ipaddr, &port)) != OK)
                return r;

        err = tcp_bind(tcp->tcp_pcb, &ipaddr, port);

        return util_convert_err(err);
}

/*
 * Callback from lwIP.  A new connection 'pcb' has arrived on the listening
 * socket identified by 'arg'.  Note that 'pcb' may be NULL in the case that
 * lwIP could not accept the connection itself.
 */
static err_t
tcpsock_event_accept(void * arg, struct tcp_pcb * pcb, err_t err)
{
        struct tcpsock *tcp = (struct tcpsock *)arg;

        assert(tcp != NULL);
        assert(tcpsock_is_listening(tcp));

        /*
         * If the given PCB is NULL, then lwIP ran out of memory allocating a
         * PCB for the new connection.  There is nothing we can do with that
         * information.  Also check 'err' just to make sure.
         */
        if (pcb == NULL || err != OK)
                return ERR_OK;

        /*
         * The TCP socket is the listening socket, but the PCB is for the
         * incoming connection.
         */
        if (tcpsock_clone(tcp, pcb) != OK) {
                /*
                 * We could not allocate the resources necessary to accept the
                 * connection.  Abort it immediately.
                 */
                tcp_abort(pcb);

                return ERR_ABRT;
        }

        /*
         * The connection has not yet been accepted, and thus should still be
         * considered on the listen queue.
         */
        tcp_backlog_delayed(pcb);

        /* Set the callback functions. */
        tcp_recv(pcb, tcpsock_event_recv);
        tcp_sent(pcb, tcpsock_event_sent);
        tcp_err(pcb, tcpsock_event_err);
        tcp_poll(pcb, tcpsock_event_poll, TCP_POLL_REG_INTERVAL);

        sockevent_raise(tcpsock_get_sock(tcp), SEV_ACCEPT);

        return ERR_OK;
}

/*
 * Put a TCP socket in listening mode.
 */
static int
tcpsock_listen(struct sock * sock, int backlog)
{
        struct tcpsock *tcp = (struct tcpsock *)sock;
        struct tcp_pcb *pcb;
        err_t err;

        /* The maximum backlog value must not exceed its field size. */
        assert(SOMAXCONN <= UINT8_MAX);

        /*
         * Allow only CLOSED sockets to enter listening mode.  If the socket
         * was already in listening mode, allow its backlog value to be
         * updated, even if it was shut down already (making this a no-op).
         */
        if (!tcpsock_is_listening(tcp) &&
            (tcp->tcp_pcb == NULL || tcp->tcp_pcb->state != CLOSED))
                return EINVAL;

        /*
         * If the socket was not already in listening mode, put it in that mode
         * now.  That involves switching PCBs as lwIP attempts to save memory
         * by replacing the original PCB with a smaller one.  If the socket was
         * already in listening mode, simply update its backlog value--this has
         * no effect on the sockets already in the backlog.
         */
        if (!tcpsock_is_listening(tcp)) {
                assert(tcp->tcp_pcb != NULL);

                /*
                 * If the socket has not been bound to a port yet, do that
                 * first.  This does mean that the listen call may fail with
                 * side effects, but that is acceptable in this case.
                 */
                if (tcp->tcp_pcb->local_port == 0) {
                        err = tcp_bind(tcp->tcp_pcb, &tcp->tcp_pcb->local_ip,
                            0 /*port*/);

                        if (err != ERR_OK)
                                return util_convert_err(err);
                }

                /*
                 * Clear the argument on the PCB that is about to be replaced,
                 * because if we do not, once the PCB is reused (which does not
                 * clear the argument), we might get weird events.  Do this
                 * before the tcp_listen() call, because we should no longer
                 * access the old PCB afterwards (even if we can).
                 */
                tcp_arg(tcp->tcp_pcb, NULL);

                pcb = tcp_listen_with_backlog_and_err(tcp->tcp_pcb, backlog,
                    &err);

                if (pcb == NULL) {
                        tcp_arg(tcp->tcp_pcb, tcp); /* oops, undo. */

                        return util_convert_err(err);
                }

                tcp_arg(pcb, tcp);
                tcp->tcp_pcb = pcb;

                tcp_accept(pcb, tcpsock_event_accept);

                /* Initialize the queue head for sockets pending acceptance. */
                TAILQ_INIT(&tcp->tcp_queue.tq_head);
        } else if (tcp->tcp_pcb != NULL)
                tcp_backlog_set(tcp->tcp_pcb, backlog);

        return OK;
}

/*
 * Callback from lwIP.  A socket connection attempt has succeeded.  Note that
 * failed socket events will trigger the tcpsock_event_err() callback instead.
 */
static err_t
tcpsock_event_connected(void * arg, struct tcp_pcb * pcb __unused, err_t err)
{
        struct tcpsock *tcp = (struct tcpsock *)arg;

        assert(tcp != NULL);
        assert(pcb == tcp->tcp_pcb);
        assert(tcpsock_get_flags(tcp) & TCPF_CONNECTING);

        /*
         * If lwIP ever changes so that this callback is called for connect
         * failures as well, then we need to change the code here accordingly.
         */
        if (err != ERR_OK)
                panic("TCP connected event with error: %d", err);

        tcpsock_clear_flag(tcp, TCPF_CONNECTING);

        sockevent_raise(tcpsock_get_sock(tcp), SEV_CONNECT | SEV_SEND);

        return ERR_OK;
}

/*
 * Connect a TCP socket to a remote address.
 */
static int
tcpsock_connect(struct sock * sock, const struct sockaddr * addr,
        socklen_t addr_len, endpoint_t user_endpt)
{
        struct tcpsock *tcp = (struct tcpsock *)sock;
        ip_addr_t dst_addr;
        uint16_t dst_port;
        err_t err;
        int r;

        /*
         * Listening sockets may not have a PCB, so we use higher-level flags
         * to throw the correct error code for those instead.
         */
        if (tcpsock_is_listening(tcp))
                return EOPNOTSUPP;

        /*
         * If there is no longer any PCB, we obviously cannot perform the
         * connection, but POSIX is not clear on which error to return.  We
         * copy NetBSD's.
         */
        if (tcp->tcp_pcb == NULL)
                return EINVAL;

        /*
         * The only state from which a connection can be initiated, is CLOSED.
         * Some of the other states require distinct error codes, though.
         */
        switch (tcp->tcp_pcb->state) {
        case CLOSED:
                break;
        case SYN_SENT:
                return EALREADY;
        case LISTEN:
                assert(0); /* we just checked.. */
        default:
                return EISCONN;
        }

        /*
         * Get the destination address, and attempt to start connecting.  If
         * the socket was not bound before, or it was bound to a port only,
         * then lwIP will select a source address for us.  We cannot do this
         * ourselves even if we wanted to: it is impossible to re-bind a TCP
         * PCB in the case it was previously bound to a port only.
         */
        if ((r = ipsock_get_dst_addr(tcpsock_get_ipsock(tcp), addr, addr_len,
            &tcp->tcp_pcb->local_ip, &dst_addr, &dst_port)) != OK)
                return r;

        err = tcp_connect(tcp->tcp_pcb, &dst_addr, dst_port,
            tcpsock_event_connected);

        /*
         * Note that various tcp_connect() error cases will leave the PCB with
         * a newly set local and remote IP address anyway.  We should be
         * careful not to rely on the addresses being as they were before.
         */
        if (err != ERR_OK)
                return util_convert_err(err);

        /* Set the other callback functions. */
        tcp_recv(tcp->tcp_pcb, tcpsock_event_recv);
        tcp_sent(tcp->tcp_pcb, tcpsock_event_sent);
        tcp_err(tcp->tcp_pcb, tcpsock_event_err);
        tcp_poll(tcp->tcp_pcb, tcpsock_event_poll, TCP_POLL_REG_INTERVAL);

        /*
         * Set a flag so that we can correct lwIP's error codes in case the
         * connection fails.
         */
        tcpsock_set_flag(tcp, TCPF_CONNECTING);

        return SUSPEND;
}

/*
 * Test whether any new connections are pending on a listening TCP socket.
 */
static int
tcpsock_test_accept(struct sock * sock)
{
        struct tcpsock *tcp = (struct tcpsock *)sock;

        /* Is this socket in listening mode at all? */
        if (!tcpsock_is_listening(tcp))
                return EINVAL;

        /* Are there any connections to accept right now? */
        if (!TAILQ_EMPTY(&tcp->tcp_queue.tq_head))
                return OK;

        /* If the socket has been shut down, we return ECONNABORTED. */
        if (tcp->tcp_pcb == NULL)
                return ECONNABORTED;

        /* Otherwise, wait for a new connection first. */
        return SUSPEND;
}

/*
 * Accept a connection on a listening TCP socket, creating a new TCP socket.
 */
static sockid_t
tcpsock_accept(struct sock * sock, struct sockaddr * addr,
        socklen_t * addr_len, endpoint_t user_endpt __unused,
        struct sock ** newsockp)
{
        struct tcpsock *listener = (struct tcpsock *)sock;
        struct tcpsock *tcp;
        int r;

        if ((r = tcpsock_test_accept(sock)) != OK)
                return r;
        /* Below, we must not assume that the listener has a PCB. */

        tcp = TAILQ_FIRST(&listener->tcp_queue.tq_head);
        assert(tcp->tcp_listener == listener);
        assert(tcp->tcp_pcb != NULL);

        TAILQ_REMOVE(&listener->tcp_queue.tq_head, tcp, tcp_queue.tq_next);
        tcp->tcp_listener = NULL;

        tcp_backlog_accepted(tcp->tcp_pcb);

        ipsock_put_addr(tcpsock_get_ipsock(tcp), addr, addr_len,
            &tcp->tcp_pcb->remote_ip, tcp->tcp_pcb->remote_port);

        /*
         * Set 'newsockp' to NULL so that libsockevent knows we already cloned
         * the socket, and it must not be reinitialized anymore.
         */
        *newsockp = NULL;
        return tcpsock_get_id(tcp);
}

/*
 * Perform preliminary checks on a send request.
 */
static int
tcpsock_pre_send(struct sock * sock, size_t len __unused,
        socklen_t ctl_len __unused, const struct sockaddr * addr __unused,
        socklen_t addr_len __unused, endpoint_t user_endpt __unused, int flags)
{

        /*
         * Reject calls with unknown flags.  Since libsockevent strips out the
         * flags it handles itself here, we only have to test for ones we can
         * not handle.  Currently, there are no send flags that we support.
         */
        if (flags != 0)
                return EOPNOTSUPP;

        return OK;
}

/*
 * Test whether the given number of data bytes can be sent on a TCP socket.
 */
static int
tcpsock_test_send(struct sock * sock, size_t min)
{
        struct tcpsock *tcp = (struct tcpsock *)sock;
        size_t sndbuf;

        if (tcp->tcp_pcb == NULL)
                return EPIPE;

        switch (tcp->tcp_pcb->state) {
        case CLOSED:                    /* new */
        case LISTEN:                    /* listening */
                return ENOTCONN;
        case SYN_SENT:                  /* connecting */
        case SYN_RCVD:                  /* simultaneous open, maybe someday? */
                return SUSPEND;
        case ESTABLISHED:               /* connected */
        case CLOSE_WAIT:                /* closed remotely */
                break;
        default:                        /* shut down locally */
                assert(tcpsock_is_shutdown(tcp, SFL_SHUT_WR));
                return EPIPE;
        }

        sndbuf = tcpsock_get_sndbuf(tcp);
        if (min > sndbuf)
                min = sndbuf;

        if (tcp->tcp_snd.ts_len + min > sndbuf)
                return SUSPEND;
        else
                return OK;
}

/*
 * Send data on a TCP socket.
 */
static int
tcpsock_send(struct sock * sock, const struct sockdriver_data * data,
        size_t len, size_t * offp, const struct sockdriver_data * ctl __unused,
        socklen_t ctl_len __unused, socklen_t * ctl_off __unused,
        const struct sockaddr * addr __unused, socklen_t addr_len __unused,
        endpoint_t user_endpt __unused, int flags __unused, size_t min)
{
        struct tcpsock *tcp = (struct tcpsock *)sock;
        struct pbuf *ptail, *pfirst, *pnext, *plast;
        size_t off, tail_off, chunk, left, sndbuf;
        int r;

        if ((r = tcpsock_test_send(sock, min)) != OK)
                return r;

        if (len == 0)
                return OK;      /* nothing to do */

        sndbuf = tcpsock_get_sndbuf(tcp);
        if (min > sndbuf)
                min = sndbuf;
        assert(min > 0);

        assert(sndbuf > tcp->tcp_snd.ts_len);
        left = sndbuf - tcp->tcp_snd.ts_len;
        if (left > len)
                left = len;

        /*
         * First see if we can fit any more data in the current tail buffer.
         * If so, we set 'ptail' to point to it and 'tail_off' to the previous
         * length of the tail buffer, while optimistically extending it to
         * include the new data.  If not, we set them to NULL/0.
         */
        if ((ptail = tcp->tcp_snd.ts_tail) != NULL &&
            ptail->len < ptail->tot_len) {
                assert(ptail->len > 0);
                tail_off = (size_t)ptail->len;

                /*
                 * Optimistically extend the head buffer to include whatever
                 * fits in it.  This is needed for util_copy_data().
                 */
                assert(ptail->tot_len > ptail->len);
                off = (size_t)ptail->tot_len - (size_t)ptail->len;
                if (off > left)
                        off = left;
                ptail->len += off;
        } else {
                ptail = NULL;
                tail_off = 0;
                off = 0;
        }

        /*
         * Then, if there is more to send, allocate new buffers as needed.  If
         * we run out of memory, work with whatever we did manage to grab.
         */
        pfirst = NULL;
        plast = NULL;
        while (off < left) {
                if (tcpsock_sendbufs >= TCP_MAX_SENDBUFS ||
                    (pnext = tcpsock_alloc_buf()) == NULL) {
                        /*
                         * Chances are that we will end up suspending this send
                         * request because of being out of buffers.  We try to
                         * resume such requests from the polling function.
                         */
                        tcpsock_set_flag(tcp, TCPF_OOM);

                        break;
                }

                tcpsock_sendbufs++;

                if (pfirst == NULL)
                        pfirst = pnext;
                else
                        plast->next = pnext;
                plast = pnext;

                chunk = (size_t)pnext->tot_len;
                if (chunk > left - off)
                        chunk = left - off;
                pnext->len = chunk;
                off += chunk;
        }

        /*
         * Copy in the data and continue, unless we did not manage to find
         * enough space to even meet the low send watermark, in which case we
         * undo any allocation and suspend the call until later.
         */
        if (off >= min) {
                /*
                 * Optimistically attach the new buffers to the tail, also for
                 * util_copy_data().  We undo all this if the copy fails.
                 */
                if (ptail != NULL) {
                        ptail->next = pfirst;

                        pnext = ptail;
                } else
                        pnext = pfirst;

                assert(pnext != NULL);

                r = util_copy_data(data, off, *offp, pnext, tail_off,
                    TRUE /*copy_in*/);
        } else
                r = SUSPEND;

        if (r != OK) {
                /* Undo the modifications made so far. */
                while (pfirst != NULL) {
                        pnext = pfirst->next;

                        assert(tcpsock_sendbufs > 0);
                        tcpsock_sendbufs--;

                        tcpsock_free_buf(pfirst);

                        pfirst = pnext;
                }

                if (ptail != NULL) {
                        ptail->next = NULL;

                        ptail->len = tail_off;
                }

                return r;
        }

        /* Attach the new buffers, if any, to the buffer tail. */
        if (pfirst != NULL) {
                if ((ptail = tcp->tcp_snd.ts_tail) != NULL) {
                        assert(ptail->len == ptail->tot_len);

                        /*
                         * Due to our earlier optimistic modifications, this
                         * may or may not be redundant.
                         */
                        ptail->next = pfirst;
                }

                assert(plast != NULL);
                tcp->tcp_snd.ts_tail = plast;

                if (tcp->tcp_snd.ts_head == NULL) {
                        tcp->tcp_snd.ts_head = pfirst;
                        assert(tcp->tcp_snd.ts_head_off == 0);
                }
                if (tcp->tcp_snd.ts_unsent == NULL) {
                        tcp->tcp_snd.ts_unsent = pfirst;
                        assert(tcp->tcp_snd.ts_unsent_off == 0);
                }
        }

        tcp->tcp_snd.ts_len += off;

        /*
         * See if we can send any of the data we just enqueued.  The socket is
         * still open as we are still processing a call from userland on it;
         * this saves us from having to deal with the cases that the following
         * calls end up freeing the socket object.
         */
        if (tcpsock_pcb_enqueue(tcp) &&
            (r = tcpsock_pcb_send(tcp, FALSE /*raise_error*/)) != OK) {
                /*
                 * That did not go well.  Return the error immediately if we
                 * had not made any progress earlier.  Otherwise, return our
                 * partial progress and leave the error to be picked up later.
                 */
                if (*offp > 0) {
                        sockevent_set_error(tcpsock_get_sock(tcp), r);

                        return OK;
                } else
                        return r;
        }

        *offp += off;
        return (off < len) ? SUSPEND : OK;
}

/*
 * Perform preliminary checks on a receive request.
 */
static int
tcpsock_pre_recv(struct sock * sock __unused, endpoint_t user_endpt __unused,
        int flags)
{

        /*
         * Reject calls with unknown flags.  Since libsockevent strips out the
         * flags it handles itself here, we only have to test for ones we can
         * not handle.
         */
        if ((flags & ~(MSG_PEEK | MSG_WAITALL)) != 0)
                return EOPNOTSUPP;

        return OK;
}

/*
 * Return TRUE if receive calls may wait for more data to come in on the
 * connection, or FALSE if we already know that that is not going to happen.
 */
static int
tcpsock_may_wait(struct tcpsock * tcp)
{

        return (tcp->tcp_pcb != NULL &&
            !(tcpsock_get_flags(tcp) & TCPF_RCVD_FIN));
}

/*
 * Test whether data can be received on a TCP socket, and if so, how many bytes
 * of data.
 */
static int
tcpsock_test_recv(struct sock * sock, size_t min, size_t * size)
{
        struct tcpsock *tcp = (struct tcpsock *)sock;
        int may_wait;

        /* If there is and never was a connection, refuse the call at all. */
        if (tcp->tcp_pcb != NULL && (tcp->tcp_pcb->state == CLOSED ||
            tcp->tcp_pcb->state == LISTEN))
                return ENOTCONN;

        /*
         * If we are certain that no more data will come in later, ignore the
         * low receive watermark.  Otherwise, bound it to the size of the
         * receive buffer, or receive calls may block forever.
         */
        if (!(may_wait = tcpsock_may_wait(tcp)))
                min = 1;
        else if (min > tcpsock_get_rcvbuf(tcp))
                min = tcpsock_get_rcvbuf(tcp);

        if (tcp->tcp_rcv.tr_len >= min) {
                if (size != NULL)
                        *size = tcp->tcp_rcv.tr_len;

                return OK;
        }

        return (may_wait) ? SUSPEND : SOCKEVENT_EOF;
}

/*
 * Receive data on a TCP socket.
 */
static int
tcpsock_recv(struct sock * sock, const struct sockdriver_data * data,
        size_t len, size_t * offp, const struct sockdriver_data * ctl __unused,
        socklen_t ctl_len __unused, socklen_t * ctl_off __unused,
        struct sockaddr * addr __unused, socklen_t * addr_len __unused,
        endpoint_t user_endpt __unused, int flags, size_t min,
        int * rflags __unused)
{
        struct tcpsock *tcp = (struct tcpsock *)sock;
        struct pbuf *ptail;
        size_t off, left;
        int r;

        /* See if we can receive at all, and if so, how much at most. */
        if ((r = tcpsock_test_recv(sock, min, NULL)) != OK)
                return r;

        if (len == 0)
                return OK;      /* nothing to do */

        off = tcp->tcp_rcv.tr_len;
        if (off > len)
                off = len;

        assert(tcp->tcp_rcv.tr_head != NULL);
        assert(tcp->tcp_rcv.tr_head_off < tcp->tcp_rcv.tr_head->len);

        /* Copy out the data to the caller. */
        if ((r = util_copy_data(data, off, *offp, tcp->tcp_rcv.tr_head,
            tcp->tcp_rcv.tr_head_off, FALSE /*copy_in*/)) != OK)
                return r;

        /* Unless peeking, remove the data from the receive queue. */
        if (!(flags & MSG_PEEK)) {
                left = off;

                /* Dequeue and free as many entire buffers as possible. */
                while ((ptail = tcp->tcp_rcv.tr_head) != NULL &&
                    left >= (size_t)ptail->len - tcp->tcp_rcv.tr_head_off) {
                        left -= (size_t)ptail->len - tcp->tcp_rcv.tr_head_off;

                        tcp->tcp_rcv.tr_head = ptail->next;
                        tcp->tcp_rcv.tr_head_off = 0;

                        if (tcp->tcp_rcv.tr_head == NULL)
                                tcp->tcp_rcv.tr_pre_tailp = NULL;
                        else if (tcp->tcp_rcv.tr_pre_tailp == &ptail->next)
                                tcp->tcp_rcv.tr_pre_tailp =
                                    &tcp->tcp_rcv.tr_head;

                        assert(tcpsock_recvbufs > 0);
                        tcpsock_recvbufs--;

                        tcpsock_free_buf(ptail);
                }

                /*
                 * If only part of the (new) head buffer is consumed, adjust
                 * the saved offset into that buffer.
                 */
                if (left > 0) {
                        assert(tcp->tcp_rcv.tr_head != NULL);
                        assert((size_t)tcp->tcp_rcv.tr_head->len -
                            tcp->tcp_rcv.tr_head_off > left);

                        tcp->tcp_rcv.tr_head_off += left;
                }

                tcp->tcp_rcv.tr_len -= off;

                if (tcp->tcp_rcv.tr_head != NULL) {
                        assert(tcp->tcp_rcv.tr_pre_tailp != NULL);
                        assert(tcp->tcp_rcv.tr_len > 0);
                } else {
                        assert(tcp->tcp_rcv.tr_pre_tailp == NULL);
                        assert(tcp->tcp_rcv.tr_len == 0);
                }

                /*
                 * The receive buffer has shrunk, so there may now be space to
                 * receive more data.
                 */
                if (tcp->tcp_pcb != NULL)
                        tcpsock_ack_recv(tcp);
        } else
                flags &= ~MSG_WAITALL; /* for the check below */

        /* Advance the current copy position, and see if we are done. */
        *offp += off;
        if ((flags & MSG_WAITALL) && off < len && tcpsock_may_wait(tcp))
                return SUSPEND;
        else
                return OK;
}

/*
 * Update the set of flag-type socket options on a TCP socket.
 */
static void
tcpsock_setsockmask(struct sock * sock, unsigned int mask)
{
        struct tcpsock *tcp = (struct tcpsock *)sock;

        if (tcp->tcp_pcb == NULL)
                return;

        if (mask & SO_REUSEADDR)
                ip_set_option(tcp->tcp_pcb, SOF_REUSEADDR);
        else
                ip_reset_option(tcp->tcp_pcb, SOF_REUSEADDR);

        if (mask & SO_KEEPALIVE)
                ip_set_option(tcp->tcp_pcb, SOF_KEEPALIVE);
        else
                ip_reset_option(tcp->tcp_pcb, SOF_KEEPALIVE);
}

/*
 * Prepare a helper structure for IP-level option processing.
 */
static void
tcpsock_get_ipopts(struct tcpsock * tcp, struct ipopts * ipopts)
{

        ipopts->local_ip = &tcp->tcp_pcb->local_ip;
        ipopts->remote_ip = &tcp->tcp_pcb->remote_ip;
        ipopts->tos = &tcp->tcp_pcb->tos;
        ipopts->ttl = &tcp->tcp_pcb->ttl;
        ipopts->sndmin = TCP_SNDBUF_MIN;
        ipopts->sndmax = TCP_SNDBUF_MAX;
        ipopts->rcvmin = TCP_RCVBUF_MIN;
        ipopts->rcvmax = TCP_RCVBUF_MAX;
}

/*
 * Set socket options on a TCP socket.
 */
static int
tcpsock_setsockopt(struct sock * sock, int level, int name,
        const struct sockdriver_data * data, socklen_t len)
{
        struct tcpsock *tcp = (struct tcpsock *)sock;
        struct ipopts ipopts;
        uint32_t uval;
        int r, val;

        if (tcp->tcp_pcb == NULL)
                return ECONNRESET;

        /* Handle TCP-level options. */
        switch (level) {
        case IPPROTO_IPV6:
                switch (name) {
                case IPV6_RECVTCLASS:
                        if ((r = sockdriver_copyin_opt(data, &val, sizeof(val),
                            len)) != OK)
                                return r;

                        /*
                         * This option is not supported for TCP sockets; it
                         * would not even make sense.  However, named(8)
                         * insists on trying to set it anyway.  We accept the
                         * request but ignore the value, not even returning
                         * what was set through getsockopt(2).
                         */
                        return OK;

                case IPV6_FAITH:
                        if ((r = sockdriver_copyin_opt(data, &val, sizeof(val),
                            len)) != OK)
                                return r;

                        /*
                         * This option is not supported at all, but to save
                         * ourselves from having to remember the current state
                         * for getsockopt(2), we also refuse to enable it.
                         */
                        if (val != 0)
                                return EINVAL;

                        return OK;
                }

                break;

        case IPPROTO_TCP:
                switch (name) {
                case TCP_NODELAY:
                        /*
                         * lwIP's listening TCP PCBs do not have this field.
                         * If this ever becomes an issue, we can create our own
                         * shadow flag and do the inheritance ourselves.
                         */
                        if (tcp->tcp_pcb->state == LISTEN)
                                return EINVAL;

                        if ((r = sockdriver_copyin_opt(data, &val, sizeof(val),
                            len)) != OK)
                                return r;

                        if (val)
                                tcp_nagle_disable(tcp->tcp_pcb);
                        else
                                tcp_nagle_enable(tcp->tcp_pcb);

                        return OK;

                case TCP_KEEPIDLE:
                case TCP_KEEPINTVL:
                        /*
                         * lwIP's listening TCP PCBs do not have these fields.
                         */
                        if (tcp->tcp_pcb->state == LISTEN)
                                return EINVAL;

                        if ((r = sockdriver_copyin_opt(data, &val, sizeof(val),
                            len)) != OK)
                                return r;

                        if (val == 0)
                                return EINVAL;

                        /*
                         * The given value is unsigned, but lwIP stores the
                         * value in milliseconds in a uint32_t field, so we
                         * have to limit large values to whatever fits in the
                         * field anyway.
                         */
                        if (val < 0 || (uint32_t)val > UINT32_MAX / 1000)
                                uval = UINT32_MAX;
                        else
                                uval = (uint32_t)val * 1000;

                        if (name == TCP_KEEPIDLE)
                                tcp->tcp_pcb->keep_idle = uval;
                        else
                                tcp->tcp_pcb->keep_intvl = uval;

                        return OK;

                case TCP_KEEPCNT:
                        /* lwIP's listening TCP PCBs do not have this field. */
                        if (tcp->tcp_pcb->state == LISTEN)
                                return EINVAL;

                        if ((r = sockdriver_copyin_opt(data, &val, sizeof(val),
                            len)) != OK)
                                return r;

                        if (val == 0)
                                return EINVAL;

                        tcp->tcp_pcb->keep_cnt = (uint32_t)val;

                        return OK;
                }

                return EOPNOTSUPP;
        }

        /* Handle all other options at the IP level. */
        tcpsock_get_ipopts(tcp, &ipopts);

        return ipsock_setsockopt(tcpsock_get_ipsock(tcp), level, name, data,
            len, &ipopts);
}

/*
 * Retrieve socket options on a TCP socket.
 */
static int
tcpsock_getsockopt(struct sock * sock, int level, int name,
        const struct sockdriver_data * data, socklen_t * len)
{
        struct tcpsock *tcp = (struct tcpsock *)sock;
        struct ipopts ipopts;
        int val;

        if (tcp->tcp_pcb == NULL)
                return ECONNRESET;

        /* Handle TCP-level options. */
        switch (level) {
        case IPPROTO_IPV6:
                switch (name) {
                case IPV6_RECVTCLASS:
                case IPV6_FAITH:
                        val = 0;

                        return sockdriver_copyout_opt(data, &val, sizeof(val),
                            len);
                }

                break;

        case IPPROTO_TCP:
                switch (name) {
                case TCP_NODELAY:
                        /* lwIP's listening TCP PCBs do not have this field. */
                        if (tcp->tcp_pcb->state == LISTEN)
                                return EINVAL;

                        val = tcp_nagle_disabled(tcp->tcp_pcb);

                        return sockdriver_copyout_opt(data, &val, sizeof(val),
                            len);

                case TCP_MAXSEG:
                        /* lwIP's listening TCP PCBs do not have this field. */
                        if (tcp->tcp_pcb->state == LISTEN)
                                return EINVAL;

                        /* This option is read-only at this time. */
                        val = tcp->tcp_pcb->mss;

                        return sockdriver_copyout_opt(data, &val, sizeof(val),
                            len);

                case TCP_KEEPIDLE:
                        /* lwIP's listening TCP PCBs do not have this field. */
                        if (tcp->tcp_pcb->state == LISTEN)
                                return EINVAL;

                        val = (int)(tcp->tcp_pcb->keep_idle / 1000);

                        return sockdriver_copyout_opt(data, &val, sizeof(val),
                            len);

                case TCP_KEEPINTVL:
                        /* lwIP's listening TCP PCBs do not have this field. */
                        if (tcp->tcp_pcb->state == LISTEN)
                                return EINVAL;

                        val = (int)(tcp->tcp_pcb->keep_intvl / 1000);

                        return sockdriver_copyout_opt(data, &val, sizeof(val),
                            len);

                case TCP_KEEPCNT:
                        /* lwIP's listening TCP PCBs do not have this field. */
                        if (tcp->tcp_pcb->state == LISTEN)
                                return EINVAL;

                        val = (int)tcp->tcp_pcb->keep_cnt;

                        return sockdriver_copyout_opt(data, &val, sizeof(val),
                            len);
                }

                return EOPNOTSUPP;
        }

        /* Handle all other options at the IP level. */
        tcpsock_get_ipopts(tcp, &ipopts);

        return ipsock_getsockopt(tcpsock_get_ipsock(tcp), level, name, data,
            len, &ipopts);
}

/*
 * Retrieve the local socket address of a TCP socket.
 */
static int
tcpsock_getsockname(struct sock * sock, struct sockaddr * addr,
        socklen_t * addr_len)
{
        struct tcpsock *tcp = (struct tcpsock *)sock;

        if (tcp->tcp_pcb == NULL)
                return EINVAL;

        ipsock_put_addr(tcpsock_get_ipsock(tcp), addr, addr_len,
            &tcp->tcp_pcb->local_ip, tcp->tcp_pcb->local_port);

        return OK;
}

/*
 * Retrieve the remote socket address of a TCP socket.
 */
static int
tcpsock_getpeername(struct sock * sock, struct sockaddr * addr,
        socklen_t * addr_len)
{
        struct tcpsock *tcp = (struct tcpsock *)sock;

        if (tcp->tcp_pcb == NULL || tcp->tcp_pcb->state == CLOSED ||
            tcp->tcp_pcb->state == LISTEN || tcp->tcp_pcb->state == SYN_SENT)
                return ENOTCONN;

        ipsock_put_addr(tcpsock_get_ipsock(tcp), addr, addr_len,
            &tcp->tcp_pcb->remote_ip, tcp->tcp_pcb->remote_port);

        return OK;
}

/*
 * Perform a TCP half-close on a TCP socket.  This operation may not complete
 * immediately due to memory conditions, in which case it will be completed at
 * a later time.
 */
static void
tcpsock_send_fin(struct tcpsock * tcp)
{

        sockevent_set_shutdown(tcpsock_get_sock(tcp), SFL_SHUT_WR);

        /*
         * Attempt to send the FIN.  If a fatal error occurs as a result, raise
         * it as an asynchronous error, because this function's callers cannot
         * do much with it.  That happens to match the way these functions are
         * used elsewhere.  In any case, as a result, the PCB may be closed.
         * However, we are never called from a situation where the socket is
         * being closed here, so the socket object will not be freed either.
         */
        if (tcpsock_pcb_enqueue(tcp)) {
                assert(!sockevent_is_closing(tcpsock_get_sock(tcp)));

                if (tcpsock_may_close(tcp))
                        tcpsock_finish_close(tcp);
                else
                        (void)tcpsock_pcb_send(tcp, TRUE /*raise_error*/);
        }
}

/*
 * Shut down a TCP socket for reading and/or writing.
 */
static int
tcpsock_shutdown(struct sock * sock, unsigned int mask)
{
        struct tcpsock *tcp = (struct tcpsock *)sock;

        /*
         * If the PCB is gone, we want to allow shutdowns for reading but not
         * writing: shutting down for writing affects the PCB, shutting down
         * for reading does not.  Also, if the PCB is in CLOSED state, we would
         * not know how to deal with subsequent operations after a shutdown for
         * writing, so forbid such calls altogether.
         */
        if ((tcp->tcp_pcb == NULL || tcp->tcp_pcb->state == CLOSED) &&
            (mask & SFL_SHUT_WR))
                return ENOTCONN;

        /*
         * Handle listening sockets as a special case.  Shutting down a
         * listening socket frees its PCB.  Sockets pending on the accept queue
         * may still be accepted, but after that, accept(2) will start
         * returning ECONNABORTED.  This feature allows multi-process server
         * applications to shut down gracefully, supposedly..
         */
        if (tcpsock_is_listening(tcp)) {
                if (tcp->tcp_pcb != NULL)
                        tcpsock_pcb_close(tcp);

                return OK;
        }

        /*
         * We control shutdown-for-reading locally, and intentially do not tell
         * lwIP about it: if we do that and also shut down for writing, the PCB
         * may disappear (now or eventually), which is not what we want.
         * Instead, we only tell lwIP to shut down for reading once we actually
         * want to get rid of the PCB, using tcp_close().  In the meantime, if
         * the socket is shut down for reading by the user, we simply discard
         * received data as fast as we can--one out of a number of possible
         * design choices there, and (reportedly) the one used by the BSDs.
         */
        if (mask & SFL_SHUT_RD)
                (void)tcpsock_clear_recv(tcp, TRUE /*ack_data*/);

        /*
         * Shutting down for writing a connecting socket simply closes its PCB.
         * Closing a PCB in SYN_SENT state simply deallocates it, so this can
         * not fail.  On the other hand, for connected sockets we want to send
         * a FIN, which may fail due to memory shortage, in which case we have
         * to try again later..
         */
        if (mask & SFL_SHUT_WR) {
                if (tcp->tcp_pcb->state == SYN_SENT)
                        tcpsock_pcb_close(tcp);
                else if (!tcpsock_is_shutdown(tcp, SFL_SHUT_WR))
                        tcpsock_send_fin(tcp);
        }

        return OK;
}

/*
 * Close a TCP socket.  Complete the operation immediately if possible, or
 * otherwise initiate the closing process and complete it later, notifying
 * libsockevent about that as well.  Depending on linger settings, this
 * function may be called twice on the same socket: the first time with the
 * 'force' flag cleared, and the second time with the 'force' flag set.
 */
static int
tcpsock_close(struct sock * sock, int force)
{
        struct tcpsock *tcp = (struct tcpsock *)sock;
        struct tcpsock *queued;
        size_t rlen;

        assert(tcp->tcp_listener == NULL);

        /*
         * If this was a listening socket, so abort and clean up any and all
         * connections on its listener queue.  Note that the listening socket
         * may or may not have a PCB at this point.
         */
        if (tcpsock_is_listening(tcp)) {
                while (!TAILQ_EMPTY(&tcp->tcp_queue.tq_head)) {
                        queued = TAILQ_FIRST(&tcp->tcp_queue.tq_head);

                        tcpsock_pcb_abort(queued);

                        (void)tcpsock_cleanup(queued, TRUE /*may_free*/);
                }
        }

        /*
         * Clear the receive queue, and make sure that we no longer add new
         * data to it.  The latter is relevant only for the case that we end up
         * returning SUSPEND below.  Remember whether there were bytes left,
         * because we should reset the connection if there were.
         */
        rlen = tcpsock_clear_recv(tcp, FALSE /*ack_data*/);

        sockevent_set_shutdown(tcpsock_get_sock(tcp), SFL_SHUT_RD);

        /*
         * If the socket is connected, perform a graceful shutdown, unless 1)
         * we are asked to force-close the socket, or 2) if the local side has
         * not consumed all data, as per RFC 1122 Sec.4.2.2.13.  Normally lwIP
         * would take care of the second point, but we may have data in our
         * receive buffer of which lwIP is not aware.
         *
         * Implementing proper linger support is somewhat difficult with lwIP.
         * In particular, we cannot reliably wait for our FIN to be ACK'ed by
         * the other side in all cases:
         *
         * - the lwIP TCP transition from states CLOSING to TIME_WAIT does not
         *   trigger any event and once in the TIME_WAIT state, the poll event
         *   no longer triggers either;
         * - the lwIP TCP transition from states FIN_WAIT_1 and FIN_WAIT_2 to
         *   TIME_WAIT will trigger a receive event, but it is not clear
         *   whether we can reliably check that our FIN was ACK'ed from there.
         *
         * That means we have to compromise.  Instead of the proper approach,
         * we complete our side of the close operation whenever:
         *
         * 1. all of or data was acknowledged, AND,
         * 2. our FIN was sent, AND,
         * 3a. our FIN was acknowledged, OR,
         * 3b. we received a FIN from the other side.
         *
         * With the addition of the rule 3b, we do not run into the above
         * reliability problems, but we may return from SO_LINGER-blocked close
         * calls too early and thus give callers a false impression of success.
         * TODO: if lwIP ever gets improved on this point, the code in this
         * module should be rewritten to make use of the improvements.
         *
         * The set of rules is basically the same as for closing the PCB early
         * as per tcpsock_may_close(), except with the check for our FIN being
         * acknowledged.  Unfortunately only the FIN_WAIT_2, TIME_WAIT, and
         * (reentered) CLOSED TCP states guarantee that there are no
         * unacknowledged data segments anymore, so we may have to wait for
         * reaching any one of these before we can actually finish closing the
         * socket with tcp_close().
         *
         * In addition, lwIP does not tell us when our FIN gets acknowledged,
         * so we have to use polling and direct access to lwIP's PCB fields
         * instead, just like lwIP's BSD API does.  There is no other way.
         * Also, we may not even be able to send the FIN right away, in which
         * case we must defer that until later.
         */
        if (tcp->tcp_pcb != NULL) {
                switch (tcp->tcp_pcb->state) {
                case CLOSE_WAIT:
                case CLOSING:
                case LAST_ACK:
                        assert(tcpsock_get_flags(tcp) & TCPF_RCVD_FIN);

                        /* FALLTHROUGH */
                case SYN_RCVD:
                case ESTABLISHED:
                case FIN_WAIT_1:
                        /* First check if we should abort the connection. */
                        if (force || rlen > 0)
                                break;

                        /*
                         * If we have not sent a FIN yet, try sending it now;
                         * if all other conditions are met for closing the
                         * socket, successful FIN transmission will complete
                         * the close.  Otherwise, perform the close check
                         * explicitly.
                         */
                        if (!tcpsock_is_shutdown(tcp, SFL_SHUT_WR))
                                tcpsock_send_fin(tcp);
                        else if (tcpsock_may_close(tcp))
                                tcpsock_pcb_close(tcp);

                        /*
                         * If at this point the PCB is gone, we managed to
                         * close the connection immediately, and the socket has
                         * already been cleaned up by now.  This may occur if
                         * there is no unacknowledged data and we already
                         * received a FIN earlier on.
                         */
                        if (tcp->tcp_pcb == NULL)
                                return OK;

                        /*
                         * Complete the close operation at a later time.
                         * Adjust the polling interval, so that we can detect
                         * completion of the close as quickly as possible.
                         */
                        tcp_poll(tcp->tcp_pcb, tcpsock_event_poll,
                            TCP_POLL_CLOSE_INTERVAL);

                        return SUSPEND;

                default:
                        /*
                         * The connection is either not yet established, or
                         * already in a state where we can close it right now.
                         */
                        tcpsock_pcb_close(tcp);
                }
        }

        /*
         * Abort the connection is the PCB is still around, and clean up the
         * socket.  We cannot let tcpsock_cleanup() free the socket object yet,
         * because we are still in the callback from libsockevent, and the
         * latter cannot handle the socket object being freed from here.
         */
        if (tcp->tcp_pcb != NULL)
                tcpsock_pcb_abort(tcp);

        (void)tcpsock_cleanup(tcp, FALSE /*may_free*/);

        return OK;
}

/*
 * Free up a closed TCP socket.
 */
static void
tcpsock_free(struct sock * sock)
{
        struct tcpsock *tcp = (struct tcpsock *)sock;

        assert(tcp->tcp_pcb == NULL);
        assert(tcp->tcp_snd.ts_len == 0);
        assert(tcp->tcp_snd.ts_head == NULL);
        assert(tcp->tcp_rcv.tr_len == 0);
        assert(tcp->tcp_rcv.tr_head == NULL);

        TAILQ_INSERT_HEAD(&tcp_freelist, tcp, tcp_queue.tq_next);
}

/* This table maps TCP states from lwIP numbers to NetBSD numbers. */
static const struct {
        int tsm_tstate;
        int tsm_sostate;
} tcpsock_statemap[] = {
        [CLOSED]        = { TCPS_CLOSED,        SS_ISDISCONNECTED       },
        [LISTEN]        = { TCPS_LISTEN,        0                       },
        [SYN_SENT]      = { TCPS_SYN_SENT,      SS_ISCONNECTING         },
        [SYN_RCVD]      = { TCPS_SYN_RECEIVED,  SS_ISCONNECTING         },
        [ESTABLISHED]   = { TCPS_ESTABLISHED,   SS_ISCONNECTED          },
        [FIN_WAIT_1]    = { TCPS_FIN_WAIT_1,    SS_ISDISCONNECTING      },
        [FIN_WAIT_2]    = { TCPS_FIN_WAIT_2,    SS_ISDISCONNECTING      },
        [CLOSE_WAIT]    = { TCPS_CLOSE_WAIT,    SS_ISCONNECTED          },
        [CLOSING]       = { TCPS_CLOSING,       SS_ISDISCONNECTING      },
        [LAST_ACK]      = { TCPS_LAST_ACK,      SS_ISDISCONNECTING      },
        [TIME_WAIT]     = { TCPS_TIME_WAIT,     SS_ISDISCONNECTED       },
};

/*
 * Fill the given kinfo_pcb sysctl(7) structure with information about the TCP
 * PCB identified by the given pointer.
 */
static void
tcpsock_get_info(struct kinfo_pcb * ki, const void * ptr)
{
        const struct tcp_pcb *pcb = (const struct tcp_pcb *)ptr;
        struct tcpsock *tcp;

        /*
         * Not all TCP PCBs have an associated tcpsock structure.  We are
         * careful enough clearing the callback argument for PCBs on any of the
         * TCP lists that we can use that callback argument to determine
         * whether there is an associated tcpsock structure, although with one
         * exception: PCBs for incoming connections that have not yet been
         * fully established (i.e., in SYN_RCVD state).  These will have the
         * callback argument of the listening socket (which itself may already
         * have been deallocated at this point) but should not be considered as
         * associated with the listening socket's tcpsock structure.
         */
        if (pcb->callback_arg != NULL && pcb->state != SYN_RCVD) {
                tcp = (struct tcpsock *)pcb->callback_arg;
                assert(tcp >= tcp_array &&
                    tcp < &tcp_array[__arraycount(tcp_array)]);

                /* TODO: change this so that sockstat(1) may work one day. */
                ki->ki_sockaddr = (uint64_t)(uintptr_t)tcpsock_get_sock(tcp);
        } else {
                /* No tcpsock.  Could also be in TIME_WAIT state etc. */
                tcp = NULL;

                ki->ki_sostate = SS_NOFDREF;
        }

        ki->ki_type = SOCK_STREAM;

        if ((unsigned int)pcb->state < __arraycount(tcpsock_statemap)) {
                ki->ki_tstate = tcpsock_statemap[pcb->state].tsm_tstate;
                /* TODO: this needs work, but does anything rely on it? */
                ki->ki_sostate |= tcpsock_statemap[pcb->state].tsm_sostate;
        }

        /* Careful with the LISTEN state here (see below). */
        ipsock_get_info(ki, &pcb->local_ip, pcb->local_port,
            &pcb->remote_ip, (pcb->state != LISTEN) ? pcb->remote_port : 0);

        /*
         * The PCBs for listening sockets are actually smaller.  Thus, for
         * listening sockets, do not attempt to access any of the fields beyond
         * those provided in the smaller structure.
         */
        if (pcb->state == LISTEN) {
                assert(tcp != NULL);
                ki->ki_refs =
                    (uint64_t)(uintptr_t)TAILQ_FIRST(&tcp->tcp_queue.tq_head);
        } else {
                if (tcp_nagle_disabled(pcb))
                        ki->ki_tflags |= NETBSD_TF_NODELAY;

                if (tcp != NULL) {
                        ki->ki_rcvq = tcp->tcp_rcv.tr_len;
                        ki->ki_sndq = tcp->tcp_snd.ts_len;

                        if (tcp->tcp_listener != NULL)
                                ki->ki_nextref = (uint64_t)(uintptr_t)
                                    TAILQ_NEXT(tcp, tcp_queue.tq_next);
                }
        }
}

/*
 * Given either NULL or a previously returned TCP PCB pointer, return the first
 * or next TCP PCB pointer, or NULL if there are no more.  The current
 * implementation supports only one concurrent iteration at once.
 */
static const void *
tcpsock_enum(const void * last)
{
        static struct {
                unsigned int i;
                const struct tcp_pcb *pcb;
        } iter;

        if (last != NULL && (iter.pcb = iter.pcb->next) != NULL)
                return (const void *)iter.pcb;

        for (iter.i = (last != NULL) ? iter.i + 1 : 0;
            iter.i < __arraycount(tcp_pcb_lists); iter.i++) {
                if ((iter.pcb = *tcp_pcb_lists[iter.i]) != NULL)
                        return (const void *)iter.pcb;
        }

        return NULL;
}

/*
 * Obtain the list of TCP protocol control blocks, for sysctl(7).
 */
static ssize_t
tcpsock_pcblist(struct rmib_call * call, struct rmib_node * node __unused,
        struct rmib_oldp * oldp, struct rmib_newp * newp __unused)
{

        return util_pcblist(call, oldp, tcpsock_enum, tcpsock_get_info);
}

static const struct sockevent_ops tcpsock_ops = {
        .sop_bind               = tcpsock_bind,
        .sop_listen             = tcpsock_listen,
        .sop_connect            = tcpsock_connect,
        .sop_accept             = tcpsock_accept,
        .sop_test_accept        = tcpsock_test_accept,
        .sop_pre_send           = tcpsock_pre_send,
        .sop_send               = tcpsock_send,
        .sop_test_send          = tcpsock_test_send,
        .sop_pre_recv           = tcpsock_pre_recv,
        .sop_recv               = tcpsock_recv,
        .sop_test_recv          = tcpsock_test_recv,
        .sop_ioctl              = ifconf_ioctl,
        .sop_setsockmask        = tcpsock_setsockmask,
        .sop_setsockopt         = tcpsock_setsockopt,
        .sop_getsockopt         = tcpsock_getsockopt,
        .sop_getsockname        = tcpsock_getsockname,
        .sop_getpeername        = tcpsock_getpeername,
        .sop_shutdown           = tcpsock_shutdown,
        .sop_close              = tcpsock_close,
        .sop_free               = tcpsock_free
};