nvmecontrol devlist: Annotate connected Fabrics hosts

[freebsd/src.git] / sys / netinet / ip_reass.c

/*-
 * Copyright (c) 2015 Gleb Smirnoff <glebius@FreeBSD.org>
 * Copyright (c) 2015 Adrian Chadd <adrian@FreeBSD.org>
 * Copyright (c) 1982, 1986, 1988, 1993
 *      The Regents of the University of California.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#include <sys/cdefs.h>
#include "opt_rss.h"

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/eventhandler.h>
#include <sys/kernel.h>
#include <sys/hash.h>
#include <sys/mbuf.h>
#include <sys/malloc.h>
#include <sys/limits.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/sysctl.h>
#include <sys/socket.h>

#include <net/if.h>
#include <net/if_var.h>
#include <net/if_private.h>
#include <net/rss_config.h>
#include <net/netisr.h>
#include <net/vnet.h>

#include <netinet/in.h>
#include <netinet/ip.h>
#include <netinet/ip_var.h>
#include <netinet/in_rss.h>
#ifdef MAC
#include <security/mac/mac_framework.h>
#endif

SYSCTL_DECL(_net_inet_ip);

/*
 * Reassembly headers are stored in hash buckets.
 */
#define IPREASS_NHASH_LOG2      10
#define IPREASS_NHASH           (1 << IPREASS_NHASH_LOG2)
#define IPREASS_HMASK           (V_ipq_hashsize - 1)

struct ipqbucket {
        TAILQ_HEAD(ipqhead, ipq) head;
        struct mtx               lock;
        struct callout           timer;
#ifdef VIMAGE
        struct vnet              *vnet;
#endif
        int                      count;
};

VNET_DEFINE_STATIC(struct ipqbucket *, ipq);
#define V_ipq           VNET(ipq)
VNET_DEFINE_STATIC(uint32_t, ipq_hashseed);
#define V_ipq_hashseed  VNET(ipq_hashseed)
VNET_DEFINE_STATIC(uint32_t, ipq_hashsize);
#define V_ipq_hashsize  VNET(ipq_hashsize)

#define IPQ_LOCK(i)     mtx_lock(&V_ipq[i].lock)
#define IPQ_TRYLOCK(i)  mtx_trylock(&V_ipq[i].lock)
#define IPQ_UNLOCK(i)   mtx_unlock(&V_ipq[i].lock)
#define IPQ_LOCK_ASSERT(i)      mtx_assert(&V_ipq[i].lock, MA_OWNED)
#define IPQ_BUCKET_LOCK_ASSERT(b)       mtx_assert(&(b)->lock, MA_OWNED)

VNET_DEFINE_STATIC(int, ipreass_maxbucketsize);
#define V_ipreass_maxbucketsize VNET(ipreass_maxbucketsize)

void            ipreass_init(void);
void            ipreass_vnet_init(void);
#ifdef VIMAGE
void            ipreass_destroy(void);
#endif
static int      sysctl_maxfragpackets(SYSCTL_HANDLER_ARGS);
static int      sysctl_maxfragbucketsize(SYSCTL_HANDLER_ARGS);
static int      sysctl_fragttl(SYSCTL_HANDLER_ARGS);
static void     ipreass_zone_change(void *);
static void     ipreass_drain_tomax(void);
static void     ipq_free(struct ipqbucket *, struct ipq *);
static struct ipq * ipq_reuse(int);
static void     ipreass_callout(void *);
static void     ipreass_reschedule(struct ipqbucket *);

static inline void
ipq_timeout(struct ipqbucket *bucket, struct ipq *fp)
{

        IPSTAT_ADD(ips_fragtimeout, fp->ipq_nfrags);
        ipq_free(bucket, fp);
}

static inline void
ipq_drop(struct ipqbucket *bucket, struct ipq *fp)
{

        IPSTAT_ADD(ips_fragdropped, fp->ipq_nfrags);
        ipq_free(bucket, fp);
        ipreass_reschedule(bucket);
}

/*
 * By default, limit the number of IP fragments across all reassembly
 * queues to  1/32 of the total number of mbuf clusters.
 *
 * Limit the total number of reassembly queues per VNET to the
 * IP fragment limit, but ensure the limit will not allow any bucket
 * to grow above 100 items. (The bucket limit is
 * IP_MAXFRAGPACKETS / (V_ipq_hashsize / 2), so the 50 is the correct
 * multiplier to reach a 100-item limit.)
 * The 100-item limit was chosen as brief testing seems to show that
 * this produces "reasonable" performance on some subset of systems
 * under DoS attack.
 */
#define IP_MAXFRAGS             (nmbclusters / 32)
#define IP_MAXFRAGPACKETS       (imin(IP_MAXFRAGS, V_ipq_hashsize * 50))

static int              maxfrags;
static u_int __exclusive_cache_line     nfrags;
SYSCTL_INT(_net_inet_ip, OID_AUTO, maxfrags, CTLFLAG_RW,
    &maxfrags, 0,
    "Maximum number of IPv4 fragments allowed across all reassembly queues");
SYSCTL_UINT(_net_inet_ip, OID_AUTO, curfrags, CTLFLAG_RD,
    &nfrags, 0,
    "Current number of IPv4 fragments across all reassembly queues");

VNET_DEFINE_STATIC(uma_zone_t, ipq_zone);
#define V_ipq_zone      VNET(ipq_zone)

SYSCTL_UINT(_net_inet_ip, OID_AUTO, reass_hashsize,
    CTLFLAG_VNET | CTLFLAG_RDTUN, &VNET_NAME(ipq_hashsize), 0,
    "Size of IP fragment reassembly hashtable");

SYSCTL_PROC(_net_inet_ip, OID_AUTO, maxfragpackets,
    CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
    NULL, 0, sysctl_maxfragpackets, "I",
    "Maximum number of IPv4 fragment reassembly queue entries");
SYSCTL_UMA_CUR(_net_inet_ip, OID_AUTO, fragpackets, CTLFLAG_VNET,
    &VNET_NAME(ipq_zone),
    "Current number of IPv4 fragment reassembly queue entries");

VNET_DEFINE_STATIC(int, noreass);
#define V_noreass       VNET(noreass)

VNET_DEFINE_STATIC(int, maxfragsperpacket);
#define V_maxfragsperpacket     VNET(maxfragsperpacket)
SYSCTL_INT(_net_inet_ip, OID_AUTO, maxfragsperpacket, CTLFLAG_VNET | CTLFLAG_RW,
    &VNET_NAME(maxfragsperpacket), 0,
    "Maximum number of IPv4 fragments allowed per packet");
SYSCTL_PROC(_net_inet_ip, OID_AUTO, maxfragbucketsize,
    CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_MPSAFE | CTLFLAG_RW, NULL, 0,
    sysctl_maxfragbucketsize, "I",
    "Maximum number of IPv4 fragment reassembly queue entries per bucket");

VNET_DEFINE_STATIC(u_int, ipfragttl) = 30;
#define V_ipfragttl     VNET(ipfragttl)
SYSCTL_PROC(_net_inet_ip, OID_AUTO, fragttl, CTLTYPE_INT | CTLFLAG_RW |
    CTLFLAG_MPSAFE | CTLFLAG_VNET, NULL, 0, sysctl_fragttl, "IU",
    "IP fragment life time on reassembly queue (seconds)");

/*
 * Take incoming datagram fragment and try to reassemble it into
 * whole datagram.  If the argument is the first fragment or one
 * in between the function will return NULL and store the mbuf
 * in the fragment chain.  If the argument is the last fragment
 * the packet will be reassembled and the pointer to the new
 * mbuf returned for further processing.  Only m_tags attached
 * to the first packet/fragment are preserved.
 * The IP header is *NOT* adjusted out of iplen.
 */
#define M_IP_FRAG       M_PROTO9
struct mbuf *
ip_reass(struct mbuf *m)
{
        struct ip *ip;
        struct mbuf *p, *q, *nq, *t;
        struct ipq *fp;
        struct ifnet *srcifp;
        struct ipqhead *head;
        int i, hlen, next, tmpmax;
        u_int8_t ecn, ecn0;
        uint32_t hash, hashkey[3];
#ifdef  RSS
        uint32_t rss_hash, rss_type;
#endif

        /*
         * If no reassembling or maxfragsperpacket are 0,
         * never accept fragments.
         * Also, drop packet if it would exceed the maximum
         * number of fragments.
         */
        tmpmax = maxfrags;
        if (V_noreass == 1 || V_maxfragsperpacket == 0 ||
            (tmpmax >= 0 && atomic_load_int(&nfrags) >= (u_int)tmpmax)) {
                IPSTAT_INC(ips_fragments);
                IPSTAT_INC(ips_fragdropped);
                m_freem(m);
                return (NULL);
        }

        ip = mtod(m, struct ip *);
        hlen = ip->ip_hl << 2;

        /*
         * Adjust ip_len to not reflect header,
         * convert offset of this to bytes.
         */
        ip->ip_len = htons(ntohs(ip->ip_len) - hlen);
        /*
         * Make sure that fragments have a data length
         * that's a non-zero multiple of 8 bytes, unless
         * this is the last fragment.
         */
        if (ip->ip_len == htons(0) ||
            ((ip->ip_off & htons(IP_MF)) && (ntohs(ip->ip_len) & 0x7) != 0)) {
                IPSTAT_INC(ips_toosmall); /* XXX */
                IPSTAT_INC(ips_fragdropped);
                m_freem(m);
                return (NULL);
        }
        if (ip->ip_off & htons(IP_MF))
                m->m_flags |= M_IP_FRAG;
        else
                m->m_flags &= ~M_IP_FRAG;
        ip->ip_off = htons(ntohs(ip->ip_off) << 3);

        /*
         * Make sure the fragment lies within a packet of valid size.
         */
        if (ntohs(ip->ip_len) + ntohs(ip->ip_off) > IP_MAXPACKET) {
                IPSTAT_INC(ips_toolong);
                IPSTAT_INC(ips_fragdropped);
                m_freem(m);
                return (NULL);
        }

        /*
         * Store receive network interface pointer for later.
         */
        srcifp = m->m_pkthdr.rcvif;

        /*
         * Attempt reassembly; if it succeeds, proceed.
         * ip_reass() will return a different mbuf.
         */
        IPSTAT_INC(ips_fragments);
        m->m_pkthdr.PH_loc.ptr = ip;

        /*
         * Presence of header sizes in mbufs
         * would confuse code below.
         */
        m->m_data += hlen;
        m->m_len -= hlen;

        hashkey[0] = ip->ip_src.s_addr;
        hashkey[1] = ip->ip_dst.s_addr;
        hashkey[2] = (uint32_t)ip->ip_p << 16;
        hashkey[2] += ip->ip_id;
        hash = jenkins_hash32(hashkey, nitems(hashkey), V_ipq_hashseed);
        hash &= IPREASS_HMASK;
        head = &V_ipq[hash].head;
        IPQ_LOCK(hash);

        /*
         * Look for queue of fragments
         * of this datagram.
         */
        TAILQ_FOREACH(fp, head, ipq_list)
                if (ip->ip_id == fp->ipq_id &&
                    ip->ip_src.s_addr == fp->ipq_src.s_addr &&
                    ip->ip_dst.s_addr == fp->ipq_dst.s_addr &&
#ifdef MAC
                    mac_ipq_match(m, fp) &&
#endif
                    ip->ip_p == fp->ipq_p)
                        break;
        /*
         * If first fragment to arrive, create a reassembly queue.
         */
        if (fp == NULL) {
                if (V_ipq[hash].count < V_ipreass_maxbucketsize)
                        fp = uma_zalloc(V_ipq_zone, M_NOWAIT);
                if (fp == NULL)
                        fp = ipq_reuse(hash);
                if (fp == NULL)
                        goto dropfrag;
#ifdef MAC
                if (mac_ipq_init(fp, M_NOWAIT) != 0) {
                        uma_zfree(V_ipq_zone, fp);
                        fp = NULL;
                        goto dropfrag;
                }
                mac_ipq_create(m, fp);
#endif
                TAILQ_INSERT_HEAD(head, fp, ipq_list);
                V_ipq[hash].count++;
                fp->ipq_nfrags = 1;
                atomic_add_int(&nfrags, 1);
                fp->ipq_expire = time_uptime + V_ipfragttl;
                fp->ipq_p = ip->ip_p;
                fp->ipq_id = ip->ip_id;
                fp->ipq_src = ip->ip_src;
                fp->ipq_dst = ip->ip_dst;
                fp->ipq_frags = m;
                if (m->m_flags & M_IP_FRAG)
                        fp->ipq_maxoff = -1;
                else
                        fp->ipq_maxoff = ntohs(ip->ip_off) + ntohs(ip->ip_len);
                m->m_nextpkt = NULL;
                if (fp == TAILQ_LAST(head, ipqhead))
                        callout_reset_sbt(&V_ipq[hash].timer,
                            SBT_1S * V_ipfragttl, SBT_1S, ipreass_callout,
                            &V_ipq[hash], 0);
                else
                        MPASS(callout_active(&V_ipq[hash].timer));
                goto done;
        } else {
                /*
                 * If we already saw the last fragment, make sure
                 * this fragment's offset looks sane. Otherwise, if
                 * this is the last fragment, record its endpoint.
                 */
                if (fp->ipq_maxoff > 0) {
                        i = ntohs(ip->ip_off) + ntohs(ip->ip_len);
                        if (((m->m_flags & M_IP_FRAG) && i >= fp->ipq_maxoff) ||
                            ((m->m_flags & M_IP_FRAG) == 0 &&
                            i != fp->ipq_maxoff)) {
                                fp = NULL;
                                goto dropfrag;
                        }
                } else if ((m->m_flags & M_IP_FRAG) == 0)
                        fp->ipq_maxoff = ntohs(ip->ip_off) + ntohs(ip->ip_len);
                fp->ipq_nfrags++;
                atomic_add_int(&nfrags, 1);
#ifdef MAC
                mac_ipq_update(m, fp);
#endif
        }

#define GETIP(m)        ((struct ip*)((m)->m_pkthdr.PH_loc.ptr))

        /*
         * Handle ECN by comparing this segment with the first one;
         * if CE is set, do not lose CE.
         * drop if CE and not-ECT are mixed for the same packet.
         */
        ecn = ip->ip_tos & IPTOS_ECN_MASK;
        ecn0 = GETIP(fp->ipq_frags)->ip_tos & IPTOS_ECN_MASK;
        if (ecn == IPTOS_ECN_CE) {
                if (ecn0 == IPTOS_ECN_NOTECT)
                        goto dropfrag;
                if (ecn0 != IPTOS_ECN_CE)
                        GETIP(fp->ipq_frags)->ip_tos |= IPTOS_ECN_CE;
        }
        if (ecn == IPTOS_ECN_NOTECT && ecn0 != IPTOS_ECN_NOTECT)
                goto dropfrag;

        /*
         * Find a segment which begins after this one does.
         */
        for (p = NULL, q = fp->ipq_frags; q; p = q, q = q->m_nextpkt)
                if (ntohs(GETIP(q)->ip_off) > ntohs(ip->ip_off))
                        break;

        /*
         * If there is a preceding segment, it may provide some of
         * our data already.  If so, drop the data from the incoming
         * segment.  If it provides all of our data, drop us, otherwise
         * stick new segment in the proper place.
         *
         * If some of the data is dropped from the preceding
         * segment, then it's checksum is invalidated.
         */
        if (p) {
                i = ntohs(GETIP(p)->ip_off) + ntohs(GETIP(p)->ip_len) -
                    ntohs(ip->ip_off);
                if (i > 0) {
                        if (i >= ntohs(ip->ip_len))
                                goto dropfrag;
                        m_adj(m, i);
                        m->m_pkthdr.csum_flags = 0;
                        ip->ip_off = htons(ntohs(ip->ip_off) + i);
                        ip->ip_len = htons(ntohs(ip->ip_len) - i);
                }
                m->m_nextpkt = p->m_nextpkt;
                p->m_nextpkt = m;
        } else {
                m->m_nextpkt = fp->ipq_frags;
                fp->ipq_frags = m;
        }

        /*
         * While we overlap succeeding segments trim them or,
         * if they are completely covered, dequeue them.
         */
        for (; q != NULL && ntohs(ip->ip_off) + ntohs(ip->ip_len) >
            ntohs(GETIP(q)->ip_off); q = nq) {
                i = (ntohs(ip->ip_off) + ntohs(ip->ip_len)) -
                    ntohs(GETIP(q)->ip_off);
                if (i < ntohs(GETIP(q)->ip_len)) {
                        GETIP(q)->ip_len = htons(ntohs(GETIP(q)->ip_len) - i);
                        GETIP(q)->ip_off = htons(ntohs(GETIP(q)->ip_off) + i);
                        m_adj(q, i);
                        q->m_pkthdr.csum_flags = 0;
                        break;
                }
                nq = q->m_nextpkt;
                m->m_nextpkt = nq;
                IPSTAT_INC(ips_fragdropped);
                fp->ipq_nfrags--;
                atomic_subtract_int(&nfrags, 1);
                m_freem(q);
        }

        /*
         * Check for complete reassembly and perform frag per packet
         * limiting.
         *
         * Frag limiting is performed here so that the nth frag has
         * a chance to complete the packet before we drop the packet.
         * As a result, n+1 frags are actually allowed per packet, but
         * only n will ever be stored. (n = maxfragsperpacket.)
         *
         */
        next = 0;
        for (p = NULL, q = fp->ipq_frags; q; p = q, q = q->m_nextpkt) {
                if (ntohs(GETIP(q)->ip_off) != next) {
                        if (fp->ipq_nfrags > V_maxfragsperpacket)
                                ipq_drop(&V_ipq[hash], fp);
                        goto done;
                }
                next += ntohs(GETIP(q)->ip_len);
        }
        /* Make sure the last packet didn't have the IP_MF flag */
        if (p->m_flags & M_IP_FRAG) {
                if (fp->ipq_nfrags > V_maxfragsperpacket)
                        ipq_drop(&V_ipq[hash], fp);
                goto done;
        }

        /*
         * Reassembly is complete.  Make sure the packet is a sane size.
         */
        q = fp->ipq_frags;
        ip = GETIP(q);
        if (next + (ip->ip_hl << 2) > IP_MAXPACKET) {
                IPSTAT_INC(ips_toolong);
                ipq_drop(&V_ipq[hash], fp);
                goto done;
        }

        /*
         * Concatenate fragments.
         */
        m = q;
        t = m->m_next;
        m->m_next = NULL;
        m_cat(m, t);
        nq = q->m_nextpkt;
        q->m_nextpkt = NULL;
        for (q = nq; q != NULL; q = nq) {
                nq = q->m_nextpkt;
                q->m_nextpkt = NULL;
                m->m_pkthdr.csum_flags &= q->m_pkthdr.csum_flags;
                m->m_pkthdr.csum_data += q->m_pkthdr.csum_data;
                m_demote_pkthdr(q);
                m_cat(m, q);
        }
        /*
         * In order to do checksumming faster we do 'end-around carry' here
         * (and not in for{} loop), though it implies we are not going to
         * reassemble more than 64k fragments.
         */
        while (m->m_pkthdr.csum_data & 0xffff0000)
                m->m_pkthdr.csum_data = (m->m_pkthdr.csum_data & 0xffff) +
                    (m->m_pkthdr.csum_data >> 16);
        atomic_subtract_int(&nfrags, fp->ipq_nfrags);
#ifdef MAC
        mac_ipq_reassemble(fp, m);
        mac_ipq_destroy(fp);
#endif

        /*
         * Create header for new ip packet by modifying header of first
         * packet;  dequeue and discard fragment reassembly header.
         * Make header visible.
         */
        ip->ip_len = htons((ip->ip_hl << 2) + next);
        ip->ip_src = fp->ipq_src;
        ip->ip_dst = fp->ipq_dst;
        TAILQ_REMOVE(head, fp, ipq_list);
        V_ipq[hash].count--;
        uma_zfree(V_ipq_zone, fp);
        m->m_len += (ip->ip_hl << 2);
        m->m_data -= (ip->ip_hl << 2);
        /* some debugging cruft by sklower, below, will go away soon */
        if (m->m_flags & M_PKTHDR) {    /* XXX this should be done elsewhere */
                m_fixhdr(m);
                /* set valid receive interface pointer */
                m->m_pkthdr.rcvif = srcifp;
        }
        IPSTAT_INC(ips_reassembled);
        ipreass_reschedule(&V_ipq[hash]);
        IPQ_UNLOCK(hash);

#ifdef  RSS
        /*
         * Query the RSS layer for the flowid / flowtype for the
         * mbuf payload.
         *
         * For now, just assume we have to calculate a new one.
         * Later on we should check to see if the assigned flowid matches
         * what RSS wants for the given IP protocol and if so, just keep it.
         *
         * We then queue into the relevant netisr so it can be dispatched
         * to the correct CPU.
         *
         * Note - this may return 1, which means the flowid in the mbuf
         * is correct for the configured RSS hash types and can be used.
         */
        if (rss_mbuf_software_hash_v4(m, 0, &rss_hash, &rss_type) == 0) {
                m->m_pkthdr.flowid = rss_hash;
                M_HASHTYPE_SET(m, rss_type);
        }

        /*
         * Queue/dispatch for reprocessing.
         *
         * Note: this is much slower than just handling the frame in the
         * current receive context.  It's likely worth investigating
         * why this is.
         */
        netisr_dispatch(NETISR_IP_DIRECT, m);
        return (NULL);
#endif

        /* Handle in-line */
        return (m);

dropfrag:
        IPSTAT_INC(ips_fragdropped);
        if (fp != NULL) {
                fp->ipq_nfrags--;
                atomic_subtract_int(&nfrags, 1);
        }
        m_freem(m);
done:
        IPQ_UNLOCK(hash);
        return (NULL);

#undef GETIP
}

/*
 * Timer expired on a bucket.
 * There should be at least one ipq to be timed out.
 */
static void
ipreass_callout(void *arg)
{
        struct ipqbucket *bucket = arg;
        struct ipq *fp;

        IPQ_BUCKET_LOCK_ASSERT(bucket);
        MPASS(atomic_load_int(&nfrags) > 0);

        CURVNET_SET(bucket->vnet);
        fp = TAILQ_LAST(&bucket->head, ipqhead);
        KASSERT(fp != NULL && fp->ipq_expire <= time_uptime,
            ("%s: stray callout on bucket %p, %ju < %ju", __func__, bucket,
            fp ? (uintmax_t)fp->ipq_expire : 0, (uintmax_t)time_uptime));

        while (fp != NULL && fp->ipq_expire <= time_uptime) {
                ipq_timeout(bucket, fp);
                fp = TAILQ_LAST(&bucket->head, ipqhead);
        }
        ipreass_reschedule(bucket);
        CURVNET_RESTORE();
}

static void
ipreass_reschedule(struct ipqbucket *bucket)
{
        struct ipq *fp;

        IPQ_BUCKET_LOCK_ASSERT(bucket);

        if ((fp = TAILQ_LAST(&bucket->head, ipqhead)) != NULL) {
                time_t t;

                /* Protect against time_uptime tick. */
                t = fp->ipq_expire - time_uptime;
                t = (t > 0) ? t : 1;
                callout_reset_sbt(&bucket->timer, SBT_1S * t, SBT_1S,
                    ipreass_callout, bucket, 0);
        } else
                callout_stop(&bucket->timer);
}

static void
ipreass_drain_vnet(void)
{
        u_int dropped = 0;

        for (int i = 0; i < V_ipq_hashsize; i++) {
                bool resched;

                IPQ_LOCK(i);
                resched = !TAILQ_EMPTY(&V_ipq[i].head);
                while(!TAILQ_EMPTY(&V_ipq[i].head)) {
                        struct ipq *fp = TAILQ_FIRST(&V_ipq[i].head);

                        dropped += fp->ipq_nfrags;
                        ipq_free(&V_ipq[i], fp);
                }
                if (resched)
                        ipreass_reschedule(&V_ipq[i]);
                KASSERT(V_ipq[i].count == 0,
                    ("%s: V_ipq[%d] count %d (V_ipq=%p)", __func__, i,
                    V_ipq[i].count, V_ipq));
                IPQ_UNLOCK(i);
        }
        IPSTAT_ADD(ips_fragdropped, dropped);
}

/*
 * Drain off all datagram fragments.
 */
static void
ipreass_drain(void)
{
        VNET_ITERATOR_DECL(vnet_iter);

        VNET_LIST_RLOCK();
        VNET_FOREACH(vnet_iter) {
                CURVNET_SET(vnet_iter);
                ipreass_drain_vnet();
                CURVNET_RESTORE();
        }
        VNET_LIST_RUNLOCK();
}


/*
 * Initialize IP reassembly structures.
 */
MALLOC_DEFINE(M_IPREASS_HASH, "IP reass", "IP packet reassembly hash headers");
void
ipreass_vnet_init(void)
{
        int max;

        V_ipq_hashsize = IPREASS_NHASH;
        TUNABLE_INT_FETCH("net.inet.ip.reass_hashsize", &V_ipq_hashsize);
        V_ipq = malloc(sizeof(struct ipqbucket) * V_ipq_hashsize,
            M_IPREASS_HASH, M_WAITOK);

        for (int i = 0; i < V_ipq_hashsize; i++) {
                TAILQ_INIT(&V_ipq[i].head);
                mtx_init(&V_ipq[i].lock, "IP reassembly", NULL,
                    MTX_DEF | MTX_DUPOK | MTX_NEW);
                callout_init_mtx(&V_ipq[i].timer, &V_ipq[i].lock, 0);
                V_ipq[i].count = 0;
#ifdef VIMAGE
                V_ipq[i].vnet = curvnet;
#endif
        }
        V_ipq_hashseed = arc4random();
        V_maxfragsperpacket = 16;
        V_ipq_zone = uma_zcreate("ipq", sizeof(struct ipq), NULL, NULL, NULL,
            NULL, UMA_ALIGN_PTR, 0);
        max = IP_MAXFRAGPACKETS;
        max = uma_zone_set_max(V_ipq_zone, max);
        V_ipreass_maxbucketsize = imax(max / (V_ipq_hashsize / 2), 1);
}

void
ipreass_init(void)
{

        maxfrags = IP_MAXFRAGS;
        EVENTHANDLER_REGISTER(nmbclusters_change, ipreass_zone_change,
            NULL, EVENTHANDLER_PRI_ANY);
        EVENTHANDLER_REGISTER(vm_lowmem, ipreass_drain, NULL,
            LOWMEM_PRI_DEFAULT);
        EVENTHANDLER_REGISTER(mbuf_lowmem, ipreass_drain, NULL,
                LOWMEM_PRI_DEFAULT);
}

/*
 * Drain off all datagram fragments belonging to
 * the given network interface.
 */
static void
ipreass_cleanup(void *arg __unused, struct ifnet *ifp)
{
        struct ipq *fp, *temp;
        struct mbuf *m;
        int i;

        KASSERT(ifp != NULL, ("%s: ifp is NULL", __func__));

        CURVNET_SET_QUIET(ifp->if_vnet);

        /*
         * Skip processing if IPv4 reassembly is not initialised or
         * torn down by ipreass_destroy().
         */
        if (V_ipq_zone == NULL) {
                CURVNET_RESTORE();
                return;
        }

        for (i = 0; i < V_ipq_hashsize; i++) {
                IPQ_LOCK(i);
                /* Scan fragment list. */
                TAILQ_FOREACH_SAFE(fp, &V_ipq[i].head, ipq_list, temp) {
                        for (m = fp->ipq_frags; m != NULL; m = m->m_nextpkt) {
                                /* clear no longer valid rcvif pointer */
                                if (m->m_pkthdr.rcvif == ifp)
                                        m->m_pkthdr.rcvif = NULL;
                        }
                }
                IPQ_UNLOCK(i);
        }
        CURVNET_RESTORE();
}
EVENTHANDLER_DEFINE(ifnet_departure_event, ipreass_cleanup, NULL, 0);

#ifdef VIMAGE
/*
 * Destroy IP reassembly structures.
 */
void
ipreass_destroy(void)
{

        ipreass_drain_vnet();
        uma_zdestroy(V_ipq_zone);
        V_ipq_zone = NULL;
        for (int i = 0; i < V_ipq_hashsize; i++)
                mtx_destroy(&V_ipq[i].lock);
        free(V_ipq, M_IPREASS_HASH);
}
#endif

/*
 * After maxnipq has been updated, propagate the change to UMA.  The UMA zone
 * max has slightly different semantics than the sysctl, for historical
 * reasons.
 */
static void
ipreass_drain_tomax(void)
{
        struct ipq *fp;
        int target;

        /*
         * Make sure each bucket is under the new limit. If
         * necessary, drop enough of the oldest elements from
         * each bucket to get under the new limit.
         */
        for (int i = 0; i < V_ipq_hashsize; i++) {
                IPQ_LOCK(i);
                while (V_ipq[i].count > V_ipreass_maxbucketsize &&
                    (fp = TAILQ_LAST(&V_ipq[i].head, ipqhead)) != NULL)
                        ipq_timeout(&V_ipq[i], fp);
                ipreass_reschedule(&V_ipq[i]);
                IPQ_UNLOCK(i);
        }

        /*
         * If we are over the maximum number of fragments,
         * drain off enough to get down to the new limit,
         * stripping off last elements on queues.  Every
         * run we strip the oldest element from each bucket.
         */
        target = uma_zone_get_max(V_ipq_zone);
        while (uma_zone_get_cur(V_ipq_zone) > target) {
                for (int i = 0; i < V_ipq_hashsize; i++) {
                        IPQ_LOCK(i);
                        fp = TAILQ_LAST(&V_ipq[i].head, ipqhead);
                        if (fp != NULL) {
                                ipq_timeout(&V_ipq[i], fp);
                                ipreass_reschedule(&V_ipq[i]);
                        }
                        IPQ_UNLOCK(i);
                }
        }
}

static void
ipreass_zone_change(void *tag)
{
        VNET_ITERATOR_DECL(vnet_iter);
        int max;

        maxfrags = IP_MAXFRAGS;
        max = IP_MAXFRAGPACKETS;
        VNET_LIST_RLOCK_NOSLEEP();
        VNET_FOREACH(vnet_iter) {
                CURVNET_SET(vnet_iter);
                max = uma_zone_set_max(V_ipq_zone, max);
                V_ipreass_maxbucketsize = imax(max / (V_ipq_hashsize / 2), 1);
                ipreass_drain_tomax();
                CURVNET_RESTORE();
        }
        VNET_LIST_RUNLOCK_NOSLEEP();
}

/*
 * Change the limit on the UMA zone, or disable the fragment allocation
 * at all.  Since 0 and -1 is a special values here, we need our own handler,
 * instead of sysctl_handle_uma_zone_max().
 */
static int
sysctl_maxfragpackets(SYSCTL_HANDLER_ARGS)
{
        int error, max;

        if (V_noreass == 0) {
                max = uma_zone_get_max(V_ipq_zone);
                if (max == 0)
                        max = -1;
        } else
                max = 0;
        error = sysctl_handle_int(oidp, &max, 0, req);
        if (error || !req->newptr)
                return (error);
        if (max > 0) {
                /*
                 * XXXRW: Might be a good idea to sanity check the argument
                 * and place an extreme upper bound.
                 */
                max = uma_zone_set_max(V_ipq_zone, max);
                V_ipreass_maxbucketsize = imax(max / (V_ipq_hashsize / 2), 1);
                ipreass_drain_tomax();
                V_noreass = 0;
        } else if (max == 0) {
                V_noreass = 1;
                ipreass_drain();
        } else if (max == -1) {
                V_noreass = 0;
                uma_zone_set_max(V_ipq_zone, 0);
                V_ipreass_maxbucketsize = INT_MAX;
        } else
                return (EINVAL);
        return (0);
}

/*
 * Seek for old fragment queue header that can be reused.  Try to
 * reuse a header from currently locked hash bucket.
 */
static struct ipq *
ipq_reuse(int start)
{
        struct ipq *fp;
        int bucket, i;

        IPQ_LOCK_ASSERT(start);

        for (i = 0; i < V_ipq_hashsize; i++) {
                bucket = (start + i) % V_ipq_hashsize;
                if (bucket != start && IPQ_TRYLOCK(bucket) == 0)
                        continue;
                fp = TAILQ_LAST(&V_ipq[bucket].head, ipqhead);
                if (fp) {
                        struct mbuf *m;

                        IPSTAT_ADD(ips_fragtimeout, fp->ipq_nfrags);
                        atomic_subtract_int(&nfrags, fp->ipq_nfrags);
                        while (fp->ipq_frags) {
                                m = fp->ipq_frags;
                                fp->ipq_frags = m->m_nextpkt;
                                m_freem(m);
                        }
                        TAILQ_REMOVE(&V_ipq[bucket].head, fp, ipq_list);
                        V_ipq[bucket].count--;
                        ipreass_reschedule(&V_ipq[bucket]);
                        if (bucket != start)
                                IPQ_UNLOCK(bucket);
                        break;
                }
                if (bucket != start)
                        IPQ_UNLOCK(bucket);
        }
        IPQ_LOCK_ASSERT(start);
        return (fp);
}

/*
 * Free a fragment reassembly header and all associated datagrams.
 */
static void
ipq_free(struct ipqbucket *bucket, struct ipq *fp)
{
        struct mbuf *q;

        atomic_subtract_int(&nfrags, fp->ipq_nfrags);
        while (fp->ipq_frags) {
                q = fp->ipq_frags;
                fp->ipq_frags = q->m_nextpkt;
                m_freem(q);
        }
        TAILQ_REMOVE(&bucket->head, fp, ipq_list);
        bucket->count--;
        uma_zfree(V_ipq_zone, fp);
}

/*
 * Get or set the maximum number of reassembly queues per bucket.
 */
static int
sysctl_maxfragbucketsize(SYSCTL_HANDLER_ARGS)
{
        int error, max;

        max = V_ipreass_maxbucketsize;
        error = sysctl_handle_int(oidp, &max, 0, req);
        if (error || !req->newptr)
                return (error);
        if (max <= 0)
                return (EINVAL);
        V_ipreass_maxbucketsize = max;
        ipreass_drain_tomax();
        return (0);
}

/*
 * Get or set the IP fragment time to live.
 */
static int
sysctl_fragttl(SYSCTL_HANDLER_ARGS)
{
        u_int ttl;
        int error;

        ttl = V_ipfragttl;
        error = sysctl_handle_int(oidp, &ttl, 0, req);
        if (error || !req->newptr)
                return (error);

        if (ttl < 1 || ttl > MAXTTL)
                return (EINVAL);

        atomic_store_int(&V_ipfragttl, ttl);
        return (0);
}