dmake: do not set MAKEFLAGS=k

[unleashed/tickless.git] / usr / src / cmd / cmd-inet / usr.lib / in.mpathd / mpd_tables.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include "mpd_defs.h"
#include "mpd_tables.h"

/*
 * Global list of phyints, phyint instances, phyint groups and the anonymous
 * group; the latter is initialized in phyint_init().
 */
struct phyint *phyints = NULL;
struct phyint_instance  *phyint_instances = NULL;
struct phyint_group *phyint_groups = NULL;
struct phyint_group *phyint_anongroup;

/*
 * Grouplist signature; initialized in phyint_init().
 */
static uint64_t phyint_grouplistsig;

static void phyint_inst_insert(struct phyint_instance *pii);
static void phyint_inst_print(struct phyint_instance *pii);

static void phyint_insert(struct phyint *pi, struct phyint_group *pg);
static void phyint_delete(struct phyint *pi);
static boolean_t phyint_is_usable(struct phyint *pi);

static void logint_print(struct logint *li);
static void logint_insert(struct phyint_instance *pii, struct logint *li);
static struct logint *logint_lookup(struct phyint_instance *pii, char *li_name);

static void target_print(struct target *tg);
static void target_insert(struct phyint_instance *pii, struct target *tg);
static struct target *target_first(struct phyint_instance *pii);
static struct target *target_select_best(struct phyint_instance *pii);
static void target_flush_hosts(struct phyint_group *pg);

static void reset_pii_probes(struct phyint_instance *pii, struct target *tg);

static boolean_t phyint_inst_v6_sockinit(struct phyint_instance *pii);
static boolean_t phyint_inst_v4_sockinit(struct phyint_instance *pii);

static int phyint_state_event(struct phyint_group *pg, struct phyint *pi);
static int phyint_group_state_event(struct phyint_group *pg);
static int phyint_group_change_event(struct phyint_group *pg, ipmp_group_op_t);
static int phyint_group_member_event(struct phyint_group *pg, struct phyint *pi,
    ipmp_if_op_t op);

static int logint_upcount(struct phyint *pi);
static uint64_t gensig(void);

/* Initialize any per-file global state.  Returns 0 on success, -1 on failure */
int
phyint_init(void)
{
        phyint_grouplistsig = gensig();
        if (track_all_phyints) {
                phyint_anongroup = phyint_group_create("");
                if (phyint_anongroup == NULL)
                        return (-1);
                phyint_group_insert(phyint_anongroup);
        }
        return (0);
}

/* Return the phyint with the given name */
struct phyint *
phyint_lookup(const char *name)
{
        struct phyint *pi;

        if (debug & D_PHYINT)
                logdebug("phyint_lookup(%s)\n", name);

        for (pi = phyints; pi != NULL; pi = pi->pi_next) {
                if (strncmp(pi->pi_name, name, sizeof (pi->pi_name)) == 0)
                        break;
        }
        return (pi);
}

/*
 * Lookup a phyint in the group that has the same hardware address as `pi', or
 * NULL if there's none.  If `online_only' is set, then only online phyints
 * are considered when matching.  Otherwise, phyints that had been offlined
 * due to a duplicate hardware address will also be considered.
 */
static struct phyint *
phyint_lookup_hwaddr(struct phyint *pi, boolean_t online_only)
{
        struct phyint *pi2;

        if (pi->pi_group == phyint_anongroup)
                return (NULL);

        for (pi2 = pi->pi_group->pg_phyint; pi2 != NULL; pi2 = pi2->pi_pgnext) {
                if (pi2 == pi)
                        continue;

                /*
                 * NOTE: even when online_only is B_FALSE, we ignore phyints
                 * that are administratively offline (rather than offline
                 * because they're dups); when they're brought back online,
                 * they'll be flagged as dups if need be.
                 */
                if (pi2->pi_state == PI_OFFLINE &&
                    (online_only || !pi2->pi_hwaddrdup))
                        continue;

                if (pi2->pi_hwaddrlen == pi->pi_hwaddrlen &&
                    bcmp(pi2->pi_hwaddr, pi->pi_hwaddr, pi->pi_hwaddrlen) == 0)
                        return (pi2);
        }
        return (NULL);
}

/*
 * Respond to DLPI notifications.  Currently, this only processes physical
 * address changes for the phyint passed via `arg' by onlining or offlining
 * phyints in the group.
 */
/* ARGSUSED */
static void
phyint_link_notify(dlpi_handle_t dh, dlpi_notifyinfo_t *dnip, void *arg)
{
        struct phyint *pi = arg;
        struct phyint *oduppi = NULL, *duppi = NULL;

        assert((dnip->dni_note & pi->pi_notes) != 0);

        if (dnip->dni_note != DL_NOTE_PHYS_ADDR)
                return;

        assert(dnip->dni_physaddrlen <= DLPI_PHYSADDR_MAX);

        /*
         * If our hardware address hasn't changed, there's nothing to do.
         */
        if (pi->pi_hwaddrlen == dnip->dni_physaddrlen &&
            bcmp(pi->pi_hwaddr, dnip->dni_physaddr, pi->pi_hwaddrlen) == 0)
                return;

        oduppi = phyint_lookup_hwaddr(pi, _B_FALSE);
        pi->pi_hwaddrlen = dnip->dni_physaddrlen;
        (void) memcpy(pi->pi_hwaddr, dnip->dni_physaddr, pi->pi_hwaddrlen);
        duppi = phyint_lookup_hwaddr(pi, _B_FALSE);

        if (oduppi != NULL || pi->pi_hwaddrdup) {
                /*
                 * Our old hardware address was a duplicate.  If we'd been
                 * offlined because of it, and our new hardware address is not
                 * a duplicate, then bring us online.  Otherwise, `oduppi'
                 * must've been the one brought offline; bring it online.
                 */
                if (pi->pi_hwaddrdup) {
                        if (duppi == NULL)
                                (void) phyint_undo_offline(pi);
                } else {
                        assert(oduppi->pi_hwaddrdup);
                        (void) phyint_undo_offline(oduppi);
                }
        }

        if (duppi != NULL && !pi->pi_hwaddrdup) {
                /*
                 * Our new hardware address was a duplicate and we're not
                 * yet flagged as a duplicate; bring us offline.
                 */
                pi->pi_hwaddrdup = _B_TRUE;
                (void) phyint_offline(pi, 0);
        }
}

/*
 * Initialize information about the underlying link for `pi', and set us
 * up to be notified about future changes.  Returns _B_TRUE on success.
 */
boolean_t
phyint_link_init(struct phyint *pi)
{
        int retval;
        uint_t notes;
        const char *errmsg;
        dlpi_notifyid_t id;

        pi->pi_notes = 0;
        retval = dlpi_open(pi->pi_name, &pi->pi_dh, 0);
        if (retval != DLPI_SUCCESS) {
                pi->pi_dh = NULL;
                errmsg = "cannot open";
                goto failed;
        }

        pi->pi_hwaddrlen = DLPI_PHYSADDR_MAX;
        retval = dlpi_get_physaddr(pi->pi_dh, DL_CURR_PHYS_ADDR, pi->pi_hwaddr,
            &pi->pi_hwaddrlen);
        if (retval != DLPI_SUCCESS) {
                errmsg = "cannot get hardware address";
                goto failed;
        }

        /*
         * Check if the link supports DLPI link state notifications.  For
         * historical reasons, the actual changes are tracked through routing
         * sockets, so we immediately disable the notification upon success.
         */
        notes = DL_NOTE_LINK_UP | DL_NOTE_LINK_DOWN;
        retval = dlpi_enabnotify(pi->pi_dh, notes, phyint_link_notify, pi, &id);
        if (retval == DLPI_SUCCESS) {
                (void) dlpi_disabnotify(pi->pi_dh, id, NULL);
                pi->pi_notes |= notes;
        }

        /*
         * Enable notification of hardware address changes to keep pi_hwaddr
         * up-to-date and track if we need to offline/undo-offline phyints.
         */
        notes = DL_NOTE_PHYS_ADDR;
        retval = dlpi_enabnotify(pi->pi_dh, notes, phyint_link_notify, pi, &id);
        if (retval == DLPI_SUCCESS && poll_add(dlpi_fd(pi->pi_dh)) == 0)
                pi->pi_notes |= notes;

        return (_B_TRUE);
failed:
        logerr("%s: %s: %s\n", pi->pi_name, errmsg, dlpi_strerror(retval));
        if (pi->pi_dh != NULL) {
                dlpi_close(pi->pi_dh);
                pi->pi_dh = NULL;
        }
        return (_B_FALSE);
}

/*
 * Close use of link on `pi'.
 */
void
phyint_link_close(struct phyint *pi)
{
        if (pi->pi_notes & DL_NOTE_PHYS_ADDR) {
                (void) poll_remove(dlpi_fd(pi->pi_dh));
                pi->pi_notes &= ~DL_NOTE_PHYS_ADDR;
        }

        /*
         * NOTE: we don't clear pi_notes here so that iflinkstate() can still
         * properly report the link state even when offline (which is possible
         * since we use IFF_RUNNING to track link state).
         */
        dlpi_close(pi->pi_dh);
        pi->pi_dh = NULL;
}

/* Return the phyint instance with the given name and the given family */
struct phyint_instance *
phyint_inst_lookup(int af, char *name)
{
        struct phyint *pi;

        if (debug & D_PHYINT)
                logdebug("phyint_inst_lookup(%s %s)\n", AF_STR(af), name);

        assert(af == AF_INET || af == AF_INET6);

        pi = phyint_lookup(name);
        if (pi == NULL)
                return (NULL);

        return (PHYINT_INSTANCE(pi, af));
}

struct phyint_group *
phyint_group_lookup(const char *pg_name)
{
        struct phyint_group *pg;

        if (debug & D_PHYINT)
                logdebug("phyint_group_lookup(%s)\n", pg_name);

        for (pg = phyint_groups; pg != NULL; pg = pg->pg_next) {
                if (strncmp(pg->pg_name, pg_name, sizeof (pg->pg_name)) == 0)
                        break;
        }
        return (pg);
}

/*
 * Insert the phyint in the linked list of all phyints. If the phyint belongs
 * to some group, insert it in the phyint group list.
 */
static void
phyint_insert(struct phyint *pi, struct phyint_group *pg)
{
        if (debug & D_PHYINT)
                logdebug("phyint_insert(%s '%s')\n", pi->pi_name, pg->pg_name);

        /* Insert the phyint at the head of the 'all phyints' list */
        pi->pi_next = phyints;
        pi->pi_prev = NULL;
        if (phyints != NULL)
                phyints->pi_prev = pi;
        phyints = pi;

        /*
         * Insert the phyint at the head of the 'phyint_group members' list
         * of the phyint group to which it belongs.
         */
        pi->pi_pgnext = NULL;
        pi->pi_pgprev = NULL;
        pi->pi_group = pg;

        pi->pi_pgnext = pg->pg_phyint;
        if (pi->pi_pgnext != NULL)
                pi->pi_pgnext->pi_pgprev = pi;
        pg->pg_phyint = pi;

        /* Refresh the group state now that this phyint has been added */
        phyint_group_refresh_state(pg);

        pg->pg_sig++;
        (void) phyint_group_member_event(pg, pi, IPMP_IF_ADD);
}

/* Insert the phyint instance in the linked list of all phyint instances. */
static void
phyint_inst_insert(struct phyint_instance *pii)
{
        if (debug & D_PHYINT) {
                logdebug("phyint_inst_insert(%s %s)\n",
                    AF_STR(pii->pii_af), pii->pii_name);
        }

        /*
         * Insert the phyint at the head of the 'all phyint instances' list.
         */
        pii->pii_next = phyint_instances;
        pii->pii_prev = NULL;
        if (phyint_instances != NULL)
                phyint_instances->pii_prev = pii;
        phyint_instances = pii;
}

/*
 * Create a new phyint with the given parameters. Also insert it into
 * the list of all phyints and the list of phyint group members by calling
 * phyint_insert().
 */
static struct phyint *
phyint_create(char *pi_name, struct phyint_group *pg, uint_t ifindex,
    uint64_t flags)
{
        struct phyint *pi;

        pi = calloc(1, sizeof (struct phyint));
        if (pi == NULL) {
                logperror("phyint_create: calloc");
                return (NULL);
        }

        /*
         * Record the phyint values.
         */
        (void) strlcpy(pi->pi_name, pi_name, sizeof (pi->pi_name));
        pi->pi_taddrthresh = getcurrentsec() + TESTADDR_CONF_TIME;
        pi->pi_ifindex = ifindex;
        pi->pi_icmpid = htons(((getpid() & 0xFF) << 8) | (ifindex & 0xFF));

        pi->pi_state = PI_INIT;
        pi->pi_flags = PHYINT_FLAGS(flags);

        /*
         * Initialize the link state.  The link state is initialized to
         * up, so that if the link is down when IPMP starts monitoring
         * the interface, it will appear as though there has been a
         * transition from the link up to link down.  This avoids
         * having to treat this situation as a special case.
         */
        INIT_LINK_STATE(pi);

        if (!phyint_link_init(pi)) {
                free(pi);
                return (NULL);
        }

        /*
         * Insert the phyint in the list of all phyints, and the
         * list of phyint group members
         */
        phyint_insert(pi, pg);

        return (pi);
}

/*
 * Create a new phyint instance belonging to the phyint 'pi' and address
 * family 'af'. Also insert it into the list of all phyint instances by
 * calling phyint_inst_insert().
 */
static struct phyint_instance *
phyint_inst_create(struct phyint *pi, int af)
{
        struct phyint_instance *pii;

        pii = calloc(1, sizeof (struct phyint_instance));
        if (pii == NULL) {
                logperror("phyint_inst_create: calloc");
                return (NULL);
        }

        /*
         * Attach the phyint instance to the phyint.
         * Set the back pointers as well
         */
        pii->pii_phyint = pi;
        if (af == AF_INET)
                pi->pi_v4 = pii;
        else
                pi->pi_v6 = pii;

        pii->pii_in_use = 1;
        pii->pii_probe_sock = -1;
        pii->pii_snxt = 1;
        pii->pii_af = af;
        pii->pii_fd_hrtime = gethrtime() +
            (FAILURE_DETECTION_QP * (hrtime_t)NANOSEC);
        pii->pii_flags = pi->pi_flags;

        /* Insert the phyint instance in the list of all phyint instances. */
        phyint_inst_insert(pii);
        return (pii);
}

/*
 * Change the state of phyint `pi' to state `state'.
 */
void
phyint_chstate(struct phyint *pi, enum pi_state state)
{
        /*
         * To simplify things, some callers always set a given state
         * regardless of the previous state of the phyint (e.g., setting
         * PI_RUNNING when it's already set).  We shouldn't bother
         * generating an event or consuming a signature for these, since
         * the actual state of the interface is unchanged.
         */
        if (pi->pi_state == state)
                return;

        pi->pi_state = state;
        phyint_changed(pi);
}

/*
 * Note that `pi' has changed state.
 */
void
phyint_changed(struct phyint *pi)
{
        pi->pi_group->pg_sig++;
        (void) phyint_state_event(pi->pi_group, pi);
}

/*
 * Insert the phyint group in the linked list of all phyint groups
 * at the head of the list
 */
void
phyint_group_insert(struct phyint_group *pg)
{
        pg->pg_next = phyint_groups;
        pg->pg_prev = NULL;
        if (phyint_groups != NULL)
                phyint_groups->pg_prev = pg;
        phyint_groups = pg;

        phyint_grouplistsig++;
        (void) phyint_group_change_event(pg, IPMP_GROUP_ADD);
}

/*
 * Create a new phyint group called 'name'.
 */
struct phyint_group *
phyint_group_create(const char *name)
{
        struct  phyint_group *pg;

        if (debug & D_PHYINT)
                logdebug("phyint_group_create(%s)\n", name);

        pg = calloc(1, sizeof (struct phyint_group));
        if (pg == NULL) {
                logperror("phyint_group_create: calloc");
                return (NULL);
        }

        (void) strlcpy(pg->pg_name, name, sizeof (pg->pg_name));
        pg->pg_sig = gensig();
        pg->pg_fdt = user_failure_detection_time;
        pg->pg_probeint = user_probe_interval;
        pg->pg_in_use = _B_TRUE;

        /*
         * Normal groups always start in the PG_FAILED state since they
         * have no active interfaces.  In contrast, anonymous groups are
         * heterogeneous and thus always PG_OK.
         */
        pg->pg_state = (name[0] == '\0' ? PG_OK : PG_FAILED);

        return (pg);
}

/*
 * Change the state of the phyint group `pg' to state `state'.
 */
void
phyint_group_chstate(struct phyint_group *pg, enum pg_state state)
{
        assert(pg != phyint_anongroup);

        /*
         * To simplify things, some callers always set a given state
         * regardless of the previous state of the group (e.g., setting
         * PG_DEGRADED when it's already set).  We shouldn't bother
         * generating an event or consuming a signature for these, since
         * the actual state of the group is unchanged.
         */
        if (pg->pg_state == state)
                return;

        pg->pg_state = state;

        switch (state) {
        case PG_FAILED:
                /*
                 * We can never know with certainty that a group has
                 * failed.  It is possible that all known targets have
                 * failed simultaneously, and new targets have come up
                 * instead. If the targets are routers then router
                 * discovery will kick in, and we will see the new routers
                 * thru routing socket messages. But if the targets are
                 * hosts, we have to discover it by multicast.  So flush
                 * all the host targets. The next probe will send out a
                 * multicast echo request. If this is a group failure, we
                 * will still not see any response, otherwise the group
                 * will be repaired after we get NUM_PROBE_REPAIRS
                 * consecutive unicast replies on any phyint.
                 */
                target_flush_hosts(pg);
                break;

        case PG_OK:
        case PG_DEGRADED:
                break;

        default:
                logerr("phyint_group_chstate: invalid group state %d; "
                    "aborting\n", state);
                abort();
        }

        pg->pg_sig++;
        (void) phyint_group_state_event(pg);
}

/*
 * Create a new phyint instance and initialize it from the values supplied by
 * the kernel. Always check for ENXIO before logging any error, because the
 * interface could have vanished after completion of SIOCGLIFCONF.
 * Return values:
 *      pointer to the phyint instance on success
 *      NULL on failure Eg. if the phyint instance is not found in the kernel
 */
struct phyint_instance *
phyint_inst_init_from_k(int af, char *pi_name)
{
        char    pg_name[LIFNAMSIZ + 1];
        int     ifsock;
        uint_t  ifindex;
        uint64_t        flags;
        struct lifreq   lifr;
        struct phyint   *pi;
        struct phyint_instance  *pii;
        boolean_t       pi_created;
        struct phyint_group     *pg;

retry:
        pii = NULL;
        pi = NULL;
        pg = NULL;
        pi_created = _B_FALSE;

        if (debug & D_PHYINT) {
                logdebug("phyint_inst_init_from_k(%s %s)\n",
                    AF_STR(af), pi_name);
        }

        assert(af == AF_INET || af == AF_INET6);

        /* Get the socket for doing ioctls */
        ifsock = (af == AF_INET) ? ifsock_v4 : ifsock_v6;

        /*
         * Get the interface flags.  Ignore virtual interfaces, IPMP
         * meta-interfaces, point-to-point interfaces, and interfaces
         * that can't support multicast.
         */
        (void) strlcpy(lifr.lifr_name, pi_name, sizeof (lifr.lifr_name));
        if (ioctl(ifsock, SIOCGLIFFLAGS, (char *)&lifr) < 0) {
                if (errno != ENXIO) {
                        logperror("phyint_inst_init_from_k:"
                            " ioctl (get flags)");
                }
                return (NULL);
        }
        flags = lifr.lifr_flags;
        if (!(flags & IFF_MULTICAST) ||
            (flags & (IFF_VIRTUAL|IFF_IPMP|IFF_POINTOPOINT)))
                return (NULL);

        /*
         * Get the ifindex for recording later in our tables, in case we need
         * to create a new phyint.
         */
        if (ioctl(ifsock, SIOCGLIFINDEX, (char *)&lifr) < 0) {
                if (errno != ENXIO) {
                        logperror("phyint_inst_init_from_k: "
                            " ioctl (get lifindex)");
                }
                return (NULL);
        }
        ifindex = lifr.lifr_index;

        /*
         * Get the phyint group name of this phyint, from the kernel.
         */
        if (ioctl(ifsock, SIOCGLIFGROUPNAME, (char *)&lifr) < 0) {
                if (errno != ENXIO) {
                        logperror("phyint_inst_init_from_k: "
                            "ioctl (get group name)");
                }
                return (NULL);
        }
        (void) strlcpy(pg_name, lifr.lifr_groupname, sizeof (pg_name));

        /*
         * If the phyint is not part of any group, pg_name is the
         * null string. If 'track_all_phyints' is false, there is no
         * need to create a phyint.
         */
        if (pg_name[0] == '\0' && !track_all_phyints) {
                /*
                 * If the IFF_FAILED, IFF_INACTIVE, or IFF_OFFLINE flags are
                 * set, reset them. These flags shouldn't be set if in.mpathd
                 * isn't tracking the interface.
                 */
                if ((flags & (IFF_FAILED | IFF_INACTIVE | IFF_OFFLINE))) {
                        lifr.lifr_flags = flags &
                            ~(IFF_FAILED | IFF_INACTIVE | IFF_OFFLINE);
                        if (ioctl(ifsock, SIOCSLIFFLAGS, (char *)&lifr) < 0) {
                                if (errno != ENXIO) {
                                        logperror("phyint_inst_init_from_k:"
                                            " ioctl (set flags)");
                                }
                        }
                }
                return (NULL);
        }

        /*
         * We need to create a new phyint instance.  We may also need to
         * create the group if e.g. the SIOCGLIFCONF loop in initifs() found
         * an underlying interface before it found its IPMP meta-interface.
         * Note that we keep any created groups even if phyint_inst_from_k()
         * fails since a group's existence is not dependent on the ability of
         * in.mpathd to the track the group's interfaces.
         */
        if ((pg = phyint_group_lookup(pg_name)) == NULL) {
                if ((pg = phyint_group_create(pg_name)) == NULL) {
                        logerr("phyint_inst_init_from_k: cannot create group "
                            "%s\n", pg_name);
                        return (NULL);
                }
                phyint_group_insert(pg);
        }

        /*
         * Lookup the phyint. If the phyint does not exist create it.
         */
        pi = phyint_lookup(pi_name);
        if (pi == NULL) {
                pi = phyint_create(pi_name, pg, ifindex, flags);
                if (pi == NULL) {
                        logerr("phyint_inst_init_from_k:"
                            " unable to create phyint %s\n", pi_name);
                        return (NULL);
                }
                pi_created = _B_TRUE;
        } else {
                /* The phyint exists already. */
                assert(pi_created == _B_FALSE);
                /*
                 * Normally we should see consistent values for the IPv4 and
                 * IPv6 instances, for phyint properties. If we don't, it
                 * means things have changed underneath us, and we should
                 * resync our tables with the kernel. Check whether the
                 * interface index has changed. If so, it is most likely
                 * the interface has been unplumbed and replumbed,
                 * while we are yet to update our tables. Do it now.
                 */
                if (pi->pi_ifindex != ifindex) {
                        phyint_inst_delete(PHYINT_INSTANCE(pi, AF_OTHER(af)));
                        goto retry;
                }
                assert(PHYINT_INSTANCE(pi, af) == NULL);

                /*
                 * If the group name seen by the IPv4 and IPv6 instances
                 * are different, it is most likely the groupname has
                 * changed, while we are yet to update our tables. Do it now.
                 */
                if (strcmp(pi->pi_group->pg_name, pg_name) != 0) {
                        phyint_inst_delete(PHYINT_INSTANCE(pi,
                            AF_OTHER(af)));
                        goto retry;
                }
        }

        /*
         * Create a new phyint instance, corresponding to the 'af'
         * passed in.
         */
        pii = phyint_inst_create(pi, af);
        if (pii == NULL) {
                logerr("phyint_inst_init_from_k: unable to create"
                    "phyint inst %s\n", pi->pi_name);
                if (pi_created)
                        phyint_delete(pi);

                return (NULL);
        }

        /*
         * NOTE: the change_pif_flags() implementation requires a phyint
         * instance before it can function, so a number of tasks that would
         * otherwise be done in phyint_create() are deferred to here.
         */
        if (pi_created) {
                /*
                 * If the interface is offline, set the state to PI_OFFLINE.
                 * Otherwise, optimistically consider this interface running.
                 * Later (in process_link_state_changes()), we will adjust
                 * this to match the current state of the link.  Further, if
                 * test addresses are subsequently assigned, we will
                 * transition to PI_NOTARGETS and then to either PI_RUNNING or
                 * PI_FAILED depending on the probe results.
                 */
                if (pi->pi_flags & IFF_OFFLINE) {
                        phyint_chstate(pi, PI_OFFLINE);
                } else {
                        /* calls phyint_chstate() */
                        phyint_transition_to_running(pi);
                }

                /*
                 * If this a standby phyint, determine whether it should be
                 * IFF_INACTIVE.
                 */
                if (pi->pi_flags & IFF_STANDBY)
                        phyint_standby_refresh_inactive(pi);

                /*
                 * If this phyint does not have a unique hardware address in its
                 * group, offline it.
                 */
                if (phyint_lookup_hwaddr(pi, _B_TRUE) != NULL) {
                        pi->pi_hwaddrdup = _B_TRUE;
                        (void) phyint_offline(pi, 0);
                }
        }

        return (pii);
}

/*
 * Bind pii_probe_sock to the address associated with pii_probe_logint.
 * This socket will be used for sending and receiving ICMP/ICMPv6 probes to
 * targets. Do the common part in this function, and complete the
 * initializations by calling the protocol specific functions
 * phyint_inst_v{4,6}_sockinit() respectively.
 *
 * Return values: _B_TRUE/_B_FALSE for success or failure respectively.
 */
boolean_t
phyint_inst_sockinit(struct phyint_instance *pii)
{
        boolean_t success;
        struct phyint_group *pg;

        if (debug & D_PHYINT) {
                logdebug("phyint_inst_sockinit(%s %s)\n",
                    AF_STR(pii->pii_af), pii->pii_name);
        }

        assert(pii->pii_probe_logint != NULL);
        assert(pii->pii_probe_logint->li_flags & IFF_UP);
        assert(pii->pii_probe_logint->li_flags & IFF_NOFAILOVER);
        assert(pii->pii_af == AF_INET || pii->pii_af == AF_INET6);

        /*
         * If the socket is already bound, close pii_probe_sock
         */
        if (pii->pii_probe_sock != -1)
                close_probe_socket(pii, _B_TRUE);

        /*
         * If the phyint is not part of a named group and track_all_phyints is
         * false, simply return.
         */
        pg = pii->pii_phyint->pi_group;
        if (pg == phyint_anongroup && !track_all_phyints) {
                if (debug & D_PHYINT)
                        logdebug("phyint_inst_sockinit: no group\n");
                return (_B_FALSE);
        }

        /*
         * Initialize the socket by calling the protocol specific function.
         * If it succeeds, add the socket to the poll list.
         */
        if (pii->pii_af == AF_INET6)
                success = phyint_inst_v6_sockinit(pii);
        else
                success = phyint_inst_v4_sockinit(pii);

        if (success && (poll_add(pii->pii_probe_sock) == 0))
                return (_B_TRUE);

        /* Something failed, cleanup and return false */
        if (pii->pii_probe_sock != -1)
                close_probe_socket(pii, _B_FALSE);

        return (_B_FALSE);
}

/*
 * IPv6 specific part in initializing the pii_probe_sock. This socket is
 * used to send/receive ICMPv6 probe packets.
 */
static boolean_t
phyint_inst_v6_sockinit(struct phyint_instance *pii)
{
        icmp6_filter_t filter;
        int hopcount = 1;
        int off = 0;
        int on = 1;
        struct  sockaddr_in6    testaddr;
        int flags;

        /*
         * Open a raw socket with ICMPv6 protocol.
         *
         * Use IPV6_BOUND_IF to make sure that probes are sent and received on
         * the specified phyint only.  Bind to the test address to ensure that
         * the responses are sent to the specified phyint.
         *
         * Set the hopcount to 1 so that probe packets are not routed.
         * Disable multicast loopback. Set the receive filter to
         * receive only ICMPv6 echo replies.
         */
        pii->pii_probe_sock = socket(pii->pii_af, SOCK_RAW, IPPROTO_ICMPV6);
        if (pii->pii_probe_sock < 0) {
                logperror_pii(pii, "phyint_inst_v6_sockinit: socket");
                return (_B_FALSE);
        }

        /*
         * Probes must not block in case of lower layer issues.
         */
        if ((flags = fcntl(pii->pii_probe_sock, F_GETFL, 0)) == -1) {
                logperror_pii(pii, "phyint_inst_v6_sockinit: fcntl"
                    " F_GETFL");
                return (_B_FALSE);
        }
        if (fcntl(pii->pii_probe_sock, F_SETFL,
            flags | O_NONBLOCK) == -1) {
                logperror_pii(pii, "phyint_inst_v6_sockinit: fcntl"
                    " F_SETFL O_NONBLOCK");
                return (_B_FALSE);
        }

        bzero(&testaddr, sizeof (testaddr));
        testaddr.sin6_family = AF_INET6;
        testaddr.sin6_port = 0;
        testaddr.sin6_addr = pii->pii_probe_logint->li_addr;

        if (bind(pii->pii_probe_sock, (struct sockaddr *)&testaddr,
            sizeof (testaddr)) < 0) {
                logperror_pii(pii, "phyint_inst_v6_sockinit: IPv6 bind");
                return (_B_FALSE);
        }

        if (setsockopt(pii->pii_probe_sock, IPPROTO_IPV6, IPV6_MULTICAST_IF,
            (char *)&pii->pii_ifindex, sizeof (uint_t)) < 0) {
                logperror_pii(pii, "phyint_inst_v6_sockinit: setsockopt"
                    " IPV6_MULTICAST_IF");
                return (_B_FALSE);
        }

        if (setsockopt(pii->pii_probe_sock, IPPROTO_IPV6, IPV6_BOUND_IF,
            &pii->pii_ifindex, sizeof (uint_t)) < 0) {
                logperror_pii(pii, "phyint_inst_v6_sockinit: setsockopt"
                    " IPV6_BOUND_IF");
                return (_B_FALSE);
        }

        if (setsockopt(pii->pii_probe_sock, IPPROTO_IPV6, IPV6_UNICAST_HOPS,
            (char *)&hopcount, sizeof (hopcount)) < 0) {
                logperror_pii(pii, "phyint_inst_v6_sockinit: setsockopt"
                    " IPV6_UNICAST_HOPS");
                return (_B_FALSE);
        }

        if (setsockopt(pii->pii_probe_sock, IPPROTO_IPV6, IPV6_MULTICAST_HOPS,
            (char *)&hopcount, sizeof (hopcount)) < 0) {
                logperror_pii(pii, "phyint_inst_v6_sockinit: setsockopt"
                    " IPV6_MULTICAST_HOPS");
                return (_B_FALSE);
        }

        if (setsockopt(pii->pii_probe_sock, IPPROTO_IPV6, IPV6_MULTICAST_LOOP,
            (char *)&off, sizeof (off)) < 0) {
                logperror_pii(pii, "phyint_inst_v6_sockinit: setsockopt"
                    " IPV6_MULTICAST_LOOP");
                return (_B_FALSE);
        }

        /*
         * Filter out so that we only receive ICMP echo replies
         */
        ICMP6_FILTER_SETBLOCKALL(&filter);
        ICMP6_FILTER_SETPASS(ICMP6_ECHO_REPLY, &filter);

        if (setsockopt(pii->pii_probe_sock, IPPROTO_ICMPV6, ICMP6_FILTER,
            (char *)&filter, sizeof (filter)) < 0) {
                logperror_pii(pii, "phyint_inst_v6_sockinit: setsockopt"
                    " ICMP6_FILTER");
                return (_B_FALSE);
        }

        /* Enable receipt of hoplimit */
        if (setsockopt(pii->pii_probe_sock, IPPROTO_IPV6, IPV6_RECVHOPLIMIT,
            &on, sizeof (on)) < 0) {
                logperror_pii(pii, "phyint_inst_v6_sockinit: setsockopt"
                    " IPV6_RECVHOPLIMIT");
                return (_B_FALSE);
        }

        /* Enable receipt of timestamp */
        if (setsockopt(pii->pii_probe_sock, SOL_SOCKET, SO_TIMESTAMP,
            &on, sizeof (on)) < 0) {
                logperror_pii(pii, "phyint_inst_v6_sockinit: setsockopt"
                    " SO_TIMESTAMP");
                return (_B_FALSE);
        }

        return (_B_TRUE);
}

/*
 * IPv4 specific part in initializing the pii_probe_sock. This socket is
 * used to send/receive ICMPv4 probe packets.
 */
static boolean_t
phyint_inst_v4_sockinit(struct phyint_instance *pii)
{
        struct sockaddr_in  testaddr;
        char    char_off = 0;
        int     ttl = 1;
        char    char_ttl = 1;
        int     on = 1;
        int     flags;

        /*
         * Open a raw socket with ICMPv4 protocol.
         *
         * Use IP_BOUND_IF to make sure that probes are sent and received on
         * the specified phyint only.  Bind to the test address to ensure that
         * the responses are sent to the specified phyint.
         *
         * Set the ttl to 1 so that probe packets are not routed.
         * Disable multicast loopback.  Enable receipt of timestamp.
         */
        pii->pii_probe_sock = socket(pii->pii_af, SOCK_RAW, IPPROTO_ICMP);
        if (pii->pii_probe_sock < 0) {
                logperror_pii(pii, "phyint_inst_v4_sockinit: socket");
                return (_B_FALSE);
        }

        /*
         * Probes must not block in case of lower layer issues.
         */
        if ((flags = fcntl(pii->pii_probe_sock, F_GETFL, 0)) == -1) {
                logperror_pii(pii, "phyint_inst_v4_sockinit: fcntl"
                    " F_GETFL");
                return (_B_FALSE);
        }
        if (fcntl(pii->pii_probe_sock, F_SETFL,
            flags | O_NONBLOCK) == -1) {
                logperror_pii(pii, "phyint_inst_v4_sockinit: fcntl"
                    " F_SETFL O_NONBLOCK");
                return (_B_FALSE);
        }

        bzero(&testaddr, sizeof (testaddr));
        testaddr.sin_family = AF_INET;
        testaddr.sin_port = 0;
        IN6_V4MAPPED_TO_INADDR(&pii->pii_probe_logint->li_addr,
            &testaddr.sin_addr);

        if (bind(pii->pii_probe_sock, (struct sockaddr *)&testaddr,
            sizeof (testaddr)) < 0) {
                logperror_pii(pii, "phyint_inst_v4_sockinit: IPv4 bind");
                return (_B_FALSE);
        }

        if (setsockopt(pii->pii_probe_sock, IPPROTO_IP, IP_BOUND_IF,
            &pii->pii_ifindex, sizeof (uint_t)) < 0) {
                logperror_pii(pii, "phyint_inst_v4_sockinit: setsockopt"
                    " IP_BOUND_IF");
                return (_B_FALSE);
        }

        if (setsockopt(pii->pii_probe_sock, IPPROTO_IP, IP_MULTICAST_IF,
            (char *)&testaddr.sin_addr, sizeof (struct in_addr)) < 0) {
                logperror_pii(pii, "phyint_inst_v4_sockinit: setsockopt"
                    " IP_MULTICAST_IF");
                return (_B_FALSE);
        }

        if (setsockopt(pii->pii_probe_sock, IPPROTO_IP, IP_TTL,
            (char *)&ttl, sizeof (ttl)) < 0) {
                logperror_pii(pii, "phyint_inst_v4_sockinit: setsockopt"
                    " IP_TTL");
                return (_B_FALSE);
        }

        if (setsockopt(pii->pii_probe_sock, IPPROTO_IP, IP_MULTICAST_LOOP,
            (char *)&char_off, sizeof (char_off)) == -1) {
                logperror_pii(pii, "phyint_inst_v4_sockinit: setsockopt"
                    " IP_MULTICAST_LOOP");
                return (_B_FALSE);
        }

        if (setsockopt(pii->pii_probe_sock, IPPROTO_IP, IP_MULTICAST_TTL,
            (char *)&char_ttl, sizeof (char_ttl)) == -1) {
                logperror_pii(pii, "phyint_inst_v4_sockinit: setsockopt"
                    " IP_MULTICAST_TTL");
                return (_B_FALSE);
        }

        if (setsockopt(pii->pii_probe_sock, SOL_SOCKET, SO_TIMESTAMP, &on,
            sizeof (on)) < 0) {
                logperror_pii(pii, "phyint_inst_v4_sockinit: setsockopt"
                    " SO_TIMESTAMP");
                return (_B_FALSE);
        }

        return (_B_TRUE);
}

/*
 * Remove the phyint group from the list of 'all phyint groups'
 * and free it.
 */
void
phyint_group_delete(struct phyint_group *pg)
{
        /*
         * The anonymous group always exists, even when empty.
         */
        if (pg == phyint_anongroup)
                return;

        if (debug & D_PHYINT)
                logdebug("phyint_group_delete('%s')\n", pg->pg_name);

        /*
         * The phyint group must be empty, and must not have any phyints.
         * The phyint group must be in the list of all phyint groups
         */
        assert(pg->pg_phyint == NULL);
        assert(phyint_groups == pg || pg->pg_prev != NULL);

        if (pg->pg_prev != NULL)
                pg->pg_prev->pg_next = pg->pg_next;
        else
                phyint_groups = pg->pg_next;

        if (pg->pg_next != NULL)
                pg->pg_next->pg_prev = pg->pg_prev;

        pg->pg_next = NULL;
        pg->pg_prev = NULL;

        phyint_grouplistsig++;
        (void) phyint_group_change_event(pg, IPMP_GROUP_REMOVE);

        addrlist_free(&pg->pg_addrs);
        free(pg);
}

/*
 * Refresh the state of `pg' based on its current members.
 */
void
phyint_group_refresh_state(struct phyint_group *pg)
{
        enum pg_state state;
        enum pg_state origstate = pg->pg_state;
        struct phyint *pi, *usablepi;
        uint_t nif = 0, nusable = 0;

        /*
         * Anonymous groups never change state.
         */
        if (pg == phyint_anongroup)
                return;

        for (pi = pg->pg_phyint; pi != NULL; pi = pi->pi_pgnext) {
                nif++;
                if (phyint_is_usable(pi)) {
                        nusable++;
                        usablepi = pi;
                }
        }

        if (nusable == 0)
                state = PG_FAILED;
        else if (nif == nusable)
                state = PG_OK;
        else
                state = PG_DEGRADED;

        phyint_group_chstate(pg, state);

        /*
         * If we're shutting down, skip logging messages since otherwise our
         * shutdown housecleaning will make us report that groups are unusable.
         */
        if (cleanup_started)
                return;

        /*
         * NOTE: We use pg_failmsg_printed rather than origstate since
         * otherwise at startup we'll log a "now usable" message when the
         * first usable phyint is added to an empty group.
         */
        if (state != PG_FAILED && pg->pg_failmsg_printed) {
                assert(origstate == PG_FAILED);
                logerr("At least 1 IP interface (%s) in group %s is now "
                    "usable\n", usablepi->pi_name, pg->pg_name);
                pg->pg_failmsg_printed = _B_FALSE;
        } else if (origstate != PG_FAILED && state == PG_FAILED) {
                logerr("All IP interfaces in group %s are now unusable\n",
                    pg->pg_name);
                pg->pg_failmsg_printed = _B_TRUE;
        }
}

/*
 * Extract information from the kernel about the desired phyint.
 * Look only for properties of the phyint and not properties of logints.
 * Take appropriate action on the changes.
 * Return codes:
 *      PI_OK
 *              The phyint exists in the kernel and matches our knowledge
 *              of the phyint.
 *      PI_DELETED
 *              The phyint has vanished in the kernel.
 *      PI_IFINDEX_CHANGED
 *              The phyint's interface index has changed.
 *              Ask the caller to delete and recreate the phyint.
 *      PI_IOCTL_ERROR
 *              Some ioctl error. Don't change anything.
 *      PI_GROUP_CHANGED
 *              The phyint has changed group.
 */
int
phyint_inst_update_from_k(struct phyint_instance *pii)
{
        struct lifreq lifr;
        int     ifsock;
        struct phyint *pi;

        pi = pii->pii_phyint;

        if (debug & D_PHYINT) {
                logdebug("phyint_inst_update_from_k(%s %s)\n",
                    AF_STR(pii->pii_af), pi->pi_name);
        }

        /*
         * Get the ifindex from the kernel, for comparison with the
         * value in our tables.
         */
        (void) strncpy(lifr.lifr_name, pi->pi_name, sizeof (lifr.lifr_name));
        lifr.lifr_name[sizeof (lifr.lifr_name) - 1] = '\0';

        ifsock = (pii->pii_af == AF_INET) ? ifsock_v4 : ifsock_v6;
        if (ioctl(ifsock, SIOCGLIFINDEX, &lifr) < 0) {
                if (errno == ENXIO) {
                        return (PI_DELETED);
                } else {
                        logperror_pii(pii, "phyint_inst_update_from_k:"
                            " ioctl (get lifindex)");
                        return (PI_IOCTL_ERROR);
                }
        }

        if (lifr.lifr_index != pi->pi_ifindex) {
                /*
                 * The index has changed. Most likely the interface has
                 * been unplumbed and replumbed. Ask the caller to take
                 * appropriate action.
                 */
                if (debug & D_PHYINT) {
                        logdebug("phyint_inst_update_from_k:"
                            " old index %d new index %d\n",
                            pi->pi_ifindex, lifr.lifr_index);
                }
                return (PI_IFINDEX_CHANGED);
        }

        /*
         * Get the group name from the kernel, for comparison with
         * the value in our tables.
         */
        if (ioctl(ifsock, SIOCGLIFGROUPNAME, &lifr) < 0) {
                if (errno == ENXIO) {
                        return (PI_DELETED);
                } else {
                        logperror_pii(pii, "phyint_inst_update_from_k:"
                            " ioctl (get groupname)");
                        return (PI_IOCTL_ERROR);
                }
        }

        /*
         * If the phyint has changed group i.e. if the phyint group name
         * returned by the kernel is different, ask the caller to delete
         * and recreate the phyint in the right group
         */
        if (strcmp(lifr.lifr_groupname, pi->pi_group->pg_name) != 0) {
                /* Groupname has changed */
                if (debug & D_PHYINT) {
                        logdebug("phyint_inst_update_from_k:"
                            " groupname change\n");
                }
                return (PI_GROUP_CHANGED);
        }

        /*
         * Get the current phyint flags from the kernel, and determine what
         * flags have changed by comparing against our tables.  Note that the
         * IFF_INACTIVE processing in initifs() relies on this call to ensure
         * that IFF_INACTIVE is really still set on the interface.
         */
        if (ioctl(ifsock, SIOCGLIFFLAGS, &lifr) < 0) {
                if (errno == ENXIO) {
                        return (PI_DELETED);
                } else {
                        logperror_pii(pii, "phyint_inst_update_from_k: "
                            " ioctl (get flags)");
                        return (PI_IOCTL_ERROR);
                }
        }

        pi->pi_flags = PHYINT_FLAGS(lifr.lifr_flags);
        if (pi->pi_v4 != NULL)
                pi->pi_v4->pii_flags = pi->pi_flags;
        if (pi->pi_v6 != NULL)
                pi->pi_v6->pii_flags = pi->pi_flags;

        /*
         * Make sure the IFF_FAILED flag is set if and only if we think
         * the interface should be failed.
         */
        if (pi->pi_flags & IFF_FAILED) {
                if (pi->pi_state == PI_RUNNING)
                        (void) change_pif_flags(pi, 0, IFF_FAILED);
        } else {
                if (pi->pi_state == PI_FAILED)
                        (void) change_pif_flags(pi, IFF_FAILED, IFF_INACTIVE);
        }

        /* No change in phyint status */
        return (PI_OK);
}

/*
 * Delete the phyint. Remove it from the list of all phyints, and the
 * list of phyint group members.
 */
static void
phyint_delete(struct phyint *pi)
{
        boolean_t active;
        struct phyint *pi2;
        struct phyint_group *pg = pi->pi_group;

        if (debug & D_PHYINT)
                logdebug("phyint_delete(%s)\n", pi->pi_name);

        /* Both IPv4 and IPv6 phyint instances must have been deleted. */
        assert(pi->pi_v4 == NULL && pi->pi_v6 == NULL);

        /*
         * The phyint must belong to a group.
         */
        assert(pg->pg_phyint == pi || pi->pi_pgprev != NULL);

        /* The phyint must be in the list of all phyints */
        assert(phyints == pi || pi->pi_prev != NULL);

        /* Remove the phyint from the phyint group list */
        pg->pg_sig++;
        (void) phyint_group_member_event(pg, pi, IPMP_IF_REMOVE);

        if (pi->pi_pgprev == NULL) {
                /* Phyint is the 1st in the phyint group list */
                pg->pg_phyint = pi->pi_pgnext;
        } else {
                pi->pi_pgprev->pi_pgnext = pi->pi_pgnext;
        }
        if (pi->pi_pgnext != NULL)
                pi->pi_pgnext->pi_pgprev = pi->pi_pgprev;
        pi->pi_pgnext = NULL;
        pi->pi_pgprev = NULL;

        /* Refresh the group state now that this phyint has been removed */
        phyint_group_refresh_state(pg);

        /* Remove the phyint from the global list of phyints */
        if (pi->pi_prev == NULL) {
                /* Phyint is the 1st in the list */
                phyints = pi->pi_next;
        } else {
                pi->pi_prev->pi_next = pi->pi_next;
        }
        if (pi->pi_next != NULL)
                pi->pi_next->pi_prev = pi->pi_prev;
        pi->pi_next = NULL;
        pi->pi_prev = NULL;

        /*
         * See if another phyint in the group had been offlined because
         * it was a dup of `pi' -- and if so, online it.
         */
        if (!pi->pi_hwaddrdup &&
            (pi2 = phyint_lookup_hwaddr(pi, _B_FALSE)) != NULL) {
                assert(pi2->pi_hwaddrdup);
                (void) phyint_undo_offline(pi2);
        }

        /*
         * If the interface was in a named group and was either an active
         * standby or the last active interface, try to activate another
         * interface to compensate.
         */
        if (pg != phyint_anongroup) {
                active = _B_FALSE;
                for (pi2 = pg->pg_phyint; pi2 != NULL; pi2 = pi2->pi_pgnext) {
                        if (phyint_is_functioning(pi2) &&
                            !(pi2->pi_flags & IFF_INACTIVE)) {
                                active = _B_TRUE;
                                break;
                        }
                }

                if (!active ||
                    (pi->pi_flags & (IFF_STANDBY|IFF_INACTIVE)) == IFF_STANDBY)
                        phyint_activate_another(pi);
        }

        phyint_link_close(pi);
        free(pi);
}

/*
 * Offline phyint `pi' if at least `minred' usable interfaces remain in the
 * group.  Returns an IPMP error code.
 */
int
phyint_offline(struct phyint *pi, uint_t minred)
{
        boolean_t was_active;
        unsigned int nusable = 0;
        struct phyint *pi2;
        struct phyint_group *pg = pi->pi_group;

        /*
         * Verify that enough usable interfaces in the group would remain.
         * As a special case, if the group has failed, allow any non-offline
         * phyints to be offlined.
         */
        if (pg != phyint_anongroup) {
                for (pi2 = pg->pg_phyint; pi2 != NULL; pi2 = pi2->pi_pgnext) {
                        if (pi2 == pi)
                                continue;
                        if (phyint_is_usable(pi2) ||
                            (GROUP_FAILED(pg) && pi2->pi_state != PI_OFFLINE))
                                nusable++;
                }
        }
        if (nusable < minred)
                return (IPMP_EMINRED);

        was_active = ((pi->pi_flags & IFF_INACTIVE) == 0);

        if (!change_pif_flags(pi, IFF_OFFLINE, IFF_INACTIVE))
                return (IPMP_FAILURE);

        /*
         * The interface is now offline, so stop probing it.  Note that
         * if_mpadm(1M) will down the test addresses, after receiving a
         * success reply from us. The routing socket message will then make us
         * close the socket used for sending probes. But it is more logical
         * that an offlined interface must not be probed, even if it has test
         * addresses.
         *
         * NOTE: stop_probing() also sets PI_OFFLINE.
         */
        stop_probing(pi);

        /*
         * If we're offlining the phyint because it has a duplicate hardware
         * address, print a warning -- and leave the link open so that we can
         * be notified of hardware address changes that make it usable again.
         * Otherwise, close the link so that we won't prevent a detach.
         */
        if (pi->pi_hwaddrdup) {
                logerr("IP interface %s has a hardware address which is not "
                    "unique in group %s; offlining\n", pi->pi_name,
                    pg->pg_name);
        } else {
                phyint_link_close(pi);
        }

        /*
         * If this phyint was preventing another phyint with a duplicate
         * hardware address from being online, bring that one online now.
         */
        if (!pi->pi_hwaddrdup &&
            (pi2 = phyint_lookup_hwaddr(pi, _B_FALSE)) != NULL) {
                assert(pi2->pi_hwaddrdup);
                (void) phyint_undo_offline(pi2);
        }

        /*
         * If this interface was active, try to activate another INACTIVE
         * interface in the group.
         */
        if (was_active)
                phyint_activate_another(pi);

        return (IPMP_SUCCESS);
}

/*
 * Undo a previous offline of `pi'.  Returns an IPMP error code.
 */
int
phyint_undo_offline(struct phyint *pi)
{
        if (pi->pi_state != PI_OFFLINE) {
                errno = EINVAL;
                return (IPMP_FAILURE);
        }

        /*
         * If necessary, reinitialize our link information and verify that its
         * hardware address is still unique across the group.
         */
        if (pi->pi_dh == NULL && !phyint_link_init(pi)) {
                errno = EIO;
                return (IPMP_FAILURE);
        }

        if (phyint_lookup_hwaddr(pi, _B_TRUE) != NULL) {
                pi->pi_hwaddrdup = _B_TRUE;
                return (IPMP_EHWADDRDUP);
        }

        if (pi->pi_hwaddrdup) {
                logerr("IP interface %s now has a unique hardware address in "
                    "group %s; onlining\n", pi->pi_name, pi->pi_group->pg_name);
                pi->pi_hwaddrdup = _B_FALSE;
        }

        if (!change_pif_flags(pi, 0, IFF_OFFLINE))
                return (IPMP_FAILURE);

        /*
         * While the interface was offline, it may have failed (e.g. the link
         * may have gone down).  phyint_inst_check_for_failure() will have
         * already set pi_flags with IFF_FAILED, so we can use that to decide
         * whether the phyint should transition to running.  Note that after
         * we transition to running, we will start sending probes again (if
         * test addresses are configured), which may also reveal that the
         * interface is in fact failed.
         */
        if (pi->pi_flags & IFF_FAILED) {
                phyint_chstate(pi, PI_FAILED);
        } else {
                /* calls phyint_chstate() */
                phyint_transition_to_running(pi);
        }

        /*
         * Give the requestor time to configure test addresses before
         * complaining that they're missing.
         */
        pi->pi_taddrthresh = getcurrentsec() + TESTADDR_CONF_TIME;

        return (IPMP_SUCCESS);
}

/*
 * Delete (unlink and free), the phyint instance.
 */
void
phyint_inst_delete(struct phyint_instance *pii)
{
        struct phyint *pi = pii->pii_phyint;

        assert(pi != NULL);

        if (debug & D_PHYINT) {
                logdebug("phyint_inst_delete(%s %s)\n",
                    AF_STR(pii->pii_af), pi->pi_name);
        }

        /*
         * If the phyint instance has associated probe targets
         * delete all the targets
         */
        while (pii->pii_targets != NULL)
                target_delete(pii->pii_targets);

        /*
         * Delete all the logints associated with this phyint
         * instance.
         */
        while (pii->pii_logint != NULL)
                logint_delete(pii->pii_logint);

        /*
         * Close the socket used to send probes to targets from this phyint.
         */
        if (pii->pii_probe_sock != -1)
                close_probe_socket(pii, _B_TRUE);

        /*
         * Phyint instance must be in the list of all phyint instances.
         * Remove phyint instance from the global list of phyint instances.
         */
        assert(phyint_instances == pii || pii->pii_prev != NULL);
        if (pii->pii_prev == NULL) {
                /* Phyint is the 1st in the list */
                phyint_instances = pii->pii_next;
        } else {
                pii->pii_prev->pii_next = pii->pii_next;
        }
        if (pii->pii_next != NULL)
                pii->pii_next->pii_prev = pii->pii_prev;
        pii->pii_next = NULL;
        pii->pii_prev = NULL;

        /*
         * Reset the phyint instance pointer in the phyint.
         * If this is the last phyint instance (being deleted) on this
         * phyint, then delete the phyint.
         */
        if (pii->pii_af == AF_INET)
                pi->pi_v4 = NULL;
        else
                pi->pi_v6 = NULL;

        if (pi->pi_v4 == NULL && pi->pi_v6 == NULL)
                phyint_delete(pi);

        free(pii);
}

static void
phyint_inst_print(struct phyint_instance *pii)
{
        struct logint *li;
        struct target *tg;
        char abuf[INET6_ADDRSTRLEN];
        int most_recent;
        int i;

        if (pii->pii_phyint == NULL) {
                logdebug("pii->pi_phyint NULL can't print\n");
                return;
        }

        logdebug("\nPhyint instance: %s %s index %u state %x flags %llx  "
            "sock %x in_use %d\n",
            AF_STR(pii->pii_af), pii->pii_name, pii->pii_ifindex,
            pii->pii_state, pii->pii_phyint->pi_flags, pii->pii_probe_sock,
            pii->pii_in_use);

        for (li = pii->pii_logint; li != NULL; li = li->li_next)
                logint_print(li);

        logdebug("\n");
        for (tg = pii->pii_targets; tg != NULL; tg = tg->tg_next)
                target_print(tg);

        if (pii->pii_targets == NULL)
                logdebug("pi_targets NULL\n");

        if (pii->pii_target_next != NULL) {
                logdebug("pi_target_next %s %s\n", AF_STR(pii->pii_af),
                    pr_addr(pii->pii_af, pii->pii_target_next->tg_address,
                    abuf, sizeof (abuf)));
        } else {
                logdebug("pi_target_next NULL\n");
        }

        if (pii->pii_rtt_target_next != NULL) {
                logdebug("pi_rtt_target_next %s %s\n", AF_STR(pii->pii_af),
                    pr_addr(pii->pii_af, pii->pii_rtt_target_next->tg_address,
                    abuf, sizeof (abuf)));
        } else {
                logdebug("pi_rtt_target_next NULL\n");
        }

        if (pii->pii_targets != NULL) {
                most_recent = PROBE_INDEX_PREV(pii->pii_probe_next);

                i = most_recent;
                do {
                        if (pii->pii_probes[i].pr_target != NULL) {
                                logdebug("#%d target %s ", i,
                                    pr_addr(pii->pii_af,
                                    pii->pii_probes[i].pr_target->tg_address,
                                    abuf, sizeof (abuf)));
                        } else {
                                logdebug("#%d target NULL ", i);
                        }
                        logdebug("time_start %lld status %d "
                            "time_ackproc %lld time_lost %u",
                            pii->pii_probes[i].pr_hrtime_start,
                            pii->pii_probes[i].pr_status,
                            pii->pii_probes[i].pr_hrtime_ackproc,
                            pii->pii_probes[i].pr_time_lost);
                        i = PROBE_INDEX_PREV(i);
                } while (i != most_recent);
        }
}

/*
 * Lookup a logint based on the logical interface name, on the given
 * phyint instance.
 */
static struct logint *
logint_lookup(struct phyint_instance *pii, char *name)
{
        struct logint *li;

        if (debug & D_LOGINT) {
                logdebug("logint_lookup(%s, %s)\n",
                    AF_STR(pii->pii_af), name);
        }

        for (li = pii->pii_logint; li != NULL; li = li->li_next) {
                if (strncmp(name, li->li_name, sizeof (li->li_name)) == 0)
                        break;
        }
        return (li);
}

/*
 * Insert a logint at the head of the list of logints of the given
 * phyint instance
 */
static void
logint_insert(struct phyint_instance *pii, struct logint *li)
{
        li->li_next = pii->pii_logint;
        li->li_prev = NULL;
        if (pii->pii_logint != NULL)
                pii->pii_logint->li_prev = li;
        pii->pii_logint = li;
        li->li_phyint_inst = pii;
}

/*
 * Create a new named logint, on the specified phyint instance.
 */
static struct logint *
logint_create(struct phyint_instance *pii, char *name)
{
        struct logint *li;

        if (debug & D_LOGINT) {
                logdebug("logint_create(%s %s %s)\n",
                    AF_STR(pii->pii_af), pii->pii_name, name);
        }

        li = calloc(1, sizeof (struct logint));
        if (li == NULL) {
                logperror("logint_create: calloc");
                return (NULL);
        }

        (void) strncpy(li->li_name, name, sizeof (li->li_name));
        li->li_name[sizeof (li->li_name) - 1] = '\0';
        logint_insert(pii, li);
        return (li);
}

/*
 * Initialize the logint based on the data returned by the kernel.
 */
void
logint_init_from_k(struct phyint_instance *pii, char *li_name)
{
        int     ifsock;
        uint64_t flags;
        uint64_t saved_flags;
        struct  logint  *li;
        struct lifreq   lifr;
        struct in6_addr test_subnet;
        struct in6_addr testaddr;
        int     test_subnet_len;
        struct sockaddr_in6     *sin6;
        struct sockaddr_in      *sin;
        char abuf[INET6_ADDRSTRLEN];
        boolean_t  ptp = _B_FALSE;
        struct in6_addr tgaddr;

        if (debug & D_LOGINT) {
                logdebug("logint_init_from_k(%s %s)\n",
                    AF_STR(pii->pii_af), li_name);
        }

        /* Get the socket for doing ioctls */
        ifsock = (pii->pii_af == AF_INET) ? ifsock_v4 : ifsock_v6;

        /*
         * Get the flags from the kernel. Also serves as a check whether
         * the logical still exists. If it doesn't exist, no need to proceed
         * any further. li_in_use will make the caller clean up the logint
         */
        (void) strncpy(lifr.lifr_name, li_name, sizeof (lifr.lifr_name));
        lifr.lifr_name[sizeof (lifr.lifr_name) - 1] = '\0';
        if (ioctl(ifsock, SIOCGLIFFLAGS, (char *)&lifr) < 0) {
                /* Interface may have vanished */
                if (errno != ENXIO) {
                        logperror_pii(pii, "logint_init_from_k: "
                            "ioctl (get flags)");
                }
                return;
        }

        flags = lifr.lifr_flags;

        /*
         * Verified the logint exists. Now lookup the logint in our tables.
         * If it does not exist, create a new logint.
         */
        li = logint_lookup(pii, li_name);
        if (li == NULL) {
                li = logint_create(pii, li_name);
                if (li == NULL) {
                        /*
                         * Pretend the interface does not exist
                         * in the kernel
                         */
                        return;
                }
        }

        /*
         * Update li->li_flags with the new flags, after saving the old
         * value. This is used later to check what flags has changed and
         * take any action
         */
        saved_flags = li->li_flags;
        li->li_flags = flags;

        /*
         * Get the address, prefix, prefixlength and update the logint.
         * Check if anything has changed. If the logint used for the
         * test address has changed, take suitable action.
         */
        if (ioctl(ifsock, SIOCGLIFADDR, (char *)&lifr) < 0) {
                /* Interface may have vanished */
                if (errno != ENXIO) {
                        logperror_li(li, "logint_init_from_k: (get addr)");
                }
                goto error;
        }

        if (pii->pii_af == AF_INET) {
                sin = (struct sockaddr_in *)&lifr.lifr_addr;
                IN6_INADDR_TO_V4MAPPED(&sin->sin_addr, &testaddr);
        } else {
                sin6 = (struct sockaddr_in6 *)&lifr.lifr_addr;
                testaddr = sin6->sin6_addr;
        }

        if (ioctl(ifsock, SIOCGLIFSUBNET, (char *)&lifr) < 0) {
                /* Interface may have vanished */
                if (errno != ENXIO)
                        logperror_li(li, "logint_init_from_k: (get subnet)");
                goto error;
        }
        if (lifr.lifr_subnet.ss_family == AF_INET6) {
                sin6 = (struct sockaddr_in6 *)&lifr.lifr_subnet;
                test_subnet = sin6->sin6_addr;
                test_subnet_len = lifr.lifr_addrlen;
        } else {
                sin = (struct sockaddr_in *)&lifr.lifr_subnet;
                IN6_INADDR_TO_V4MAPPED(&sin->sin_addr, &test_subnet);
                test_subnet_len = lifr.lifr_addrlen + (IPV6_ABITS - IP_ABITS);
        }

        /*
         * If this is the logint corresponding to the test address used for
         * sending probes, then if anything significant has changed we need to
         * determine the test address again.  We ignore changes to the
         * IFF_FAILED and IFF_RUNNING flags since those happen as a matter of
         * course.
         */
        if (pii->pii_probe_logint == li) {
                if (((li->li_flags ^ saved_flags) &
                    ~(IFF_FAILED | IFF_RUNNING)) != 0 ||
                    !IN6_ARE_ADDR_EQUAL(&testaddr, &li->li_addr) ||
                    (!ptp && !IN6_ARE_ADDR_EQUAL(&test_subnet,
                    &li->li_subnet)) ||
                    (!ptp && test_subnet_len != li->li_subnet_len) ||
                    (ptp && !IN6_ARE_ADDR_EQUAL(&tgaddr, &li->li_dstaddr))) {
                        /*
                         * Something significant that affects the testaddress
                         * has changed. Redo the testaddress selection later on
                         * in select_test_ifs(). For now do the cleanup and
                         * set pii_probe_logint to NULL.
                         */
                        if (pii->pii_probe_sock != -1)
                                close_probe_socket(pii, _B_TRUE);
                        pii->pii_probe_logint = NULL;
                }
        }


        /* Update the logint with the values obtained from the kernel.  */
        li->li_addr = testaddr;
        li->li_in_use = 1;
        if (ptp) {
                li->li_dstaddr = tgaddr;
                li->li_subnet_len = (pii->pii_af == AF_INET) ?
                    IP_ABITS : IPV6_ABITS;
        } else {
                li->li_subnet = test_subnet;
                li->li_subnet_len = test_subnet_len;
        }

        if (debug & D_LOGINT)
                logint_print(li);

        return;

error:
        logerr("logint_init_from_k: IGNORED %s %s %s addr %s\n",
            AF_STR(pii->pii_af), pii->pii_name, li->li_name,
            pr_addr(pii->pii_af, testaddr, abuf, sizeof (abuf)));
        logint_delete(li);
}

/*
 * Delete (unlink and free) a logint.
 */
void
logint_delete(struct logint *li)
{
        struct phyint_instance *pii;

        pii = li->li_phyint_inst;
        assert(pii != NULL);

        if (debug & D_LOGINT) {
                int af;
                char abuf[INET6_ADDRSTRLEN];

                af = pii->pii_af;
                logdebug("logint_delete(%s %s %s/%u)\n",
                    AF_STR(af), li->li_name,
                    pr_addr(af, li->li_addr, abuf, sizeof (abuf)),
                    li->li_subnet_len);
        }

        /* logint must be in the list of logints */
        assert(pii->pii_logint == li || li->li_prev != NULL);

        /* Remove the logint from the list of logints  */
        if (li->li_prev == NULL) {
                /* logint is the 1st in the list */
                pii->pii_logint = li->li_next;
        } else {
                li->li_prev->li_next = li->li_next;
        }
        if (li->li_next != NULL)
                li->li_next->li_prev = li->li_prev;
        li->li_next = NULL;
        li->li_prev = NULL;

        /*
         * If this logint is also being used for probing, then close the
         * associated socket, if it exists.
         */
        if (pii->pii_probe_logint == li) {
                if (pii->pii_probe_sock != -1)
                        close_probe_socket(pii, _B_TRUE);
                pii->pii_probe_logint = NULL;
        }

        free(li);
}

static void
logint_print(struct logint *li)
{
        char abuf[INET6_ADDRSTRLEN];
        int af = li->li_phyint_inst->pii_af;

        logdebug("logint: %s %s addr %s/%u", AF_STR(af), li->li_name,
            pr_addr(af, li->li_addr, abuf, sizeof (abuf)), li->li_subnet_len);

        logdebug("\tFlags: %llx in_use %d\n", li->li_flags, li->li_in_use);
}

char *
pr_addr(int af, struct in6_addr addr, char *abuf, int len)
{
        struct in_addr  addr_v4;

        if (af == AF_INET) {
                IN6_V4MAPPED_TO_INADDR(&addr, &addr_v4);
                (void) inet_ntop(AF_INET, (void *)&addr_v4, abuf, len);
        } else {
                (void) inet_ntop(AF_INET6, (void *)&addr, abuf, len);
        }
        return (abuf);
}

/*
 * Fill in the sockaddr_storage pointed to by `ssp' with the IP address
 * represented by the [`af',`addr'] pair.  Needed because in.mpathd internally
 * stores all addresses as in6_addrs, but we don't want to expose that.
 */
void
addr2storage(int af, const struct in6_addr *addr, struct sockaddr_storage *ssp)
{
        struct sockaddr_in *sinp = (struct sockaddr_in *)ssp;
        struct sockaddr_in6 *sin6p = (struct sockaddr_in6 *)ssp;

        assert(af == AF_INET || af == AF_INET6);

        switch (af) {
        case AF_INET:
                (void) memset(sinp, 0, sizeof (*sinp));
                sinp->sin_family = AF_INET;
                IN6_V4MAPPED_TO_INADDR(addr, &sinp->sin_addr);
                break;
        case AF_INET6:
                (void) memset(sin6p, 0, sizeof (*sin6p));
                sin6p->sin6_family = AF_INET6;
                sin6p->sin6_addr = *addr;
                break;
        }
}

/* Lookup target on its address */
struct target *
target_lookup(struct phyint_instance *pii, struct in6_addr addr)
{
        struct target *tg;

        if (debug & D_TARGET) {
                char abuf[INET6_ADDRSTRLEN];

                logdebug("target_lookup(%s %s): addr %s\n",
                    AF_STR(pii->pii_af), pii->pii_name,
                    pr_addr(pii->pii_af, addr, abuf, sizeof (abuf)));
        }

        for (tg = pii->pii_targets; tg != NULL; tg = tg->tg_next) {
                if (IN6_ARE_ADDR_EQUAL(&tg->tg_address, &addr))
                        break;
        }
        return (tg);
}

/*
 * Find and return the next active target, for the next probe.
 * If no active targets are available, return NULL.
 */
struct target *
target_next(struct target *tg)
{
        struct  phyint_instance *pii = tg->tg_phyint_inst;
        struct  target  *marker = tg;
        hrtime_t now;

        now = gethrtime();

        /*
         * Target must be in the list of targets for this phyint
         * instance.
         */
        assert(pii->pii_targets == tg || tg->tg_prev != NULL);
        assert(pii->pii_targets != NULL);

        /* Return the next active target */
        do {
                /*
                 * Go to the next target. If we hit the end,
                 * reset the ptr to the head
                 */
                tg = tg->tg_next;
                if (tg == NULL)
                        tg = pii->pii_targets;

                assert(TG_STATUS_VALID(tg->tg_status));

                switch (tg->tg_status) {
                case TG_ACTIVE:
                        return (tg);

                case TG_UNUSED:
                        assert(pii->pii_targets_are_routers);
                        if (pii->pii_ntargets < MAX_PROBE_TARGETS) {
                                /*
                                 * Bubble up the unused target to active
                                 */
                                tg->tg_status = TG_ACTIVE;
                                pii->pii_ntargets++;
                                return (tg);
                        }
                        break;

                case TG_SLOW:
                        assert(pii->pii_targets_are_routers);
                        if (tg->tg_latime + MIN_RECOVERY_TIME < now) {
                                /*
                                 * Bubble up the slow target to unused
                                 */
                                tg->tg_status = TG_UNUSED;
                        }
                        break;

                case TG_DEAD:
                        assert(pii->pii_targets_are_routers);
                        if (tg->tg_latime + MIN_RECOVERY_TIME < now) {
                                /*
                                 * Bubble up the dead target to slow
                                 */
                                tg->tg_status = TG_SLOW;
                                tg->tg_latime = now;
                        }
                        break;
                }

        } while (tg != marker);

        return (NULL);
}

/*
 * Select the best available target, that is not already TG_ACTIVE,
 * for the caller. The caller will determine whether it wants to
 * make the returned target TG_ACTIVE.
 * The selection order is as follows.
 * 1. pick a TG_UNSED target, if it exists.
 * 2. else pick a TG_SLOW target that has recovered, if it exists
 * 3. else pick any TG_SLOW target, if it exists
 * 4. else pick a TG_DEAD target that has recovered, if it exists
 * 5. else pick any TG_DEAD target, if it exists
 * 6. else return null
 */
static struct target *
target_select_best(struct phyint_instance *pii)
{
        struct target *tg;
        struct target *slow = NULL;
        struct target *dead = NULL;
        struct target *slow_recovered = NULL;
        struct target *dead_recovered = NULL;
        hrtime_t now;

        now = gethrtime();

        for (tg = pii->pii_targets; tg != NULL; tg = tg->tg_next) {
                assert(TG_STATUS_VALID(tg->tg_status));

                switch (tg->tg_status) {
                case TG_UNUSED:
                        return (tg);

                case TG_SLOW:
                        if (tg->tg_latime + MIN_RECOVERY_TIME < now) {
                                slow_recovered = tg;
                                /*
                                 * Promote the slow_recovered to unused
                                 */
                                tg->tg_status = TG_UNUSED;
                        } else {
                                slow = tg;
                        }
                        break;

                case TG_DEAD:
                        if (tg->tg_latime + MIN_RECOVERY_TIME < now) {
                                dead_recovered = tg;
                                /*
                                 * Promote the dead_recovered to slow
                                 */
                                tg->tg_status = TG_SLOW;
                                tg->tg_latime = now;
                        } else {
                                dead = tg;
                        }
                        break;

                default:
                        break;
                }
        }

        if (slow_recovered != NULL)
                return (slow_recovered);
        else if (slow != NULL)
                return (slow);
        else if (dead_recovered != NULL)
                return (dead_recovered);
        else
                return (dead);
}

/*
 * Some target was deleted. If we don't have even MIN_PROBE_TARGETS
 * that are active, pick the next best below.
 */
static void
target_activate_all(struct phyint_instance *pii)
{
        struct target *tg;

        assert(pii->pii_ntargets == 0);
        assert(pii->pii_target_next == NULL);
        assert(pii->pii_rtt_target_next == NULL);
        assert(pii->pii_targets_are_routers);

        while (pii->pii_ntargets < MIN_PROBE_TARGETS) {
                tg = target_select_best(pii);
                if (tg == NULL) {
                        /* We are out of targets */
                        return;
                }

                assert(TG_STATUS_VALID(tg->tg_status));
                assert(tg->tg_status != TG_ACTIVE);
                tg->tg_status = TG_ACTIVE;
                pii->pii_ntargets++;
                if (pii->pii_target_next == NULL) {
                        pii->pii_target_next = tg;
                        pii->pii_rtt_target_next = tg;
                }
        }
}

static struct target *
target_first(struct phyint_instance *pii)
{
        struct target *tg;

        for (tg = pii->pii_targets; tg != NULL; tg = tg->tg_next) {
                assert(TG_STATUS_VALID(tg->tg_status));
                if (tg->tg_status == TG_ACTIVE)
                        break;
        }

        return (tg);
}

/*
 * Create a default target entry.
 */
void
target_create(struct phyint_instance *pii, struct in6_addr addr,
    boolean_t is_router)
{
        struct target *tg;
        struct phyint *pi;
        struct logint *li;

        if (debug & D_TARGET) {
                char abuf[INET6_ADDRSTRLEN];

                logdebug("target_create(%s %s, %s)\n",
                    AF_STR(pii->pii_af), pii->pii_name,
                    pr_addr(pii->pii_af, addr, abuf, sizeof (abuf)));
        }

        /*
         * If the test address is not yet initialized, do not add
         * any target, since we cannot determine whether the target
         * belongs to the same subnet as the test address.
         */
        li = pii->pii_probe_logint;
        if (li == NULL)
                return;

        /*
         * If there are multiple subnets associated with an interface, then
         * add the target to this phyint instance only if it belongs to the
         * same subnet as the test address.  This assures us that we will
         * be able to reach this target through our routing table.
         */
        if (!prefix_equal(li->li_subnet, addr, li->li_subnet_len))
                return;

        if (pii->pii_targets != NULL) {
                assert(pii->pii_ntargets <= MAX_PROBE_TARGETS);
                if (is_router) {
                        if (!pii->pii_targets_are_routers) {
                                /*
                                 * Prefer router over hosts. Using hosts is a
                                 * fallback mechanism, hence delete all host
                                 * targets.
                                 */
                                while (pii->pii_targets != NULL)
                                        target_delete(pii->pii_targets);
                        }
                } else {
                        /*
                         * Routers take precedence over hosts. If this
                         * is a router list and we are trying to add a
                         * host, just return. If this is a host list
                         * and if we have sufficient targets, just return
                         */
                        if (pii->pii_targets_are_routers ||
                            pii->pii_ntargets == MAX_PROBE_TARGETS)
                                return;
                }
        }

        tg = calloc(1, sizeof (struct target));
        if (tg == NULL) {
                logperror("target_create: calloc");
                return;
        }

        tg->tg_phyint_inst = pii;
        tg->tg_address = addr;
        tg->tg_in_use = 1;
        tg->tg_rtt_sa = -1;
        tg->tg_num_deferred = 0;

        /*
         * If this is the first target, set 'pii_targets_are_routers'
         * The list of targets is either a list of hosts or list or
         * routers, but not a mix.
         */
        if (pii->pii_targets == NULL) {
                assert(pii->pii_ntargets == 0);
                assert(pii->pii_target_next == NULL);
                assert(pii->pii_rtt_target_next == NULL);
                pii->pii_targets_are_routers = is_router ? 1 : 0;
        }

        if (pii->pii_ntargets == MAX_PROBE_TARGETS) {
                assert(pii->pii_targets_are_routers);
                assert(pii->pii_target_next != NULL);
                assert(pii->pii_rtt_target_next != NULL);
                tg->tg_status = TG_UNUSED;
        } else {
                if (pii->pii_ntargets == 0) {
                        assert(pii->pii_target_next == NULL);
                        pii->pii_target_next = tg;
                        pii->pii_rtt_target_next = tg;
                }
                pii->pii_ntargets++;
                tg->tg_status = TG_ACTIVE;
        }

        target_insert(pii, tg);

        /*
         * Change state to PI_RUNNING if this phyint instance is capable of
         * sending and receiving probes -- that is, if we know of at least 1
         * target, and this phyint instance is probe-capable.  For more
         * details, see the phyint state diagram in mpd_probe.c.
         */
        pi = pii->pii_phyint;
        if (pi->pi_state == PI_NOTARGETS && PROBE_CAPABLE(pii)) {
                if (pi->pi_flags & IFF_FAILED)
                        phyint_chstate(pi, PI_FAILED);
                else
                        phyint_chstate(pi, PI_RUNNING);
        }
}

/*
 * Add the target address named by `addr' to phyint instance `pii' if it does
 * not already exist.  If the target is a router, `is_router' should be set to
 * B_TRUE.
 */
void
target_add(struct phyint_instance *pii, struct in6_addr addr,
    boolean_t is_router)
{
        struct target *tg;

        if (pii == NULL)
                return;

        tg = target_lookup(pii, addr);

        /*
         * If the target does not exist, create it; target_create() will set
         * tg_in_use to true.  Even if it exists already, if it's a router
         * target and we'd previously learned of it through multicast, then we
         * need to recreate it as a router target.  Otherwise, just set
         * tg_in_use to to true so that init_router_targets() won't delete it.
         */
        if (tg == NULL || (is_router && !pii->pii_targets_are_routers))
                target_create(pii, addr, is_router);
        else if (is_router)
                tg->tg_in_use = 1;
}

/*
 * Insert target at head of linked list of targets for the associated
 * phyint instance
 */
static void
target_insert(struct phyint_instance *pii, struct target *tg)
{
        tg->tg_next = pii->pii_targets;
        tg->tg_prev = NULL;
        if (tg->tg_next != NULL)
                tg->tg_next->tg_prev = tg;
        pii->pii_targets = tg;
}

/*
 * Delete a target (unlink and free).
 */
void
target_delete(struct target *tg)
{
        int af;
        struct phyint_instance  *pii;
        struct phyint_instance  *pii_other;

        pii = tg->tg_phyint_inst;
        af = pii->pii_af;

        if (debug & D_TARGET) {
                char abuf[INET6_ADDRSTRLEN];

                logdebug("target_delete(%s %s, %s)\n",
                    AF_STR(af), pii->pii_name,
                    pr_addr(af, tg->tg_address, abuf, sizeof (abuf)));
        }

        /*
         * Target must be in the list of targets for this phyint
         * instance.
         */
        assert(pii->pii_targets == tg || tg->tg_prev != NULL);

        /*
         * Reset all references to 'tg' in the probe information
         * for this phyint.
         */
        reset_pii_probes(pii, tg);

        /*
         * Remove this target from the list of targets of this
         * phyint instance.
         */
        if (tg->tg_prev == NULL) {
                pii->pii_targets = tg->tg_next;
        } else {
                tg->tg_prev->tg_next = tg->tg_next;
        }

        if (tg->tg_next != NULL)
                tg->tg_next->tg_prev = tg->tg_prev;

        tg->tg_next = NULL;
        tg->tg_prev = NULL;

        if (tg->tg_status == TG_ACTIVE)
                pii->pii_ntargets--;

        /*
         * Adjust the next target to probe, if it points to
         * to the currently deleted target.
         */
        if (pii->pii_target_next == tg)
                pii->pii_target_next = target_first(pii);

        if (pii->pii_rtt_target_next == tg)
                pii->pii_rtt_target_next = target_first(pii);

        free(tg);

        /*
         * The number of active targets pii_ntargets == 0 iff
         * the next active target pii->pii_target_next == NULL
         */
        if (pii->pii_ntargets != 0) {
                assert(pii->pii_target_next != NULL);
                assert(pii->pii_rtt_target_next != NULL);
                assert(pii->pii_target_next->tg_status == TG_ACTIVE);
                assert(pii->pii_rtt_target_next->tg_status == TG_ACTIVE);
                return;
        }

        /* At this point, we don't have any active targets. */
        assert(pii->pii_target_next == NULL);
        assert(pii->pii_rtt_target_next == NULL);

        if (pii->pii_targets_are_routers) {
                /*
                 * Activate any TG_SLOW or TG_DEAD router targets,
                 * since we don't have any other targets
                 */
                target_activate_all(pii);

                if (pii->pii_ntargets != 0) {
                        assert(pii->pii_target_next != NULL);
                        assert(pii->pii_rtt_target_next != NULL);
                        assert(pii->pii_target_next->tg_status == TG_ACTIVE);
                        assert(pii->pii_rtt_target_next->tg_status ==
                            TG_ACTIVE);
                        return;
                }
        }

        /*
         * If we still don't have any active targets, the list must
         * must be really empty. There aren't even TG_SLOW or TG_DEAD
         * targets. Zero out the probe stats since it will not be
         * relevant any longer.
         */
        assert(pii->pii_targets == NULL);
        pii->pii_targets_are_routers = _B_FALSE;
        clear_pii_probe_stats(pii);
        pii_other = phyint_inst_other(pii);

        /*
         * If there are no targets on both instances and the interface would
         * otherwise be considered PI_RUNNING, go back to PI_NOTARGETS state,
         * since we cannot probe this phyint any more.  For more details,
         * please see phyint state diagram in mpd_probe.c.
         */
        if (!PROBE_CAPABLE(pii_other) && LINK_UP(pii->pii_phyint) &&
            pii->pii_phyint->pi_state != PI_OFFLINE)
                phyint_chstate(pii->pii_phyint, PI_NOTARGETS);
}

/*
 * Flush the target list of every phyint in the group, if the list
 * is a host target list. This is called if group failure is suspected.
 * If all targets have failed, multicast will subsequently discover new
 * targets. Else it is a group failure.
 * Note: This function is a no-op if the list is a router target list.
 */
static void
target_flush_hosts(struct phyint_group *pg)
{
        struct phyint *pi;
        struct phyint_instance *pii;

        if (debug & D_TARGET)
                logdebug("target_flush_hosts(%s)\n", pg->pg_name);

        for (pi = pg->pg_phyint; pi != NULL; pi = pi->pi_pgnext) {
                pii = pi->pi_v4;
                if (pii != NULL && !pii->pii_targets_are_routers) {
                        /*
                         * Delete all the targets. When the list becomes
                         * empty, target_delete() will set pii->pii_targets
                         * to NULL.
                         */
                        while (pii->pii_targets != NULL)
                                target_delete(pii->pii_targets);
                }
                pii = pi->pi_v6;
                if (pii != NULL && !pii->pii_targets_are_routers) {
                        /*
                         * Delete all the targets. When the list becomes
                         * empty, target_delete() will set pii->pii_targets
                         * to NULL.
                         */
                        while (pii->pii_targets != NULL)
                                target_delete(pii->pii_targets);
                }
        }
}

/*
 * Reset all references to 'target' in the probe info, as this target is
 * being deleted. The pr_target field is guaranteed to be non-null if
 * pr_status is PR_UNACKED. So we change the pr_status to PR_LOST, so that
 * pr_target will not be accessed unconditionally.
 */
static void
reset_pii_probes(struct phyint_instance *pii, struct target *tg)
{
        int i;

        for (i = 0; i < PROBE_STATS_COUNT; i++) {
                if (pii->pii_probes[i].pr_target == tg) {
                        if (pii->pii_probes[i].pr_status == PR_UNACKED) {
                                probe_chstate(&pii->pii_probes[i], pii,
                                    PR_LOST);
                        }
                        pii->pii_probes[i].pr_target = NULL;
                }
        }
}

/*
 * Clear the probe statistics array.
 */
void
clear_pii_probe_stats(struct phyint_instance *pii)
{
        bzero(pii->pii_probes, sizeof (struct probe_stats) * PROBE_STATS_COUNT);
        /* Reset the next probe index in the probe stats array */
        pii->pii_probe_next = 0;
}

static void
target_print(struct target *tg)
{
        char    abuf[INET6_ADDRSTRLEN];
        char    buf[128];
        char    buf2[128];
        int     af;
        int     i;

        af = tg->tg_phyint_inst->pii_af;

        logdebug("Target on %s %s addr %s\n"
            "status %d rtt_sa %lld rtt_sd %lld crtt %d tg_in_use %d\n",
            AF_STR(af), tg->tg_phyint_inst->pii_name,
            pr_addr(af, tg->tg_address, abuf, sizeof (abuf)),
            tg->tg_status, tg->tg_rtt_sa, tg->tg_rtt_sd,
            tg->tg_crtt, tg->tg_in_use);

        buf[0] = '\0';
        for (i = 0; i < tg->tg_num_deferred; i++) {
                (void) snprintf(buf2, sizeof (buf2), " %dms",
                    tg->tg_deferred[i]);
                (void) strlcat(buf, buf2, sizeof (buf));
        }
        logdebug("deferred rtts:%s\n", buf);
}

void
phyint_inst_print_all(void)
{
        struct phyint_instance *pii;

        for (pii = phyint_instances; pii != NULL; pii = pii->pii_next) {
                phyint_inst_print(pii);
        }
}

/*
 * Compare two prefixes that have the same prefix length.
 * Fails if the prefix length is unreasonable.
 */
boolean_t
prefix_equal(struct in6_addr p1, struct in6_addr p2, uint_t prefix_len)
{
        uchar_t mask;
        int j;

        if (prefix_len > IPV6_ABITS)
                return (_B_FALSE);

        for (j = 0; prefix_len > 8; prefix_len -= 8, j++)
                if (p1.s6_addr[j] != p2.s6_addr[j])
                        return (_B_FALSE);

        /* Make the N leftmost bits one */
        mask = 0xff << (8 - prefix_len);
        if ((p1.s6_addr[j] & mask) != (p2.s6_addr[j] & mask))
                return (_B_FALSE);

        return (_B_TRUE);
}

/*
 * Get the number of UP logints on phyint `pi'.
 */
static int
logint_upcount(struct phyint *pi)
{
        struct  logint  *li;
        int count = 0;

        if (pi->pi_v4 != NULL) {
                for (li = pi->pi_v4->pii_logint; li != NULL; li = li->li_next) {
                        if (li->li_flags & IFF_UP)
                                count++;
                }
        }

        if (pi->pi_v6 != NULL) {
                for (li = pi->pi_v6->pii_logint; li != NULL; li = li->li_next) {
                        if (li->li_flags & IFF_UP)
                                count++;
                }
        }

        return (count);
}

/*
 * Get the phyint instance with the other (IPv4 / IPv6) protocol
 */
struct phyint_instance *
phyint_inst_other(struct phyint_instance *pii)
{
        if (pii->pii_af == AF_INET)
                return (pii->pii_phyint->pi_v6);
        else
                return (pii->pii_phyint->pi_v4);
}

/*
 * Check whether a phyint is functioning.
 */
boolean_t
phyint_is_functioning(struct phyint *pi)
{
        if (pi->pi_state == PI_RUNNING)
                return (_B_TRUE);
        return (pi->pi_state == PI_NOTARGETS && !(pi->pi_flags & IFF_FAILED));
}

/*
 * Check whether a phyint is usable.
 */
boolean_t
phyint_is_usable(struct phyint *pi)
{
        if (logint_upcount(pi) == 0)
                return (_B_FALSE);
        return (phyint_is_functioning(pi));
}

/*
 * Post an EC_IPMP sysevent of subclass `subclass' and attributes `nvl'.
 * Before sending the event, it prepends the current version of the IPMP
 * sysevent API.  Returns 0 on success, -1 on failure (in either case,
 * `nvl' is freed).
 */
static int
post_event(const char *subclass, nvlist_t *nvl)
{
        static evchan_t *evchp = NULL;

        /*
         * Initialize the event channel if we haven't already done so.
         */
        if (evchp == NULL) {
                errno = sysevent_evc_bind(IPMP_EVENT_CHAN, &evchp, EVCH_CREAT);
                if (errno != 0) {
                        logerr("cannot create event channel `%s': %s\n",
                            IPMP_EVENT_CHAN, strerror(errno));
                        goto failed;
                }
        }

        errno = nvlist_add_uint32(nvl, IPMP_EVENT_VERSION,
            IPMP_EVENT_CUR_VERSION);
        if (errno != 0) {
                logerr("cannot create `%s' event: %s", subclass,
                    strerror(errno));
                goto failed;
        }

        errno = sysevent_evc_publish(evchp, EC_IPMP, subclass, "com.sun",
            "in.mpathd", nvl, EVCH_NOSLEEP);
        if (errno != 0) {
                logerr("cannot send `%s' event: %s\n", subclass,
                    strerror(errno));
                goto failed;
        }

        nvlist_free(nvl);
        return (0);
failed:
        nvlist_free(nvl);
        return (-1);
}

/*
 * Return the external IPMP state associated with phyint `pi'.
 */
static ipmp_if_state_t
ifstate(struct phyint *pi)
{
        switch (pi->pi_state) {
        case PI_INIT:
                return (IPMP_IF_UNKNOWN);

        case PI_NOTARGETS:
                if (pi->pi_flags & IFF_FAILED)
                        return (IPMP_IF_FAILED);
                return (IPMP_IF_UNKNOWN);

        case PI_OFFLINE:
                return (IPMP_IF_OFFLINE);

        case PI_FAILED:
                return (IPMP_IF_FAILED);

        case PI_RUNNING:
                return (IPMP_IF_OK);
        }

        logerr("ifstate: unknown state %d; aborting\n", pi->pi_state);
        abort();
        /* NOTREACHED */
}

/*
 * Return the external IPMP interface type associated with phyint `pi'.
 */
static ipmp_if_type_t
iftype(struct phyint *pi)
{
        if (pi->pi_flags & IFF_STANDBY)
                return (IPMP_IF_STANDBY);
        else
                return (IPMP_IF_NORMAL);
}

/*
 * Return the external IPMP link state associated with phyint `pi'.
 */
static ipmp_if_linkstate_t
iflinkstate(struct phyint *pi)
{
        if (!(pi->pi_notes & (DL_NOTE_LINK_UP|DL_NOTE_LINK_DOWN)))
                return (IPMP_LINK_UNKNOWN);

        return (LINK_DOWN(pi) ? IPMP_LINK_DOWN : IPMP_LINK_UP);
}

/*
 * Return the external IPMP probe state associated with phyint `pi'.
 */
static ipmp_if_probestate_t
ifprobestate(struct phyint *pi)
{
        if (!PROBE_ENABLED(pi->pi_v4) && !PROBE_ENABLED(pi->pi_v6))
                return (IPMP_PROBE_DISABLED);

        if (pi->pi_state == PI_FAILED)
                return (IPMP_PROBE_FAILED);

        if (!PROBE_CAPABLE(pi->pi_v4) && !PROBE_CAPABLE(pi->pi_v6))
                return (IPMP_PROBE_UNKNOWN);

        return (IPMP_PROBE_OK);
}

/*
 * Return the external IPMP target mode associated with phyint instance `pii'.
 */
static ipmp_if_targmode_t
iftargmode(struct phyint_instance *pii)
{
        if (!PROBE_ENABLED(pii))
                return (IPMP_TARG_DISABLED);
        else if (pii->pii_targets_are_routers)
                return (IPMP_TARG_ROUTES);
        else
                return (IPMP_TARG_MULTICAST);
}

/*
 * Return the external IPMP flags associated with phyint `pi'.
 */
static ipmp_if_flags_t
ifflags(struct phyint *pi)
{
        ipmp_if_flags_t flags = 0;

        if (logint_upcount(pi) == 0)
                flags |= IPMP_IFFLAG_DOWN;
        if (pi->pi_flags & IFF_INACTIVE)
                flags |= IPMP_IFFLAG_INACTIVE;
        if (pi->pi_hwaddrdup)
                flags |= IPMP_IFFLAG_HWADDRDUP;
        if (phyint_is_functioning(pi) && flags == 0)
                flags |= IPMP_IFFLAG_ACTIVE;

        return (flags);
}

/*
 * Store the test address used on phyint instance `pii' in `ssp'.  If there's
 * no test address, 0.0.0.0 is stored.
 */
static struct sockaddr_storage *
iftestaddr(struct phyint_instance *pii, struct sockaddr_storage *ssp)
{
        if (PROBE_ENABLED(pii))
                addr2storage(pii->pii_af, &pii->pii_probe_logint->li_addr, ssp);
        else
                addr2storage(AF_INET6, &in6addr_any, ssp);

        return (ssp);
}

/*
 * Return the external IPMP group state associated with phyint group `pg'.
 */
static ipmp_group_state_t
groupstate(struct phyint_group *pg)
{
        switch (pg->pg_state) {
        case PG_FAILED:
                return (IPMP_GROUP_FAILED);
        case PG_DEGRADED:
                return (IPMP_GROUP_DEGRADED);
        case PG_OK:
                return (IPMP_GROUP_OK);
        }

        logerr("groupstate: unknown state %d; aborting\n", pg->pg_state);
        abort();
        /* NOTREACHED */
}

/*
 * Return the external IPMP probe state associated with probe `ps'.
 */
static ipmp_probe_state_t
probestate(struct probe_stats *ps)
{
        switch (ps->pr_status) {
        case PR_UNUSED:
        case PR_LOST:
                return (IPMP_PROBE_LOST);
        case PR_UNACKED:
                return (IPMP_PROBE_SENT);
        case PR_ACKED:
                return (IPMP_PROBE_ACKED);
        }

        logerr("probestate: unknown state %d; aborting\n", ps->pr_status);
        abort();
        /* NOTREACHED */
}

/*
 * Generate an ESC_IPMP_PROBE_STATE sysevent for the probe described by `pr'
 * on phyint instance `pii'.  Returns 0 on success, -1 on failure.
 */
int
probe_state_event(struct probe_stats *pr, struct phyint_instance *pii)
{
        nvlist_t *nvl;
        hrtime_t proc_time = 0, recv_time = 0;
        struct sockaddr_storage ss;
        struct target *tg = pr->pr_target;
        int64_t rttavg, rttdev;

        errno = nvlist_alloc(&nvl, NV_UNIQUE_NAME, 0);
        if (errno != 0) {
                logperror("cannot create `interface change' event");
                return (-1);
        }

        errno = nvlist_add_uint32(nvl, IPMP_PROBE_ID, pr->pr_id);
        if (errno != 0)
                goto failed;

        errno = nvlist_add_string(nvl, IPMP_IF_NAME, pii->pii_phyint->pi_name);
        if (errno != 0)
                goto failed;

        errno = nvlist_add_uint32(nvl, IPMP_PROBE_STATE, probestate(pr));
        if (errno != 0)
                goto failed;

        errno = nvlist_add_hrtime(nvl, IPMP_PROBE_START_TIME,
            pr->pr_hrtime_start);
        if (errno != 0)
                goto failed;

        errno = nvlist_add_hrtime(nvl, IPMP_PROBE_SENT_TIME,
            pr->pr_hrtime_sent);
        if (errno != 0)
                goto failed;

        if (pr->pr_status == PR_ACKED) {
                recv_time = pr->pr_hrtime_ackrecv;
                proc_time = pr->pr_hrtime_ackproc;
        }

        errno = nvlist_add_hrtime(nvl, IPMP_PROBE_ACKRECV_TIME, recv_time);
        if (errno != 0)
                goto failed;

        errno = nvlist_add_hrtime(nvl, IPMP_PROBE_ACKPROC_TIME, proc_time);
        if (errno != 0)
                goto failed;

        if (tg != NULL)
                addr2storage(pii->pii_af, &tg->tg_address, &ss);
        else
                addr2storage(pii->pii_af, &in6addr_any, &ss);

        errno = nvlist_add_byte_array(nvl, IPMP_PROBE_TARGET, (uchar_t *)&ss,
            sizeof (ss));
        if (errno != 0)
                goto failed;

        rttavg = (tg != NULL) ? (tg->tg_rtt_sa / 8) : 0;
        errno = nvlist_add_int64(nvl, IPMP_PROBE_TARGET_RTTAVG, rttavg);
        if (errno != 0)
                goto failed;

        rttdev = (tg != NULL) ? (tg->tg_rtt_sd / 4) : 0;
        errno = nvlist_add_int64(nvl, IPMP_PROBE_TARGET_RTTDEV, rttdev);
        if (errno != 0)
                goto failed;

        return (post_event(ESC_IPMP_PROBE_STATE, nvl));
failed:
        logperror("cannot create `probe state' event");
        nvlist_free(nvl);
        return (-1);
}

/*
 * Generate an ESC_IPMP_GROUP_STATE sysevent for phyint group `pg'.
 * Returns 0 on success, -1 on failure.
 */
static int
phyint_group_state_event(struct phyint_group *pg)
{
        nvlist_t        *nvl;

        errno = nvlist_alloc(&nvl, NV_UNIQUE_NAME, 0);
        if (errno != 0) {
                logperror("cannot create `group state change' event");
                return (-1);
        }

        errno = nvlist_add_string(nvl, IPMP_GROUP_NAME, pg->pg_name);
        if (errno != 0)
                goto failed;

        errno = nvlist_add_uint64(nvl, IPMP_GROUP_SIGNATURE, pg->pg_sig);
        if (errno != 0)
                goto failed;

        errno = nvlist_add_uint32(nvl, IPMP_GROUP_STATE, groupstate(pg));
        if (errno != 0)
                goto failed;

        return (post_event(ESC_IPMP_GROUP_STATE, nvl));
failed:
        logperror("cannot create `group state change' event");
        nvlist_free(nvl);
        return (-1);
}

/*
 * Generate an ESC_IPMP_GROUP_CHANGE sysevent of type `op' for phyint group
 * `pg'.  Returns 0 on success, -1 on failure.
 */
static int
phyint_group_change_event(struct phyint_group *pg, ipmp_group_op_t op)
{
        nvlist_t *nvl;

        errno = nvlist_alloc(&nvl, NV_UNIQUE_NAME, 0);
        if (errno != 0) {
                logperror("cannot create `group change' event");
                return (-1);
        }

        errno = nvlist_add_string(nvl, IPMP_GROUP_NAME, pg->pg_name);
        if (errno != 0)
                goto failed;

        errno = nvlist_add_uint64(nvl, IPMP_GROUP_SIGNATURE, pg->pg_sig);
        if (errno != 0)
                goto failed;

        errno = nvlist_add_uint64(nvl, IPMP_GROUPLIST_SIGNATURE,
            phyint_grouplistsig);
        if (errno != 0)
                goto failed;

        errno = nvlist_add_uint32(nvl, IPMP_GROUP_OPERATION, op);
        if (errno != 0)
                goto failed;

        return (post_event(ESC_IPMP_GROUP_CHANGE, nvl));
failed:
        logperror("cannot create `group change' event");
        nvlist_free(nvl);
        return (-1);
}

/*
 * Generate an ESC_IPMP_GROUP_MEMBER_CHANGE sysevent for phyint `pi' in
 * group `pg'.  Returns 0 on success, -1 on failure.
 */
static int
phyint_group_member_event(struct phyint_group *pg, struct phyint *pi,
    ipmp_if_op_t op)
{
        nvlist_t *nvl;

        errno = nvlist_alloc(&nvl, NV_UNIQUE_NAME, 0);
        if (errno != 0) {
                logperror("cannot create `group member change' event");
                return (-1);
        }

        errno = nvlist_add_string(nvl, IPMP_GROUP_NAME, pg->pg_name);
        if (errno != 0)
                goto failed;

        errno = nvlist_add_uint64(nvl, IPMP_GROUP_SIGNATURE, pg->pg_sig);
        if (errno != 0)
                goto failed;

        errno = nvlist_add_uint32(nvl, IPMP_IF_OPERATION, op);
        if (errno != 0)
                goto failed;

        errno = nvlist_add_string(nvl, IPMP_IF_NAME, pi->pi_name);
        if (errno != 0)
                goto failed;

        errno = nvlist_add_uint32(nvl, IPMP_IF_TYPE, iftype(pi));
        if (errno != 0)
                goto failed;

        errno = nvlist_add_uint32(nvl, IPMP_IF_STATE, ifstate(pi));
        if (errno != 0)
                goto failed;

        return (post_event(ESC_IPMP_GROUP_MEMBER_CHANGE, nvl));
failed:
        logperror("cannot create `group member change' event");
        nvlist_free(nvl);
        return (-1);

}

/*
 * Generate an ESC_IPMP_IF_CHANGE sysevent for phyint `pi' in group `pg'.
 * Returns 0 on success, -1 on failure.
 */
static int
phyint_state_event(struct phyint_group *pg, struct phyint *pi)
{
        nvlist_t *nvl;

        errno = nvlist_alloc(&nvl, NV_UNIQUE_NAME, 0);
        if (errno != 0) {
                logperror("cannot create `interface change' event");
                return (-1);
        }

        errno = nvlist_add_string(nvl, IPMP_GROUP_NAME, pg->pg_name);
        if (errno != 0)
                goto failed;

        errno = nvlist_add_uint64(nvl, IPMP_GROUP_SIGNATURE, pg->pg_sig);
        if (errno != 0)
                goto failed;

        errno = nvlist_add_string(nvl, IPMP_IF_NAME, pi->pi_name);
        if (errno != 0)
                goto failed;

        errno = nvlist_add_uint32(nvl, IPMP_IF_TYPE, iftype(pi));
        if (errno != 0)
                goto failed;

        errno = nvlist_add_uint32(nvl, IPMP_IF_STATE, ifstate(pi));
        if (errno != 0)
                goto failed;

        return (post_event(ESC_IPMP_IF_CHANGE, nvl));
failed:
        logperror("cannot create `interface change' event");
        nvlist_free(nvl);
        return (-1);

}

/*
 * Generate a signature for use.  The signature is conceptually divided
 * into two pieces: a random 16-bit "generation number" and a 48-bit
 * monotonically increasing integer.  The generation number protects
 * against stale updates to entities (e.g., IPMP groups) that have been
 * deleted and since recreated.
 */
static uint64_t
gensig(void)
{
        static int seeded = 0;

        if (seeded == 0) {
                srand48((long)gethrtime());
                seeded++;
        }

        return ((uint64_t)lrand48() << 48 | 1);
}

/*
 * Store the information associated with group `grname' into a dynamically
 * allocated structure pointed to by `*grinfopp'.  Returns an IPMP error code.
 */
unsigned int
getgroupinfo(const char *grname, ipmp_groupinfo_t **grinfopp)
{
        struct phyint           *pi;
        struct phyint_group     *pg;
        char                    (*ifs)[LIFNAMSIZ];
        unsigned int            i, j;
        unsigned int            nif = 0, naddr = 0;
        lifgroupinfo_t          lifgr;
        addrlist_t              *addrp;
        struct sockaddr_storage *addrs;
        int                     fdt = 0;

        pg = phyint_group_lookup(grname);
        if (pg == NULL)
                return (IPMP_EUNKGROUP);

        /*
         * Tally up the number of interfaces, allocate an array to hold them,
         * and insert their names into the array.  While we're at it, if any
         * interface is actually enabled to send probes, save the group fdt.
         */
        for (pi = pg->pg_phyint; pi != NULL; pi = pi->pi_pgnext)
                nif++;

        ifs = alloca(nif * sizeof (*ifs));
        for (i = 0, pi = pg->pg_phyint; pi != NULL; pi = pi->pi_pgnext, i++) {
                assert(i < nif);
                (void) strlcpy(ifs[i], pi->pi_name, LIFNAMSIZ);
                if (PROBE_ENABLED(pi->pi_v4) || PROBE_ENABLED(pi->pi_v6))
                        fdt = pg->pg_fdt;
        }
        assert(i == nif);

        /*
         * If this is the anonymous group, there's no other information to
         * collect (since there's no IPMP interface).
         */
        if (pg == phyint_anongroup) {
                *grinfopp = ipmp_groupinfo_create(pg->pg_name, pg->pg_sig, fdt,
                    groupstate(pg), nif, ifs, "", "", "", "", 0, NULL);
                return (*grinfopp == NULL ? IPMP_ENOMEM : IPMP_SUCCESS);
        }

        /*
         * Grab some additional information about the group from the kernel.
         * (NOTE: since SIOCGLIFGROUPINFO does not look up by interface name,
         * we can use ifsock_v4 even for a V6-only group.)
         */
        (void) strlcpy(lifgr.gi_grname, grname, LIFGRNAMSIZ);
        if (ioctl(ifsock_v4, SIOCGLIFGROUPINFO, &lifgr) == -1) {
                if (errno == ENOENT)
                        return (IPMP_EUNKGROUP);

                logperror("getgroupinfo: SIOCGLIFGROUPINFO");
                return (IPMP_FAILURE);
        }

        /*
         * Tally up the number of data addresses, allocate an array to hold
         * them, and insert their values into the array.
         */
        for (addrp = pg->pg_addrs; addrp != NULL; addrp = addrp->al_next)
                naddr++;

        addrs = alloca(naddr * sizeof (*addrs));
        i = 0;
        for (addrp = pg->pg_addrs; addrp != NULL; addrp = addrp->al_next) {
                /*
                 * It's possible to have duplicate addresses (if some are
                 * down).  Weed the dups out to avoid confusing consumers.
                 * (If groups start having tons of addresses, we'll need a
                 * better algorithm here.)
                 */
                for (j = 0; j < i; j++) {
                        if (sockaddrcmp(&addrs[j], &addrp->al_addr))
                                break;
                }
                if (j == i) {
                        assert(i < naddr);
                        addrs[i++] = addrp->al_addr;
                }
        }
        naddr = i;

        *grinfopp = ipmp_groupinfo_create(pg->pg_name, pg->pg_sig, fdt,
            groupstate(pg), nif, ifs, lifgr.gi_grifname, lifgr.gi_m4ifname,
            lifgr.gi_m6ifname, lifgr.gi_bcifname, naddr, addrs);
        return (*grinfopp == NULL ? IPMP_ENOMEM : IPMP_SUCCESS);
}

/*
 * Store the target information associated with phyint instance `pii' into a
 * dynamically allocated structure pointed to by `*targinfopp'.  Returns an
 * IPMP error code.
 */
unsigned int
gettarginfo(struct phyint_instance *pii, const char *name,
    ipmp_targinfo_t **targinfopp)
{
        uint_t ntarg = 0;
        struct target *tg;
        struct sockaddr_storage ss;
        struct sockaddr_storage *targs = NULL;

        if (PROBE_CAPABLE(pii)) {
                targs = alloca(pii->pii_ntargets * sizeof (*targs));
                tg = pii->pii_target_next;
                do {
                        if (tg->tg_status == TG_ACTIVE) {
                                assert(ntarg < pii->pii_ntargets);
                                addr2storage(pii->pii_af, &tg->tg_address,
                                    &targs[ntarg++]);
                        }
                        if ((tg = tg->tg_next) == NULL)
                                tg = pii->pii_targets;
                } while (tg != pii->pii_target_next);

                assert(ntarg == pii->pii_ntargets);
        }

        *targinfopp = ipmp_targinfo_create(name, iftestaddr(pii, &ss),
            iftargmode(pii), ntarg, targs);
        return (*targinfopp == NULL ? IPMP_ENOMEM : IPMP_SUCCESS);
}

/*
 * Store the information associated with interface `ifname' into a dynamically
 * allocated structure pointed to by `*ifinfopp'.  Returns an IPMP error code.
 */
unsigned int
getifinfo(const char *ifname, ipmp_ifinfo_t **ifinfopp)
{
        int             retval;
        struct phyint   *pi;
        ipmp_targinfo_t *targinfo4;
        ipmp_targinfo_t *targinfo6;

        pi = phyint_lookup(ifname);
        if (pi == NULL)
                return (IPMP_EUNKIF);

        if ((retval = gettarginfo(pi->pi_v4, pi->pi_name, &targinfo4)) != 0 ||
            (retval = gettarginfo(pi->pi_v6, pi->pi_name, &targinfo6)) != 0)
                goto out;

        *ifinfopp = ipmp_ifinfo_create(pi->pi_name, pi->pi_group->pg_name,
            ifstate(pi), iftype(pi), iflinkstate(pi), ifprobestate(pi),
            ifflags(pi), targinfo4, targinfo6);
        retval = (*ifinfopp == NULL ? IPMP_ENOMEM : IPMP_SUCCESS);
out:
        if (targinfo4 != NULL)
                ipmp_freetarginfo(targinfo4);
        if (targinfo6 != NULL)
                ipmp_freetarginfo(targinfo6);
        return (retval);
}

/*
 * Store the current list of IPMP groups into a dynamically allocated
 * structure pointed to by `*grlistpp'.  Returns an IPMP error code.
 */
unsigned int
getgrouplist(ipmp_grouplist_t **grlistpp)
{
        struct phyint_group     *pg;
        char                    (*groups)[LIFGRNAMSIZ];
        unsigned int            i, ngroup;

        /*
         * Tally up the number of groups, allocate an array to hold them, and
         * insert their names into the array.
         */
        for (ngroup = 0, pg = phyint_groups; pg != NULL; pg = pg->pg_next)
                ngroup++;

        groups = alloca(ngroup * sizeof (*groups));
        for (i = 0, pg = phyint_groups; pg != NULL; pg = pg->pg_next, i++) {
                assert(i < ngroup);
                (void) strlcpy(groups[i], pg->pg_name, LIFGRNAMSIZ);
        }
        assert(i == ngroup);

        *grlistpp = ipmp_grouplist_create(phyint_grouplistsig, ngroup, groups);
        return (*grlistpp == NULL ? IPMP_ENOMEM : IPMP_SUCCESS);
}

/*
 * Store the address information for `ssp' (in group `grname') into a
 * dynamically allocated structure pointed to by `*adinfopp'.  Returns an IPMP
 * error code.  (We'd call this function getaddrinfo(), but it would conflict
 * with getaddrinfo(3SOCKET)).
 */
unsigned int
getgraddrinfo(const char *grname, struct sockaddr_storage *ssp,
    ipmp_addrinfo_t **adinfopp)
{
        int ifsock;
        addrlist_t *addrp, *addrmatchp = NULL;
        ipmp_addr_state_t state;
        const char *binding = "";
        struct lifreq lifr;
        struct phyint_group *pg;

        if ((pg = phyint_group_lookup(grname)) == NULL)
                return (IPMP_EUNKADDR);

        /*
         * Walk through the data addresses, and find a match.  Note that since
         * some of the addresses may be down, more than one may match.  We
         * prefer an up address (if one exists).
         */
        for (addrp = pg->pg_addrs; addrp != NULL; addrp = addrp->al_next) {
                if (sockaddrcmp(ssp, &addrp->al_addr)) {
                        addrmatchp = addrp;
                        if (addrmatchp->al_flags & IFF_UP)
                                break;
                }
        }

        if (addrmatchp == NULL)
                return (IPMP_EUNKADDR);

        state = (addrmatchp->al_flags & IFF_UP) ? IPMP_ADDR_UP : IPMP_ADDR_DOWN;
        if (state == IPMP_ADDR_UP) {
                ifsock = (ssp->ss_family == AF_INET) ? ifsock_v4 : ifsock_v6;
                (void) strlcpy(lifr.lifr_name, addrmatchp->al_name, LIFNAMSIZ);
                if (ioctl(ifsock, SIOCGLIFBINDING, &lifr) >= 0)
                        binding = lifr.lifr_binding;
        }

        *adinfopp = ipmp_addrinfo_create(ssp, state, pg->pg_name, binding);
        return (*adinfopp == NULL ? IPMP_ENOMEM : IPMP_SUCCESS);
}

/*
 * Store a snapshot of the IPMP subsystem into a dynamically allocated
 * structure pointed to by `*snapp'.  Returns an IPMP error code.
 */
unsigned int
getsnap(ipmp_snap_t **snapp)
{
        ipmp_grouplist_t        *grlistp;
        ipmp_groupinfo_t        *grinfop;
        ipmp_addrinfo_t         *adinfop;
        ipmp_addrlist_t         *adlistp;
        ipmp_ifinfo_t           *ifinfop;
        ipmp_snap_t             *snap;
        struct phyint           *pi;
        unsigned int            i, j;
        int                     retval;

        snap = ipmp_snap_create();
        if (snap == NULL)
                return (IPMP_ENOMEM);

        /*
         * Add group list.
         */
        retval = getgrouplist(&snap->sn_grlistp);
        if (retval != IPMP_SUCCESS)
                goto failed;

        /*
         * Add information for each group in the list, along with all of its
         * data addresses.
         */
        grlistp = snap->sn_grlistp;
        for (i = 0; i < grlistp->gl_ngroup; i++) {
                retval = getgroupinfo(grlistp->gl_groups[i], &grinfop);
                if (retval != IPMP_SUCCESS)
                        goto failed;

                retval = ipmp_snap_addgroupinfo(snap, grinfop);
                if (retval != IPMP_SUCCESS) {
                        ipmp_freegroupinfo(grinfop);
                        goto failed;
                }

                adlistp = grinfop->gr_adlistp;
                for (j = 0; j < adlistp->al_naddr; j++) {
                        retval = getgraddrinfo(grinfop->gr_name,
                            &adlistp->al_addrs[j], &adinfop);
                        if (retval != IPMP_SUCCESS)
                                goto failed;

                        retval = ipmp_snap_addaddrinfo(snap, adinfop);
                        if (retval != IPMP_SUCCESS) {
                                ipmp_freeaddrinfo(adinfop);
                                goto failed;
                        }
                }
        }

        /*
         * Add information for each configured phyint.
         */
        for (pi = phyints; pi != NULL; pi = pi->pi_next) {
                retval = getifinfo(pi->pi_name, &ifinfop);
                if (retval != IPMP_SUCCESS)
                        goto failed;

                retval = ipmp_snap_addifinfo(snap, ifinfop);
                if (retval != IPMP_SUCCESS) {
                        ipmp_freeifinfo(ifinfop);
                        goto failed;
                }
        }

        *snapp = snap;
        return (IPMP_SUCCESS);
failed:
        ipmp_snap_free(snap);
        return (retval);
}