flush: only detect lack of flush support in one place

[zfs.git] / cmd / zed / agents / zfs_diagnosis.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or https://opensource.org/licenses/CDDL-1.0.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
 * Copyright 2015 Nexenta Systems, Inc.  All rights reserved.
 * Copyright (c) 2016, Intel Corporation.
 * Copyright (c) 2023, Klara Inc.
 */

#include <stddef.h>
#include <string.h>
#include <libuutil.h>
#include <libzfs.h>
#include <sys/types.h>
#include <sys/time.h>
#include <sys/fs/zfs.h>
#include <sys/fm/protocol.h>
#include <sys/fm/fs/zfs.h>
#include <sys/zio.h>

#include "zfs_agents.h"
#include "fmd_api.h"

/*
 * Default values for the serd engine when processing checksum or io errors. The
 * semantics are N <events> in T <seconds>.
 */
#define DEFAULT_CHECKSUM_N      10      /* events */
#define DEFAULT_CHECKSUM_T      600     /* seconds */
#define DEFAULT_IO_N            10      /* events */
#define DEFAULT_IO_T            600     /* seconds */
#define DEFAULT_SLOW_IO_N       10      /* events */
#define DEFAULT_SLOW_IO_T       30      /* seconds */

#define CASE_GC_TIMEOUT_SECS    43200   /* 12 hours */

/*
 * Our serd engines are named in the following format:
 *     'zfs_<pool_guid>_<vdev_guid>_{checksum,io,slow_io}'
 * This #define reserves enough space for two 64-bit hex values plus the
 * length of the longest string.
 */
#define MAX_SERDLEN     (16 * 2 + sizeof ("zfs___checksum"))

/*
 * On-disk case structure.  This must maintain backwards compatibility with
 * previous versions of the DE.  By default, any members appended to the end
 * will be filled with zeros if they don't exist in a previous version.
 */
typedef struct zfs_case_data {
        uint64_t        zc_version;
        uint64_t        zc_ena;
        uint64_t        zc_pool_guid;
        uint64_t        zc_vdev_guid;
        uint64_t        zc_parent_guid;
        int             zc_pool_state;
        char            zc_serd_checksum[MAX_SERDLEN];
        char            zc_serd_io[MAX_SERDLEN];
        char            zc_serd_slow_io[MAX_SERDLEN];
        int             zc_has_remove_timer;
} zfs_case_data_t;

/*
 * Time-of-day
 */
typedef struct er_timeval {
        uint64_t        ertv_sec;
        uint64_t        ertv_nsec;
} er_timeval_t;

/*
 * In-core case structure.
 */
typedef struct zfs_case {
        boolean_t       zc_present;
        uint32_t        zc_version;
        zfs_case_data_t zc_data;
        fmd_case_t      *zc_case;
        uu_list_node_t  zc_node;
        id_t            zc_remove_timer;
        char            *zc_fru;
        er_timeval_t    zc_when;
} zfs_case_t;

#define CASE_DATA                       "data"
#define CASE_FRU                        "fru"
#define CASE_DATA_VERSION_INITIAL       1
#define CASE_DATA_VERSION_SERD          2

typedef struct zfs_de_stats {
        fmd_stat_t      old_drops;
        fmd_stat_t      dev_drops;
        fmd_stat_t      vdev_drops;
        fmd_stat_t      import_drops;
        fmd_stat_t      resource_drops;
} zfs_de_stats_t;

zfs_de_stats_t zfs_stats = {
        { "old_drops", FMD_TYPE_UINT64, "ereports dropped (from before load)" },
        { "dev_drops", FMD_TYPE_UINT64, "ereports dropped (dev during open)"},
        { "vdev_drops", FMD_TYPE_UINT64, "ereports dropped (weird vdev types)"},
        { "import_drops", FMD_TYPE_UINT64, "ereports dropped (during import)" },
        { "resource_drops", FMD_TYPE_UINT64, "resource related ereports" }
};

/* wait 15 seconds after a removal */
static hrtime_t zfs_remove_timeout = SEC2NSEC(15);

uu_list_pool_t *zfs_case_pool;
uu_list_t *zfs_cases;

#define ZFS_MAKE_RSRC(type)     \
    FM_RSRC_CLASS "." ZFS_ERROR_CLASS "." type
#define ZFS_MAKE_EREPORT(type)  \
    FM_EREPORT_CLASS "." ZFS_ERROR_CLASS "." type

static void zfs_purge_cases(fmd_hdl_t *hdl);

/*
 * Write out the persistent representation of an active case.
 */
static void
zfs_case_serialize(zfs_case_t *zcp)
{
        zcp->zc_data.zc_version = CASE_DATA_VERSION_SERD;
}

/*
 * Read back the persistent representation of an active case.
 */
static zfs_case_t *
zfs_case_unserialize(fmd_hdl_t *hdl, fmd_case_t *cp)
{
        zfs_case_t *zcp;

        zcp = fmd_hdl_zalloc(hdl, sizeof (zfs_case_t), FMD_SLEEP);
        zcp->zc_case = cp;

        fmd_buf_read(hdl, cp, CASE_DATA, &zcp->zc_data,
            sizeof (zcp->zc_data));

        if (zcp->zc_data.zc_version > CASE_DATA_VERSION_SERD) {
                fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t));
                return (NULL);
        }

        /*
         * fmd_buf_read() will have already zeroed out the remainder of the
         * buffer, so we don't have to do anything special if the version
         * doesn't include the SERD engine name.
         */

        if (zcp->zc_data.zc_has_remove_timer)
                zcp->zc_remove_timer = fmd_timer_install(hdl, zcp,
                    NULL, zfs_remove_timeout);

        uu_list_node_init(zcp, &zcp->zc_node, zfs_case_pool);
        (void) uu_list_insert_before(zfs_cases, NULL, zcp);

        fmd_case_setspecific(hdl, cp, zcp);

        return (zcp);
}

/*
 * Return count of other unique SERD cases under same vdev parent
 */
static uint_t
zfs_other_serd_cases(fmd_hdl_t *hdl, const zfs_case_data_t *zfs_case)
{
        zfs_case_t *zcp;
        uint_t cases = 0;
        static hrtime_t next_check = 0;

        /*
         * Note that plumbing in some external GC would require adding locking,
         * since most of this module code is not thread safe and assumes there
         * is only one thread running against the module. So we perform GC here
         * inline periodically so that future delay induced faults will be
         * possible once the issue causing multiple vdev delays is resolved.
         */
        if (gethrestime_sec() > next_check) {
                /* Periodically purge old SERD entries and stale cases */
                fmd_serd_gc(hdl);
                zfs_purge_cases(hdl);
                next_check = gethrestime_sec() + CASE_GC_TIMEOUT_SECS;
        }

        for (zcp = uu_list_first(zfs_cases); zcp != NULL;
            zcp = uu_list_next(zfs_cases, zcp)) {
                zfs_case_data_t *zcd = &zcp->zc_data;

                /*
                 * must be same pool and parent vdev but different leaf vdev
                 */
                if (zcd->zc_pool_guid != zfs_case->zc_pool_guid ||
                    zcd->zc_parent_guid != zfs_case->zc_parent_guid ||
                    zcd->zc_vdev_guid == zfs_case->zc_vdev_guid) {
                        continue;
                }

                /*
                 * Check if there is another active serd case besides zfs_case
                 *
                 * Only one serd engine will be assigned to the case
                 */
                if (zcd->zc_serd_checksum[0] == zfs_case->zc_serd_checksum[0] &&
                    fmd_serd_active(hdl, zcd->zc_serd_checksum)) {
                        cases++;
                }
                if (zcd->zc_serd_io[0] == zfs_case->zc_serd_io[0] &&
                    fmd_serd_active(hdl, zcd->zc_serd_io)) {
                        cases++;
                }
                if (zcd->zc_serd_slow_io[0] == zfs_case->zc_serd_slow_io[0] &&
                    fmd_serd_active(hdl, zcd->zc_serd_slow_io)) {
                        cases++;
                }
        }
        return (cases);
}

/*
 * Iterate over any active cases.  If any cases are associated with a pool or
 * vdev which is no longer present on the system, close the associated case.
 */
static void
zfs_mark_vdev(uint64_t pool_guid, nvlist_t *vd, er_timeval_t *loaded)
{
        uint64_t vdev_guid = 0;
        uint_t c, children;
        nvlist_t **child;
        zfs_case_t *zcp;

        (void) nvlist_lookup_uint64(vd, ZPOOL_CONFIG_GUID, &vdev_guid);

        /*
         * Mark any cases associated with this (pool, vdev) pair.
         */
        for (zcp = uu_list_first(zfs_cases); zcp != NULL;
            zcp = uu_list_next(zfs_cases, zcp)) {
                if (zcp->zc_data.zc_pool_guid == pool_guid &&
                    zcp->zc_data.zc_vdev_guid == vdev_guid) {
                        zcp->zc_present = B_TRUE;
                        zcp->zc_when = *loaded;
                }
        }

        /*
         * Iterate over all children.
         */
        if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_CHILDREN, &child,
            &children) == 0) {
                for (c = 0; c < children; c++)
                        zfs_mark_vdev(pool_guid, child[c], loaded);
        }

        if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_L2CACHE, &child,
            &children) == 0) {
                for (c = 0; c < children; c++)
                        zfs_mark_vdev(pool_guid, child[c], loaded);
        }

        if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_SPARES, &child,
            &children) == 0) {
                for (c = 0; c < children; c++)
                        zfs_mark_vdev(pool_guid, child[c], loaded);
        }
}

static int
zfs_mark_pool(zpool_handle_t *zhp, void *unused)
{
        (void) unused;
        zfs_case_t *zcp;
        uint64_t pool_guid;
        uint64_t *tod;
        er_timeval_t loaded = { 0 };
        nvlist_t *config, *vd;
        uint_t nelem = 0;
        int ret;

        pool_guid = zpool_get_prop_int(zhp, ZPOOL_PROP_GUID, NULL);
        /*
         * Mark any cases associated with just this pool.
         */
        for (zcp = uu_list_first(zfs_cases); zcp != NULL;
            zcp = uu_list_next(zfs_cases, zcp)) {
                if (zcp->zc_data.zc_pool_guid == pool_guid &&
                    zcp->zc_data.zc_vdev_guid == 0)
                        zcp->zc_present = B_TRUE;
        }

        if ((config = zpool_get_config(zhp, NULL)) == NULL) {
                zpool_close(zhp);
                return (-1);
        }

        (void) nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_LOADED_TIME,
            &tod, &nelem);
        if (nelem == 2) {
                loaded.ertv_sec = tod[0];
                loaded.ertv_nsec = tod[1];
                for (zcp = uu_list_first(zfs_cases); zcp != NULL;
                    zcp = uu_list_next(zfs_cases, zcp)) {
                        if (zcp->zc_data.zc_pool_guid == pool_guid &&
                            zcp->zc_data.zc_vdev_guid == 0) {
                                zcp->zc_when = loaded;
                        }
                }
        }

        ret = nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &vd);
        if (ret) {
                zpool_close(zhp);
                return (-1);
        }

        zfs_mark_vdev(pool_guid, vd, &loaded);

        zpool_close(zhp);

        return (0);
}

struct load_time_arg {
        uint64_t lt_guid;
        er_timeval_t *lt_time;
        boolean_t lt_found;
};

static int
zpool_find_load_time(zpool_handle_t *zhp, void *arg)
{
        struct load_time_arg *lta = arg;
        uint64_t pool_guid;
        uint64_t *tod;
        nvlist_t *config;
        uint_t nelem;

        if (lta->lt_found) {
                zpool_close(zhp);
                return (0);
        }

        pool_guid = zpool_get_prop_int(zhp, ZPOOL_PROP_GUID, NULL);
        if (pool_guid != lta->lt_guid) {
                zpool_close(zhp);
                return (0);
        }

        if ((config = zpool_get_config(zhp, NULL)) == NULL) {
                zpool_close(zhp);
                return (-1);
        }

        if (nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_LOADED_TIME,
            &tod, &nelem) == 0 && nelem == 2) {
                lta->lt_found = B_TRUE;
                lta->lt_time->ertv_sec = tod[0];
                lta->lt_time->ertv_nsec = tod[1];
        }

        zpool_close(zhp);

        return (0);
}

static void
zfs_purge_cases(fmd_hdl_t *hdl)
{
        zfs_case_t *zcp;
        uu_list_walk_t *walk;
        libzfs_handle_t *zhdl = fmd_hdl_getspecific(hdl);

        /*
         * There is no way to open a pool by GUID, or lookup a vdev by GUID.  No
         * matter what we do, we're going to have to stomach an O(vdevs * cases)
         * algorithm.  In reality, both quantities are likely so small that
         * neither will matter. Given that iterating over pools is more
         * expensive than iterating over the in-memory case list, we opt for a
         * 'present' flag in each case that starts off cleared.  We then iterate
         * over all pools, marking those that are still present, and removing
         * those that aren't found.
         *
         * Note that we could also construct an FMRI and rely on
         * fmd_nvl_fmri_present(), but this would end up doing the same search.
         */

        /*
         * Mark the cases as not present.
         */
        for (zcp = uu_list_first(zfs_cases); zcp != NULL;
            zcp = uu_list_next(zfs_cases, zcp))
                zcp->zc_present = B_FALSE;

        /*
         * Iterate over all pools and mark the pools and vdevs found.  If this
         * fails (most probably because we're out of memory), then don't close
         * any of the cases and we cannot be sure they are accurate.
         */
        if (zpool_iter(zhdl, zfs_mark_pool, NULL) != 0)
                return;

        /*
         * Remove those cases which were not found.
         */
        walk = uu_list_walk_start(zfs_cases, UU_WALK_ROBUST);
        while ((zcp = uu_list_walk_next(walk)) != NULL) {
                if (!zcp->zc_present)
                        fmd_case_close(hdl, zcp->zc_case);
        }
        uu_list_walk_end(walk);
}

/*
 * Construct the name of a serd engine given the pool/vdev GUID and type (io or
 * checksum).
 */
static void
zfs_serd_name(char *buf, uint64_t pool_guid, uint64_t vdev_guid,
    const char *type)
{
        (void) snprintf(buf, MAX_SERDLEN, "zfs_%llx_%llx_%s",
            (long long unsigned int)pool_guid,
            (long long unsigned int)vdev_guid, type);
}

static void
zfs_case_retire(fmd_hdl_t *hdl, zfs_case_t *zcp)
{
        fmd_hdl_debug(hdl, "retiring case");

        fmd_case_close(hdl, zcp->zc_case);
}

/*
 * Solve a given ZFS case.  This first checks to make sure the diagnosis is
 * still valid, as well as cleaning up any pending timer associated with the
 * case.
 */
static void
zfs_case_solve(fmd_hdl_t *hdl, zfs_case_t *zcp, const char *faultname)
{
        nvlist_t *detector, *fault;
        boolean_t serialize;
        nvlist_t *fru = NULL;
        fmd_hdl_debug(hdl, "solving fault '%s'", faultname);

        /*
         * Construct the detector from the case data.  The detector is in the
         * ZFS scheme, and is either the pool or the vdev, depending on whether
         * this is a vdev or pool fault.
         */
        detector = fmd_nvl_alloc(hdl, FMD_SLEEP);

        (void) nvlist_add_uint8(detector, FM_VERSION, ZFS_SCHEME_VERSION0);
        (void) nvlist_add_string(detector, FM_FMRI_SCHEME, FM_FMRI_SCHEME_ZFS);
        (void) nvlist_add_uint64(detector, FM_FMRI_ZFS_POOL,
            zcp->zc_data.zc_pool_guid);
        if (zcp->zc_data.zc_vdev_guid != 0) {
                (void) nvlist_add_uint64(detector, FM_FMRI_ZFS_VDEV,
                    zcp->zc_data.zc_vdev_guid);
        }

        fault = fmd_nvl_create_fault(hdl, faultname, 100, detector,
            fru, detector);
        fmd_case_add_suspect(hdl, zcp->zc_case, fault);

        nvlist_free(fru);

        fmd_case_solve(hdl, zcp->zc_case);

        serialize = B_FALSE;
        if (zcp->zc_data.zc_has_remove_timer) {
                fmd_timer_remove(hdl, zcp->zc_remove_timer);
                zcp->zc_data.zc_has_remove_timer = 0;
                serialize = B_TRUE;
        }
        if (serialize)
                zfs_case_serialize(zcp);

        nvlist_free(detector);
}

static boolean_t
timeval_earlier(er_timeval_t *a, er_timeval_t *b)
{
        return (a->ertv_sec < b->ertv_sec ||
            (a->ertv_sec == b->ertv_sec && a->ertv_nsec < b->ertv_nsec));
}

static void
zfs_ereport_when(fmd_hdl_t *hdl, nvlist_t *nvl, er_timeval_t *when)
{
        (void) hdl;
        int64_t *tod;
        uint_t  nelem;

        if (nvlist_lookup_int64_array(nvl, FM_EREPORT_TIME, &tod,
            &nelem) == 0 && nelem == 2) {
                when->ertv_sec = tod[0];
                when->ertv_nsec = tod[1];
        } else {
                when->ertv_sec = when->ertv_nsec = UINT64_MAX;
        }
}

/*
 * Record the specified event in the SERD engine and return a
 * boolean value indicating whether or not the engine fired as
 * the result of inserting this event.
 *
 * When the pool has similar active cases on other vdevs, then
 * the fired state is disregarded and the case is retired.
 */
static int
zfs_fm_serd_record(fmd_hdl_t *hdl, const char *name, fmd_event_t *ep,
    zfs_case_t *zcp, const char *err_type)
{
        int fired = fmd_serd_record(hdl, name, ep);
        int peers = 0;

        if (fired && (peers = zfs_other_serd_cases(hdl, &zcp->zc_data)) > 0) {
                fmd_hdl_debug(hdl, "pool %llu is tracking %d other %s cases "
                    "-- skip faulting the vdev %llu",
                    (u_longlong_t)zcp->zc_data.zc_pool_guid,
                    peers, err_type,
                    (u_longlong_t)zcp->zc_data.zc_vdev_guid);
                zfs_case_retire(hdl, zcp);
                fired = 0;
        }

        return (fired);
}

/*
 * Main fmd entry point.
 */
static void
zfs_fm_recv(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl, const char *class)
{
        zfs_case_t *zcp, *dcp;
        int32_t pool_state;
        uint64_t ena, pool_guid, vdev_guid, parent_guid;
        uint64_t checksum_n, checksum_t;
        uint64_t io_n, io_t;
        er_timeval_t pool_load;
        er_timeval_t er_when;
        nvlist_t *detector;
        boolean_t pool_found = B_FALSE;
        boolean_t isresource;
        const char *type;

        /*
         * We subscribe to notifications for vdev or pool removal.  In these
         * cases, there may be cases that no longer apply.  Purge any cases
         * that no longer apply.
         */
        if (fmd_nvl_class_match(hdl, nvl, "sysevent.fs.zfs.*")) {
                fmd_hdl_debug(hdl, "purging orphaned cases from %s",
                    strrchr(class, '.') + 1);
                zfs_purge_cases(hdl);
                zfs_stats.resource_drops.fmds_value.ui64++;
                return;
        }

        isresource = fmd_nvl_class_match(hdl, nvl, "resource.fs.zfs.*");

        if (isresource) {
                /*
                 * For resources, we don't have a normal payload.
                 */
                if (nvlist_lookup_uint64(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID,
                    &vdev_guid) != 0)
                        pool_state = SPA_LOAD_OPEN;
                else
                        pool_state = SPA_LOAD_NONE;
                detector = NULL;
        } else {
                (void) nvlist_lookup_nvlist(nvl,
                    FM_EREPORT_DETECTOR, &detector);
                (void) nvlist_lookup_int32(nvl,
                    FM_EREPORT_PAYLOAD_ZFS_POOL_CONTEXT, &pool_state);
        }

        /*
         * We also ignore all ereports generated during an import of a pool,
         * since the only possible fault (.pool) would result in import failure,
         * and hence no persistent fault.  Some day we may want to do something
         * with these ereports, so we continue generating them internally.
         */
        if (pool_state == SPA_LOAD_IMPORT) {
                zfs_stats.import_drops.fmds_value.ui64++;
                fmd_hdl_debug(hdl, "ignoring '%s' during import", class);
                return;
        }

        /*
         * Device I/O errors are ignored during pool open.
         */
        if (pool_state == SPA_LOAD_OPEN &&
            (fmd_nvl_class_match(hdl, nvl,
            ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM)) ||
            fmd_nvl_class_match(hdl, nvl,
            ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO)) ||
            fmd_nvl_class_match(hdl, nvl,
            ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE)))) {
                fmd_hdl_debug(hdl, "ignoring '%s' during pool open", class);
                zfs_stats.dev_drops.fmds_value.ui64++;
                return;
        }

        /*
         * We ignore ereports for anything except disks and files.
         */
        if (nvlist_lookup_string(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_TYPE,
            &type) == 0) {
                if (strcmp(type, VDEV_TYPE_DISK) != 0 &&
                    strcmp(type, VDEV_TYPE_FILE) != 0) {
                        zfs_stats.vdev_drops.fmds_value.ui64++;
                        return;
                }
        }

        /*
         * Determine if this ereport corresponds to an open case.
         * Each vdev or pool can have a single case.
         */
        (void) nvlist_lookup_uint64(nvl,
            FM_EREPORT_PAYLOAD_ZFS_POOL_GUID, &pool_guid);
        if (nvlist_lookup_uint64(nvl,
            FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID, &vdev_guid) != 0)
                vdev_guid = 0;
        if (nvlist_lookup_uint64(nvl,
            FM_EREPORT_PAYLOAD_ZFS_PARENT_GUID, &parent_guid) != 0)
                parent_guid = 0;
        if (nvlist_lookup_uint64(nvl, FM_EREPORT_ENA, &ena) != 0)
                ena = 0;

        zfs_ereport_when(hdl, nvl, &er_when);

        for (zcp = uu_list_first(zfs_cases); zcp != NULL;
            zcp = uu_list_next(zfs_cases, zcp)) {
                if (zcp->zc_data.zc_pool_guid == pool_guid) {
                        pool_found = B_TRUE;
                        pool_load = zcp->zc_when;
                }
                if (zcp->zc_data.zc_vdev_guid == vdev_guid)
                        break;
        }

        /*
         * Avoid falsely accusing a pool of being faulty.  Do so by
         * not replaying ereports that were generated prior to the
         * current import.  If the failure that generated them was
         * transient because the device was actually removed but we
         * didn't receive the normal asynchronous notification, we
         * don't want to mark it as faulted and potentially panic. If
         * there is still a problem we'd expect not to be able to
         * import the pool, or that new ereports will be generated
         * once the pool is used.
         */
        if (pool_found && timeval_earlier(&er_when, &pool_load)) {
                fmd_hdl_debug(hdl, "ignoring pool %llx, "
                    "ereport time %lld.%lld, pool load time = %lld.%lld",
                    pool_guid, er_when.ertv_sec, er_when.ertv_nsec,
                    pool_load.ertv_sec, pool_load.ertv_nsec);
                zfs_stats.old_drops.fmds_value.ui64++;
                return;
        }

        if (!pool_found) {
                /*
                 * Haven't yet seen this pool, but same situation
                 * may apply.
                 */
                libzfs_handle_t *zhdl = fmd_hdl_getspecific(hdl);
                struct load_time_arg la;

                la.lt_guid = pool_guid;
                la.lt_time = &pool_load;
                la.lt_found = B_FALSE;

                if (zhdl != NULL &&
                    zpool_iter(zhdl, zpool_find_load_time, &la) == 0 &&
                    la.lt_found == B_TRUE) {
                        pool_found = B_TRUE;

                        if (timeval_earlier(&er_when, &pool_load)) {
                                fmd_hdl_debug(hdl, "ignoring pool %llx, "
                                    "ereport time %lld.%lld, "
                                    "pool load time = %lld.%lld",
                                    pool_guid, er_when.ertv_sec,
                                    er_when.ertv_nsec, pool_load.ertv_sec,
                                    pool_load.ertv_nsec);
                                zfs_stats.old_drops.fmds_value.ui64++;
                                return;
                        }
                }
        }

        if (zcp == NULL) {
                fmd_case_t *cs;
                zfs_case_data_t data = { 0 };

                /*
                 * If this is one of our 'fake' resource ereports, and there is
                 * no case open, simply discard it.
                 */
                if (isresource) {
                        zfs_stats.resource_drops.fmds_value.ui64++;
                        fmd_hdl_debug(hdl, "discarding '%s for vdev %llu",
                            class, vdev_guid);
                        return;
                }

                /*
                 * Skip tracking some ereports
                 */
                if (strcmp(class,
                    ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_DATA)) == 0 ||
                    strcmp(class,
                    ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CONFIG_CACHE_WRITE)) == 0) {
                        zfs_stats.resource_drops.fmds_value.ui64++;
                        return;
                }

                /*
                 * Open a new case.
                 */
                cs = fmd_case_open(hdl, NULL);

                fmd_hdl_debug(hdl, "opening case for vdev %llu due to '%s'",
                    vdev_guid, class);

                /*
                 * Initialize the case buffer.  To commonize code, we actually
                 * create the buffer with existing data, and then call
                 * zfs_case_unserialize() to instantiate the in-core structure.
                 */
                fmd_buf_create(hdl, cs, CASE_DATA, sizeof (zfs_case_data_t));

                data.zc_version = CASE_DATA_VERSION_SERD;
                data.zc_ena = ena;
                data.zc_pool_guid = pool_guid;
                data.zc_vdev_guid = vdev_guid;
                data.zc_parent_guid = parent_guid;
                data.zc_pool_state = (int)pool_state;

                fmd_buf_write(hdl, cs, CASE_DATA, &data, sizeof (data));

                zcp = zfs_case_unserialize(hdl, cs);
                assert(zcp != NULL);
                if (pool_found)
                        zcp->zc_when = pool_load;
        }

        if (isresource) {
                fmd_hdl_debug(hdl, "resource event '%s'", class);

                if (fmd_nvl_class_match(hdl, nvl,
                    ZFS_MAKE_RSRC(FM_RESOURCE_AUTOREPLACE))) {
                        /*
                         * The 'resource.fs.zfs.autoreplace' event indicates
                         * that the pool was loaded with the 'autoreplace'
                         * property set.  In this case, any pending device
                         * failures should be ignored, as the asynchronous
                         * autoreplace handling will take care of them.
                         */
                        fmd_case_close(hdl, zcp->zc_case);
                } else if (fmd_nvl_class_match(hdl, nvl,
                    ZFS_MAKE_RSRC(FM_RESOURCE_REMOVED))) {
                        /*
                         * The 'resource.fs.zfs.removed' event indicates that
                         * device removal was detected, and the device was
                         * closed asynchronously.  If this is the case, we
                         * assume that any recent I/O errors were due to the
                         * device removal, not any fault of the device itself.
                         * We reset the SERD engine, and cancel any pending
                         * timers.
                         */
                        if (zcp->zc_data.zc_has_remove_timer) {
                                fmd_timer_remove(hdl, zcp->zc_remove_timer);
                                zcp->zc_data.zc_has_remove_timer = 0;
                                zfs_case_serialize(zcp);
                        }
                        if (zcp->zc_data.zc_serd_io[0] != '\0')
                                fmd_serd_reset(hdl, zcp->zc_data.zc_serd_io);
                        if (zcp->zc_data.zc_serd_checksum[0] != '\0')
                                fmd_serd_reset(hdl,
                                    zcp->zc_data.zc_serd_checksum);
                        if (zcp->zc_data.zc_serd_slow_io[0] != '\0')
                                fmd_serd_reset(hdl,
                                    zcp->zc_data.zc_serd_slow_io);
                } else if (fmd_nvl_class_match(hdl, nvl,
                    ZFS_MAKE_RSRC(FM_RESOURCE_STATECHANGE))) {
                        uint64_t state = 0;

                        if (zcp != NULL &&
                            nvlist_lookup_uint64(nvl,
                            FM_EREPORT_PAYLOAD_ZFS_VDEV_STATE, &state) == 0 &&
                            state == VDEV_STATE_HEALTHY) {
                                fmd_hdl_debug(hdl, "closing case after a "
                                    "device statechange to healthy");
                                fmd_case_close(hdl, zcp->zc_case);
                        }
                }
                zfs_stats.resource_drops.fmds_value.ui64++;
                return;
        }

        /*
         * Associate the ereport with this case.
         */
        fmd_case_add_ereport(hdl, zcp->zc_case, ep);

        /*
         * Don't do anything else if this case is already solved.
         */
        if (fmd_case_solved(hdl, zcp->zc_case))
                return;

        if (vdev_guid)
                fmd_hdl_debug(hdl, "error event '%s', vdev %llu", class,
                    vdev_guid);
        else
                fmd_hdl_debug(hdl, "error event '%s'", class);

        /*
         * Determine if we should solve the case and generate a fault.  We solve
         * a case if:
         *
         *      a. A pool failed to open (ereport.fs.zfs.pool)
         *      b. A device failed to open (ereport.fs.zfs.pool) while a pool
         *         was up and running.
         *
         * We may see a series of ereports associated with a pool open, all
         * chained together by the same ENA.  If the pool open succeeds, then
         * we'll see no further ereports.  To detect when a pool open has
         * succeeded, we associate a timer with the event.  When it expires, we
         * close the case.
         */
        if (fmd_nvl_class_match(hdl, nvl,
            ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_POOL))) {
                /*
                 * Pool level fault.  Before solving the case, go through and
                 * close any open device cases that may be pending.
                 */
                for (dcp = uu_list_first(zfs_cases); dcp != NULL;
                    dcp = uu_list_next(zfs_cases, dcp)) {
                        if (dcp->zc_data.zc_pool_guid ==
                            zcp->zc_data.zc_pool_guid &&
                            dcp->zc_data.zc_vdev_guid != 0)
                                fmd_case_close(hdl, dcp->zc_case);
                }

                zfs_case_solve(hdl, zcp, "fault.fs.zfs.pool");
        } else if (fmd_nvl_class_match(hdl, nvl,
            ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_LOG_REPLAY))) {
                /*
                 * Pool level fault for reading the intent logs.
                 */
                zfs_case_solve(hdl, zcp, "fault.fs.zfs.log_replay");
        } else if (fmd_nvl_class_match(hdl, nvl, "ereport.fs.zfs.vdev.*")) {
                /*
                 * Device fault.
                 */
                zfs_case_solve(hdl, zcp, "fault.fs.zfs.device");
        } else if (fmd_nvl_class_match(hdl, nvl,
            ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO)) ||
            fmd_nvl_class_match(hdl, nvl,
            ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM)) ||
            fmd_nvl_class_match(hdl, nvl,
            ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO_FAILURE)) ||
            fmd_nvl_class_match(hdl, nvl,
            ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_DELAY)) ||
            fmd_nvl_class_match(hdl, nvl,
            ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE))) {
                const char *failmode = NULL;
                boolean_t checkremove = B_FALSE;
                uint32_t pri = 0;

                /*
                 * If this is a checksum or I/O error, then toss it into the
                 * appropriate SERD engine and check to see if it has fired.
                 * Ideally, we want to do something more sophisticated,
                 * (persistent errors for a single data block, etc).  For now,
                 * a single SERD engine is sufficient.
                 */
                if (fmd_nvl_class_match(hdl, nvl,
                    ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO))) {
                        if (zcp->zc_data.zc_serd_io[0] == '\0') {
                                if (nvlist_lookup_uint64(nvl,
                                    FM_EREPORT_PAYLOAD_ZFS_VDEV_IO_N,
                                    &io_n) != 0) {
                                        io_n = DEFAULT_IO_N;
                                }
                                if (nvlist_lookup_uint64(nvl,
                                    FM_EREPORT_PAYLOAD_ZFS_VDEV_IO_T,
                                    &io_t) != 0) {
                                        io_t = DEFAULT_IO_T;
                                }
                                zfs_serd_name(zcp->zc_data.zc_serd_io,
                                    pool_guid, vdev_guid, "io");
                                fmd_serd_create(hdl, zcp->zc_data.zc_serd_io,
                                    io_n,
                                    SEC2NSEC(io_t));
                                zfs_case_serialize(zcp);
                        }
                        if (zfs_fm_serd_record(hdl, zcp->zc_data.zc_serd_io,
                            ep, zcp, "io error")) {
                                checkremove = B_TRUE;
                        }
                } else if (fmd_nvl_class_match(hdl, nvl,
                    ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_DELAY))) {
                        uint64_t slow_io_n, slow_io_t;

                        /*
                         * Create a slow io SERD engine when the VDEV has the
                         * 'vdev_slow_io_n' and 'vdev_slow_io_n' properties.
                         */
                        if (zcp->zc_data.zc_serd_slow_io[0] == '\0' &&
                            nvlist_lookup_uint64(nvl,
                            FM_EREPORT_PAYLOAD_ZFS_VDEV_SLOW_IO_N,
                            &slow_io_n) == 0 &&
                            nvlist_lookup_uint64(nvl,
                            FM_EREPORT_PAYLOAD_ZFS_VDEV_SLOW_IO_T,
                            &slow_io_t) == 0) {
                                zfs_serd_name(zcp->zc_data.zc_serd_slow_io,
                                    pool_guid, vdev_guid, "slow_io");
                                fmd_serd_create(hdl,
                                    zcp->zc_data.zc_serd_slow_io,
                                    slow_io_n,
                                    SEC2NSEC(slow_io_t));
                                zfs_case_serialize(zcp);
                        }
                        /* Pass event to SERD engine and see if this triggers */
                        if (zcp->zc_data.zc_serd_slow_io[0] != '\0' &&
                            zfs_fm_serd_record(hdl,
                            zcp->zc_data.zc_serd_slow_io, ep, zcp, "slow io")) {
                                zfs_case_solve(hdl, zcp,
                                    "fault.fs.zfs.vdev.slow_io");
                        }
                } else if (fmd_nvl_class_match(hdl, nvl,
                    ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM))) {
                        uint64_t flags = 0;
                        int32_t flags32 = 0;
                        /*
                         * We ignore ereports for checksum errors generated by
                         * scrub/resilver I/O to avoid potentially further
                         * degrading the pool while it's being repaired.
                         *
                         * Note that FM_EREPORT_PAYLOAD_ZFS_ZIO_FLAGS used to
                         * be int32. To allow newer zed to work on older
                         * kernels, if we don't find the flags, we look for
                         * the older ones too.
                         */
                        if (((nvlist_lookup_uint32(nvl,
                            FM_EREPORT_PAYLOAD_ZFS_ZIO_PRIORITY, &pri) == 0) &&
                            (pri == ZIO_PRIORITY_SCRUB ||
                            pri == ZIO_PRIORITY_REBUILD)) ||
                            ((nvlist_lookup_uint64(nvl,
                            FM_EREPORT_PAYLOAD_ZFS_ZIO_FLAGS, &flags) == 0) &&
                            (flags & (ZIO_FLAG_SCRUB | ZIO_FLAG_RESILVER))) ||
                            ((nvlist_lookup_int32(nvl,
                            FM_EREPORT_PAYLOAD_ZFS_ZIO_FLAGS, &flags32) == 0) &&
                            (flags32 & (ZIO_FLAG_SCRUB | ZIO_FLAG_RESILVER)))) {
                                fmd_hdl_debug(hdl, "ignoring '%s' for "
                                    "scrub/resilver I/O", class);
                                return;
                        }

                        if (zcp->zc_data.zc_serd_checksum[0] == '\0') {
                                if (nvlist_lookup_uint64(nvl,
                                    FM_EREPORT_PAYLOAD_ZFS_VDEV_CKSUM_N,
                                    &checksum_n) != 0) {
                                        checksum_n = DEFAULT_CHECKSUM_N;
                                }
                                if (nvlist_lookup_uint64(nvl,
                                    FM_EREPORT_PAYLOAD_ZFS_VDEV_CKSUM_T,
                                    &checksum_t) != 0) {
                                        checksum_t = DEFAULT_CHECKSUM_T;
                                }

                                zfs_serd_name(zcp->zc_data.zc_serd_checksum,
                                    pool_guid, vdev_guid, "checksum");
                                fmd_serd_create(hdl,
                                    zcp->zc_data.zc_serd_checksum,
                                    checksum_n,
                                    SEC2NSEC(checksum_t));
                                zfs_case_serialize(zcp);
                        }
                        if (zfs_fm_serd_record(hdl,
                            zcp->zc_data.zc_serd_checksum, ep, zcp,
                            "checksum")) {
                                zfs_case_solve(hdl, zcp,
                                    "fault.fs.zfs.vdev.checksum");
                        }
                } else if (fmd_nvl_class_match(hdl, nvl,
                    ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO_FAILURE)) &&
                    (nvlist_lookup_string(nvl,
                    FM_EREPORT_PAYLOAD_ZFS_POOL_FAILMODE, &failmode) == 0) &&
                    failmode != NULL) {
                        if (strncmp(failmode, FM_EREPORT_FAILMODE_CONTINUE,
                            strlen(FM_EREPORT_FAILMODE_CONTINUE)) == 0) {
                                zfs_case_solve(hdl, zcp,
                                    "fault.fs.zfs.io_failure_continue");
                        } else if (strncmp(failmode, FM_EREPORT_FAILMODE_WAIT,
                            strlen(FM_EREPORT_FAILMODE_WAIT)) == 0) {
                                zfs_case_solve(hdl, zcp,
                                    "fault.fs.zfs.io_failure_wait");
                        }
                } else if (fmd_nvl_class_match(hdl, nvl,
                    ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE))) {
#ifndef __linux__
                        /* This causes an unexpected fault diagnosis on linux */
                        checkremove = B_TRUE;
#endif
                }

                /*
                 * Because I/O errors may be due to device removal, we postpone
                 * any diagnosis until we're sure that we aren't about to
                 * receive a 'resource.fs.zfs.removed' event.
                 */
                if (checkremove) {
                        if (zcp->zc_data.zc_has_remove_timer)
                                fmd_timer_remove(hdl, zcp->zc_remove_timer);
                        zcp->zc_remove_timer = fmd_timer_install(hdl, zcp, NULL,
                            zfs_remove_timeout);
                        if (!zcp->zc_data.zc_has_remove_timer) {
                                zcp->zc_data.zc_has_remove_timer = 1;
                                zfs_case_serialize(zcp);
                        }
                }
        }
}

/*
 * The timeout is fired when we diagnosed an I/O error, and it was not due to
 * device removal (which would cause the timeout to be cancelled).
 */
static void
zfs_fm_timeout(fmd_hdl_t *hdl, id_t id, void *data)
{
        zfs_case_t *zcp = data;

        if (id == zcp->zc_remove_timer)
                zfs_case_solve(hdl, zcp, "fault.fs.zfs.vdev.io");
}

/*
 * The specified case has been closed and any case-specific
 * data structures should be deallocated.
 */
static void
zfs_fm_close(fmd_hdl_t *hdl, fmd_case_t *cs)
{
        zfs_case_t *zcp = fmd_case_getspecific(hdl, cs);

        if (zcp->zc_data.zc_serd_checksum[0] != '\0')
                fmd_serd_destroy(hdl, zcp->zc_data.zc_serd_checksum);
        if (zcp->zc_data.zc_serd_io[0] != '\0')
                fmd_serd_destroy(hdl, zcp->zc_data.zc_serd_io);
        if (zcp->zc_data.zc_serd_slow_io[0] != '\0')
                fmd_serd_destroy(hdl, zcp->zc_data.zc_serd_slow_io);
        if (zcp->zc_data.zc_has_remove_timer)
                fmd_timer_remove(hdl, zcp->zc_remove_timer);

        uu_list_remove(zfs_cases, zcp);
        uu_list_node_fini(zcp, &zcp->zc_node, zfs_case_pool);
        fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t));
}

static const fmd_hdl_ops_t fmd_ops = {
        zfs_fm_recv,    /* fmdo_recv */
        zfs_fm_timeout, /* fmdo_timeout */
        zfs_fm_close,   /* fmdo_close */
        NULL,           /* fmdo_stats */
        NULL,   /* fmdo_gc */
};

static const fmd_prop_t fmd_props[] = {
        { NULL, 0, NULL }
};

static const fmd_hdl_info_t fmd_info = {
        "ZFS Diagnosis Engine", "1.0", &fmd_ops, fmd_props
};

void
_zfs_diagnosis_init(fmd_hdl_t *hdl)
{
        libzfs_handle_t *zhdl;

        if ((zhdl = libzfs_init()) == NULL)
                return;

        if ((zfs_case_pool = uu_list_pool_create("zfs_case_pool",
            sizeof (zfs_case_t), offsetof(zfs_case_t, zc_node),
            NULL, UU_LIST_POOL_DEBUG)) == NULL) {
                libzfs_fini(zhdl);
                return;
        }

        if ((zfs_cases = uu_list_create(zfs_case_pool, NULL,
            UU_LIST_DEBUG)) == NULL) {
                uu_list_pool_destroy(zfs_case_pool);
                libzfs_fini(zhdl);
                return;
        }

        if (fmd_hdl_register(hdl, FMD_API_VERSION, &fmd_info) != 0) {
                uu_list_destroy(zfs_cases);
                uu_list_pool_destroy(zfs_case_pool);
                libzfs_fini(zhdl);
                return;
        }

        fmd_hdl_setspecific(hdl, zhdl);

        (void) fmd_stat_create(hdl, FMD_STAT_NOALLOC, sizeof (zfs_stats) /
            sizeof (fmd_stat_t), (fmd_stat_t *)&zfs_stats);
}

void
_zfs_diagnosis_fini(fmd_hdl_t *hdl)
{
        zfs_case_t *zcp;
        uu_list_walk_t *walk;
        libzfs_handle_t *zhdl;

        /*
         * Remove all active cases.
         */
        walk = uu_list_walk_start(zfs_cases, UU_WALK_ROBUST);
        while ((zcp = uu_list_walk_next(walk)) != NULL) {
                fmd_hdl_debug(hdl, "removing case ena %llu",
                    (long long unsigned)zcp->zc_data.zc_ena);
                uu_list_remove(zfs_cases, zcp);
                uu_list_node_fini(zcp, &zcp->zc_node, zfs_case_pool);
                fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t));
        }
        uu_list_walk_end(walk);

        uu_list_destroy(zfs_cases);
        uu_list_pool_destroy(zfs_case_pool);

        zhdl = fmd_hdl_getspecific(hdl);
        libzfs_fini(zhdl);
}