4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or https://opensource.org/licenses/CDDL-1.0.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2012, 2015 by Delphix. All rights reserved.
24 * Copyright (c) 2017, Intel Corporation.
25 * Copyright (c) 2023-2024, Klara Inc.
31 * This userland component takes a set of options and uses libzpool to translate
32 * from a user-visible object type and name to an internal representation.
33 * There are two basic types of faults: device faults and data faults.
38 * Errors can be injected into a particular vdev using the '-d' option. This
39 * option takes a path or vdev GUID to uniquely identify the device within a
40 * pool. There are four types of errors that can be injected, IO, ENXIO,
41 * ECHILD, and EILSEQ. These can be controlled through the '-e' option and the
42 * default is ENXIO. For EIO failures, any attempt to read data from the device
43 * will return EIO, but a subsequent attempt to reopen the device will succeed.
44 * For ENXIO failures, any attempt to read from the device will return EIO, but
45 * any attempt to reopen the device will also return ENXIO. The EILSEQ failures
46 * only apply to read operations (-T read) and will flip a bit after the device
47 * has read the original data.
49 * For label faults, the -L option must be specified. This allows faults
50 * to be injected into either the nvlist, uberblock, pad1, or pad2 region
51 * of all the labels for the specified device.
53 * This form of the command looks like:
55 * zinject -d device [-e errno] [-L <uber | nvlist | pad1 | pad2>] pool
60 * We begin with a tuple of the form:
62 * <type,level,range,object>
64 * type A string describing the type of data to target. Each type
65 * implicitly describes how to interpret 'object'. Currently,
66 * the following values are supported:
68 * data User data for a file
69 * dnode Dnode for a file or directory
71 * The following MOS objects are special. Instead of injecting
72 * errors on a particular object or blkid, we inject errors across
73 * all objects of the given type.
75 * mos Any data in the MOS
76 * mosdir object directory
77 * config pool configuration
81 * errlog persistent error log
83 * level Object level. Defaults to '0', not applicable to all types. If
84 * a range is given, this corresponds to the indirect block
85 * corresponding to the specific range.
87 * range A numerical range [start,end) within the object. Defaults to
88 * the full size of the file.
90 * object A string describing the logical location of the object. For
91 * files and directories (currently the only supported types),
92 * this is the path of the object on disk.
94 * This is translated, via libzpool, into the following internal representation:
96 * <type,objset,object,level,range>
98 * These types should be self-explanatory. This tuple is then passed to the
99 * kernel via a special ioctl() to initiate fault injection for the given
100 * object. Note that 'type' is not strictly necessary for fault injection, but
101 * is used when translating existing faults into a human-readable string.
104 * The command itself takes one of the forms:
107 * zinject <-a | -u pool>
108 * zinject -c <id|all>
109 * zinject [-q] <-t type> [-f freq] [-u] [-a] [-m] [-e errno] [-l level]
110 * [-r range] <object>
111 * zinject [-f freq] [-a] [-m] [-u] -b objset:object:level:start:end pool
113 * With no arguments, the command prints all currently registered injection
114 * handlers, with their numeric identifiers.
116 * The '-c' option will clear the given handler, or all handlers if 'all' is
119 * The '-e' option takes a string describing the errno to simulate. This must
120 * be one of 'io', 'checksum', 'decompress', or 'decrypt'. In most cases this
121 * will result in the same behavior, but RAID-Z will produce a different set of
122 * ereports for this situation.
124 * The '-a', '-u', and '-m' flags toggle internal flush behavior. If '-a' is
125 * specified, then the ARC cache is flushed appropriately. If '-u' is
126 * specified, then the underlying SPA is unloaded. Either of these flags can be
127 * specified independently of any other handlers. The '-m' flag automatically
128 * does an unmount and remount of the underlying dataset to aid in flushing the
131 * The '-f' flag controls the frequency of errors injected, expressed as a
132 * real number percentage between 0.0001 and 100. The default is 100.
134 * The this form is responsible for actually injecting the handler into the
135 * framework. It takes the arguments described above, translates them to the
136 * internal tuple using libzpool, and then issues an ioctl() to register the
139 * The final form can target a specific bookmark, regardless of whether a
140 * human-readable interface has been designed. It allows developers to specify
141 * a particular block by number.
152 #include <sys/fs/zfs.h>
153 #include <sys/mount.h>
157 #undef verify /* both libzfs.h and zfs_context.h want to define this */
161 libzfs_handle_t
*g_zfs
;
164 static const char *const errtable
[TYPE_INVAL
] = {
181 name_to_type(const char *arg
)
184 for (i
= 0; i
< TYPE_INVAL
; i
++)
185 if (strcmp(errtable
[i
], arg
) == 0)
192 type_to_name(uint64_t type
)
195 case DMU_OT_OBJECT_DIRECTORY
:
197 case DMU_OT_OBJECT_ARRAY
:
199 case DMU_OT_PACKED_NVLIST
:
203 case DMU_OT_SPACE_MAP
:
205 case DMU_OT_ERROR_LOG
:
216 static const struct errstr errstrtable
[] = {
218 { ECKSUM
, "checksum" },
219 { EINVAL
, "decompress" },
220 { EACCES
, "decrypt" },
223 { EILSEQ
, "corrupt" },
229 str_to_err(const char *str
)
231 for (int i
= 0; errstrtable
[i
].str
!= NULL
; i
++)
232 if (strcasecmp(errstrtable
[i
].str
, str
) == 0)
233 return (errstrtable
[i
].err
);
239 for (int i
= 0; errstrtable
[i
].str
!= NULL
; i
++)
240 if (errstrtable
[i
].err
== err
)
241 return (errstrtable
[i
].str
);
242 return ("[unknown]");
246 * Print usage message.
256 "\t\tList all active injection records.\n"
258 "\tzinject -c <id|all>\n"
260 "\t\tClear the particular record (if given a numeric ID), or\n"
261 "\t\tall records if 'all' is specified.\n"
263 "\tzinject -p <function name> pool\n"
264 "\t\tInject a panic fault at the specified function. Only \n"
265 "\t\tfunctions which call spa_vdev_config_exit(), or \n"
266 "\t\tspa_vdev_exit() will trigger a panic.\n"
268 "\tzinject -d device [-e errno] [-L <nvlist|uber|pad1|pad2>] [-F]\n"
269 "\t\t[-T <read|write|free|claim|flush|all>] [-f frequency] pool\n\n"
270 "\t\tInject a fault into a particular device or the device's\n"
271 "\t\tlabel. Label injection can either be 'nvlist', 'uber',\n "
272 "\t\t'pad1', or 'pad2'.\n"
273 "\t\t'errno' can be 'nxio' (the default), 'io', 'dtl',\n"
274 "\t\t'corrupt' (bit flip), or 'noop' (successfully do nothing).\n"
275 "\t\t'frequency' is a value between 0.0001 and 100.0 that limits\n"
276 "\t\tdevice error injection to a percentage of the IOs.\n"
278 "\tzinject -d device -A <degrade|fault> -D <delay secs> pool\n"
279 "\t\tPerform a specific action on a particular device.\n"
281 "\tzinject -d device -D latency:lanes pool\n"
283 "\t\tAdd an artificial delay to IO requests on a particular\n"
284 "\t\tdevice, such that the requests take a minimum of 'latency'\n"
285 "\t\tmilliseconds to complete. Each delay has an associated\n"
286 "\t\tnumber of 'lanes' which defines the number of concurrent\n"
287 "\t\tIO requests that can be processed.\n"
289 "\t\tFor example, with a single lane delay of 10 ms (-D 10:1),\n"
290 "\t\tthe device will only be able to service a single IO request\n"
291 "\t\tat a time with each request taking 10 ms to complete. So,\n"
292 "\t\tif only a single request is submitted every 10 ms, the\n"
293 "\t\taverage latency will be 10 ms; but if more than one request\n"
294 "\t\tis submitted every 10 ms, the average latency will be more\n"
297 "\t\tSimilarly, if a delay of 10 ms is specified to have two\n"
298 "\t\tlanes (-D 10:2), then the device will be able to service\n"
299 "\t\ttwo requests at a time, each with a minimum latency of\n"
300 "\t\t10 ms. So, if two requests are submitted every 10 ms, then\n"
301 "\t\tthe average latency will be 10 ms; but if more than two\n"
302 "\t\trequests are submitted every 10 ms, the average latency\n"
303 "\t\twill be more than 10 ms.\n"
305 "\t\tAlso note, these delays are additive. So two invocations\n"
306 "\t\tof '-D 10:1', is roughly equivalent to a single invocation\n"
307 "\t\tof '-D 10:2'. This also means, one can specify multiple\n"
308 "\t\tlanes with differing target latencies. For example, an\n"
309 "\t\tinvocation of '-D 10:1' followed by '-D 25:2' will\n"
310 "\t\tcreate 3 lanes on the device; one lane with a latency\n"
311 "\t\tof 10 ms and two lanes with a 25 ms latency.\n"
313 "\tzinject -P import|export -s <seconds> pool\n"
314 "\t\tAdd an artificial delay to a future pool import or export,\n"
315 "\t\tsuch that the operation takes a minimum of supplied seconds\n"
318 "\tzinject -I [-s <seconds> | -g <txgs>] pool\n"
319 "\t\tCause the pool to stop writing blocks yet not\n"
320 "\t\treport errors for a duration. Simulates buggy hardware\n"
321 "\t\tthat fails to honor cache flush requests.\n"
322 "\t\tDefault duration is 30 seconds. The machine is panicked\n"
323 "\t\tat the end of the duration.\n"
325 "\tzinject -b objset:object:level:blkid pool\n"
327 "\t\tInject an error into pool 'pool' with the numeric bookmark\n"
328 "\t\tspecified by the remaining tuple. Each number is in\n"
329 "\t\thexadecimal, and only one block can be specified.\n"
331 "\tzinject [-q] <-t type> [-C dvas] [-e errno] [-l level]\n"
332 "\t\t[-r range] [-a] [-m] [-u] [-f freq] <object>\n"
334 "\t\tInject an error into the object specified by the '-t' option\n"
335 "\t\tand the object descriptor. The 'object' parameter is\n"
336 "\t\tinterpreted depending on the '-t' option.\n"
338 "\t\t-q\tQuiet mode. Only print out the handler number added.\n"
339 "\t\t-e\tInject a specific error. Must be one of 'io',\n"
340 "\t\t\t'checksum', 'decompress', or 'decrypt'. Default is 'io'.\n"
341 "\t\t-C\tInject the given error only into specific DVAs. The\n"
342 "\t\t\tDVAs should be specified as a list of 0-indexed DVAs\n"
343 "\t\t\tseparated by commas (ex. '0,2').\n"
344 "\t\t-l\tInject error at a particular block level. Default is "
346 "\t\t-m\tAutomatically remount underlying filesystem.\n"
347 "\t\t-r\tInject error over a particular logical range of an\n"
348 "\t\t\tobject. Will be translated to the appropriate blkid\n"
349 "\t\t\trange according to the object's properties.\n"
350 "\t\t-a\tFlush the ARC cache. Can be specified without any\n"
351 "\t\t\tassociated object.\n"
352 "\t\t-u\tUnload the associated pool. Can be specified with only\n"
353 "\t\t\ta pool object.\n"
354 "\t\t-f\tOnly inject errors a fraction of the time. Expressed as\n"
355 "\t\t\ta percentage between 0.0001 and 100.\n"
357 "\t-t data\t\tInject an error into the plain file contents of a\n"
358 "\t\t\tfile. The object must be specified as a complete path\n"
359 "\t\t\tto a file on a ZFS filesystem.\n"
361 "\t-t dnode\tInject an error into the metadnode in the block\n"
362 "\t\t\tcorresponding to the dnode for a file or directory. The\n"
363 "\t\t\t'-r' option is incompatible with this mode. The object\n"
364 "\t\t\tis specified as a complete path to a file or directory\n"
365 "\t\t\ton a ZFS filesystem.\n"
367 "\t-t <mos>\tInject errors into the MOS for objects of the given\n"
368 "\t\t\ttype. Valid types are: mos, mosdir, config, bpobj,\n"
369 "\t\t\tspacemap, metaslab, errlog. The only valid <object> is\n"
370 "\t\t\tthe poolname.\n");
374 iter_handlers(int (*func
)(int, const char *, zinject_record_t
*, void *),
377 zfs_cmd_t zc
= {"\0"};
380 while (zfs_ioctl(g_zfs
, ZFS_IOC_INJECT_LIST_NEXT
, &zc
) == 0)
381 if ((ret
= func((int)zc
.zc_guid
, zc
.zc_name
,
382 &zc
.zc_inject_record
, data
)) != 0)
385 if (errno
!= ENOENT
) {
386 (void) fprintf(stderr
, "Unable to list handlers: %s\n",
395 print_data_handler(int id
, const char *pool
, zinject_record_t
*record
,
400 if (record
->zi_guid
!= 0 || record
->zi_func
[0] != '\0' ||
401 record
->zi_duration
!= 0) {
406 (void) printf("%3s %-15s %-6s %-6s %-8s %3s %-4s "
407 "%-15s\n", "ID", "POOL", "OBJSET", "OBJECT", "TYPE",
408 "LVL", "DVAs", "RANGE");
409 (void) printf("--- --------------- ------ "
410 "------ -------- --- ---- ---------------\n");
415 (void) printf("%3d %-15s %-6llu %-6llu %-8s %-3d 0x%02x ",
416 id
, pool
, (u_longlong_t
)record
->zi_objset
,
417 (u_longlong_t
)record
->zi_object
, type_to_name(record
->zi_type
),
418 record
->zi_level
, record
->zi_dvas
);
421 if (record
->zi_start
== 0 &&
422 record
->zi_end
== -1ULL)
423 (void) printf("all\n");
425 (void) printf("[%llu, %llu]\n", (u_longlong_t
)record
->zi_start
,
426 (u_longlong_t
)record
->zi_end
);
432 print_device_handler(int id
, const char *pool
, zinject_record_t
*record
,
435 static const char *iotypestr
[] = {
436 "null", "read", "write", "free", "claim", "flush", "trim", "all",
441 if (record
->zi_guid
== 0 || record
->zi_func
[0] != '\0')
444 if (record
->zi_cmd
== ZINJECT_DELAY_IO
)
448 (void) printf("%3s %-15s %-16s %-5s %-10s %-9s\n",
449 "ID", "POOL", "GUID", "TYPE", "ERROR", "FREQ");
451 "--- --------------- ---------------- "
452 "----- ---------- ---------\n");
457 double freq
= record
->zi_freq
== 0 ? 100.0f
:
458 (((double)record
->zi_freq
) / ZI_PERCENTAGE_MAX
) * 100.0f
;
460 (void) printf("%3d %-15s %llx %-5s %-10s %8.4f%%\n", id
, pool
,
461 (u_longlong_t
)record
->zi_guid
, iotypestr
[record
->zi_iotype
],
462 err_to_str(record
->zi_error
), freq
);
468 print_delay_handler(int id
, const char *pool
, zinject_record_t
*record
,
473 if (record
->zi_guid
== 0 || record
->zi_func
[0] != '\0')
476 if (record
->zi_cmd
!= ZINJECT_DELAY_IO
)
480 (void) printf("%3s %-15s %-15s %-15s %s\n",
481 "ID", "POOL", "DELAY (ms)", "LANES", "GUID");
482 (void) printf("--- --------------- --------------- "
483 "--------------- ----------------\n");
488 (void) printf("%3d %-15s %-15llu %-15llu %llx\n", id
, pool
,
489 (u_longlong_t
)NSEC2MSEC(record
->zi_timer
),
490 (u_longlong_t
)record
->zi_nlanes
,
491 (u_longlong_t
)record
->zi_guid
);
497 print_panic_handler(int id
, const char *pool
, zinject_record_t
*record
,
502 if (record
->zi_func
[0] == '\0')
506 (void) printf("%3s %-15s %s\n", "ID", "POOL", "FUNCTION");
507 (void) printf("--- --------------- ----------------\n");
512 (void) printf("%3d %-15s %s\n", id
, pool
, record
->zi_func
);
518 print_pool_delay_handler(int id
, const char *pool
, zinject_record_t
*record
,
523 if (record
->zi_cmd
!= ZINJECT_DELAY_IMPORT
&&
524 record
->zi_cmd
!= ZINJECT_DELAY_EXPORT
) {
529 (void) printf("%3s %-19s %-11s %s\n",
530 "ID", "POOL", "DELAY (sec)", "COMMAND");
531 (void) printf("--- ------------------- -----------"
537 (void) printf("%3d %-19s %-11llu %s\n",
538 id
, pool
, (u_longlong_t
)record
->zi_duration
,
539 record
->zi_cmd
== ZINJECT_DELAY_IMPORT
? "import": "export");
545 * Print all registered error handlers. Returns the number of handlers
549 print_all_handlers(void)
551 int count
= 0, total
= 0;
553 (void) iter_handlers(print_device_handler
, &count
);
560 (void) iter_handlers(print_delay_handler
, &count
);
567 (void) iter_handlers(print_data_handler
, &count
);
574 (void) iter_handlers(print_pool_delay_handler
, &count
);
581 (void) iter_handlers(print_panic_handler
, &count
);
583 return (count
+ total
);
587 cancel_one_handler(int id
, const char *pool
, zinject_record_t
*record
,
590 (void) pool
, (void) record
, (void) data
;
591 zfs_cmd_t zc
= {"\0"};
593 zc
.zc_guid
= (uint64_t)id
;
595 if (zfs_ioctl(g_zfs
, ZFS_IOC_CLEAR_FAULT
, &zc
) != 0) {
596 (void) fprintf(stderr
, "failed to remove handler %d: %s\n",
597 id
, strerror(errno
));
605 * Remove all fault injection handlers.
608 cancel_all_handlers(void)
610 int ret
= iter_handlers(cancel_one_handler
, NULL
);
613 (void) printf("removed all registered handlers\n");
619 * Remove a specific fault injection handler.
622 cancel_handler(int id
)
624 zfs_cmd_t zc
= {"\0"};
626 zc
.zc_guid
= (uint64_t)id
;
628 if (zfs_ioctl(g_zfs
, ZFS_IOC_CLEAR_FAULT
, &zc
) != 0) {
629 (void) fprintf(stderr
, "failed to remove handler %d: %s\n",
630 id
, strerror(errno
));
634 (void) printf("removed handler %d\n", id
);
640 * Register a new fault injection handler.
643 register_handler(const char *pool
, int flags
, zinject_record_t
*record
,
646 zfs_cmd_t zc
= {"\0"};
648 (void) strlcpy(zc
.zc_name
, pool
, sizeof (zc
.zc_name
));
649 zc
.zc_inject_record
= *record
;
652 if (zfs_ioctl(g_zfs
, ZFS_IOC_INJECT_FAULT
, &zc
) != 0) {
653 const char *errmsg
= strerror(errno
);
657 errmsg
= "block level exceeds max level of object";
660 if (record
->zi_cmd
== ZINJECT_DELAY_IMPORT
)
661 errmsg
= "pool already imported";
662 if (record
->zi_cmd
== ZINJECT_DELAY_EXPORT
)
663 errmsg
= "a handler already exists";
666 /* import delay injector running on older zfs module */
667 if (record
->zi_cmd
== ZINJECT_DELAY_IMPORT
)
668 errmsg
= "import delay injector not supported";
673 (void) fprintf(stderr
, "failed to add handler: %s\n", errmsg
);
677 if (flags
& ZINJECT_NULL
)
681 (void) printf("%llu\n", (u_longlong_t
)zc
.zc_guid
);
683 (void) printf("Added handler %llu with the following "
684 "properties:\n", (u_longlong_t
)zc
.zc_guid
);
685 (void) printf(" pool: %s\n", pool
);
686 if (record
->zi_guid
) {
687 (void) printf(" vdev: %llx\n",
688 (u_longlong_t
)record
->zi_guid
);
689 } else if (record
->zi_func
[0] != '\0') {
690 (void) printf(" panic function: %s\n",
692 } else if (record
->zi_duration
> 0) {
693 (void) printf(" time: %lld seconds\n",
694 (u_longlong_t
)record
->zi_duration
);
695 } else if (record
->zi_duration
< 0) {
696 (void) printf(" txgs: %lld \n",
697 (u_longlong_t
)-record
->zi_duration
);
698 } else if (record
->zi_timer
> 0) {
699 (void) printf(" timer: %lld ms\n",
700 (u_longlong_t
)NSEC2MSEC(record
->zi_timer
));
702 (void) printf("objset: %llu\n",
703 (u_longlong_t
)record
->zi_objset
);
704 (void) printf("object: %llu\n",
705 (u_longlong_t
)record
->zi_object
);
706 (void) printf(" type: %llu\n",
707 (u_longlong_t
)record
->zi_type
);
708 (void) printf(" level: %d\n", record
->zi_level
);
709 if (record
->zi_start
== 0 &&
710 record
->zi_end
== -1ULL)
711 (void) printf(" range: all\n");
713 (void) printf(" range: [%llu, %llu)\n",
714 (u_longlong_t
)record
->zi_start
,
715 (u_longlong_t
)record
->zi_end
);
716 (void) printf(" dvas: 0x%x\n", record
->zi_dvas
);
724 perform_action(const char *pool
, zinject_record_t
*record
, int cmd
)
726 zfs_cmd_t zc
= {"\0"};
728 ASSERT(cmd
== VDEV_STATE_DEGRADED
|| cmd
== VDEV_STATE_FAULTED
);
729 (void) strlcpy(zc
.zc_name
, pool
, sizeof (zc
.zc_name
));
730 zc
.zc_guid
= record
->zi_guid
;
733 if (zfs_ioctl(g_zfs
, ZFS_IOC_VDEV_SET_STATE
, &zc
) == 0)
740 parse_delay(char *str
, uint64_t *delay
, uint64_t *nlanes
)
742 unsigned long scan_delay
;
743 unsigned long scan_nlanes
;
745 if (sscanf(str
, "%lu:%lu", &scan_delay
, &scan_nlanes
) != 2)
749 * We explicitly disallow a delay of zero here, because we key
750 * off this value being non-zero in translate_device(), to
751 * determine if the fault is a ZINJECT_DELAY_IO fault or not.
757 * The units for the CLI delay parameter is milliseconds, but
758 * the data passed to the kernel is interpreted as nanoseconds.
759 * Thus we scale the milliseconds to nanoseconds here, and this
760 * nanosecond value is used to pass the delay to the kernel.
762 *delay
= MSEC2NSEC(scan_delay
);
763 *nlanes
= scan_nlanes
;
769 parse_frequency(const char *str
, uint32_t *percent
)
774 val
= strtod(str
, &post
);
775 if (post
== NULL
|| *post
!= '\0')
778 /* valid range is [0.0001, 100.0] */
780 if (val
< 0.000001f
|| val
> 1.0f
)
783 /* convert to an integer for use by kernel */
784 *percent
= ((uint32_t)(val
* ZI_PERCENTAGE_MAX
));
790 * This function converts a string specifier for DVAs into a bit mask.
791 * The dva's provided by the user should be 0 indexed and separated by
792 * a comma. For example:
793 * "1" -> 0b0010 (0x2)
794 * "0,1" -> 0b0011 (0x3)
795 * "0,1,2" -> 0b0111 (0x7)
798 parse_dvas(const char *str
, uint32_t *dvas_out
)
802 boolean_t need_delim
= B_FALSE
;
804 /* max string length is 5 ("0,1,2") */
805 if (strlen(str
) > 5 || strlen(str
) == 0)
813 /* check for pipe between DVAs */
817 /* check if this DVA has been set already */
818 if (mask
& (1 << ((*c
) - '0')))
821 mask
|= (1 << ((*c
) - '0'));
825 need_delim
= B_FALSE
;
828 /* check for invalid character */
834 /* check for dangling delimiter */
843 main(int argc
, char **argv
)
855 int io_type
= ZIO_TYPES
;
856 int action
= VDEV_STATE_UNKNOWN
;
857 err_type_t type
= TYPE_INVAL
;
858 err_type_t label
= TYPE_INVAL
;
859 zinject_record_t record
= { 0 };
860 char pool
[MAXNAMELEN
] = "";
861 char dataset
[MAXNAMELEN
] = "";
862 zfs_handle_t
*zhp
= NULL
;
870 if ((g_zfs
= libzfs_init()) == NULL
) {
871 (void) fprintf(stderr
, "%s\n", libzfs_error_init(errno
));
875 libzfs_print_on_error(g_zfs
, B_TRUE
);
877 if ((zfs_fd
= open(ZFS_DEV
, O_RDWR
)) < 0) {
878 (void) fprintf(stderr
, "failed to open ZFS device\n");
885 * No arguments. Print the available handlers. If there are no
886 * available handlers, direct the user to '-h' for help
889 if (print_all_handlers() == 0) {
890 (void) printf("No handlers registered.\n");
891 (void) printf("Run 'zinject -h' for usage "
898 while ((c
= getopt(argc
, argv
,
899 ":aA:b:C:d:D:f:Fg:qhIc:t:T:l:mr:s:e:uL:p:P:")) != -1) {
902 flags
|= ZINJECT_FLUSH_ARC
;
905 if (strcasecmp(optarg
, "degrade") == 0) {
906 action
= VDEV_STATE_DEGRADED
;
907 } else if (strcasecmp(optarg
, "fault") == 0) {
908 action
= VDEV_STATE_FAULTED
;
910 (void) fprintf(stderr
, "invalid action '%s': "
911 "must be 'degrade' or 'fault'\n", optarg
);
924 ret
= parse_dvas(optarg
, &dvas
);
926 (void) fprintf(stderr
, "invalid DVA list '%s': "
927 "DVAs should be 0 indexed and separated by "
928 "commas.\n", optarg
);
939 ret
= parse_delay(optarg
, &record
.zi_timer
,
943 (void) fprintf(stderr
, "invalid i/o delay "
944 "value: '%s'\n", optarg
);
951 error
= str_to_err(optarg
);
953 (void) fprintf(stderr
, "invalid error type "
954 "'%s': must be one of: io decompress "
955 "decrypt nxio dtl corrupt noop\n",
963 ret
= parse_frequency(optarg
, &record
.zi_freq
);
965 (void) fprintf(stderr
, "%sfrequency value must "
966 "be in the range [0.0001, 100.0]\n",
967 ret
== EINVAL
? "invalid value: " :
968 ret
== ERANGE
? "out of range: " : "");
974 record
.zi_failfast
= B_TRUE
;
978 record
.zi_duration
= (int)strtol(optarg
, &end
, 10);
979 if (record
.zi_duration
<= 0 || *end
!= '\0') {
980 (void) fprintf(stderr
, "invalid duration '%s': "
981 "must be a positive integer\n", optarg
);
986 /* store duration of txgs as its negative */
987 record
.zi_duration
*= -1;
994 /* default duration, if one hasn't yet been defined */
996 if (dur_secs
== 0 && dur_txg
== 0)
997 record
.zi_duration
= 30;
1000 level
= (int)strtol(optarg
, &end
, 10);
1002 (void) fprintf(stderr
, "invalid level '%s': "
1003 "must be an integer\n", optarg
);
1013 (void) strlcpy(record
.zi_func
, optarg
,
1014 sizeof (record
.zi_func
));
1015 record
.zi_cmd
= ZINJECT_PANIC
;
1018 if (strcasecmp(optarg
, "import") == 0) {
1019 record
.zi_cmd
= ZINJECT_DELAY_IMPORT
;
1020 } else if (strcasecmp(optarg
, "export") == 0) {
1021 record
.zi_cmd
= ZINJECT_DELAY_EXPORT
;
1023 (void) fprintf(stderr
, "invalid command '%s': "
1024 "must be 'import' or 'export'\n", optarg
);
1035 flags
|= ZINJECT_CALC_RANGE
;
1039 record
.zi_duration
= (int)strtol(optarg
, &end
, 10);
1040 if (record
.zi_duration
<= 0 || *end
!= '\0') {
1041 (void) fprintf(stderr
, "invalid duration '%s': "
1042 "must be a positive integer\n", optarg
);
1049 if (strcasecmp(optarg
, "read") == 0) {
1050 io_type
= ZIO_TYPE_READ
;
1051 } else if (strcasecmp(optarg
, "write") == 0) {
1052 io_type
= ZIO_TYPE_WRITE
;
1053 } else if (strcasecmp(optarg
, "free") == 0) {
1054 io_type
= ZIO_TYPE_FREE
;
1055 } else if (strcasecmp(optarg
, "claim") == 0) {
1056 io_type
= ZIO_TYPE_CLAIM
;
1057 } else if (strcasecmp(optarg
, "flush") == 0) {
1058 io_type
= ZIO_TYPE_FLUSH
;
1059 } else if (strcasecmp(optarg
, "all") == 0) {
1060 io_type
= ZIO_TYPES
;
1062 (void) fprintf(stderr
, "invalid I/O type "
1063 "'%s': must be 'read', 'write', 'free', "
1064 "'claim', 'flush' or 'all'\n", optarg
);
1071 if ((type
= name_to_type(optarg
)) == TYPE_INVAL
&&
1073 (void) fprintf(stderr
, "invalid type '%s'\n",
1081 flags
|= ZINJECT_UNLOAD_SPA
;
1084 if ((label
= name_to_type(optarg
)) == TYPE_INVAL
&&
1085 !LABEL_TYPE(type
)) {
1086 (void) fprintf(stderr
, "invalid label type "
1094 (void) fprintf(stderr
, "option -%c requires an "
1095 "operand\n", optopt
);
1100 (void) fprintf(stderr
, "invalid option '%c'\n",
1111 if (record
.zi_duration
!= 0 && record
.zi_cmd
== 0)
1112 record
.zi_cmd
= ZINJECT_IGNORED_WRITES
;
1114 if (cancel
!= NULL
) {
1116 * '-c' is invalid with any other options.
1118 if (raw
!= NULL
|| range
!= NULL
|| type
!= TYPE_INVAL
||
1119 level
!= 0 || record
.zi_cmd
!= ZINJECT_UNINITIALIZED
||
1120 record
.zi_freq
> 0 || dvas
!= 0) {
1121 (void) fprintf(stderr
, "cancel (-c) incompatible with "
1122 "any other options\n");
1128 (void) fprintf(stderr
, "extraneous argument to '-c'\n");
1134 if (strcmp(cancel
, "all") == 0) {
1135 return (cancel_all_handlers());
1137 int id
= (int)strtol(cancel
, &end
, 10);
1139 (void) fprintf(stderr
, "invalid handle id '%s':"
1140 " must be an integer or 'all'\n", cancel
);
1145 return (cancel_handler(id
));
1149 if (device
!= NULL
) {
1151 * Device (-d) injection uses a completely different mechanism
1152 * for doing injection, so handle it separately here.
1154 if (raw
!= NULL
|| range
!= NULL
|| type
!= TYPE_INVAL
||
1155 level
!= 0 || record
.zi_cmd
!= ZINJECT_UNINITIALIZED
||
1157 (void) fprintf(stderr
, "device (-d) incompatible with "
1158 "data error injection\n");
1165 (void) fprintf(stderr
, "device (-d) injection requires "
1166 "a single pool name\n");
1172 (void) strlcpy(pool
, argv
[0], sizeof (pool
));
1175 if (error
== ECKSUM
) {
1176 (void) fprintf(stderr
, "device error type must be "
1177 "'io', 'nxio' or 'corrupt'\n");
1182 if (error
== EILSEQ
&&
1183 (record
.zi_freq
== 0 || io_type
!= ZIO_TYPE_READ
)) {
1184 (void) fprintf(stderr
, "device corrupt errors require "
1185 "io type read and a frequency value\n");
1190 record
.zi_iotype
= io_type
;
1191 if (translate_device(pool
, device
, label
, &record
) != 0) {
1196 if (record
.zi_nlanes
) {
1199 case ZIO_TYPE_WRITE
:
1203 (void) fprintf(stderr
, "I/O type for a delay "
1204 "must be 'read' or 'write'\n");
1214 if (action
!= VDEV_STATE_UNKNOWN
)
1215 return (perform_action(pool
, &record
, action
));
1217 } else if (raw
!= NULL
) {
1218 if (range
!= NULL
|| type
!= TYPE_INVAL
|| level
!= 0 ||
1219 record
.zi_cmd
!= ZINJECT_UNINITIALIZED
||
1220 record
.zi_freq
> 0 || dvas
!= 0) {
1221 (void) fprintf(stderr
, "raw (-b) format with "
1222 "any other options\n");
1229 (void) fprintf(stderr
, "raw (-b) format expects a "
1230 "single pool name\n");
1236 (void) strlcpy(pool
, argv
[0], sizeof (pool
));
1239 if (error
== ENXIO
) {
1240 (void) fprintf(stderr
, "data error type must be "
1241 "'checksum' or 'io'\n");
1246 record
.zi_cmd
= ZINJECT_DATA_FAULT
;
1247 if (translate_raw(raw
, &record
) != 0) {
1253 } else if (record
.zi_cmd
== ZINJECT_PANIC
) {
1254 if (raw
!= NULL
|| range
!= NULL
|| type
!= TYPE_INVAL
||
1255 level
!= 0 || device
!= NULL
|| record
.zi_freq
> 0 ||
1257 (void) fprintf(stderr
, "%s incompatible with other "
1258 "options\n", "import|export delay (-P)");
1264 if (argc
< 1 || argc
> 2) {
1265 (void) fprintf(stderr
, "panic (-p) injection requires "
1266 "a single pool name and an optional id\n");
1272 (void) strlcpy(pool
, argv
[0], sizeof (pool
));
1273 if (argv
[1] != NULL
)
1274 record
.zi_type
= atoi(argv
[1]);
1276 } else if (record
.zi_cmd
== ZINJECT_DELAY_IMPORT
||
1277 record
.zi_cmd
== ZINJECT_DELAY_EXPORT
) {
1278 if (raw
!= NULL
|| range
!= NULL
|| type
!= TYPE_INVAL
||
1279 level
!= 0 || device
!= NULL
|| record
.zi_freq
> 0 ||
1281 (void) fprintf(stderr
, "%s incompatible with other "
1282 "options\n", "import|export delay (-P)");
1288 if (argc
!= 1 || record
.zi_duration
<= 0) {
1289 (void) fprintf(stderr
, "import|export delay (-P) "
1290 "injection requires a duration (-s) and a single "
1297 (void) strlcpy(pool
, argv
[0], sizeof (pool
));
1298 } else if (record
.zi_cmd
== ZINJECT_IGNORED_WRITES
) {
1299 if (raw
!= NULL
|| range
!= NULL
|| type
!= TYPE_INVAL
||
1300 level
!= 0 || record
.zi_freq
> 0 || dvas
!= 0) {
1301 (void) fprintf(stderr
, "hardware failure (-I) "
1302 "incompatible with other options\n");
1308 if (nowrites
== 0) {
1309 (void) fprintf(stderr
, "-s or -g meaningless "
1310 "without -I (ignore writes)\n");
1314 } else if (dur_secs
&& dur_txg
) {
1315 (void) fprintf(stderr
, "choose a duration either "
1316 "in seconds (-s) or a number of txgs (-g) "
1321 } else if (argc
!= 1) {
1322 (void) fprintf(stderr
, "ignore writes (-I) "
1323 "injection requires a single pool name\n");
1329 (void) strlcpy(pool
, argv
[0], sizeof (pool
));
1331 } else if (type
== TYPE_INVAL
) {
1333 (void) fprintf(stderr
, "at least one of '-b', '-d', "
1334 "'-t', '-a', '-p', '-I' or '-u' "
1335 "must be specified\n");
1341 if (argc
== 1 && (flags
& ZINJECT_UNLOAD_SPA
)) {
1342 (void) strlcpy(pool
, argv
[0], sizeof (pool
));
1344 } else if (argc
!= 0) {
1345 (void) fprintf(stderr
, "extraneous argument for "
1352 flags
|= ZINJECT_NULL
;
1355 (void) fprintf(stderr
, "missing object\n");
1361 if (error
== ENXIO
|| error
== EILSEQ
) {
1362 (void) fprintf(stderr
, "data error type must be "
1363 "'checksum' or 'io'\n");
1369 if (error
== EACCES
|| error
== EINVAL
) {
1370 (void) fprintf(stderr
, "the '-C' option may "
1371 "not be used with logical data errors "
1372 "'decrypt' and 'decompress'\n");
1377 record
.zi_dvas
= dvas
;
1380 if (error
== EACCES
) {
1381 if (type
!= TYPE_DATA
) {
1382 (void) fprintf(stderr
, "decryption errors "
1383 "may only be injected for 'data' types\n");
1388 record
.zi_cmd
= ZINJECT_DECRYPT_FAULT
;
1390 * Internally, ZFS actually uses ECKSUM for decryption
1391 * errors since EACCES is used to indicate the key was
1396 record
.zi_cmd
= ZINJECT_DATA_FAULT
;
1399 if (translate_record(type
, argv
[0], range
, level
, &record
, pool
,
1409 * If this is pool-wide metadata, unmount everything. The ioctl() will
1410 * unload the pool, so that we trigger spa-wide reopen of metadata next
1411 * time we access the pool.
1413 if (dataset
[0] != '\0' && domount
) {
1414 if ((zhp
= zfs_open(g_zfs
, dataset
,
1415 ZFS_TYPE_DATASET
)) == NULL
) {
1419 if (zfs_unmount(zhp
, NULL
, 0) != 0) {
1425 record
.zi_error
= error
;
1427 ret
= register_handler(pool
, flags
, &record
, quiet
);
1429 if (dataset
[0] != '\0' && domount
)
1430 ret
= (zfs_mount(zhp
, NULL
, 0) != 0);