?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
ztest: update expectation for sparing a special device
[zfs.git] / cmd / ztest.c
blob721abe981dc820c5fd53ca4b0a5af56fc2248799
1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or https://opensource.org/licenses/CDDL-1.0.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2011, 2018 by Delphix. All rights reserved.
24 * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
25 * Copyright (c) 2013 Steven Hartland. All rights reserved.
26 * Copyright (c) 2014 Integros [integros.com]
27 * Copyright 2017 Joyent, Inc.
28 * Copyright (c) 2017, Intel Corporation.
29 */
30
31 /*
32 * The objective of this program is to provide a DMU/ZAP/SPA stress test
33 * that runs entirely in userland, is easy to use, and easy to extend.
34 *
35 * The overall design of the ztest program is as follows:
36 *
37 * (1) For each major functional area (e.g. adding vdevs to a pool,
38 * creating and destroying datasets, reading and writing objects, etc)
39 * we have a simple routine to test that functionality. These
40 * individual routines do not have to do anything "stressful".
41 *
42 * (2) We turn these simple functionality tests into a stress test by
43 * running them all in parallel, with as many threads as desired,
44 * and spread across as many datasets, objects, and vdevs as desired.
45 *
46 * (3) While all this is happening, we inject faults into the pool to
47 * verify that self-healing data really works.
48 *
49 * (4) Every time we open a dataset, we change its checksum and compression
50 * functions. Thus even individual objects vary from block to block
51 * in which checksum they use and whether they're compressed.
52 *
53 * (5) To verify that we never lose on-disk consistency after a crash,
54 * we run the entire test in a child of the main process.
55 * At random times, the child self-immolates with a SIGKILL.
56 * This is the software equivalent of pulling the power cord.
57 * The parent then runs the test again, using the existing
58 * storage pool, as many times as desired. If backwards compatibility
59 * testing is enabled ztest will sometimes run the "older" version
60 * of ztest after a SIGKILL.
61 *
62 * (6) To verify that we don't have future leaks or temporal incursions,
63 * many of the functional tests record the transaction group number
64 * as part of their data. When reading old data, they verify that
65 * the transaction group number is less than the current, open txg.
66 * If you add a new test, please do this if applicable.
67 *
68 * (7) Threads are created with a reduced stack size, for sanity checking.
69 * Therefore, it's important not to allocate huge buffers on the stack.
70 *
71 * When run with no arguments, ztest runs for about five minutes and
72 * produces no output if successful. To get a little bit of information,
73 * specify -V. To get more information, specify -VV, and so on.
74 *
75 * To turn this into an overnight stress test, use -T to specify run time.
76 *
77 * You can ask more vdevs [-v], datasets [-d], or threads [-t]
78 * to increase the pool capacity, fanout, and overall stress level.
79 *
80 * Use the -k option to set the desired frequency of kills.
81 *
82 * When ztest invokes itself it passes all relevant information through a
83 * temporary file which is mmap-ed in the child process. This allows shared
84 * memory to survive the exec syscall. The ztest_shared_hdr_t struct is always
85 * stored at offset 0 of this file and contains information on the size and
86 * number of shared structures in the file. The information stored in this file
87 * must remain backwards compatible with older versions of ztest so that
88 * ztest can invoke them during backwards compatibility testing (-B).
89 */
90
91 #include <sys/zfs_context.h>
92 #include <sys/spa.h>
93 #include <sys/dmu.h>
94 #include <sys/txg.h>
95 #include <sys/dbuf.h>
96 #include <sys/zap.h>
97 #include <sys/dmu_objset.h>
98 #include <sys/poll.h>
99 #include <sys/stat.h>
100 #include <sys/time.h>
101 #include <sys/wait.h>
102 #include <sys/mman.h>
103 #include <sys/resource.h>
104 #include <sys/zio.h>
105 #include <sys/zil.h>
106 #include <sys/zil_impl.h>
107 #include <sys/vdev_draid.h>
108 #include <sys/vdev_impl.h>
109 #include <sys/vdev_file.h>
110 #include <sys/vdev_initialize.h>
111 #include <sys/vdev_raidz.h>
112 #include <sys/vdev_trim.h>
113 #include <sys/spa_impl.h>
114 #include <sys/metaslab_impl.h>
115 #include <sys/dsl_prop.h>
116 #include <sys/dsl_dataset.h>
117 #include <sys/dsl_destroy.h>
118 #include <sys/dsl_scan.h>
119 #include <sys/zio_checksum.h>
120 #include <sys/zfs_refcount.h>
121 #include <sys/zfeature.h>
122 #include <sys/dsl_userhold.h>
123 #include <sys/abd.h>
124 #include <sys/blake3.h>
125 #include <stdio.h>
126 #include <stdlib.h>
127 #include <unistd.h>
128 #include <getopt.h>
129 #include <signal.h>
130 #include <umem.h>
131 #include <ctype.h>
132 #include <math.h>
133 #include <sys/fs/zfs.h>
134 #include <zfs_fletcher.h>
135 #include <libnvpair.h>
136 #include <libzutil.h>
137 #include <sys/crypto/icp.h>
138 #if (__GLIBC__ && !__UCLIBC__)
139 #include <execinfo.h> /* for backtrace() */
140 #endif
141
142 static int ztest_fd_data = -1;
143 static int ztest_fd_rand = -1;
144
145 typedef struct ztest_shared_hdr {
146 uint64_t zh_hdr_size;
147 uint64_t zh_opts_size;
148 uint64_t zh_size;
149 uint64_t zh_stats_size;
150 uint64_t zh_stats_count;
151 uint64_t zh_ds_size;
152 uint64_t zh_ds_count;
153 } ztest_shared_hdr_t;
154
155 static ztest_shared_hdr_t *ztest_shared_hdr;
156
157 enum ztest_class_state {
158 ZTEST_VDEV_CLASS_OFF,
159 ZTEST_VDEV_CLASS_ON,
160 ZTEST_VDEV_CLASS_RND
161 };
162
163 #define ZO_GVARS_MAX_ARGLEN ((size_t)64)
164 #define ZO_GVARS_MAX_COUNT ((size_t)10)
165
166 typedef struct ztest_shared_opts {
167 char zo_pool[ZFS_MAX_DATASET_NAME_LEN];
168 char zo_dir[ZFS_MAX_DATASET_NAME_LEN];
169 char zo_alt_ztest[MAXNAMELEN];
170 char zo_alt_libpath[MAXNAMELEN];
171 uint64_t zo_vdevs;
172 uint64_t zo_vdevtime;
173 size_t zo_vdev_size;
174 int zo_ashift;
175 int zo_mirrors;
176 int zo_raid_children;
177 int zo_raid_parity;
178 char zo_raid_type[8];
179 int zo_draid_data;
180 int zo_draid_spares;
181 int zo_datasets;
182 int zo_threads;
183 uint64_t zo_passtime;
184 uint64_t zo_killrate;
185 int zo_verbose;
186 int zo_init;
187 uint64_t zo_time;
188 uint64_t zo_maxloops;
189 uint64_t zo_metaslab_force_ganging;
190 int zo_mmp_test;
191 int zo_special_vdevs;
192 int zo_dump_dbgmsg;
193 int zo_gvars_count;
194 char zo_gvars[ZO_GVARS_MAX_COUNT][ZO_GVARS_MAX_ARGLEN];
195 } ztest_shared_opts_t;
196
197 /* Default values for command line options. */
198 #define DEFAULT_POOL "ztest"
199 #define DEFAULT_VDEV_DIR "/tmp"
200 #define DEFAULT_VDEV_COUNT 5
201 #define DEFAULT_VDEV_SIZE (SPA_MINDEVSIZE * 4) /* 256m default size */
202 #define DEFAULT_VDEV_SIZE_STR "256M"
203 #define DEFAULT_ASHIFT SPA_MINBLOCKSHIFT
204 #define DEFAULT_MIRRORS 2
205 #define DEFAULT_RAID_CHILDREN 4
206 #define DEFAULT_RAID_PARITY 1
207 #define DEFAULT_DRAID_DATA 4
208 #define DEFAULT_DRAID_SPARES 1
209 #define DEFAULT_DATASETS_COUNT 7
210 #define DEFAULT_THREADS 23
211 #define DEFAULT_RUN_TIME 300 /* 300 seconds */
212 #define DEFAULT_RUN_TIME_STR "300 sec"
213 #define DEFAULT_PASS_TIME 60 /* 60 seconds */
214 #define DEFAULT_PASS_TIME_STR "60 sec"
215 #define DEFAULT_KILL_RATE 70 /* 70% kill rate */
216 #define DEFAULT_KILLRATE_STR "70%"
217 #define DEFAULT_INITS 1
218 #define DEFAULT_MAX_LOOPS 50 /* 5 minutes */
219 #define DEFAULT_FORCE_GANGING (64 << 10)
220 #define DEFAULT_FORCE_GANGING_STR "64K"
221
222 /* Simplifying assumption: -1 is not a valid default. */
223 #define NO_DEFAULT -1
224
225 static const ztest_shared_opts_t ztest_opts_defaults = {
226 .zo_pool = DEFAULT_POOL,
227 .zo_dir = DEFAULT_VDEV_DIR,
228 .zo_alt_ztest = { '\0' },
229 .zo_alt_libpath = { '\0' },
230 .zo_vdevs = DEFAULT_VDEV_COUNT,
231 .zo_ashift = DEFAULT_ASHIFT,
232 .zo_mirrors = DEFAULT_MIRRORS,
233 .zo_raid_children = DEFAULT_RAID_CHILDREN,
234 .zo_raid_parity = DEFAULT_RAID_PARITY,
235 .zo_raid_type = VDEV_TYPE_RAIDZ,
236 .zo_vdev_size = DEFAULT_VDEV_SIZE,
237 .zo_draid_data = DEFAULT_DRAID_DATA, /* data drives */
238 .zo_draid_spares = DEFAULT_DRAID_SPARES, /* distributed spares */
239 .zo_datasets = DEFAULT_DATASETS_COUNT,
240 .zo_threads = DEFAULT_THREADS,
241 .zo_passtime = DEFAULT_PASS_TIME,
242 .zo_killrate = DEFAULT_KILL_RATE,
243 .zo_verbose = 0,
244 .zo_mmp_test = 0,
245 .zo_init = DEFAULT_INITS,
246 .zo_time = DEFAULT_RUN_TIME,
247 .zo_maxloops = DEFAULT_MAX_LOOPS, /* max loops during spa_freeze() */
248 .zo_metaslab_force_ganging = DEFAULT_FORCE_GANGING,
249 .zo_special_vdevs = ZTEST_VDEV_CLASS_RND,
250 .zo_gvars_count = 0,
251 };
252
253 extern uint64_t metaslab_force_ganging;
254 extern uint64_t metaslab_df_alloc_threshold;
255 extern uint64_t zfs_deadman_synctime_ms;
256 extern uint_t metaslab_preload_limit;
257 extern int zfs_compressed_arc_enabled;
258 extern int zfs_abd_scatter_enabled;
259 extern uint_t dmu_object_alloc_chunk_shift;
260 extern boolean_t zfs_force_some_double_word_sm_entries;
261 extern unsigned long zio_decompress_fail_fraction;
262 extern unsigned long zfs_reconstruct_indirect_damage_fraction;
263
264
265 static ztest_shared_opts_t *ztest_shared_opts;
266 static ztest_shared_opts_t ztest_opts;
267 static const char *const ztest_wkeydata = "abcdefghijklmnopqrstuvwxyz012345";
268
269 typedef struct ztest_shared_ds {
270 uint64_t zd_seq;
271 } ztest_shared_ds_t;
272
273 static ztest_shared_ds_t *ztest_shared_ds;
274 #define ZTEST_GET_SHARED_DS(d) (&ztest_shared_ds[d])
275
276 #define BT_MAGIC 0x123456789abcdefULL
277 #define MAXFAULTS(zs) \
278 (MAX((zs)->zs_mirrors, 1) * (ztest_opts.zo_raid_parity + 1) - 1)
279
280 enum ztest_io_type {
281 ZTEST_IO_WRITE_TAG,
282 ZTEST_IO_WRITE_PATTERN,
283 ZTEST_IO_WRITE_ZEROES,
284 ZTEST_IO_TRUNCATE,
285 ZTEST_IO_SETATTR,
286 ZTEST_IO_REWRITE,
287 ZTEST_IO_TYPES
288 };
289
290 typedef struct ztest_block_tag {
291 uint64_t bt_magic;
292 uint64_t bt_objset;
293 uint64_t bt_object;
294 uint64_t bt_dnodesize;
295 uint64_t bt_offset;
296 uint64_t bt_gen;
297 uint64_t bt_txg;
298 uint64_t bt_crtxg;
299 } ztest_block_tag_t;
300
301 typedef struct bufwad {
302 uint64_t bw_index;
303 uint64_t bw_txg;
304 uint64_t bw_data;
305 } bufwad_t;
306
307 /*
308 * It would be better to use a rangelock_t per object. Unfortunately
309 * the rangelock_t is not a drop-in replacement for rl_t, because we
310 * still need to map from object ID to rangelock_t.
311 */
312 typedef enum {
313 RL_READER,
314 RL_WRITER,
315 RL_APPEND
316 } rl_type_t;
317
318 typedef struct rll {
319 void *rll_writer;
320 int rll_readers;
321 kmutex_t rll_lock;
322 kcondvar_t rll_cv;
323 } rll_t;
324
325 typedef struct rl {
326 uint64_t rl_object;
327 uint64_t rl_offset;
328 uint64_t rl_size;
329 rll_t *rl_lock;
330 } rl_t;
331
332 #define ZTEST_RANGE_LOCKS 64
333 #define ZTEST_OBJECT_LOCKS 64
334
335 /*
336 * Object descriptor. Used as a template for object lookup/create/remove.
337 */
338 typedef struct ztest_od {
339 uint64_t od_dir;
340 uint64_t od_object;
341 dmu_object_type_t od_type;
342 dmu_object_type_t od_crtype;
343 uint64_t od_blocksize;
344 uint64_t od_crblocksize;
345 uint64_t od_crdnodesize;
346 uint64_t od_gen;
347 uint64_t od_crgen;
348 char od_name[ZFS_MAX_DATASET_NAME_LEN];
349 } ztest_od_t;
350
351 /*
352 * Per-dataset state.
353 */
354 typedef struct ztest_ds {
355 ztest_shared_ds_t *zd_shared;
356 objset_t *zd_os;
357 pthread_rwlock_t zd_zilog_lock;
358 zilog_t *zd_zilog;
359 ztest_od_t *zd_od; /* debugging aid */
360 char zd_name[ZFS_MAX_DATASET_NAME_LEN];
361 kmutex_t zd_dirobj_lock;
362 rll_t zd_object_lock[ZTEST_OBJECT_LOCKS];
363 rll_t zd_range_lock[ZTEST_RANGE_LOCKS];
364 } ztest_ds_t;
365
366 /*
367 * Per-iteration state.
368 */
369 typedef void ztest_func_t(ztest_ds_t *zd, uint64_t id);
370
371 typedef struct ztest_info {
372 ztest_func_t *zi_func; /* test function */
373 uint64_t zi_iters; /* iterations per execution */
374 uint64_t *zi_interval; /* execute every <interval> seconds */
375 const char *zi_funcname; /* name of test function */
376 } ztest_info_t;
377
378 typedef struct ztest_shared_callstate {
379 uint64_t zc_count; /* per-pass count */
380 uint64_t zc_time; /* per-pass time */
381 uint64_t zc_next; /* next time to call this function */
382 } ztest_shared_callstate_t;
383
384 static ztest_shared_callstate_t *ztest_shared_callstate;
385 #define ZTEST_GET_SHARED_CALLSTATE(c) (&ztest_shared_callstate[c])
386
387 ztest_func_t ztest_dmu_read_write;
388 ztest_func_t ztest_dmu_write_parallel;
389 ztest_func_t ztest_dmu_object_alloc_free;
390 ztest_func_t ztest_dmu_object_next_chunk;
391 ztest_func_t ztest_dmu_commit_callbacks;
392 ztest_func_t ztest_zap;
393 ztest_func_t ztest_zap_parallel;
394 ztest_func_t ztest_zil_commit;
395 ztest_func_t ztest_zil_remount;
396 ztest_func_t ztest_dmu_read_write_zcopy;
397 ztest_func_t ztest_dmu_objset_create_destroy;
398 ztest_func_t ztest_dmu_prealloc;
399 ztest_func_t ztest_fzap;
400 ztest_func_t ztest_dmu_snapshot_create_destroy;
401 ztest_func_t ztest_dsl_prop_get_set;
402 ztest_func_t ztest_spa_prop_get_set;
403 ztest_func_t ztest_spa_create_destroy;
404 ztest_func_t ztest_fault_inject;
405 ztest_func_t ztest_dmu_snapshot_hold;
406 ztest_func_t ztest_mmp_enable_disable;
407 ztest_func_t ztest_scrub;
408 ztest_func_t ztest_dsl_dataset_promote_busy;
409 ztest_func_t ztest_vdev_attach_detach;
410 ztest_func_t ztest_vdev_LUN_growth;
411 ztest_func_t ztest_vdev_add_remove;
412 ztest_func_t ztest_vdev_class_add;
413 ztest_func_t ztest_vdev_aux_add_remove;
414 ztest_func_t ztest_split_pool;
415 ztest_func_t ztest_reguid;
416 ztest_func_t ztest_spa_upgrade;
417 ztest_func_t ztest_device_removal;
418 ztest_func_t ztest_spa_checkpoint_create_discard;
419 ztest_func_t ztest_initialize;
420 ztest_func_t ztest_trim;
421 ztest_func_t ztest_blake3;
422 ztest_func_t ztest_fletcher;
423 ztest_func_t ztest_fletcher_incr;
424 ztest_func_t ztest_verify_dnode_bt;
425
426 static uint64_t zopt_always = 0ULL * NANOSEC; /* all the time */
427 static uint64_t zopt_incessant = 1ULL * NANOSEC / 10; /* every 1/10 second */
428 static uint64_t zopt_often = 1ULL * NANOSEC; /* every second */
429 static uint64_t zopt_sometimes = 10ULL * NANOSEC; /* every 10 seconds */
430 static uint64_t zopt_rarely = 60ULL * NANOSEC; /* every 60 seconds */
431
432 #define ZTI_INIT(func, iters, interval) \
433 { .zi_func = (func), \
434 .zi_iters = (iters), \
435 .zi_interval = (interval), \
436 .zi_funcname = # func }
437
438 static ztest_info_t ztest_info[] = {
439 ZTI_INIT(ztest_dmu_read_write, 1, &zopt_always),
440 ZTI_INIT(ztest_dmu_write_parallel, 10, &zopt_always),
441 ZTI_INIT(ztest_dmu_object_alloc_free, 1, &zopt_always),
442 ZTI_INIT(ztest_dmu_object_next_chunk, 1, &zopt_sometimes),
443 ZTI_INIT(ztest_dmu_commit_callbacks, 1, &zopt_always),
444 ZTI_INIT(ztest_zap, 30, &zopt_always),
445 ZTI_INIT(ztest_zap_parallel, 100, &zopt_always),
446 ZTI_INIT(ztest_split_pool, 1, &zopt_always),
447 ZTI_INIT(ztest_zil_commit, 1, &zopt_incessant),
448 ZTI_INIT(ztest_zil_remount, 1, &zopt_sometimes),
449 ZTI_INIT(ztest_dmu_read_write_zcopy, 1, &zopt_often),
450 ZTI_INIT(ztest_dmu_objset_create_destroy, 1, &zopt_often),
451 ZTI_INIT(ztest_dsl_prop_get_set, 1, &zopt_often),
452 ZTI_INIT(ztest_spa_prop_get_set, 1, &zopt_sometimes),
453 #if 0
454 ZTI_INIT(ztest_dmu_prealloc, 1, &zopt_sometimes),
455 #endif
456 ZTI_INIT(ztest_fzap, 1, &zopt_sometimes),
457 ZTI_INIT(ztest_dmu_snapshot_create_destroy, 1, &zopt_sometimes),
458 ZTI_INIT(ztest_spa_create_destroy, 1, &zopt_sometimes),
459 ZTI_INIT(ztest_fault_inject, 1, &zopt_sometimes),
460 ZTI_INIT(ztest_dmu_snapshot_hold, 1, &zopt_sometimes),
461 ZTI_INIT(ztest_mmp_enable_disable, 1, &zopt_sometimes),
462 ZTI_INIT(ztest_reguid, 1, &zopt_rarely),
463 ZTI_INIT(ztest_scrub, 1, &zopt_rarely),
464 ZTI_INIT(ztest_spa_upgrade, 1, &zopt_rarely),
465 ZTI_INIT(ztest_dsl_dataset_promote_busy, 1, &zopt_rarely),
466 ZTI_INIT(ztest_vdev_attach_detach, 1, &zopt_sometimes),
467 ZTI_INIT(ztest_vdev_LUN_growth, 1, &zopt_rarely),
468 ZTI_INIT(ztest_vdev_add_remove, 1, &ztest_opts.zo_vdevtime),
469 ZTI_INIT(ztest_vdev_class_add, 1, &ztest_opts.zo_vdevtime),
470 ZTI_INIT(ztest_vdev_aux_add_remove, 1, &ztest_opts.zo_vdevtime),
471 ZTI_INIT(ztest_device_removal, 1, &zopt_sometimes),
472 ZTI_INIT(ztest_spa_checkpoint_create_discard, 1, &zopt_rarely),
473 ZTI_INIT(ztest_initialize, 1, &zopt_sometimes),
474 ZTI_INIT(ztest_trim, 1, &zopt_sometimes),
475 ZTI_INIT(ztest_blake3, 1, &zopt_rarely),
476 ZTI_INIT(ztest_fletcher, 1, &zopt_rarely),
477 ZTI_INIT(ztest_fletcher_incr, 1, &zopt_rarely),
478 ZTI_INIT(ztest_verify_dnode_bt, 1, &zopt_sometimes),
479 };
480
481 #define ZTEST_FUNCS (sizeof (ztest_info) / sizeof (ztest_info_t))
482
483 /*
484 * The following struct is used to hold a list of uncalled commit callbacks.
485 * The callbacks are ordered by txg number.
486 */
487 typedef struct ztest_cb_list {
488 kmutex_t zcl_callbacks_lock;
489 list_t zcl_callbacks;
490 } ztest_cb_list_t;
491
492 /*
493 * Stuff we need to share writably between parent and child.
494 */
495 typedef struct ztest_shared {
496 boolean_t zs_do_init;
497 hrtime_t zs_proc_start;
498 hrtime_t zs_proc_stop;
499 hrtime_t zs_thread_start;
500 hrtime_t zs_thread_stop;
501 hrtime_t zs_thread_kill;
502 uint64_t zs_enospc_count;
503 uint64_t zs_vdev_next_leaf;
504 uint64_t zs_vdev_aux;
505 uint64_t zs_alloc;
506 uint64_t zs_space;
507 uint64_t zs_splits;
508 uint64_t zs_mirrors;
509 uint64_t zs_metaslab_sz;
510 uint64_t zs_metaslab_df_alloc_threshold;
511 uint64_t zs_guid;
512 } ztest_shared_t;
513
514 #define ID_PARALLEL -1ULL
515
516 static char ztest_dev_template[] = "%s/%s.%llua";
517 static char ztest_aux_template[] = "%s/%s.%s.%llu";
518 static ztest_shared_t *ztest_shared;
519
520 static spa_t *ztest_spa = NULL;
521 static ztest_ds_t *ztest_ds;
522
523 static kmutex_t ztest_vdev_lock;
524 static boolean_t ztest_device_removal_active = B_FALSE;
525 static boolean_t ztest_pool_scrubbed = B_FALSE;
526 static kmutex_t ztest_checkpoint_lock;
527
528 /*
529 * The ztest_name_lock protects the pool and dataset namespace used by
530 * the individual tests. To modify the namespace, consumers must grab
531 * this lock as writer. Grabbing the lock as reader will ensure that the
532 * namespace does not change while the lock is held.
533 */
534 static pthread_rwlock_t ztest_name_lock;
535
536 static boolean_t ztest_dump_core = B_TRUE;
537 static boolean_t ztest_exiting;
538
539 /* Global commit callback list */
540 static ztest_cb_list_t zcl;
541 /* Commit cb delay */
542 static uint64_t zc_min_txg_delay = UINT64_MAX;
543 static int zc_cb_counter = 0;
544
545 /*
546 * Minimum number of commit callbacks that need to be registered for us to check
547 * whether the minimum txg delay is acceptable.
548 */
549 #define ZTEST_COMMIT_CB_MIN_REG 100
550
551 /*
552 * If a number of txgs equal to this threshold have been created after a commit
553 * callback has been registered but not called, then we assume there is an
554 * implementation bug.
555 */
556 #define ZTEST_COMMIT_CB_THRESH (TXG_CONCURRENT_STATES + 1000)
557
558 enum ztest_object {
559 ZTEST_META_DNODE = 0,
560 ZTEST_DIROBJ,
561 ZTEST_OBJECTS
562 };
563
564 static __attribute__((noreturn)) void usage(boolean_t requested);
565 static int ztest_scrub_impl(spa_t *spa);
566
567 /*
568 * These libumem hooks provide a reasonable set of defaults for the allocator's
569 * debugging facilities.
570 */
571 const char *
572 _umem_debug_init(void)
573 {
574 return ("default,verbose"); /* $UMEM_DEBUG setting */
575 }
576
577 const char *
578 _umem_logging_init(void)
579 {
580 return ("fail,contents"); /* $UMEM_LOGGING setting */
581 }
582
583 static void
584 dump_debug_buffer(void)
585 {
586 ssize_t ret __attribute__((unused));
587
588 if (!ztest_opts.zo_dump_dbgmsg)
589 return;
590
591 /*
592 * We use write() instead of printf() so that this function
593 * is safe to call from a signal handler.
594 */
595 ret = write(STDOUT_FILENO, "\n", 1);
596 zfs_dbgmsg_print("ztest");
597 }
598
599 #define BACKTRACE_SZ 100
600
601 static void sig_handler(int signo)
602 {
603 struct sigaction action;
604 #if (__GLIBC__ && !__UCLIBC__) /* backtrace() is a GNU extension */
605 int nptrs;
606 void *buffer[BACKTRACE_SZ];
607
608 nptrs = backtrace(buffer, BACKTRACE_SZ);
609 backtrace_symbols_fd(buffer, nptrs, STDERR_FILENO);
610 #endif
611 dump_debug_buffer();
612
613 /*
614 * Restore default action and re-raise signal so SIGSEGV and
615 * SIGABRT can trigger a core dump.
616 */
617 action.sa_handler = SIG_DFL;
618 sigemptyset(&action.sa_mask);
619 action.sa_flags = 0;
620 (void) sigaction(signo, &action, NULL);
621 raise(signo);
622 }
623
624 #define FATAL_MSG_SZ 1024
625
626 static const char *fatal_msg;
627
628 static __attribute__((format(printf, 2, 3))) __attribute__((noreturn)) void
629 fatal(int do_perror, const char *message, ...)
630 {
631 va_list args;
632 int save_errno = errno;
633 char *buf;
634
635 (void) fflush(stdout);
636 buf = umem_alloc(FATAL_MSG_SZ, UMEM_NOFAIL);
637 if (buf == NULL)
638 goto out;
639
640 va_start(args, message);
641 (void) sprintf(buf, "ztest: ");
642 /* LINTED */
643 (void) vsprintf(buf + strlen(buf), message, args);
644 va_end(args);
645 if (do_perror) {
646 (void) snprintf(buf + strlen(buf), FATAL_MSG_SZ - strlen(buf),
647 ": %s", strerror(save_errno));
648 }
649 (void) fprintf(stderr, "%s\n", buf);
650 fatal_msg = buf; /* to ease debugging */
651
652 out:
653 if (ztest_dump_core)
654 abort();
655 else
656 dump_debug_buffer();
657
658 exit(3);
659 }
660
661 static int
662 str2shift(const char *buf)
663 {
664 const char *ends = "BKMGTPEZ";
665 int i;
666
667 if (buf[0] == '\0')
668 return (0);
669 for (i = 0; i < strlen(ends); i++) {
670 if (toupper(buf[0]) == ends[i])
671 break;
672 }
673 if (i == strlen(ends)) {
674 (void) fprintf(stderr, "ztest: invalid bytes suffix: %s\n",
675 buf);
676 usage(B_FALSE);
677 }
678 if (buf[1] == '\0' || (toupper(buf[1]) == 'B' && buf[2] == '\0')) {
679 return (10*i);
680 }
681 (void) fprintf(stderr, "ztest: invalid bytes suffix: %s\n", buf);
682 usage(B_FALSE);
683 }
684
685 static uint64_t
686 nicenumtoull(const char *buf)
687 {
688 char *end;
689 uint64_t val;
690
691 val = strtoull(buf, &end, 0);
692 if (end == buf) {
693 (void) fprintf(stderr, "ztest: bad numeric value: %s\n", buf);
694 usage(B_FALSE);
695 } else if (end[0] == '.') {
696 double fval = strtod(buf, &end);
697 fval *= pow(2, str2shift(end));
698 /*
699 * UINT64_MAX is not exactly representable as a double.
700 * The closest representation is UINT64_MAX + 1, so we
701 * use a >= comparison instead of > for the bounds check.
702 */
703 if (fval >= (double)UINT64_MAX) {
704 (void) fprintf(stderr, "ztest: value too large: %s\n",
705 buf);
706 usage(B_FALSE);
707 }
708 val = (uint64_t)fval;
709 } else {
710 int shift = str2shift(end);
711 if (shift >= 64 || (val << shift) >> shift != val) {
712 (void) fprintf(stderr, "ztest: value too large: %s\n",
713 buf);
714 usage(B_FALSE);
715 }
716 val <<= shift;
717 }
718 return (val);
719 }
720
721 typedef struct ztest_option {
722 const char short_opt;
723 const char *long_opt;
724 const char *long_opt_param;
725 const char *comment;
726 unsigned int default_int;
727 const char *default_str;
728 } ztest_option_t;
729
730 /*
731 * The following option_table is used for generating the usage info as well as
732 * the long and short option information for calling getopt_long().
733 */
734 static ztest_option_t option_table[] = {
735 { 'v', "vdevs", "INTEGER", "Number of vdevs", DEFAULT_VDEV_COUNT,
736 NULL},
737 { 's', "vdev-size", "INTEGER", "Size of each vdev",
738 NO_DEFAULT, DEFAULT_VDEV_SIZE_STR},
739 { 'a', "alignment-shift", "INTEGER",
740 "Alignment shift; use 0 for random", DEFAULT_ASHIFT, NULL},
741 { 'm', "mirror-copies", "INTEGER", "Number of mirror copies",
742 DEFAULT_MIRRORS, NULL},
743 { 'r', "raid-disks", "INTEGER", "Number of raidz/draid disks",
744 DEFAULT_RAID_CHILDREN, NULL},
745 { 'R', "raid-parity", "INTEGER", "Raid parity",
746 DEFAULT_RAID_PARITY, NULL},
747 { 'K', "raid-kind", "raidz|draid|random", "Raid kind",
748 NO_DEFAULT, "random"},
749 { 'D', "draid-data", "INTEGER", "Number of draid data drives",
750 DEFAULT_DRAID_DATA, NULL},
751 { 'S', "draid-spares", "INTEGER", "Number of draid spares",
752 DEFAULT_DRAID_SPARES, NULL},
753 { 'd', "datasets", "INTEGER", "Number of datasets",
754 DEFAULT_DATASETS_COUNT, NULL},
755 { 't', "threads", "INTEGER", "Number of ztest threads",
756 DEFAULT_THREADS, NULL},
757 { 'g', "gang-block-threshold", "INTEGER",
758 "Metaslab gang block threshold",
759 NO_DEFAULT, DEFAULT_FORCE_GANGING_STR},
760 { 'i', "init-count", "INTEGER", "Number of times to initialize pool",
761 DEFAULT_INITS, NULL},
762 { 'k', "kill-percentage", "INTEGER", "Kill percentage",
763 NO_DEFAULT, DEFAULT_KILLRATE_STR},
764 { 'p', "pool-name", "STRING", "Pool name",
765 NO_DEFAULT, DEFAULT_POOL},
766 { 'f', "vdev-file-directory", "PATH", "File directory for vdev files",
767 NO_DEFAULT, DEFAULT_VDEV_DIR},
768 { 'M', "multi-host", NULL,
769 "Multi-host; simulate pool imported on remote host",
770 NO_DEFAULT, NULL},
771 { 'E', "use-existing-pool", NULL,
772 "Use existing pool instead of creating new one", NO_DEFAULT, NULL},
773 { 'T', "run-time", "INTEGER", "Total run time",
774 NO_DEFAULT, DEFAULT_RUN_TIME_STR},
775 { 'P', "pass-time", "INTEGER", "Time per pass",
776 NO_DEFAULT, DEFAULT_PASS_TIME_STR},
777 { 'F', "freeze-loops", "INTEGER", "Max loops in spa_freeze()",
778 DEFAULT_MAX_LOOPS, NULL},
779 { 'B', "alt-ztest", "PATH", "Alternate ztest path",
780 NO_DEFAULT, NULL},
781 { 'C', "vdev-class-state", "on|off|random", "vdev class state",
782 NO_DEFAULT, "random"},
783 { 'o', "option", "\"OPTION=INTEGER\"",
784 "Set global variable to an unsigned 32-bit integer value",
785 NO_DEFAULT, NULL},
786 { 'G', "dump-debug-msg", NULL,
787 "Dump zfs_dbgmsg buffer before exiting due to an error",
788 NO_DEFAULT, NULL},
789 { 'V', "verbose", NULL,
790 "Verbose (use multiple times for ever more verbosity)",
791 NO_DEFAULT, NULL},
792 { 'h', "help", NULL, "Show this help",
793 NO_DEFAULT, NULL},
794 {0, 0, 0, 0, 0, 0}
795 };
796
797 static struct option *long_opts = NULL;
798 static char *short_opts = NULL;
799
800 static void
801 init_options(void)
802 {
803 ASSERT3P(long_opts, ==, NULL);
804 ASSERT3P(short_opts, ==, NULL);
805
806 int count = sizeof (option_table) / sizeof (option_table[0]);
807 long_opts = umem_alloc(sizeof (struct option) * count, UMEM_NOFAIL);
808
809 short_opts = umem_alloc(sizeof (char) * 2 * count, UMEM_NOFAIL);
810 int short_opt_index = 0;
811
812 for (int i = 0; i < count; i++) {
813 long_opts[i].val = option_table[i].short_opt;
814 long_opts[i].name = option_table[i].long_opt;
815 long_opts[i].has_arg = option_table[i].long_opt_param != NULL
816 ? required_argument : no_argument;
817 long_opts[i].flag = NULL;
818 short_opts[short_opt_index++] = option_table[i].short_opt;
819 if (option_table[i].long_opt_param != NULL) {
820 short_opts[short_opt_index++] = ':';
821 }
822 }
823 }
824
825 static void
826 fini_options(void)
827 {
828 int count = sizeof (option_table) / sizeof (option_table[0]);
829
830 umem_free(long_opts, sizeof (struct option) * count);
831 umem_free(short_opts, sizeof (char) * 2 * count);
832
833 long_opts = NULL;
834 short_opts = NULL;
835 }
836
837 static __attribute__((noreturn)) void
838 usage(boolean_t requested)
839 {
840 char option[80];
841 FILE *fp = requested ? stdout : stderr;
842
843 (void) fprintf(fp, "Usage: %s [OPTIONS...]\n", DEFAULT_POOL);
844 for (int i = 0; option_table[i].short_opt != 0; i++) {
845 if (option_table[i].long_opt_param != NULL) {
846 (void) sprintf(option, " -%c --%s=%s",
847 option_table[i].short_opt,
848 option_table[i].long_opt,
849 option_table[i].long_opt_param);
850 } else {
851 (void) sprintf(option, " -%c --%s",
852 option_table[i].short_opt,
853 option_table[i].long_opt);
854 }
855 (void) fprintf(fp, " %-40s%s", option,
856 option_table[i].comment);
857
858 if (option_table[i].long_opt_param != NULL) {
859 if (option_table[i].default_str != NULL) {
860 (void) fprintf(fp, " (default: %s)",
861 option_table[i].default_str);
862 } else if (option_table[i].default_int != NO_DEFAULT) {
863 (void) fprintf(fp, " (default: %u)",
864 option_table[i].default_int);
865 }
866 }
867 (void) fprintf(fp, "\n");
868 }
869 exit(requested ? 0 : 1);
870 }
871
872 static uint64_t
873 ztest_random(uint64_t range)
874 {
875 uint64_t r;
876
877 ASSERT3S(ztest_fd_rand, >=, 0);
878
879 if (range == 0)
880 return (0);
881
882 if (read(ztest_fd_rand, &r, sizeof (r)) != sizeof (r))
883 fatal(B_TRUE, "short read from /dev/urandom");
884
885 return (r % range);
886 }
887
888 static void
889 ztest_parse_name_value(const char *input, ztest_shared_opts_t *zo)
890 {
891 char name[32];
892 char *value;
893 int state = ZTEST_VDEV_CLASS_RND;
894
895 (void) strlcpy(name, input, sizeof (name));
896
897 value = strchr(name, '=');
898 if (value == NULL) {
899 (void) fprintf(stderr, "missing value in property=value "
900 "'-C' argument (%s)\n", input);
901 usage(B_FALSE);
902 }
903 *(value) = '\0';
904 value++;
905
906 if (strcmp(value, "on") == 0) {
907 state = ZTEST_VDEV_CLASS_ON;
908 } else if (strcmp(value, "off") == 0) {
909 state = ZTEST_VDEV_CLASS_OFF;
910 } else if (strcmp(value, "random") == 0) {
911 state = ZTEST_VDEV_CLASS_RND;
912 } else {
913 (void) fprintf(stderr, "invalid property value '%s'\n", value);
914 usage(B_FALSE);
915 }
916
917 if (strcmp(name, "special") == 0) {
918 zo->zo_special_vdevs = state;
919 } else {
920 (void) fprintf(stderr, "invalid property name '%s'\n", name);
921 usage(B_FALSE);
922 }
923 if (zo->zo_verbose >= 3)
924 (void) printf("%s vdev state is '%s'\n", name, value);
925 }
926
927 static void
928 process_options(int argc, char **argv)
929 {
930 char *path;
931 ztest_shared_opts_t *zo = &ztest_opts;
932
933 int opt;
934 uint64_t value;
935 const char *raid_kind = "random";
936
937 memcpy(zo, &ztest_opts_defaults, sizeof (*zo));
938
939 init_options();
940
941 while ((opt = getopt_long(argc, argv, short_opts, long_opts,
942 NULL)) != EOF) {
943 value = 0;
944 switch (opt) {
945 case 'v':
946 case 's':
947 case 'a':
948 case 'm':
949 case 'r':
950 case 'R':
951 case 'D':
952 case 'S':
953 case 'd':
954 case 't':
955 case 'g':
956 case 'i':
957 case 'k':
958 case 'T':
959 case 'P':
960 case 'F':
961 value = nicenumtoull(optarg);
962 }
963 switch (opt) {
964 case 'v':
965 zo->zo_vdevs = value;
966 break;
967 case 's':
968 zo->zo_vdev_size = MAX(SPA_MINDEVSIZE, value);
969 break;
970 case 'a':
971 zo->zo_ashift = value;
972 break;
973 case 'm':
974 zo->zo_mirrors = value;
975 break;
976 case 'r':
977 zo->zo_raid_children = MAX(1, value);
978 break;
979 case 'R':
980 zo->zo_raid_parity = MIN(MAX(value, 1), 3);
981 break;
982 case 'K':
983 raid_kind = optarg;
984 break;
985 case 'D':
986 zo->zo_draid_data = MAX(1, value);
987 break;
988 case 'S':
989 zo->zo_draid_spares = MAX(1, value);
990 break;
991 case 'd':
992 zo->zo_datasets = MAX(1, value);
993 break;
994 case 't':
995 zo->zo_threads = MAX(1, value);
996 break;
997 case 'g':
998 zo->zo_metaslab_force_ganging =
999 MAX(SPA_MINBLOCKSIZE << 1, value);
1000 break;
1001 case 'i':
1002 zo->zo_init = value;
1003 break;
1004 case 'k':
1005 zo->zo_killrate = value;
1006 break;
1007 case 'p':
1008 (void) strlcpy(zo->zo_pool, optarg,
1009 sizeof (zo->zo_pool));
1010 break;
1011 case 'f':
1012 path = realpath(optarg, NULL);
1013 if (path == NULL) {
1014 (void) fprintf(stderr, "error: %s: %s\n",
1015 optarg, strerror(errno));
1016 usage(B_FALSE);
1017 } else {
1018 (void) strlcpy(zo->zo_dir, path,
1019 sizeof (zo->zo_dir));
1020 free(path);
1021 }
1022 break;
1023 case 'M':
1024 zo->zo_mmp_test = 1;
1025 break;
1026 case 'V':
1027 zo->zo_verbose++;
1028 break;
1029 case 'E':
1030 zo->zo_init = 0;
1031 break;
1032 case 'T':
1033 zo->zo_time = value;
1034 break;
1035 case 'P':
1036 zo->zo_passtime = MAX(1, value);
1037 break;
1038 case 'F':
1039 zo->zo_maxloops = MAX(1, value);
1040 break;
1041 case 'B':
1042 (void) strlcpy(zo->zo_alt_ztest, optarg,
1043 sizeof (zo->zo_alt_ztest));
1044 break;
1045 case 'C':
1046 ztest_parse_name_value(optarg, zo);
1047 break;
1048 case 'o':
1049 if (zo->zo_gvars_count >= ZO_GVARS_MAX_COUNT) {
1050 (void) fprintf(stderr,
1051 "max global var count (%zu) exceeded\n",
1052 ZO_GVARS_MAX_COUNT);
1053 usage(B_FALSE);
1054 }
1055 char *v = zo->zo_gvars[zo->zo_gvars_count];
1056 if (strlcpy(v, optarg, ZO_GVARS_MAX_ARGLEN) >=
1057 ZO_GVARS_MAX_ARGLEN) {
1058 (void) fprintf(stderr,
1059 "global var option '%s' is too long\n",
1060 optarg);
1061 usage(B_FALSE);
1062 }
1063 zo->zo_gvars_count++;
1064 break;
1065 case 'G':
1066 zo->zo_dump_dbgmsg = 1;
1067 break;
1068 case 'h':
1069 usage(B_TRUE);
1070 break;
1071 case '?':
1072 default:
1073 usage(B_FALSE);
1074 break;
1075 }
1076 }
1077
1078 fini_options();
1079
1080 /* When raid choice is 'random' add a draid pool 50% of the time */
1081 if (strcmp(raid_kind, "random") == 0) {
1082 raid_kind = (ztest_random(2) == 0) ? "draid" : "raidz";
1083
1084 if (ztest_opts.zo_verbose >= 3)
1085 (void) printf("choosing RAID type '%s'\n", raid_kind);
1086 }
1087
1088 if (strcmp(raid_kind, "draid") == 0) {
1089 uint64_t min_devsize;
1090
1091 /* With fewer disk use 256M, otherwise 128M is OK */
1092 min_devsize = (ztest_opts.zo_raid_children < 16) ?
1093 (256ULL << 20) : (128ULL << 20);
1094
1095 /* No top-level mirrors with dRAID for now */
1096 zo->zo_mirrors = 0;
1097
1098 /* Use more appropriate defaults for dRAID */
1099 if (zo->zo_vdevs == ztest_opts_defaults.zo_vdevs)
1100 zo->zo_vdevs = 1;
1101 if (zo->zo_raid_children ==
1102 ztest_opts_defaults.zo_raid_children)
1103 zo->zo_raid_children = 16;
1104 if (zo->zo_ashift < 12)
1105 zo->zo_ashift = 12;
1106 if (zo->zo_vdev_size < min_devsize)
1107 zo->zo_vdev_size = min_devsize;
1108
1109 if (zo->zo_draid_data + zo->zo_raid_parity >
1110 zo->zo_raid_children - zo->zo_draid_spares) {
1111 (void) fprintf(stderr, "error: too few draid "
1112 "children (%d) for stripe width (%d)\n",
1113 zo->zo_raid_children,
1114 zo->zo_draid_data + zo->zo_raid_parity);
1115 usage(B_FALSE);
1116 }
1117
1118 (void) strlcpy(zo->zo_raid_type, VDEV_TYPE_DRAID,
1119 sizeof (zo->zo_raid_type));
1120
1121 } else /* using raidz */ {
1122 ASSERT0(strcmp(raid_kind, "raidz"));
1123
1124 zo->zo_raid_parity = MIN(zo->zo_raid_parity,
1125 zo->zo_raid_children - 1);
1126 }
1127
1128 zo->zo_vdevtime =
1129 (zo->zo_vdevs > 0 ? zo->zo_time * NANOSEC / zo->zo_vdevs :
1130 UINT64_MAX >> 2);
1131
1132 if (*zo->zo_alt_ztest) {
1133 const char *invalid_what = "ztest";
1134 char *val = zo->zo_alt_ztest;
1135 if (0 != access(val, X_OK) ||
1136 (strrchr(val, '/') == NULL && (errno == EINVAL)))
1137 goto invalid;
1138
1139 int dirlen = strrchr(val, '/') - val;
1140 strlcpy(zo->zo_alt_libpath, val,
1141 MIN(sizeof (zo->zo_alt_libpath), dirlen + 1));
1142 invalid_what = "library path", val = zo->zo_alt_libpath;
1143 if (strrchr(val, '/') == NULL && (errno == EINVAL))
1144 goto invalid;
1145 *strrchr(val, '/') = '\0';
1146 strlcat(val, "/lib", sizeof (zo->zo_alt_libpath));
1147
1148 if (0 != access(zo->zo_alt_libpath, X_OK))
1149 goto invalid;
1150 return;
1151
1152 invalid:
1153 ztest_dump_core = B_FALSE;
1154 fatal(B_TRUE, "invalid alternate %s %s", invalid_what, val);
1155 }
1156 }
1157
1158 static void
1159 ztest_kill(ztest_shared_t *zs)
1160 {
1161 zs->zs_alloc = metaslab_class_get_alloc(spa_normal_class(ztest_spa));
1162 zs->zs_space = metaslab_class_get_space(spa_normal_class(ztest_spa));
1163
1164 /*
1165 * Before we kill ourselves, make sure that the config is updated.
1166 * See comment above spa_write_cachefile().
1167 */
1168 mutex_enter(&spa_namespace_lock);
1169 spa_write_cachefile(ztest_spa, B_FALSE, B_FALSE, B_FALSE);
1170 mutex_exit(&spa_namespace_lock);
1171
1172 (void) raise(SIGKILL);
1173 }
1174
1175 static void
1176 ztest_record_enospc(const char *s)
1177 {
1178 (void) s;
1179 ztest_shared->zs_enospc_count++;
1180 }
1181
1182 static uint64_t
1183 ztest_get_ashift(void)
1184 {
1185 if (ztest_opts.zo_ashift == 0)
1186 return (SPA_MINBLOCKSHIFT + ztest_random(5));
1187 return (ztest_opts.zo_ashift);
1188 }
1189
1190 static boolean_t
1191 ztest_is_draid_spare(const char *name)
1192 {
1193 uint64_t spare_id = 0, parity = 0, vdev_id = 0;
1194
1195 if (sscanf(name, VDEV_TYPE_DRAID "%"PRIu64"-%"PRIu64"-%"PRIu64"",
1196 &parity, &vdev_id, &spare_id) == 3) {
1197 return (B_TRUE);
1198 }
1199
1200 return (B_FALSE);
1201 }
1202
1203 static nvlist_t *
1204 make_vdev_file(const char *path, const char *aux, const char *pool,
1205 size_t size, uint64_t ashift)
1206 {
1207 char *pathbuf = NULL;
1208 uint64_t vdev;
1209 nvlist_t *file;
1210 boolean_t draid_spare = B_FALSE;
1211
1212
1213 if (ashift == 0)
1214 ashift = ztest_get_ashift();
1215
1216 if (path == NULL) {
1217 pathbuf = umem_alloc(MAXPATHLEN, UMEM_NOFAIL);
1218 path = pathbuf;
1219
1220 if (aux != NULL) {
1221 vdev = ztest_shared->zs_vdev_aux;
1222 (void) snprintf(pathbuf, MAXPATHLEN,
1223 ztest_aux_template, ztest_opts.zo_dir,
1224 pool == NULL ? ztest_opts.zo_pool : pool,
1225 aux, vdev);
1226 } else {
1227 vdev = ztest_shared->zs_vdev_next_leaf++;
1228 (void) snprintf(pathbuf, MAXPATHLEN,
1229 ztest_dev_template, ztest_opts.zo_dir,
1230 pool == NULL ? ztest_opts.zo_pool : pool, vdev);
1231 }
1232 } else {
1233 draid_spare = ztest_is_draid_spare(path);
1234 }
1235
1236 if (size != 0 && !draid_spare) {
1237 int fd = open(path, O_RDWR | O_CREAT | O_TRUNC, 0666);
1238 if (fd == -1)
1239 fatal(B_TRUE, "can't open %s", path);
1240 if (ftruncate(fd, size) != 0)
1241 fatal(B_TRUE, "can't ftruncate %s", path);
1242 (void) close(fd);
1243 }
1244
1245 file = fnvlist_alloc();
1246 fnvlist_add_string(file, ZPOOL_CONFIG_TYPE,
1247 draid_spare ? VDEV_TYPE_DRAID_SPARE : VDEV_TYPE_FILE);
1248 fnvlist_add_string(file, ZPOOL_CONFIG_PATH, path);
1249 fnvlist_add_uint64(file, ZPOOL_CONFIG_ASHIFT, ashift);
1250 umem_free(pathbuf, MAXPATHLEN);
1251
1252 return (file);
1253 }
1254
1255 static nvlist_t *
1256 make_vdev_raid(const char *path, const char *aux, const char *pool, size_t size,
1257 uint64_t ashift, int r)
1258 {
1259 nvlist_t *raid, **child;
1260 int c;
1261
1262 if (r < 2)
1263 return (make_vdev_file(path, aux, pool, size, ashift));
1264 child = umem_alloc(r * sizeof (nvlist_t *), UMEM_NOFAIL);
1265
1266 for (c = 0; c < r; c++)
1267 child[c] = make_vdev_file(path, aux, pool, size, ashift);
1268
1269 raid = fnvlist_alloc();
1270 fnvlist_add_string(raid, ZPOOL_CONFIG_TYPE,
1271 ztest_opts.zo_raid_type);
1272 fnvlist_add_uint64(raid, ZPOOL_CONFIG_NPARITY,
1273 ztest_opts.zo_raid_parity);
1274 fnvlist_add_nvlist_array(raid, ZPOOL_CONFIG_CHILDREN,
1275 (const nvlist_t **)child, r);
1276
1277 if (strcmp(ztest_opts.zo_raid_type, VDEV_TYPE_DRAID) == 0) {
1278 uint64_t ndata = ztest_opts.zo_draid_data;
1279 uint64_t nparity = ztest_opts.zo_raid_parity;
1280 uint64_t nspares = ztest_opts.zo_draid_spares;
1281 uint64_t children = ztest_opts.zo_raid_children;
1282 uint64_t ngroups = 1;
1283
1284 /*
1285 * Calculate the minimum number of groups required to fill a
1286 * slice. This is the LCM of the stripe width (data + parity)
1287 * and the number of data drives (children - spares).
1288 */
1289 while (ngroups * (ndata + nparity) % (children - nspares) != 0)
1290 ngroups++;
1291
1292 /* Store the basic dRAID configuration. */
1293 fnvlist_add_uint64(raid, ZPOOL_CONFIG_DRAID_NDATA, ndata);
1294 fnvlist_add_uint64(raid, ZPOOL_CONFIG_DRAID_NSPARES, nspares);
1295 fnvlist_add_uint64(raid, ZPOOL_CONFIG_DRAID_NGROUPS, ngroups);
1296 }
1297
1298 for (c = 0; c < r; c++)
1299 fnvlist_free(child[c]);
1300
1301 umem_free(child, r * sizeof (nvlist_t *));
1302
1303 return (raid);
1304 }
1305
1306 static nvlist_t *
1307 make_vdev_mirror(const char *path, const char *aux, const char *pool,
1308 size_t size, uint64_t ashift, int r, int m)
1309 {
1310 nvlist_t *mirror, **child;
1311 int c;
1312
1313 if (m < 1)
1314 return (make_vdev_raid(path, aux, pool, size, ashift, r));
1315
1316 child = umem_alloc(m * sizeof (nvlist_t *), UMEM_NOFAIL);
1317
1318 for (c = 0; c < m; c++)
1319 child[c] = make_vdev_raid(path, aux, pool, size, ashift, r);
1320
1321 mirror = fnvlist_alloc();
1322 fnvlist_add_string(mirror, ZPOOL_CONFIG_TYPE, VDEV_TYPE_MIRROR);
1323 fnvlist_add_nvlist_array(mirror, ZPOOL_CONFIG_CHILDREN,
1324 (const nvlist_t **)child, m);
1325
1326 for (c = 0; c < m; c++)
1327 fnvlist_free(child[c]);
1328
1329 umem_free(child, m * sizeof (nvlist_t *));
1330
1331 return (mirror);
1332 }
1333
1334 static nvlist_t *
1335 make_vdev_root(const char *path, const char *aux, const char *pool, size_t size,
1336 uint64_t ashift, const char *class, int r, int m, int t)
1337 {
1338 nvlist_t *root, **child;
1339 int c;
1340 boolean_t log;
1341
1342 ASSERT3S(t, >, 0);
1343
1344 log = (class != NULL && strcmp(class, "log") == 0);
1345
1346 child = umem_alloc(t * sizeof (nvlist_t *), UMEM_NOFAIL);
1347
1348 for (c = 0; c < t; c++) {
1349 child[c] = make_vdev_mirror(path, aux, pool, size, ashift,
1350 r, m);
1351 fnvlist_add_uint64(child[c], ZPOOL_CONFIG_IS_LOG, log);
1352
1353 if (class != NULL && class[0] != '\0') {
1354 ASSERT(m > 1 || log); /* expecting a mirror */
1355 fnvlist_add_string(child[c],
1356 ZPOOL_CONFIG_ALLOCATION_BIAS, class);
1357 }
1358 }
1359
1360 root = fnvlist_alloc();
1361 fnvlist_add_string(root, ZPOOL_CONFIG_TYPE, VDEV_TYPE_ROOT);
1362 fnvlist_add_nvlist_array(root, aux ? aux : ZPOOL_CONFIG_CHILDREN,
1363 (const nvlist_t **)child, t);
1364
1365 for (c = 0; c < t; c++)
1366 fnvlist_free(child[c]);
1367
1368 umem_free(child, t * sizeof (nvlist_t *));
1369
1370 return (root);
1371 }
1372
1373 /*
1374 * Find a random spa version. Returns back a random spa version in the
1375 * range [initial_version, SPA_VERSION_FEATURES].
1376 */
1377 static uint64_t
1378 ztest_random_spa_version(uint64_t initial_version)
1379 {
1380 uint64_t version = initial_version;
1381
1382 if (version <= SPA_VERSION_BEFORE_FEATURES) {
1383 version = version +
1384 ztest_random(SPA_VERSION_BEFORE_FEATURES - version + 1);
1385 }
1386
1387 if (version > SPA_VERSION_BEFORE_FEATURES)
1388 version = SPA_VERSION_FEATURES;
1389
1390 ASSERT(SPA_VERSION_IS_SUPPORTED(version));
1391 return (version);
1392 }
1393
1394 static int
1395 ztest_random_blocksize(void)
1396 {
1397 ASSERT3U(ztest_spa->spa_max_ashift, !=, 0);
1398
1399 /*
1400 * Choose a block size >= the ashift.
1401 * If the SPA supports new MAXBLOCKSIZE, test up to 1MB blocks.
1402 */
1403 int maxbs = SPA_OLD_MAXBLOCKSHIFT;
1404 if (spa_maxblocksize(ztest_spa) == SPA_MAXBLOCKSIZE)
1405 maxbs = 20;
1406 uint64_t block_shift =
1407 ztest_random(maxbs - ztest_spa->spa_max_ashift + 1);
1408 return (1 << (SPA_MINBLOCKSHIFT + block_shift));
1409 }
1410
1411 static int
1412 ztest_random_dnodesize(void)
1413 {
1414 int slots;
1415 int max_slots = spa_maxdnodesize(ztest_spa) >> DNODE_SHIFT;
1416
1417 if (max_slots == DNODE_MIN_SLOTS)
1418 return (DNODE_MIN_SIZE);
1419
1420 /*
1421 * Weight the random distribution more heavily toward smaller
1422 * dnode sizes since that is more likely to reflect real-world
1423 * usage.
1424 */
1425 ASSERT3U(max_slots, >, 4);
1426 switch (ztest_random(10)) {
1427 case 0:
1428 slots = 5 + ztest_random(max_slots - 4);
1429 break;
1430 case 1 ... 4:
1431 slots = 2 + ztest_random(3);
1432 break;
1433 default:
1434 slots = 1;
1435 break;
1436 }
1437
1438 return (slots << DNODE_SHIFT);
1439 }
1440
1441 static int
1442 ztest_random_ibshift(void)
1443 {
1444 return (DN_MIN_INDBLKSHIFT +
1445 ztest_random(DN_MAX_INDBLKSHIFT - DN_MIN_INDBLKSHIFT + 1));
1446 }
1447
1448 static uint64_t
1449 ztest_random_vdev_top(spa_t *spa, boolean_t log_ok)
1450 {
1451 uint64_t top;
1452 vdev_t *rvd = spa->spa_root_vdev;
1453 vdev_t *tvd;
1454
1455 ASSERT3U(spa_config_held(spa, SCL_ALL, RW_READER), !=, 0);
1456
1457 do {
1458 top = ztest_random(rvd->vdev_children);
1459 tvd = rvd->vdev_child[top];
1460 } while (!vdev_is_concrete(tvd) || (tvd->vdev_islog && !log_ok) ||
1461 tvd->vdev_mg == NULL || tvd->vdev_mg->mg_class == NULL);
1462
1463 return (top);
1464 }
1465
1466 static uint64_t
1467 ztest_random_dsl_prop(zfs_prop_t prop)
1468 {
1469 uint64_t value;
1470
1471 do {
1472 value = zfs_prop_random_value(prop, ztest_random(-1ULL));
1473 } while (prop == ZFS_PROP_CHECKSUM && value == ZIO_CHECKSUM_OFF);
1474
1475 return (value);
1476 }
1477
1478 static int
1479 ztest_dsl_prop_set_uint64(char *osname, zfs_prop_t prop, uint64_t value,
1480 boolean_t inherit)
1481 {
1482 const char *propname = zfs_prop_to_name(prop);
1483 const char *valname;
1484 char *setpoint;
1485 uint64_t curval;
1486 int error;
1487
1488 error = dsl_prop_set_int(osname, propname,
1489 (inherit ? ZPROP_SRC_NONE : ZPROP_SRC_LOCAL), value);
1490
1491 if (error == ENOSPC) {
1492 ztest_record_enospc(FTAG);
1493 return (error);
1494 }
1495 ASSERT0(error);
1496
1497 setpoint = umem_alloc(MAXPATHLEN, UMEM_NOFAIL);
1498 VERIFY0(dsl_prop_get_integer(osname, propname, &curval, setpoint));
1499
1500 if (ztest_opts.zo_verbose >= 6) {
1501 int err;
1502
1503 err = zfs_prop_index_to_string(prop, curval, &valname);
1504 if (err)
1505 (void) printf("%s %s = %llu at '%s'\n", osname,
1506 propname, (unsigned long long)curval, setpoint);
1507 else
1508 (void) printf("%s %s = %s at '%s'\n",
1509 osname, propname, valname, setpoint);
1510 }
1511 umem_free(setpoint, MAXPATHLEN);
1512
1513 return (error);
1514 }
1515
1516 static int
1517 ztest_spa_prop_set_uint64(zpool_prop_t prop, uint64_t value)
1518 {
1519 spa_t *spa = ztest_spa;
1520 nvlist_t *props = NULL;
1521 int error;
1522
1523 props = fnvlist_alloc();
1524 fnvlist_add_uint64(props, zpool_prop_to_name(prop), value);
1525
1526 error = spa_prop_set(spa, props);
1527
1528 fnvlist_free(props);
1529
1530 if (error == ENOSPC) {
1531 ztest_record_enospc(FTAG);
1532 return (error);
1533 }
1534 ASSERT0(error);
1535
1536 return (error);
1537 }
1538
1539 static int
1540 ztest_dmu_objset_own(const char *name, dmu_objset_type_t type,
1541 boolean_t readonly, boolean_t decrypt, const void *tag, objset_t **osp)
1542 {
1543 int err;
1544 char *cp = NULL;
1545 char ddname[ZFS_MAX_DATASET_NAME_LEN];
1546
1547 strlcpy(ddname, name, sizeof (ddname));
1548 cp = strchr(ddname, '@');
1549 if (cp != NULL)
1550 *cp = '\0';
1551
1552 err = dmu_objset_own(name, type, readonly, decrypt, tag, osp);
1553 while (decrypt && err == EACCES) {
1554 dsl_crypto_params_t *dcp;
1555 nvlist_t *crypto_args = fnvlist_alloc();
1556
1557 fnvlist_add_uint8_array(crypto_args, "wkeydata",
1558 (uint8_t *)ztest_wkeydata, WRAPPING_KEY_LEN);
1559 VERIFY0(dsl_crypto_params_create_nvlist(DCP_CMD_NONE, NULL,
1560 crypto_args, &dcp));
1561 err = spa_keystore_load_wkey(ddname, dcp, B_FALSE);
1562 /*
1563 * Note: if there was an error loading, the wkey was not
1564 * consumed, and needs to be freed.
1565 */
1566 dsl_crypto_params_free(dcp, (err != 0));
1567 fnvlist_free(crypto_args);
1568
1569 if (err == EINVAL) {
1570 /*
1571 * We couldn't load a key for this dataset so try
1572 * the parent. This loop will eventually hit the
1573 * encryption root since ztest only makes clones
1574 * as children of their origin datasets.
1575 */
1576 cp = strrchr(ddname, '/');
1577 if (cp == NULL)
1578 return (err);
1579
1580 *cp = '\0';
1581 err = EACCES;
1582 continue;
1583 } else if (err != 0) {
1584 break;
1585 }
1586
1587 err = dmu_objset_own(name, type, readonly, decrypt, tag, osp);
1588 break;
1589 }
1590
1591 return (err);
1592 }
1593
1594 static void
1595 ztest_rll_init(rll_t *rll)
1596 {
1597 rll->rll_writer = NULL;
1598 rll->rll_readers = 0;
1599 mutex_init(&rll->rll_lock, NULL, MUTEX_DEFAULT, NULL);
1600 cv_init(&rll->rll_cv, NULL, CV_DEFAULT, NULL);
1601 }
1602
1603 static void
1604 ztest_rll_destroy(rll_t *rll)
1605 {
1606 ASSERT3P(rll->rll_writer, ==, NULL);
1607 ASSERT0(rll->rll_readers);
1608 mutex_destroy(&rll->rll_lock);
1609 cv_destroy(&rll->rll_cv);
1610 }
1611
1612 static void
1613 ztest_rll_lock(rll_t *rll, rl_type_t type)
1614 {
1615 mutex_enter(&rll->rll_lock);
1616
1617 if (type == RL_READER) {
1618 while (rll->rll_writer != NULL)
1619 (void) cv_wait(&rll->rll_cv, &rll->rll_lock);
1620 rll->rll_readers++;
1621 } else {
1622 while (rll->rll_writer != NULL || rll->rll_readers)
1623 (void) cv_wait(&rll->rll_cv, &rll->rll_lock);
1624 rll->rll_writer = curthread;
1625 }
1626
1627 mutex_exit(&rll->rll_lock);
1628 }
1629
1630 static void
1631 ztest_rll_unlock(rll_t *rll)
1632 {
1633 mutex_enter(&rll->rll_lock);
1634
1635 if (rll->rll_writer) {
1636 ASSERT0(rll->rll_readers);
1637 rll->rll_writer = NULL;
1638 } else {
1639 ASSERT3S(rll->rll_readers, >, 0);
1640 ASSERT3P(rll->rll_writer, ==, NULL);
1641 rll->rll_readers--;
1642 }
1643
1644 if (rll->rll_writer == NULL && rll->rll_readers == 0)
1645 cv_broadcast(&rll->rll_cv);
1646
1647 mutex_exit(&rll->rll_lock);
1648 }
1649
1650 static void
1651 ztest_object_lock(ztest_ds_t *zd, uint64_t object, rl_type_t type)
1652 {
1653 rll_t *rll = &zd->zd_object_lock[object & (ZTEST_OBJECT_LOCKS - 1)];
1654
1655 ztest_rll_lock(rll, type);
1656 }
1657
1658 static void
1659 ztest_object_unlock(ztest_ds_t *zd, uint64_t object)
1660 {
1661 rll_t *rll = &zd->zd_object_lock[object & (ZTEST_OBJECT_LOCKS - 1)];
1662
1663 ztest_rll_unlock(rll);
1664 }
1665
1666 static rl_t *
1667 ztest_range_lock(ztest_ds_t *zd, uint64_t object, uint64_t offset,
1668 uint64_t size, rl_type_t type)
1669 {
1670 uint64_t hash = object ^ (offset % (ZTEST_RANGE_LOCKS + 1));
1671 rll_t *rll = &zd->zd_range_lock[hash & (ZTEST_RANGE_LOCKS - 1)];
1672 rl_t *rl;
1673
1674 rl = umem_alloc(sizeof (*rl), UMEM_NOFAIL);
1675 rl->rl_object = object;
1676 rl->rl_offset = offset;
1677 rl->rl_size = size;
1678 rl->rl_lock = rll;
1679
1680 ztest_rll_lock(rll, type);
1681
1682 return (rl);
1683 }
1684
1685 static void
1686 ztest_range_unlock(rl_t *rl)
1687 {
1688 rll_t *rll = rl->rl_lock;
1689
1690 ztest_rll_unlock(rll);
1691
1692 umem_free(rl, sizeof (*rl));
1693 }
1694
1695 static void
1696 ztest_zd_init(ztest_ds_t *zd, ztest_shared_ds_t *szd, objset_t *os)
1697 {
1698 zd->zd_os = os;
1699 zd->zd_zilog = dmu_objset_zil(os);
1700 zd->zd_shared = szd;
1701 dmu_objset_name(os, zd->zd_name);
1702 int l;
1703
1704 if (zd->zd_shared != NULL)
1705 zd->zd_shared->zd_seq = 0;
1706
1707 VERIFY0(pthread_rwlock_init(&zd->zd_zilog_lock, NULL));
1708 mutex_init(&zd->zd_dirobj_lock, NULL, MUTEX_DEFAULT, NULL);
1709
1710 for (l = 0; l < ZTEST_OBJECT_LOCKS; l++)
1711 ztest_rll_init(&zd->zd_object_lock[l]);
1712
1713 for (l = 0; l < ZTEST_RANGE_LOCKS; l++)
1714 ztest_rll_init(&zd->zd_range_lock[l]);
1715 }
1716
1717 static void
1718 ztest_zd_fini(ztest_ds_t *zd)
1719 {
1720 int l;
1721
1722 mutex_destroy(&zd->zd_dirobj_lock);
1723 (void) pthread_rwlock_destroy(&zd->zd_zilog_lock);
1724
1725 for (l = 0; l < ZTEST_OBJECT_LOCKS; l++)
1726 ztest_rll_destroy(&zd->zd_object_lock[l]);
1727
1728 for (l = 0; l < ZTEST_RANGE_LOCKS; l++)
1729 ztest_rll_destroy(&zd->zd_range_lock[l]);
1730 }
1731
1732 #define TXG_MIGHTWAIT (ztest_random(10) == 0 ? TXG_NOWAIT : TXG_WAIT)
1733
1734 static uint64_t
1735 ztest_tx_assign(dmu_tx_t *tx, uint64_t txg_how, const char *tag)
1736 {
1737 uint64_t txg;
1738 int error;
1739
1740 /*
1741 * Attempt to assign tx to some transaction group.
1742 */
1743 error = dmu_tx_assign(tx, txg_how);
1744 if (error) {
1745 if (error == ERESTART) {
1746 ASSERT3U(txg_how, ==, TXG_NOWAIT);
1747 dmu_tx_wait(tx);
1748 } else {
1749 ASSERT3U(error, ==, ENOSPC);
1750 ztest_record_enospc(tag);
1751 }
1752 dmu_tx_abort(tx);
1753 return (0);
1754 }
1755 txg = dmu_tx_get_txg(tx);
1756 ASSERT3U(txg, !=, 0);
1757 return (txg);
1758 }
1759
1760 static void
1761 ztest_bt_generate(ztest_block_tag_t *bt, objset_t *os, uint64_t object,
1762 uint64_t dnodesize, uint64_t offset, uint64_t gen, uint64_t txg,
1763 uint64_t crtxg)
1764 {
1765 bt->bt_magic = BT_MAGIC;
1766 bt->bt_objset = dmu_objset_id(os);
1767 bt->bt_object = object;
1768 bt->bt_dnodesize = dnodesize;
1769 bt->bt_offset = offset;
1770 bt->bt_gen = gen;
1771 bt->bt_txg = txg;
1772 bt->bt_crtxg = crtxg;
1773 }
1774
1775 static void
1776 ztest_bt_verify(ztest_block_tag_t *bt, objset_t *os, uint64_t object,
1777 uint64_t dnodesize, uint64_t offset, uint64_t gen, uint64_t txg,
1778 uint64_t crtxg)
1779 {
1780 ASSERT3U(bt->bt_magic, ==, BT_MAGIC);
1781 ASSERT3U(bt->bt_objset, ==, dmu_objset_id(os));
1782 ASSERT3U(bt->bt_object, ==, object);
1783 ASSERT3U(bt->bt_dnodesize, ==, dnodesize);
1784 ASSERT3U(bt->bt_offset, ==, offset);
1785 ASSERT3U(bt->bt_gen, <=, gen);
1786 ASSERT3U(bt->bt_txg, <=, txg);
1787 ASSERT3U(bt->bt_crtxg, ==, crtxg);
1788 }
1789
1790 static ztest_block_tag_t *
1791 ztest_bt_bonus(dmu_buf_t *db)
1792 {
1793 dmu_object_info_t doi;
1794 ztest_block_tag_t *bt;
1795
1796 dmu_object_info_from_db(db, &doi);
1797 ASSERT3U(doi.doi_bonus_size, <=, db->db_size);
1798 ASSERT3U(doi.doi_bonus_size, >=, sizeof (*bt));
1799 bt = (void *)((char *)db->db_data + doi.doi_bonus_size - sizeof (*bt));
1800
1801 return (bt);
1802 }
1803
1804 /*
1805 * Generate a token to fill up unused bonus buffer space. Try to make
1806 * it unique to the object, generation, and offset to verify that data
1807 * is not getting overwritten by data from other dnodes.
1808 */
1809 #define ZTEST_BONUS_FILL_TOKEN(obj, ds, gen, offset) \
1810 (((ds) << 48) | ((gen) << 32) | ((obj) << 8) | (offset))
1811
1812 /*
1813 * Fill up the unused bonus buffer region before the block tag with a
1814 * verifiable pattern. Filling the whole bonus area with non-zero data
1815 * helps ensure that all dnode traversal code properly skips the
1816 * interior regions of large dnodes.
1817 */
1818 static void
1819 ztest_fill_unused_bonus(dmu_buf_t *db, void *end, uint64_t obj,
1820 objset_t *os, uint64_t gen)
1821 {
1822 uint64_t *bonusp;
1823
1824 ASSERT(IS_P2ALIGNED((char *)end - (char *)db->db_data, 8));
1825
1826 for (bonusp = db->db_data; bonusp < (uint64_t *)end; bonusp++) {
1827 uint64_t token = ZTEST_BONUS_FILL_TOKEN(obj, dmu_objset_id(os),
1828 gen, bonusp - (uint64_t *)db->db_data);
1829 *bonusp = token;
1830 }
1831 }
1832
1833 /*
1834 * Verify that the unused area of a bonus buffer is filled with the
1835 * expected tokens.
1836 */
1837 static void
1838 ztest_verify_unused_bonus(dmu_buf_t *db, void *end, uint64_t obj,
1839 objset_t *os, uint64_t gen)
1840 {
1841 uint64_t *bonusp;
1842
1843 for (bonusp = db->db_data; bonusp < (uint64_t *)end; bonusp++) {
1844 uint64_t token = ZTEST_BONUS_FILL_TOKEN(obj, dmu_objset_id(os),
1845 gen, bonusp - (uint64_t *)db->db_data);
1846 VERIFY3U(*bonusp, ==, token);
1847 }
1848 }
1849
1850 /*
1851 * ZIL logging ops
1852 */
1853
1854 #define lrz_type lr_mode
1855 #define lrz_blocksize lr_uid
1856 #define lrz_ibshift lr_gid
1857 #define lrz_bonustype lr_rdev
1858 #define lrz_dnodesize lr_crtime[1]
1859
1860 static void
1861 ztest_log_create(ztest_ds_t *zd, dmu_tx_t *tx, lr_create_t *lr)
1862 {
1863 char *name = (void *)(lr + 1); /* name follows lr */
1864 size_t namesize = strlen(name) + 1;
1865 itx_t *itx;
1866
1867 if (zil_replaying(zd->zd_zilog, tx))
1868 return;
1869
1870 itx = zil_itx_create(TX_CREATE, sizeof (*lr) + namesize);
1871 memcpy(&itx->itx_lr + 1, &lr->lr_common + 1,
1872 sizeof (*lr) + namesize - sizeof (lr_t));
1873
1874 zil_itx_assign(zd->zd_zilog, itx, tx);
1875 }
1876
1877 static void
1878 ztest_log_remove(ztest_ds_t *zd, dmu_tx_t *tx, lr_remove_t *lr, uint64_t object)
1879 {
1880 char *name = (void *)(lr + 1); /* name follows lr */
1881 size_t namesize = strlen(name) + 1;
1882 itx_t *itx;
1883
1884 if (zil_replaying(zd->zd_zilog, tx))
1885 return;
1886
1887 itx = zil_itx_create(TX_REMOVE, sizeof (*lr) + namesize);
1888 memcpy(&itx->itx_lr + 1, &lr->lr_common + 1,
1889 sizeof (*lr) + namesize - sizeof (lr_t));
1890
1891 itx->itx_oid = object;
1892 zil_itx_assign(zd->zd_zilog, itx, tx);
1893 }
1894
1895 static void
1896 ztest_log_write(ztest_ds_t *zd, dmu_tx_t *tx, lr_write_t *lr)
1897 {
1898 itx_t *itx;
1899 itx_wr_state_t write_state = ztest_random(WR_NUM_STATES);
1900
1901 if (zil_replaying(zd->zd_zilog, tx))
1902 return;
1903
1904 if (lr->lr_length > zil_max_log_data(zd->zd_zilog))
1905 write_state = WR_INDIRECT;
1906
1907 itx = zil_itx_create(TX_WRITE,
1908 sizeof (*lr) + (write_state == WR_COPIED ? lr->lr_length : 0));
1909
1910 if (write_state == WR_COPIED &&
1911 dmu_read(zd->zd_os, lr->lr_foid, lr->lr_offset, lr->lr_length,
1912 ((lr_write_t *)&itx->itx_lr) + 1, DMU_READ_NO_PREFETCH) != 0) {
1913 zil_itx_destroy(itx);
1914 itx = zil_itx_create(TX_WRITE, sizeof (*lr));
1915 write_state = WR_NEED_COPY;
1916 }
1917 itx->itx_private = zd;
1918 itx->itx_wr_state = write_state;
1919 itx->itx_sync = (ztest_random(8) == 0);
1920
1921 memcpy(&itx->itx_lr + 1, &lr->lr_common + 1,
1922 sizeof (*lr) - sizeof (lr_t));
1923
1924 zil_itx_assign(zd->zd_zilog, itx, tx);
1925 }
1926
1927 static void
1928 ztest_log_truncate(ztest_ds_t *zd, dmu_tx_t *tx, lr_truncate_t *lr)
1929 {
1930 itx_t *itx;
1931
1932 if (zil_replaying(zd->zd_zilog, tx))
1933 return;
1934
1935 itx = zil_itx_create(TX_TRUNCATE, sizeof (*lr));
1936 memcpy(&itx->itx_lr + 1, &lr->lr_common + 1,
1937 sizeof (*lr) - sizeof (lr_t));
1938
1939 itx->itx_sync = B_FALSE;
1940 zil_itx_assign(zd->zd_zilog, itx, tx);
1941 }
1942
1943 static void
1944 ztest_log_setattr(ztest_ds_t *zd, dmu_tx_t *tx, lr_setattr_t *lr)
1945 {
1946 itx_t *itx;
1947
1948 if (zil_replaying(zd->zd_zilog, tx))
1949 return;
1950
1951 itx = zil_itx_create(TX_SETATTR, sizeof (*lr));
1952 memcpy(&itx->itx_lr + 1, &lr->lr_common + 1,
1953 sizeof (*lr) - sizeof (lr_t));
1954
1955 itx->itx_sync = B_FALSE;
1956 zil_itx_assign(zd->zd_zilog, itx, tx);
1957 }
1958
1959 /*
1960 * ZIL replay ops
1961 */
1962 static int
1963 ztest_replay_create(void *arg1, void *arg2, boolean_t byteswap)
1964 {
1965 ztest_ds_t *zd = arg1;
1966 lr_create_t *lr = arg2;
1967 char *name = (void *)(lr + 1); /* name follows lr */
1968 objset_t *os = zd->zd_os;
1969 ztest_block_tag_t *bbt;
1970 dmu_buf_t *db;
1971 dmu_tx_t *tx;
1972 uint64_t txg;
1973 int error = 0;
1974 int bonuslen;
1975
1976 if (byteswap)
1977 byteswap_uint64_array(lr, sizeof (*lr));
1978
1979 ASSERT3U(lr->lr_doid, ==, ZTEST_DIROBJ);
1980 ASSERT3S(name[0], !=, '\0');
1981
1982 tx = dmu_tx_create(os);
1983
1984 dmu_tx_hold_zap(tx, lr->lr_doid, B_TRUE, name);
1985
1986 if (lr->lrz_type == DMU_OT_ZAP_OTHER) {
1987 dmu_tx_hold_zap(tx, DMU_NEW_OBJECT, B_TRUE, NULL);
1988 } else {
1989 dmu_tx_hold_bonus(tx, DMU_NEW_OBJECT);
1990 }
1991
1992 txg = ztest_tx_assign(tx, TXG_WAIT, FTAG);
1993 if (txg == 0)
1994 return (ENOSPC);
1995
1996 ASSERT3U(dmu_objset_zil(os)->zl_replay, ==, !!lr->lr_foid);
1997 bonuslen = DN_BONUS_SIZE(lr->lrz_dnodesize);
1998
1999 if (lr->lrz_type == DMU_OT_ZAP_OTHER) {
2000 if (lr->lr_foid == 0) {
2001 lr->lr_foid = zap_create_dnsize(os,
2002 lr->lrz_type, lr->lrz_bonustype,
2003 bonuslen, lr->lrz_dnodesize, tx);
2004 } else {
2005 error = zap_create_claim_dnsize(os, lr->lr_foid,
2006 lr->lrz_type, lr->lrz_bonustype,
2007 bonuslen, lr->lrz_dnodesize, tx);
2008 }
2009 } else {
2010 if (lr->lr_foid == 0) {
2011 lr->lr_foid = dmu_object_alloc_dnsize(os,
2012 lr->lrz_type, 0, lr->lrz_bonustype,
2013 bonuslen, lr->lrz_dnodesize, tx);
2014 } else {
2015 error = dmu_object_claim_dnsize(os, lr->lr_foid,
2016 lr->lrz_type, 0, lr->lrz_bonustype,
2017 bonuslen, lr->lrz_dnodesize, tx);
2018 }
2019 }
2020
2021 if (error) {
2022 ASSERT3U(error, ==, EEXIST);
2023 ASSERT(zd->zd_zilog->zl_replay);
2024 dmu_tx_commit(tx);
2025 return (error);
2026 }
2027
2028 ASSERT3U(lr->lr_foid, !=, 0);
2029
2030 if (lr->lrz_type != DMU_OT_ZAP_OTHER)
2031 VERIFY0(dmu_object_set_blocksize(os, lr->lr_foid,
2032 lr->lrz_blocksize, lr->lrz_ibshift, tx));
2033
2034 VERIFY0(dmu_bonus_hold(os, lr->lr_foid, FTAG, &db));
2035 bbt = ztest_bt_bonus(db);
2036 dmu_buf_will_dirty(db, tx);
2037 ztest_bt_generate(bbt, os, lr->lr_foid, lr->lrz_dnodesize, -1ULL,
2038 lr->lr_gen, txg, txg);
2039 ztest_fill_unused_bonus(db, bbt, lr->lr_foid, os, lr->lr_gen);
2040 dmu_buf_rele(db, FTAG);
2041
2042 VERIFY0(zap_add(os, lr->lr_doid, name, sizeof (uint64_t), 1,
2043 &lr->lr_foid, tx));
2044
2045 (void) ztest_log_create(zd, tx, lr);
2046
2047 dmu_tx_commit(tx);
2048
2049 return (0);
2050 }
2051
2052 static int
2053 ztest_replay_remove(void *arg1, void *arg2, boolean_t byteswap)
2054 {
2055 ztest_ds_t *zd = arg1;
2056 lr_remove_t *lr = arg2;
2057 char *name = (void *)(lr + 1); /* name follows lr */
2058 objset_t *os = zd->zd_os;
2059 dmu_object_info_t doi;
2060 dmu_tx_t *tx;
2061 uint64_t object, txg;
2062
2063 if (byteswap)
2064 byteswap_uint64_array(lr, sizeof (*lr));
2065
2066 ASSERT3U(lr->lr_doid, ==, ZTEST_DIROBJ);
2067 ASSERT3S(name[0], !=, '\0');
2068
2069 VERIFY0(
2070 zap_lookup(os, lr->lr_doid, name, sizeof (object), 1, &object));
2071 ASSERT3U(object, !=, 0);
2072
2073 ztest_object_lock(zd, object, RL_WRITER);
2074
2075 VERIFY0(dmu_object_info(os, object, &doi));
2076
2077 tx = dmu_tx_create(os);
2078
2079 dmu_tx_hold_zap(tx, lr->lr_doid, B_FALSE, name);
2080 dmu_tx_hold_free(tx, object, 0, DMU_OBJECT_END);
2081
2082 txg = ztest_tx_assign(tx, TXG_WAIT, FTAG);
2083 if (txg == 0) {
2084 ztest_object_unlock(zd, object);
2085 return (ENOSPC);
2086 }
2087
2088 if (doi.doi_type == DMU_OT_ZAP_OTHER) {
2089 VERIFY0(zap_destroy(os, object, tx));
2090 } else {
2091 VERIFY0(dmu_object_free(os, object, tx));
2092 }
2093
2094 VERIFY0(zap_remove(os, lr->lr_doid, name, tx));
2095
2096 (void) ztest_log_remove(zd, tx, lr, object);
2097
2098 dmu_tx_commit(tx);
2099
2100 ztest_object_unlock(zd, object);
2101
2102 return (0);
2103 }
2104
2105 static int
2106 ztest_replay_write(void *arg1, void *arg2, boolean_t byteswap)
2107 {
2108 ztest_ds_t *zd = arg1;
2109 lr_write_t *lr = arg2;
2110 objset_t *os = zd->zd_os;
2111 void *data = lr + 1; /* data follows lr */
2112 uint64_t offset, length;
2113 ztest_block_tag_t *bt = data;
2114 ztest_block_tag_t *bbt;
2115 uint64_t gen, txg, lrtxg, crtxg;
2116 dmu_object_info_t doi;
2117 dmu_tx_t *tx;
2118 dmu_buf_t *db;
2119 arc_buf_t *abuf = NULL;
2120 rl_t *rl;
2121
2122 if (byteswap)
2123 byteswap_uint64_array(lr, sizeof (*lr));
2124
2125 offset = lr->lr_offset;
2126 length = lr->lr_length;
2127
2128 /* If it's a dmu_sync() block, write the whole block */
2129 if (lr->lr_common.lrc_reclen == sizeof (lr_write_t)) {
2130 uint64_t blocksize = BP_GET_LSIZE(&lr->lr_blkptr);
2131 if (length < blocksize) {
2132 offset -= offset % blocksize;
2133 length = blocksize;
2134 }
2135 }
2136
2137 if (bt->bt_magic == BSWAP_64(BT_MAGIC))
2138 byteswap_uint64_array(bt, sizeof (*bt));
2139
2140 if (bt->bt_magic != BT_MAGIC)
2141 bt = NULL;
2142
2143 ztest_object_lock(zd, lr->lr_foid, RL_READER);
2144 rl = ztest_range_lock(zd, lr->lr_foid, offset, length, RL_WRITER);
2145
2146 VERIFY0(dmu_bonus_hold(os, lr->lr_foid, FTAG, &db));
2147
2148 dmu_object_info_from_db(db, &doi);
2149
2150 bbt = ztest_bt_bonus(db);
2151 ASSERT3U(bbt->bt_magic, ==, BT_MAGIC);
2152 gen = bbt->bt_gen;
2153 crtxg = bbt->bt_crtxg;
2154 lrtxg = lr->lr_common.lrc_txg;
2155
2156 tx = dmu_tx_create(os);
2157
2158 dmu_tx_hold_write(tx, lr->lr_foid, offset, length);
2159
2160 if (ztest_random(8) == 0 && length == doi.doi_data_block_size &&
2161 P2PHASE(offset, length) == 0)
2162 abuf = dmu_request_arcbuf(db, length);
2163
2164 txg = ztest_tx_assign(tx, TXG_WAIT, FTAG);
2165 if (txg == 0) {
2166 if (abuf != NULL)
2167 dmu_return_arcbuf(abuf);
2168 dmu_buf_rele(db, FTAG);
2169 ztest_range_unlock(rl);
2170 ztest_object_unlock(zd, lr->lr_foid);
2171 return (ENOSPC);
2172 }
2173
2174 if (bt != NULL) {
2175 /*
2176 * Usually, verify the old data before writing new data --
2177 * but not always, because we also want to verify correct
2178 * behavior when the data was not recently read into cache.
2179 */
2180 ASSERT(doi.doi_data_block_size);
2181 ASSERT0(offset % doi.doi_data_block_size);
2182 if (ztest_random(4) != 0) {
2183 int prefetch = ztest_random(2) ?
2184 DMU_READ_PREFETCH : DMU_READ_NO_PREFETCH;
2185 ztest_block_tag_t rbt;
2186
2187 VERIFY(dmu_read(os, lr->lr_foid, offset,
2188 sizeof (rbt), &rbt, prefetch) == 0);
2189 if (rbt.bt_magic == BT_MAGIC) {
2190 ztest_bt_verify(&rbt, os, lr->lr_foid, 0,
2191 offset, gen, txg, crtxg);
2192 }
2193 }
2194
2195 /*
2196 * Writes can appear to be newer than the bonus buffer because
2197 * the ztest_get_data() callback does a dmu_read() of the
2198 * open-context data, which may be different than the data
2199 * as it was when the write was generated.
2200 */
2201 if (zd->zd_zilog->zl_replay) {
2202 ztest_bt_verify(bt, os, lr->lr_foid, 0, offset,
2203 MAX(gen, bt->bt_gen), MAX(txg, lrtxg),
2204 bt->bt_crtxg);
2205 }
2206
2207 /*
2208 * Set the bt's gen/txg to the bonus buffer's gen/txg
2209 * so that all of the usual ASSERTs will work.
2210 */
2211 ztest_bt_generate(bt, os, lr->lr_foid, 0, offset, gen, txg,
2212 crtxg);
2213 }
2214
2215 if (abuf == NULL) {
2216 dmu_write(os, lr->lr_foid, offset, length, data, tx);
2217 } else {
2218 memcpy(abuf->b_data, data, length);
2219 VERIFY0(dmu_assign_arcbuf_by_dbuf(db, offset, abuf, tx));
2220 }
2221
2222 (void) ztest_log_write(zd, tx, lr);
2223
2224 dmu_buf_rele(db, FTAG);
2225
2226 dmu_tx_commit(tx);
2227
2228 ztest_range_unlock(rl);
2229 ztest_object_unlock(zd, lr->lr_foid);
2230
2231 return (0);
2232 }
2233
2234 static int
2235 ztest_replay_truncate(void *arg1, void *arg2, boolean_t byteswap)
2236 {
2237 ztest_ds_t *zd = arg1;
2238 lr_truncate_t *lr = arg2;
2239 objset_t *os = zd->zd_os;
2240 dmu_tx_t *tx;
2241 uint64_t txg;
2242 rl_t *rl;
2243
2244 if (byteswap)
2245 byteswap_uint64_array(lr, sizeof (*lr));
2246
2247 ztest_object_lock(zd, lr->lr_foid, RL_READER);
2248 rl = ztest_range_lock(zd, lr->lr_foid, lr->lr_offset, lr->lr_length,
2249 RL_WRITER);
2250
2251 tx = dmu_tx_create(os);
2252
2253 dmu_tx_hold_free(tx, lr->lr_foid, lr->lr_offset, lr->lr_length);
2254
2255 txg = ztest_tx_assign(tx, TXG_WAIT, FTAG);
2256 if (txg == 0) {
2257 ztest_range_unlock(rl);
2258 ztest_object_unlock(zd, lr->lr_foid);
2259 return (ENOSPC);
2260 }
2261
2262 VERIFY0(dmu_free_range(os, lr->lr_foid, lr->lr_offset,
2263 lr->lr_length, tx));
2264
2265 (void) ztest_log_truncate(zd, tx, lr);
2266
2267 dmu_tx_commit(tx);
2268
2269 ztest_range_unlock(rl);
2270 ztest_object_unlock(zd, lr->lr_foid);
2271
2272 return (0);
2273 }
2274
2275 static int
2276 ztest_replay_setattr(void *arg1, void *arg2, boolean_t byteswap)
2277 {
2278 ztest_ds_t *zd = arg1;
2279 lr_setattr_t *lr = arg2;
2280 objset_t *os = zd->zd_os;
2281 dmu_tx_t *tx;
2282 dmu_buf_t *db;
2283 ztest_block_tag_t *bbt;
2284 uint64_t txg, lrtxg, crtxg, dnodesize;
2285
2286 if (byteswap)
2287 byteswap_uint64_array(lr, sizeof (*lr));
2288
2289 ztest_object_lock(zd, lr->lr_foid, RL_WRITER);
2290
2291 VERIFY0(dmu_bonus_hold(os, lr->lr_foid, FTAG, &db));
2292
2293 tx = dmu_tx_create(os);
2294 dmu_tx_hold_bonus(tx, lr->lr_foid);
2295
2296 txg = ztest_tx_assign(tx, TXG_WAIT, FTAG);
2297 if (txg == 0) {
2298 dmu_buf_rele(db, FTAG);
2299 ztest_object_unlock(zd, lr->lr_foid);
2300 return (ENOSPC);
2301 }
2302
2303 bbt = ztest_bt_bonus(db);
2304 ASSERT3U(bbt->bt_magic, ==, BT_MAGIC);
2305 crtxg = bbt->bt_crtxg;
2306 lrtxg = lr->lr_common.lrc_txg;
2307 dnodesize = bbt->bt_dnodesize;
2308
2309 if (zd->zd_zilog->zl_replay) {
2310 ASSERT3U(lr->lr_size, !=, 0);
2311 ASSERT3U(lr->lr_mode, !=, 0);
2312 ASSERT3U(lrtxg, !=, 0);
2313 } else {
2314 /*
2315 * Randomly change the size and increment the generation.
2316 */
2317 lr->lr_size = (ztest_random(db->db_size / sizeof (*bbt)) + 1) *
2318 sizeof (*bbt);
2319 lr->lr_mode = bbt->bt_gen + 1;
2320 ASSERT0(lrtxg);
2321 }
2322
2323 /*
2324 * Verify that the current bonus buffer is not newer than our txg.
2325 */
2326 ztest_bt_verify(bbt, os, lr->lr_foid, dnodesize, -1ULL, lr->lr_mode,
2327 MAX(txg, lrtxg), crtxg);
2328
2329 dmu_buf_will_dirty(db, tx);
2330
2331 ASSERT3U(lr->lr_size, >=, sizeof (*bbt));
2332 ASSERT3U(lr->lr_size, <=, db->db_size);
2333 VERIFY0(dmu_set_bonus(db, lr->lr_size, tx));
2334 bbt = ztest_bt_bonus(db);
2335
2336 ztest_bt_generate(bbt, os, lr->lr_foid, dnodesize, -1ULL, lr->lr_mode,
2337 txg, crtxg);
2338 ztest_fill_unused_bonus(db, bbt, lr->lr_foid, os, bbt->bt_gen);
2339 dmu_buf_rele(db, FTAG);
2340
2341 (void) ztest_log_setattr(zd, tx, lr);
2342
2343 dmu_tx_commit(tx);
2344
2345 ztest_object_unlock(zd, lr->lr_foid);
2346
2347 return (0);
2348 }
2349
2350 static zil_replay_func_t *ztest_replay_vector[TX_MAX_TYPE] = {
2351 NULL, /* 0 no such transaction type */
2352 ztest_replay_create, /* TX_CREATE */
2353 NULL, /* TX_MKDIR */
2354 NULL, /* TX_MKXATTR */
2355 NULL, /* TX_SYMLINK */
2356 ztest_replay_remove, /* TX_REMOVE */
2357 NULL, /* TX_RMDIR */
2358 NULL, /* TX_LINK */
2359 NULL, /* TX_RENAME */
2360 ztest_replay_write, /* TX_WRITE */
2361 ztest_replay_truncate, /* TX_TRUNCATE */
2362 ztest_replay_setattr, /* TX_SETATTR */
2363 NULL, /* TX_ACL */
2364 NULL, /* TX_CREATE_ACL */
2365 NULL, /* TX_CREATE_ATTR */
2366 NULL, /* TX_CREATE_ACL_ATTR */
2367 NULL, /* TX_MKDIR_ACL */
2368 NULL, /* TX_MKDIR_ATTR */
2369 NULL, /* TX_MKDIR_ACL_ATTR */
2370 NULL, /* TX_WRITE2 */
2371 NULL, /* TX_SETSAXATTR */
2372 NULL, /* TX_RENAME_EXCHANGE */
2373 NULL, /* TX_RENAME_WHITEOUT */
2374 };
2375
2376 /*
2377 * ZIL get_data callbacks
2378 */
2379
2380 static void
2381 ztest_get_done(zgd_t *zgd, int error)
2382 {
2383 (void) error;
2384 ztest_ds_t *zd = zgd->zgd_private;
2385 uint64_t object = ((rl_t *)zgd->zgd_lr)->rl_object;
2386
2387 if (zgd->zgd_db)
2388 dmu_buf_rele(zgd->zgd_db, zgd);
2389
2390 ztest_range_unlock((rl_t *)zgd->zgd_lr);
2391 ztest_object_unlock(zd, object);
2392
2393 umem_free(zgd, sizeof (*zgd));
2394 }
2395
2396 static int
2397 ztest_get_data(void *arg, uint64_t arg2, lr_write_t *lr, char *buf,
2398 struct lwb *lwb, zio_t *zio)
2399 {
2400 (void) arg2;
2401 ztest_ds_t *zd = arg;
2402 objset_t *os = zd->zd_os;
2403 uint64_t object = lr->lr_foid;
2404 uint64_t offset = lr->lr_offset;
2405 uint64_t size = lr->lr_length;
2406 uint64_t txg = lr->lr_common.lrc_txg;
2407 uint64_t crtxg;
2408 dmu_object_info_t doi;
2409 dmu_buf_t *db;
2410 zgd_t *zgd;
2411 int error;
2412
2413 ASSERT3P(lwb, !=, NULL);
2414 ASSERT3P(zio, !=, NULL);
2415 ASSERT3U(size, !=, 0);
2416
2417 ztest_object_lock(zd, object, RL_READER);
2418 error = dmu_bonus_hold(os, object, FTAG, &db);
2419 if (error) {
2420 ztest_object_unlock(zd, object);
2421 return (error);
2422 }
2423
2424 crtxg = ztest_bt_bonus(db)->bt_crtxg;
2425
2426 if (crtxg == 0 || crtxg > txg) {
2427 dmu_buf_rele(db, FTAG);
2428 ztest_object_unlock(zd, object);
2429 return (ENOENT);
2430 }
2431
2432 dmu_object_info_from_db(db, &doi);
2433 dmu_buf_rele(db, FTAG);
2434 db = NULL;
2435
2436 zgd = umem_zalloc(sizeof (*zgd), UMEM_NOFAIL);
2437 zgd->zgd_lwb = lwb;
2438 zgd->zgd_private = zd;
2439
2440 if (buf != NULL) { /* immediate write */
2441 zgd->zgd_lr = (struct zfs_locked_range *)ztest_range_lock(zd,
2442 object, offset, size, RL_READER);
2443
2444 error = dmu_read(os, object, offset, size, buf,
2445 DMU_READ_NO_PREFETCH);
2446 ASSERT0(error);
2447 } else {
2448 size = doi.doi_data_block_size;
2449 if (ISP2(size)) {
2450 offset = P2ALIGN(offset, size);
2451 } else {
2452 ASSERT3U(offset, <, size);
2453 offset = 0;
2454 }
2455
2456 zgd->zgd_lr = (struct zfs_locked_range *)ztest_range_lock(zd,
2457 object, offset, size, RL_READER);
2458
2459 error = dmu_buf_hold(os, object, offset, zgd, &db,
2460 DMU_READ_NO_PREFETCH);
2461
2462 if (error == 0) {
2463 blkptr_t *bp = &lr->lr_blkptr;
2464
2465 zgd->zgd_db = db;
2466 zgd->zgd_bp = bp;
2467
2468 ASSERT3U(db->db_offset, ==, offset);
2469 ASSERT3U(db->db_size, ==, size);
2470
2471 error = dmu_sync(zio, lr->lr_common.lrc_txg,
2472 ztest_get_done, zgd);
2473
2474 if (error == 0)
2475 return (0);
2476 }
2477 }
2478
2479 ztest_get_done(zgd, error);
2480
2481 return (error);
2482 }
2483
2484 static void *
2485 ztest_lr_alloc(size_t lrsize, char *name)
2486 {
2487 char *lr;
2488 size_t namesize = name ? strlen(name) + 1 : 0;
2489
2490 lr = umem_zalloc(lrsize + namesize, UMEM_NOFAIL);
2491
2492 if (name)
2493 memcpy(lr + lrsize, name, namesize);
2494
2495 return (lr);
2496 }
2497
2498 static void
2499 ztest_lr_free(void *lr, size_t lrsize, char *name)
2500 {
2501 size_t namesize = name ? strlen(name) + 1 : 0;
2502
2503 umem_free(lr, lrsize + namesize);
2504 }
2505
2506 /*
2507 * Lookup a bunch of objects. Returns the number of objects not found.
2508 */
2509 static int
2510 ztest_lookup(ztest_ds_t *zd, ztest_od_t *od, int count)
2511 {
2512 int missing = 0;
2513 int error;
2514 int i;
2515
2516 ASSERT(MUTEX_HELD(&zd->zd_dirobj_lock));
2517
2518 for (i = 0; i < count; i++, od++) {
2519 od->od_object = 0;
2520 error = zap_lookup(zd->zd_os, od->od_dir, od->od_name,
2521 sizeof (uint64_t), 1, &od->od_object);
2522 if (error) {
2523 ASSERT3S(error, ==, ENOENT);
2524 ASSERT0(od->od_object);
2525 missing++;
2526 } else {
2527 dmu_buf_t *db;
2528 ztest_block_tag_t *bbt;
2529 dmu_object_info_t doi;
2530
2531 ASSERT3U(od->od_object, !=, 0);
2532 ASSERT0(missing); /* there should be no gaps */
2533
2534 ztest_object_lock(zd, od->od_object, RL_READER);
2535 VERIFY0(dmu_bonus_hold(zd->zd_os, od->od_object,
2536 FTAG, &db));
2537 dmu_object_info_from_db(db, &doi);
2538 bbt = ztest_bt_bonus(db);
2539 ASSERT3U(bbt->bt_magic, ==, BT_MAGIC);
2540 od->od_type = doi.doi_type;
2541 od->od_blocksize = doi.doi_data_block_size;
2542 od->od_gen = bbt->bt_gen;
2543 dmu_buf_rele(db, FTAG);
2544 ztest_object_unlock(zd, od->od_object);
2545 }
2546 }
2547
2548 return (missing);
2549 }
2550
2551 static int
2552 ztest_create(ztest_ds_t *zd, ztest_od_t *od, int count)
2553 {
2554 int missing = 0;
2555 int i;
2556
2557 ASSERT(MUTEX_HELD(&zd->zd_dirobj_lock));
2558
2559 for (i = 0; i < count; i++, od++) {
2560 if (missing) {
2561 od->od_object = 0;
2562 missing++;
2563 continue;
2564 }
2565
2566 lr_create_t *lr = ztest_lr_alloc(sizeof (*lr), od->od_name);
2567
2568 lr->lr_doid = od->od_dir;
2569 lr->lr_foid = 0; /* 0 to allocate, > 0 to claim */
2570 lr->lrz_type = od->od_crtype;
2571 lr->lrz_blocksize = od->od_crblocksize;
2572 lr->lrz_ibshift = ztest_random_ibshift();
2573 lr->lrz_bonustype = DMU_OT_UINT64_OTHER;
2574 lr->lrz_dnodesize = od->od_crdnodesize;
2575 lr->lr_gen = od->od_crgen;
2576 lr->lr_crtime[0] = time(NULL);
2577
2578 if (ztest_replay_create(zd, lr, B_FALSE) != 0) {
2579 ASSERT0(missing);
2580 od->od_object = 0;
2581 missing++;
2582 } else {
2583 od->od_object = lr->lr_foid;
2584 od->od_type = od->od_crtype;
2585 od->od_blocksize = od->od_crblocksize;
2586 od->od_gen = od->od_crgen;
2587 ASSERT3U(od->od_object, !=, 0);
2588 }
2589
2590 ztest_lr_free(lr, sizeof (*lr), od->od_name);
2591 }
2592
2593 return (missing);
2594 }
2595
2596 static int
2597 ztest_remove(ztest_ds_t *zd, ztest_od_t *od, int count)
2598 {
2599 int missing = 0;
2600 int error;
2601 int i;
2602
2603 ASSERT(MUTEX_HELD(&zd->zd_dirobj_lock));
2604
2605 od += count - 1;
2606
2607 for (i = count - 1; i >= 0; i--, od--) {
2608 if (missing) {
2609 missing++;
2610 continue;
2611 }
2612
2613 /*
2614 * No object was found.
2615 */
2616 if (od->od_object == 0)
2617 continue;
2618
2619 lr_remove_t *lr = ztest_lr_alloc(sizeof (*lr), od->od_name);
2620
2621 lr->lr_doid = od->od_dir;
2622
2623 if ((error = ztest_replay_remove(zd, lr, B_FALSE)) != 0) {
2624 ASSERT3U(error, ==, ENOSPC);
2625 missing++;
2626 } else {
2627 od->od_object = 0;
2628 }
2629 ztest_lr_free(lr, sizeof (*lr), od->od_name);
2630 }
2631
2632 return (missing);
2633 }
2634
2635 static int
2636 ztest_write(ztest_ds_t *zd, uint64_t object, uint64_t offset, uint64_t size,
2637 void *data)
2638 {
2639 lr_write_t *lr;
2640 int error;
2641
2642 lr = ztest_lr_alloc(sizeof (*lr) + size, NULL);
2643
2644 lr->lr_foid = object;
2645 lr->lr_offset = offset;
2646 lr->lr_length = size;
2647 lr->lr_blkoff = 0;
2648 BP_ZERO(&lr->lr_blkptr);
2649
2650 memcpy(lr + 1, data, size);
2651
2652 error = ztest_replay_write(zd, lr, B_FALSE);
2653
2654 ztest_lr_free(lr, sizeof (*lr) + size, NULL);
2655
2656 return (error);
2657 }
2658
2659 static int
2660 ztest_truncate(ztest_ds_t *zd, uint64_t object, uint64_t offset, uint64_t size)
2661 {
2662 lr_truncate_t *lr;
2663 int error;
2664
2665 lr = ztest_lr_alloc(sizeof (*lr), NULL);
2666
2667 lr->lr_foid = object;
2668 lr->lr_offset = offset;
2669 lr->lr_length = size;
2670
2671 error = ztest_replay_truncate(zd, lr, B_FALSE);
2672
2673 ztest_lr_free(lr, sizeof (*lr), NULL);
2674
2675 return (error);
2676 }
2677
2678 static int
2679 ztest_setattr(ztest_ds_t *zd, uint64_t object)
2680 {
2681 lr_setattr_t *lr;
2682 int error;
2683
2684 lr = ztest_lr_alloc(sizeof (*lr), NULL);
2685
2686 lr->lr_foid = object;
2687 lr->lr_size = 0;
2688 lr->lr_mode = 0;
2689
2690 error = ztest_replay_setattr(zd, lr, B_FALSE);
2691
2692 ztest_lr_free(lr, sizeof (*lr), NULL);
2693
2694 return (error);
2695 }
2696
2697 static void
2698 ztest_prealloc(ztest_ds_t *zd, uint64_t object, uint64_t offset, uint64_t size)
2699 {
2700 objset_t *os = zd->zd_os;
2701 dmu_tx_t *tx;
2702 uint64_t txg;
2703 rl_t *rl;
2704
2705 txg_wait_synced(dmu_objset_pool(os), 0);
2706
2707 ztest_object_lock(zd, object, RL_READER);
2708 rl = ztest_range_lock(zd, object, offset, size, RL_WRITER);
2709
2710 tx = dmu_tx_create(os);
2711
2712 dmu_tx_hold_write(tx, object, offset, size);
2713
2714 txg = ztest_tx_assign(tx, TXG_WAIT, FTAG);
2715
2716 if (txg != 0) {
2717 dmu_prealloc(os, object, offset, size, tx);
2718 dmu_tx_commit(tx);
2719 txg_wait_synced(dmu_objset_pool(os), txg);
2720 } else {
2721 (void) dmu_free_long_range(os, object, offset, size);
2722 }
2723
2724 ztest_range_unlock(rl);
2725 ztest_object_unlock(zd, object);
2726 }
2727
2728 static void
2729 ztest_io(ztest_ds_t *zd, uint64_t object, uint64_t offset)
2730 {
2731 int err;
2732 ztest_block_tag_t wbt;
2733 dmu_object_info_t doi;
2734 enum ztest_io_type io_type;
2735 uint64_t blocksize;
2736 void *data;
2737
2738 VERIFY0(dmu_object_info(zd->zd_os, object, &doi));
2739 blocksize = doi.doi_data_block_size;
2740 data = umem_alloc(blocksize, UMEM_NOFAIL);
2741
2742 /*
2743 * Pick an i/o type at random, biased toward writing block tags.
2744 */
2745 io_type = ztest_random(ZTEST_IO_TYPES);
2746 if (ztest_random(2) == 0)
2747 io_type = ZTEST_IO_WRITE_TAG;
2748
2749 (void) pthread_rwlock_rdlock(&zd->zd_zilog_lock);
2750
2751 switch (io_type) {
2752
2753 case ZTEST_IO_WRITE_TAG:
2754 ztest_bt_generate(&wbt, zd->zd_os, object, doi.doi_dnodesize,
2755 offset, 0, 0, 0);
2756 (void) ztest_write(zd, object, offset, sizeof (wbt), &wbt);
2757 break;
2758
2759 case ZTEST_IO_WRITE_PATTERN:
2760 (void) memset(data, 'a' + (object + offset) % 5, blocksize);
2761 if (ztest_random(2) == 0) {
2762 /*
2763 * Induce fletcher2 collisions to ensure that
2764 * zio_ddt_collision() detects and resolves them
2765 * when using fletcher2-verify for deduplication.
2766 */
2767 ((uint64_t *)data)[0] ^= 1ULL << 63;
2768 ((uint64_t *)data)[4] ^= 1ULL << 63;
2769 }
2770 (void) ztest_write(zd, object, offset, blocksize, data);
2771 break;
2772
2773 case ZTEST_IO_WRITE_ZEROES:
2774 memset(data, 0, blocksize);
2775 (void) ztest_write(zd, object, offset, blocksize, data);
2776 break;
2777
2778 case ZTEST_IO_TRUNCATE:
2779 (void) ztest_truncate(zd, object, offset, blocksize);
2780 break;
2781
2782 case ZTEST_IO_SETATTR:
2783 (void) ztest_setattr(zd, object);
2784 break;
2785 default:
2786 break;
2787
2788 case ZTEST_IO_REWRITE:
2789 (void) pthread_rwlock_rdlock(&ztest_name_lock);
2790 err = ztest_dsl_prop_set_uint64(zd->zd_name,
2791 ZFS_PROP_CHECKSUM, spa_dedup_checksum(ztest_spa),
2792 B_FALSE);
2793 VERIFY(err == 0 || err == ENOSPC);
2794 err = ztest_dsl_prop_set_uint64(zd->zd_name,
2795 ZFS_PROP_COMPRESSION,
2796 ztest_random_dsl_prop(ZFS_PROP_COMPRESSION),
2797 B_FALSE);
2798 VERIFY(err == 0 || err == ENOSPC);
2799 (void) pthread_rwlock_unlock(&ztest_name_lock);
2800
2801 VERIFY0(dmu_read(zd->zd_os, object, offset, blocksize, data,
2802 DMU_READ_NO_PREFETCH));
2803
2804 (void) ztest_write(zd, object, offset, blocksize, data);
2805 break;
2806 }
2807
2808 (void) pthread_rwlock_unlock(&zd->zd_zilog_lock);
2809
2810 umem_free(data, blocksize);
2811 }
2812
2813 /*
2814 * Initialize an object description template.
2815 */
2816 static void
2817 ztest_od_init(ztest_od_t *od, uint64_t id, const char *tag, uint64_t index,
2818 dmu_object_type_t type, uint64_t blocksize, uint64_t dnodesize,
2819 uint64_t gen)
2820 {
2821 od->od_dir = ZTEST_DIROBJ;
2822 od->od_object = 0;
2823
2824 od->od_crtype = type;
2825 od->od_crblocksize = blocksize ? blocksize : ztest_random_blocksize();
2826 od->od_crdnodesize = dnodesize ? dnodesize : ztest_random_dnodesize();
2827 od->od_crgen = gen;
2828
2829 od->od_type = DMU_OT_NONE;
2830 od->od_blocksize = 0;
2831 od->od_gen = 0;
2832
2833 (void) snprintf(od->od_name, sizeof (od->od_name),
2834 "%s(%"PRId64")[%"PRIu64"]",
2835 tag, id, index);
2836 }
2837
2838 /*
2839 * Lookup or create the objects for a test using the od template.
2840 * If the objects do not all exist, or if 'remove' is specified,
2841 * remove any existing objects and create new ones. Otherwise,
2842 * use the existing objects.
2843 */
2844 static int
2845 ztest_object_init(ztest_ds_t *zd, ztest_od_t *od, size_t size, boolean_t remove)
2846 {
2847 int count = size / sizeof (*od);
2848 int rv = 0;
2849
2850 mutex_enter(&zd->zd_dirobj_lock);
2851 if ((ztest_lookup(zd, od, count) != 0 || remove) &&
2852 (ztest_remove(zd, od, count) != 0 ||
2853 ztest_create(zd, od, count) != 0))
2854 rv = -1;
2855 zd->zd_od = od;
2856 mutex_exit(&zd->zd_dirobj_lock);
2857
2858 return (rv);
2859 }
2860
2861 void
2862 ztest_zil_commit(ztest_ds_t *zd, uint64_t id)
2863 {
2864 (void) id;
2865 zilog_t *zilog = zd->zd_zilog;
2866
2867 (void) pthread_rwlock_rdlock(&zd->zd_zilog_lock);
2868
2869 zil_commit(zilog, ztest_random(ZTEST_OBJECTS));
2870
2871 /*
2872 * Remember the committed values in zd, which is in parent/child
2873 * shared memory. If we die, the next iteration of ztest_run()
2874 * will verify that the log really does contain this record.
2875 */
2876 mutex_enter(&zilog->zl_lock);
2877 ASSERT3P(zd->zd_shared, !=, NULL);
2878 ASSERT3U(zd->zd_shared->zd_seq, <=, zilog->zl_commit_lr_seq);
2879 zd->zd_shared->zd_seq = zilog->zl_commit_lr_seq;
2880 mutex_exit(&zilog->zl_lock);
2881
2882 (void) pthread_rwlock_unlock(&zd->zd_zilog_lock);
2883 }
2884
2885 /*
2886 * This function is designed to simulate the operations that occur during a
2887 * mount/unmount operation. We hold the dataset across these operations in an
2888 * attempt to expose any implicit assumptions about ZIL management.
2889 */
2890 void
2891 ztest_zil_remount(ztest_ds_t *zd, uint64_t id)
2892 {
2893 (void) id;
2894 objset_t *os = zd->zd_os;
2895
2896 /*
2897 * We hold the ztest_vdev_lock so we don't cause problems with
2898 * other threads that wish to remove a log device, such as
2899 * ztest_device_removal().
2900 */
2901 mutex_enter(&ztest_vdev_lock);
2902
2903 /*
2904 * We grab the zd_dirobj_lock to ensure that no other thread is
2905 * updating the zil (i.e. adding in-memory log records) and the
2906 * zd_zilog_lock to block any I/O.
2907 */
2908 mutex_enter(&zd->zd_dirobj_lock);
2909 (void) pthread_rwlock_wrlock(&zd->zd_zilog_lock);
2910
2911 /* zfsvfs_teardown() */
2912 zil_close(zd->zd_zilog);
2913
2914 /* zfsvfs_setup() */
2915 VERIFY3P(zil_open(os, ztest_get_data, NULL), ==, zd->zd_zilog);
2916 zil_replay(os, zd, ztest_replay_vector);
2917
2918 (void) pthread_rwlock_unlock(&zd->zd_zilog_lock);
2919 mutex_exit(&zd->zd_dirobj_lock);
2920 mutex_exit(&ztest_vdev_lock);
2921 }
2922
2923 /*
2924 * Verify that we can't destroy an active pool, create an existing pool,
2925 * or create a pool with a bad vdev spec.
2926 */
2927 void
2928 ztest_spa_create_destroy(ztest_ds_t *zd, uint64_t id)
2929 {
2930 (void) zd, (void) id;
2931 ztest_shared_opts_t *zo = &ztest_opts;
2932 spa_t *spa;
2933 nvlist_t *nvroot;
2934
2935 if (zo->zo_mmp_test)
2936 return;
2937
2938 /*
2939 * Attempt to create using a bad file.
2940 */
2941 nvroot = make_vdev_root("/dev/bogus", NULL, NULL, 0, 0, NULL, 0, 0, 1);
2942 VERIFY3U(ENOENT, ==,
2943 spa_create("ztest_bad_file", nvroot, NULL, NULL, NULL));
2944 fnvlist_free(nvroot);
2945
2946 /*
2947 * Attempt to create using a bad mirror.
2948 */
2949 nvroot = make_vdev_root("/dev/bogus", NULL, NULL, 0, 0, NULL, 0, 2, 1);
2950 VERIFY3U(ENOENT, ==,
2951 spa_create("ztest_bad_mirror", nvroot, NULL, NULL, NULL));
2952 fnvlist_free(nvroot);
2953
2954 /*
2955 * Attempt to create an existing pool. It shouldn't matter
2956 * what's in the nvroot; we should fail with EEXIST.
2957 */
2958 (void) pthread_rwlock_rdlock(&ztest_name_lock);
2959 nvroot = make_vdev_root("/dev/bogus", NULL, NULL, 0, 0, NULL, 0, 0, 1);
2960 VERIFY3U(EEXIST, ==,
2961 spa_create(zo->zo_pool, nvroot, NULL, NULL, NULL));
2962 fnvlist_free(nvroot);
2963
2964 /*
2965 * We open a reference to the spa and then we try to export it
2966 * expecting one of the following errors:
2967 *
2968 * EBUSY
2969 * Because of the reference we just opened.
2970 *
2971 * ZFS_ERR_EXPORT_IN_PROGRESS
2972 * For the case that there is another ztest thread doing
2973 * an export concurrently.
2974 */
2975 VERIFY0(spa_open(zo->zo_pool, &spa, FTAG));
2976 int error = spa_destroy(zo->zo_pool);
2977 if (error != EBUSY && error != ZFS_ERR_EXPORT_IN_PROGRESS) {
2978 fatal(B_FALSE, "spa_destroy(%s) returned unexpected value %d",
2979 spa->spa_name, error);
2980 }
2981 spa_close(spa, FTAG);
2982
2983 (void) pthread_rwlock_unlock(&ztest_name_lock);
2984 }
2985
2986 /*
2987 * Start and then stop the MMP threads to ensure the startup and shutdown code
2988 * works properly. Actual protection and property-related code tested via ZTS.
2989 */
2990 void
2991 ztest_mmp_enable_disable(ztest_ds_t *zd, uint64_t id)
2992 {
2993 (void) zd, (void) id;
2994 ztest_shared_opts_t *zo = &ztest_opts;
2995 spa_t *spa = ztest_spa;
2996
2997 if (zo->zo_mmp_test)
2998 return;
2999
3000 /*
3001 * Since enabling MMP involves setting a property, it could not be done
3002 * while the pool is suspended.
3003 */
3004 if (spa_suspended(spa))
3005 return;
3006
3007 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
3008 mutex_enter(&spa->spa_props_lock);
3009
3010 zfs_multihost_fail_intervals = 0;
3011
3012 if (!spa_multihost(spa)) {
3013 spa->spa_multihost = B_TRUE;
3014 mmp_thread_start(spa);
3015 }
3016
3017 mutex_exit(&spa->spa_props_lock);
3018 spa_config_exit(spa, SCL_CONFIG, FTAG);
3019
3020 txg_wait_synced(spa_get_dsl(spa), 0);
3021 mmp_signal_all_threads();
3022 txg_wait_synced(spa_get_dsl(spa), 0);
3023
3024 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
3025 mutex_enter(&spa->spa_props_lock);
3026
3027 if (spa_multihost(spa)) {
3028 mmp_thread_stop(spa);
3029 spa->spa_multihost = B_FALSE;
3030 }
3031
3032 mutex_exit(&spa->spa_props_lock);
3033 spa_config_exit(spa, SCL_CONFIG, FTAG);
3034 }
3035
3036 void
3037 ztest_spa_upgrade(ztest_ds_t *zd, uint64_t id)
3038 {
3039 (void) zd, (void) id;
3040 spa_t *spa;
3041 uint64_t initial_version = SPA_VERSION_INITIAL;
3042 uint64_t version, newversion;
3043 nvlist_t *nvroot, *props;
3044 char *name;
3045
3046 if (ztest_opts.zo_mmp_test)
3047 return;
3048
3049 /* dRAID added after feature flags, skip upgrade test. */
3050 if (strcmp(ztest_opts.zo_raid_type, VDEV_TYPE_DRAID) == 0)
3051 return;
3052
3053 mutex_enter(&ztest_vdev_lock);
3054 name = kmem_asprintf("%s_upgrade", ztest_opts.zo_pool);
3055
3056 /*
3057 * Clean up from previous runs.
3058 */
3059 (void) spa_destroy(name);
3060
3061 nvroot = make_vdev_root(NULL, NULL, name, ztest_opts.zo_vdev_size, 0,
3062 NULL, ztest_opts.zo_raid_children, ztest_opts.zo_mirrors, 1);
3063
3064 /*
3065 * If we're configuring a RAIDZ device then make sure that the
3066 * initial version is capable of supporting that feature.
3067 */
3068 switch (ztest_opts.zo_raid_parity) {
3069 case 0:
3070 case 1:
3071 initial_version = SPA_VERSION_INITIAL;
3072 break;
3073 case 2:
3074 initial_version = SPA_VERSION_RAIDZ2;
3075 break;
3076 case 3:
3077 initial_version = SPA_VERSION_RAIDZ3;
3078 break;
3079 }
3080
3081 /*
3082 * Create a pool with a spa version that can be upgraded. Pick
3083 * a value between initial_version and SPA_VERSION_BEFORE_FEATURES.
3084 */
3085 do {
3086 version = ztest_random_spa_version(initial_version);
3087 } while (version > SPA_VERSION_BEFORE_FEATURES);
3088
3089 props = fnvlist_alloc();
3090 fnvlist_add_uint64(props,
3091 zpool_prop_to_name(ZPOOL_PROP_VERSION), version);
3092 VERIFY0(spa_create(name, nvroot, props, NULL, NULL));
3093 fnvlist_free(nvroot);
3094 fnvlist_free(props);
3095
3096 VERIFY0(spa_open(name, &spa, FTAG));
3097 VERIFY3U(spa_version(spa), ==, version);
3098 newversion = ztest_random_spa_version(version + 1);
3099
3100 if (ztest_opts.zo_verbose >= 4) {
3101 (void) printf("upgrading spa version from "
3102 "%"PRIu64" to %"PRIu64"\n",
3103 version, newversion);
3104 }
3105
3106 spa_upgrade(spa, newversion);
3107 VERIFY3U(spa_version(spa), >, version);
3108 VERIFY3U(spa_version(spa), ==, fnvlist_lookup_uint64(spa->spa_config,
3109 zpool_prop_to_name(ZPOOL_PROP_VERSION)));
3110 spa_close(spa, FTAG);
3111
3112 kmem_strfree(name);
3113 mutex_exit(&ztest_vdev_lock);
3114 }
3115
3116 static void
3117 ztest_spa_checkpoint(spa_t *spa)
3118 {
3119 ASSERT(MUTEX_HELD(&ztest_checkpoint_lock));
3120
3121 int error = spa_checkpoint(spa->spa_name);
3122
3123 switch (error) {
3124 case 0:
3125 case ZFS_ERR_DEVRM_IN_PROGRESS:
3126 case ZFS_ERR_DISCARDING_CHECKPOINT:
3127 case ZFS_ERR_CHECKPOINT_EXISTS:
3128 break;
3129 case ENOSPC:
3130 ztest_record_enospc(FTAG);
3131 break;
3132 default:
3133 fatal(B_FALSE, "spa_checkpoint(%s) = %d", spa->spa_name, error);
3134 }
3135 }
3136
3137 static void
3138 ztest_spa_discard_checkpoint(spa_t *spa)
3139 {
3140 ASSERT(MUTEX_HELD(&ztest_checkpoint_lock));
3141
3142 int error = spa_checkpoint_discard(spa->spa_name);
3143
3144 switch (error) {
3145 case 0:
3146 case ZFS_ERR_DISCARDING_CHECKPOINT:
3147 case ZFS_ERR_NO_CHECKPOINT:
3148 break;
3149 default:
3150 fatal(B_FALSE, "spa_discard_checkpoint(%s) = %d",
3151 spa->spa_name, error);
3152 }
3153
3154 }
3155
3156 void
3157 ztest_spa_checkpoint_create_discard(ztest_ds_t *zd, uint64_t id)
3158 {
3159 (void) zd, (void) id;
3160 spa_t *spa = ztest_spa;
3161
3162 mutex_enter(&ztest_checkpoint_lock);
3163 if (ztest_random(2) == 0) {
3164 ztest_spa_checkpoint(spa);
3165 } else {
3166 ztest_spa_discard_checkpoint(spa);
3167 }
3168 mutex_exit(&ztest_checkpoint_lock);
3169 }
3170
3171
3172 static vdev_t *
3173 vdev_lookup_by_path(vdev_t *vd, const char *path)
3174 {
3175 vdev_t *mvd;
3176 int c;
3177
3178 if (vd->vdev_path != NULL && strcmp(path, vd->vdev_path) == 0)
3179 return (vd);
3180
3181 for (c = 0; c < vd->vdev_children; c++)
3182 if ((mvd = vdev_lookup_by_path(vd->vdev_child[c], path)) !=
3183 NULL)
3184 return (mvd);
3185
3186 return (NULL);
3187 }
3188
3189 static int
3190 spa_num_top_vdevs(spa_t *spa)
3191 {
3192 vdev_t *rvd = spa->spa_root_vdev;
3193 ASSERT3U(spa_config_held(spa, SCL_VDEV, RW_READER), ==, SCL_VDEV);
3194 return (rvd->vdev_children);
3195 }
3196
3197 /*
3198 * Verify that vdev_add() works as expected.
3199 */
3200 void
3201 ztest_vdev_add_remove(ztest_ds_t *zd, uint64_t id)
3202 {
3203 (void) zd, (void) id;
3204 ztest_shared_t *zs = ztest_shared;
3205 spa_t *spa = ztest_spa;
3206 uint64_t leaves;
3207 uint64_t guid;
3208 nvlist_t *nvroot;
3209 int error;
3210
3211 if (ztest_opts.zo_mmp_test)
3212 return;
3213
3214 mutex_enter(&ztest_vdev_lock);
3215 leaves = MAX(zs->zs_mirrors + zs->zs_splits, 1) *
3216 ztest_opts.zo_raid_children;
3217
3218 spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);
3219
3220 ztest_shared->zs_vdev_next_leaf = spa_num_top_vdevs(spa) * leaves;
3221
3222 /*
3223 * If we have slogs then remove them 1/4 of the time.
3224 */
3225 if (spa_has_slogs(spa) && ztest_random(4) == 0) {
3226 metaslab_group_t *mg;
3227
3228 /*
3229 * find the first real slog in log allocation class
3230 */
3231 mg = spa_log_class(spa)->mc_allocator[0].mca_rotor;
3232 while (!mg->mg_vd->vdev_islog)
3233 mg = mg->mg_next;
3234
3235 guid = mg->mg_vd->vdev_guid;
3236
3237 spa_config_exit(spa, SCL_VDEV, FTAG);
3238
3239 /*
3240 * We have to grab the zs_name_lock as writer to
3241 * prevent a race between removing a slog (dmu_objset_find)
3242 * and destroying a dataset. Removing the slog will
3243 * grab a reference on the dataset which may cause
3244 * dsl_destroy_head() to fail with EBUSY thus
3245 * leaving the dataset in an inconsistent state.
3246 */
3247 pthread_rwlock_wrlock(&ztest_name_lock);
3248 error = spa_vdev_remove(spa, guid, B_FALSE);
3249 pthread_rwlock_unlock(&ztest_name_lock);
3250
3251 switch (error) {
3252 case 0:
3253 case EEXIST: /* Generic zil_reset() error */
3254 case EBUSY: /* Replay required */
3255 case EACCES: /* Crypto key not loaded */
3256 case ZFS_ERR_CHECKPOINT_EXISTS:
3257 case ZFS_ERR_DISCARDING_CHECKPOINT:
3258 break;
3259 default:
3260 fatal(B_FALSE, "spa_vdev_remove() = %d", error);
3261 }
3262 } else {
3263 spa_config_exit(spa, SCL_VDEV, FTAG);
3264
3265 /*
3266 * Make 1/4 of the devices be log devices
3267 */
3268 nvroot = make_vdev_root(NULL, NULL, NULL,
3269 ztest_opts.zo_vdev_size, 0, (ztest_random(4) == 0) ?
3270 "log" : NULL, ztest_opts.zo_raid_children, zs->zs_mirrors,
3271 1);
3272
3273 error = spa_vdev_add(spa, nvroot);
3274 fnvlist_free(nvroot);
3275
3276 switch (error) {
3277 case 0:
3278 break;
3279 case ENOSPC:
3280 ztest_record_enospc("spa_vdev_add");
3281 break;
3282 default:
3283 fatal(B_FALSE, "spa_vdev_add() = %d", error);
3284 }
3285 }
3286
3287 mutex_exit(&ztest_vdev_lock);
3288 }
3289
3290 void
3291 ztest_vdev_class_add(ztest_ds_t *zd, uint64_t id)
3292 {
3293 (void) zd, (void) id;
3294 ztest_shared_t *zs = ztest_shared;
3295 spa_t *spa = ztest_spa;
3296 uint64_t leaves;
3297 nvlist_t *nvroot;
3298 const char *class = (ztest_random(2) == 0) ?
3299 VDEV_ALLOC_BIAS_SPECIAL : VDEV_ALLOC_BIAS_DEDUP;
3300 int error;
3301
3302 /*
3303 * By default add a special vdev 50% of the time
3304 */
3305 if ((ztest_opts.zo_special_vdevs == ZTEST_VDEV_CLASS_OFF) ||
3306 (ztest_opts.zo_special_vdevs == ZTEST_VDEV_CLASS_RND &&
3307 ztest_random(2) == 0)) {
3308 return;
3309 }
3310
3311 mutex_enter(&ztest_vdev_lock);
3312
3313 /* Only test with mirrors */
3314 if (zs->zs_mirrors < 2) {
3315 mutex_exit(&ztest_vdev_lock);
3316 return;
3317 }
3318
3319 /* requires feature@allocation_classes */
3320 if (!spa_feature_is_enabled(spa, SPA_FEATURE_ALLOCATION_CLASSES)) {
3321 mutex_exit(&ztest_vdev_lock);
3322 return;
3323 }
3324
3325 leaves = MAX(zs->zs_mirrors + zs->zs_splits, 1) *
3326 ztest_opts.zo_raid_children;
3327
3328 spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);
3329 ztest_shared->zs_vdev_next_leaf = spa_num_top_vdevs(spa) * leaves;
3330 spa_config_exit(spa, SCL_VDEV, FTAG);
3331
3332 nvroot = make_vdev_root(NULL, NULL, NULL, ztest_opts.zo_vdev_size, 0,
3333 class, ztest_opts.zo_raid_children, zs->zs_mirrors, 1);
3334
3335 error = spa_vdev_add(spa, nvroot);
3336 fnvlist_free(nvroot);
3337
3338 if (error == ENOSPC)
3339 ztest_record_enospc("spa_vdev_add");
3340 else if (error != 0)
3341 fatal(B_FALSE, "spa_vdev_add() = %d", error);
3342
3343 /*
3344 * 50% of the time allow small blocks in the special class
3345 */
3346 if (error == 0 &&
3347 spa_special_class(spa)->mc_groups == 1 && ztest_random(2) == 0) {
3348 if (ztest_opts.zo_verbose >= 3)
3349 (void) printf("Enabling special VDEV small blocks\n");
3350 (void) ztest_dsl_prop_set_uint64(zd->zd_name,
3351 ZFS_PROP_SPECIAL_SMALL_BLOCKS, 32768, B_FALSE);
3352 }
3353
3354 mutex_exit(&ztest_vdev_lock);
3355
3356 if (ztest_opts.zo_verbose >= 3) {
3357 metaslab_class_t *mc;
3358
3359 if (strcmp(class, VDEV_ALLOC_BIAS_SPECIAL) == 0)
3360 mc = spa_special_class(spa);
3361 else
3362 mc = spa_dedup_class(spa);
3363 (void) printf("Added a %s mirrored vdev (of %d)\n",
3364 class, (int)mc->mc_groups);
3365 }
3366 }
3367
3368 /*
3369 * Verify that adding/removing aux devices (l2arc, hot spare) works as expected.
3370 */
3371 void
3372 ztest_vdev_aux_add_remove(ztest_ds_t *zd, uint64_t id)
3373 {
3374 (void) zd, (void) id;
3375 ztest_shared_t *zs = ztest_shared;
3376 spa_t *spa = ztest_spa;
3377 vdev_t *rvd = spa->spa_root_vdev;
3378 spa_aux_vdev_t *sav;
3379 const char *aux;
3380 char *path;
3381 uint64_t guid = 0;
3382 int error, ignore_err = 0;
3383
3384 if (ztest_opts.zo_mmp_test)
3385 return;
3386
3387 path = umem_alloc(MAXPATHLEN, UMEM_NOFAIL);
3388
3389 if (ztest_random(2) == 0) {
3390 sav = &spa->spa_spares;
3391 aux = ZPOOL_CONFIG_SPARES;
3392 } else {
3393 sav = &spa->spa_l2cache;
3394 aux = ZPOOL_CONFIG_L2CACHE;
3395 }
3396
3397 mutex_enter(&ztest_vdev_lock);
3398
3399 spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);
3400
3401 if (sav->sav_count != 0 && ztest_random(4) == 0) {
3402 /*
3403 * Pick a random device to remove.
3404 */
3405 vdev_t *svd = sav->sav_vdevs[ztest_random(sav->sav_count)];
3406
3407 /* dRAID spares cannot be removed; try anyways to see ENOTSUP */
3408 if (strstr(svd->vdev_path, VDEV_TYPE_DRAID) != NULL)
3409 ignore_err = ENOTSUP;
3410
3411 guid = svd->vdev_guid;
3412 } else {
3413 /*
3414 * Find an unused device we can add.
3415 */
3416 zs->zs_vdev_aux = 0;
3417 for (;;) {
3418 int c;
3419 (void) snprintf(path, MAXPATHLEN, ztest_aux_template,
3420 ztest_opts.zo_dir, ztest_opts.zo_pool, aux,
3421 zs->zs_vdev_aux);
3422 for (c = 0; c < sav->sav_count; c++)
3423 if (strcmp(sav->sav_vdevs[c]->vdev_path,
3424 path) == 0)
3425 break;
3426 if (c == sav->sav_count &&
3427 vdev_lookup_by_path(rvd, path) == NULL)
3428 break;
3429 zs->zs_vdev_aux++;
3430 }
3431 }
3432
3433 spa_config_exit(spa, SCL_VDEV, FTAG);
3434
3435 if (guid == 0) {
3436 /*
3437 * Add a new device.
3438 */
3439 nvlist_t *nvroot = make_vdev_root(NULL, aux, NULL,
3440 (ztest_opts.zo_vdev_size * 5) / 4, 0, NULL, 0, 0, 1);
3441 error = spa_vdev_add(spa, nvroot);
3442
3443 switch (error) {
3444 case 0:
3445 break;
3446 default:
3447 fatal(B_FALSE, "spa_vdev_add(%p) = %d", nvroot, error);
3448 }
3449 fnvlist_free(nvroot);
3450 } else {
3451 /*
3452 * Remove an existing device. Sometimes, dirty its
3453 * vdev state first to make sure we handle removal
3454 * of devices that have pending state changes.
3455 */
3456 if (ztest_random(2) == 0)
3457 (void) vdev_online(spa, guid, 0, NULL);
3458
3459 error = spa_vdev_remove(spa, guid, B_FALSE);
3460
3461 switch (error) {
3462 case 0:
3463 case EBUSY:
3464 case ZFS_ERR_CHECKPOINT_EXISTS:
3465 case ZFS_ERR_DISCARDING_CHECKPOINT:
3466 break;
3467 default:
3468 if (error != ignore_err)
3469 fatal(B_FALSE,
3470 "spa_vdev_remove(%"PRIu64") = %d",
3471 guid, error);
3472 }
3473 }
3474
3475 mutex_exit(&ztest_vdev_lock);
3476
3477 umem_free(path, MAXPATHLEN);
3478 }
3479
3480 /*
3481 * split a pool if it has mirror tlvdevs
3482 */
3483 void
3484 ztest_split_pool(ztest_ds_t *zd, uint64_t id)
3485 {
3486 (void) zd, (void) id;
3487 ztest_shared_t *zs = ztest_shared;
3488 spa_t *spa = ztest_spa;
3489 vdev_t *rvd = spa->spa_root_vdev;
3490 nvlist_t *tree, **child, *config, *split, **schild;
3491 uint_t c, children, schildren = 0, lastlogid = 0;
3492 int error = 0;
3493
3494 if (ztest_opts.zo_mmp_test)
3495 return;
3496
3497 mutex_enter(&ztest_vdev_lock);
3498
3499 /* ensure we have a usable config; mirrors of raidz aren't supported */
3500 if (zs->zs_mirrors < 3 || ztest_opts.zo_raid_children > 1) {
3501 mutex_exit(&ztest_vdev_lock);
3502 return;
3503 }
3504
3505 /* clean up the old pool, if any */
3506 (void) spa_destroy("splitp");
3507
3508 spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);
3509
3510 /* generate a config from the existing config */
3511 mutex_enter(&spa->spa_props_lock);
3512 tree = fnvlist_lookup_nvlist(spa->spa_config, ZPOOL_CONFIG_VDEV_TREE);
3513 mutex_exit(&spa->spa_props_lock);
3514
3515 VERIFY0(nvlist_lookup_nvlist_array(tree, ZPOOL_CONFIG_CHILDREN,
3516 &child, &children));
3517
3518 schild = umem_alloc(rvd->vdev_children * sizeof (nvlist_t *),
3519 UMEM_NOFAIL);
3520 for (c = 0; c < children; c++) {
3521 vdev_t *tvd = rvd->vdev_child[c];
3522 nvlist_t **mchild;
3523 uint_t mchildren;
3524
3525 if (tvd->vdev_islog || tvd->vdev_ops == &vdev_hole_ops) {
3526 schild[schildren] = fnvlist_alloc();
3527 fnvlist_add_string(schild[schildren],
3528 ZPOOL_CONFIG_TYPE, VDEV_TYPE_HOLE);
3529 fnvlist_add_uint64(schild[schildren],
3530 ZPOOL_CONFIG_IS_HOLE, 1);
3531 if (lastlogid == 0)
3532 lastlogid = schildren;
3533 ++schildren;
3534 continue;
3535 }
3536 lastlogid = 0;
3537 VERIFY0(nvlist_lookup_nvlist_array(child[c],
3538 ZPOOL_CONFIG_CHILDREN, &mchild, &mchildren));
3539 schild[schildren++] = fnvlist_dup(mchild[0]);
3540 }
3541
3542 /* OK, create a config that can be used to split */
3543 split = fnvlist_alloc();
3544 fnvlist_add_string(split, ZPOOL_CONFIG_TYPE, VDEV_TYPE_ROOT);
3545 fnvlist_add_nvlist_array(split, ZPOOL_CONFIG_CHILDREN,
3546 (const nvlist_t **)schild, lastlogid != 0 ? lastlogid : schildren);
3547
3548 config = fnvlist_alloc();
3549 fnvlist_add_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, split);
3550
3551 for (c = 0; c < schildren; c++)
3552 fnvlist_free(schild[c]);
3553 umem_free(schild, rvd->vdev_children * sizeof (nvlist_t *));
3554 fnvlist_free(split);
3555
3556 spa_config_exit(spa, SCL_VDEV, FTAG);
3557
3558 (void) pthread_rwlock_wrlock(&ztest_name_lock);
3559 error = spa_vdev_split_mirror(spa, "splitp", config, NULL, B_FALSE);
3560 (void) pthread_rwlock_unlock(&ztest_name_lock);
3561
3562 fnvlist_free(config);
3563
3564 if (error == 0) {
3565 (void) printf("successful split - results:\n");
3566 mutex_enter(&spa_namespace_lock);
3567 show_pool_stats(spa);
3568 show_pool_stats(spa_lookup("splitp"));
3569 mutex_exit(&spa_namespace_lock);
3570 ++zs->zs_splits;
3571 --zs->zs_mirrors;
3572 }
3573 mutex_exit(&ztest_vdev_lock);
3574 }
3575
3576 /*
3577 * Verify that we can attach and detach devices.
3578 */
3579 void
3580 ztest_vdev_attach_detach(ztest_ds_t *zd, uint64_t id)
3581 {
3582 (void) zd, (void) id;
3583 ztest_shared_t *zs = ztest_shared;
3584 spa_t *spa = ztest_spa;
3585 spa_aux_vdev_t *sav = &spa->spa_spares;
3586 vdev_t *rvd = spa->spa_root_vdev;
3587 vdev_t *oldvd, *newvd, *pvd;
3588 nvlist_t *root;
3589 uint64_t leaves;
3590 uint64_t leaf, top;
3591 uint64_t ashift = ztest_get_ashift();
3592 uint64_t oldguid, pguid;
3593 uint64_t oldsize, newsize;
3594 char *oldpath, *newpath;
3595 int replacing;
3596 int oldvd_has_siblings = B_FALSE;
3597 int newvd_is_spare = B_FALSE;
3598 int newvd_is_dspare = B_FALSE;
3599 int oldvd_is_log;
3600 int oldvd_is_special;
3601 int error, expected_error;
3602
3603 if (ztest_opts.zo_mmp_test)
3604 return;
3605
3606 oldpath = umem_alloc(MAXPATHLEN, UMEM_NOFAIL);
3607 newpath = umem_alloc(MAXPATHLEN, UMEM_NOFAIL);
3608
3609 mutex_enter(&ztest_vdev_lock);
3610 leaves = MAX(zs->zs_mirrors, 1) * ztest_opts.zo_raid_children;
3611
3612 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER);
3613
3614 /*
3615 * If a vdev is in the process of being removed, its removal may
3616 * finish while we are in progress, leading to an unexpected error
3617 * value. Don't bother trying to attach while we are in the middle
3618 * of removal.
3619 */
3620 if (ztest_device_removal_active) {
3621 spa_config_exit(spa, SCL_ALL, FTAG);
3622 goto out;
3623 }
3624
3625 /*
3626 * Decide whether to do an attach or a replace.
3627 */
3628 replacing = ztest_random(2);
3629
3630 /*
3631 * Pick a random top-level vdev.
3632 */
3633 top = ztest_random_vdev_top(spa, B_TRUE);
3634
3635 /*
3636 * Pick a random leaf within it.
3637 */
3638 leaf = ztest_random(leaves);
3639
3640 /*
3641 * Locate this vdev.
3642 */
3643 oldvd = rvd->vdev_child[top];
3644
3645 /* pick a child from the mirror */
3646 if (zs->zs_mirrors >= 1) {
3647 ASSERT3P(oldvd->vdev_ops, ==, &vdev_mirror_ops);
3648 ASSERT3U(oldvd->vdev_children, >=, zs->zs_mirrors);
3649 oldvd = oldvd->vdev_child[leaf / ztest_opts.zo_raid_children];
3650 }
3651
3652 /* pick a child out of the raidz group */
3653 if (ztest_opts.zo_raid_children > 1) {
3654 if (strcmp(oldvd->vdev_ops->vdev_op_type, "raidz") == 0)
3655 ASSERT3P(oldvd->vdev_ops, ==, &vdev_raidz_ops);
3656 else
3657 ASSERT3P(oldvd->vdev_ops, ==, &vdev_draid_ops);
3658 ASSERT3U(oldvd->vdev_children, ==, ztest_opts.zo_raid_children);
3659 oldvd = oldvd->vdev_child[leaf % ztest_opts.zo_raid_children];
3660 }
3661
3662 /*
3663 * If we're already doing an attach or replace, oldvd may be a
3664 * mirror vdev -- in which case, pick a random child.
3665 */
3666 while (oldvd->vdev_children != 0) {
3667 oldvd_has_siblings = B_TRUE;
3668 ASSERT3U(oldvd->vdev_children, >=, 2);
3669 oldvd = oldvd->vdev_child[ztest_random(oldvd->vdev_children)];
3670 }
3671
3672 oldguid = oldvd->vdev_guid;
3673 oldsize = vdev_get_min_asize(oldvd);
3674 oldvd_is_log = oldvd->vdev_top->vdev_islog;
3675 oldvd_is_special =
3676 oldvd->vdev_top->vdev_alloc_bias == VDEV_BIAS_SPECIAL ||
3677 oldvd->vdev_top->vdev_alloc_bias == VDEV_BIAS_DEDUP;
3678 (void) strlcpy(oldpath, oldvd->vdev_path, MAXPATHLEN);
3679 pvd = oldvd->vdev_parent;
3680 pguid = pvd->vdev_guid;
3681
3682 /*
3683 * If oldvd has siblings, then half of the time, detach it. Prior
3684 * to the detach the pool is scrubbed in order to prevent creating
3685 * unrepairable blocks as a result of the data corruption injection.
3686 */
3687 if (oldvd_has_siblings && ztest_random(2) == 0) {
3688 spa_config_exit(spa, SCL_ALL, FTAG);
3689
3690 error = ztest_scrub_impl(spa);
3691 if (error)
3692 goto out;
3693
3694 error = spa_vdev_detach(spa, oldguid, pguid, B_FALSE);
3695 if (error != 0 && error != ENODEV && error != EBUSY &&
3696 error != ENOTSUP && error != ZFS_ERR_CHECKPOINT_EXISTS &&
3697 error != ZFS_ERR_DISCARDING_CHECKPOINT)
3698 fatal(B_FALSE, "detach (%s) returned %d",
3699 oldpath, error);
3700 goto out;
3701 }
3702
3703 /*
3704 * For the new vdev, choose with equal probability between the two
3705 * standard paths (ending in either 'a' or 'b') or a random hot spare.
3706 */
3707 if (sav->sav_count != 0 && ztest_random(3) == 0) {
3708 newvd = sav->sav_vdevs[ztest_random(sav->sav_count)];
3709 newvd_is_spare = B_TRUE;
3710
3711 if (newvd->vdev_ops == &vdev_draid_spare_ops)
3712 newvd_is_dspare = B_TRUE;
3713
3714 (void) strlcpy(newpath, newvd->vdev_path, MAXPATHLEN);
3715 } else {
3716 (void) snprintf(newpath, MAXPATHLEN, ztest_dev_template,
3717 ztest_opts.zo_dir, ztest_opts.zo_pool,
3718 top * leaves + leaf);
3719 if (ztest_random(2) == 0)
3720 newpath[strlen(newpath) - 1] = 'b';
3721 newvd = vdev_lookup_by_path(rvd, newpath);
3722 }
3723
3724 if (newvd) {
3725 /*
3726 * Reopen to ensure the vdev's asize field isn't stale.
3727 */
3728 vdev_reopen(newvd);
3729 newsize = vdev_get_min_asize(newvd);
3730 } else {
3731 /*
3732 * Make newsize a little bigger or smaller than oldsize.
3733 * If it's smaller, the attach should fail.
3734 * If it's larger, and we're doing a replace,
3735 * we should get dynamic LUN growth when we're done.
3736 */
3737 newsize = 10 * oldsize / (9 + ztest_random(3));
3738 }
3739
3740 /*
3741 * If pvd is not a mirror or root, the attach should fail with ENOTSUP,
3742 * unless it's a replace; in that case any non-replacing parent is OK.
3743 *
3744 * If newvd is already part of the pool, it should fail with EBUSY.
3745 *
3746 * If newvd is too small, it should fail with EOVERFLOW.
3747 *
3748 * If newvd is a distributed spare and it's being attached to a
3749 * dRAID which is not its parent it should fail with EINVAL.
3750 */
3751 if (pvd->vdev_ops != &vdev_mirror_ops &&
3752 pvd->vdev_ops != &vdev_root_ops && (!replacing ||
3753 pvd->vdev_ops == &vdev_replacing_ops ||
3754 pvd->vdev_ops == &vdev_spare_ops))
3755 expected_error = ENOTSUP;
3756 else if (newvd_is_spare &&
3757 (!replacing || oldvd_is_log || oldvd_is_special))
3758 expected_error = ENOTSUP;
3759 else if (newvd == oldvd)
3760 expected_error = replacing ? 0 : EBUSY;
3761 else if (vdev_lookup_by_path(rvd, newpath) != NULL)
3762 expected_error = EBUSY;
3763 else if (!newvd_is_dspare && newsize < oldsize)
3764 expected_error = EOVERFLOW;
3765 else if (ashift > oldvd->vdev_top->vdev_ashift)
3766 expected_error = EDOM;
3767 else if (newvd_is_dspare && pvd != vdev_draid_spare_get_parent(newvd))
3768 expected_error = ENOTSUP;
3769 else
3770 expected_error = 0;
3771
3772 spa_config_exit(spa, SCL_ALL, FTAG);
3773
3774 /*
3775 * Build the nvlist describing newpath.
3776 */
3777 root = make_vdev_root(newpath, NULL, NULL, newvd == NULL ? newsize : 0,
3778 ashift, NULL, 0, 0, 1);
3779
3780 /*
3781 * When supported select either a healing or sequential resilver.
3782 */
3783 boolean_t rebuilding = B_FALSE;
3784 if (pvd->vdev_ops == &vdev_mirror_ops ||
3785 pvd->vdev_ops == &vdev_root_ops) {
3786 rebuilding = !!ztest_random(2);
3787 }
3788
3789 error = spa_vdev_attach(spa, oldguid, root, replacing, rebuilding);
3790
3791 fnvlist_free(root);
3792
3793 /*
3794 * If our parent was the replacing vdev, but the replace completed,
3795 * then instead of failing with ENOTSUP we may either succeed,
3796 * fail with ENODEV, or fail with EOVERFLOW.
3797 */
3798 if (expected_error == ENOTSUP &&
3799 (error == 0 || error == ENODEV || error == EOVERFLOW))
3800 expected_error = error;
3801
3802 /*
3803 * If someone grew the LUN, the replacement may be too small.
3804 */
3805 if (error == EOVERFLOW || error == EBUSY)
3806 expected_error = error;
3807
3808 if (error == ZFS_ERR_CHECKPOINT_EXISTS ||
3809 error == ZFS_ERR_DISCARDING_CHECKPOINT ||
3810 error == ZFS_ERR_RESILVER_IN_PROGRESS ||
3811 error == ZFS_ERR_REBUILD_IN_PROGRESS)
3812 expected_error = error;
3813
3814 if (error != expected_error && expected_error != EBUSY) {
3815 fatal(B_FALSE, "attach (%s %"PRIu64", %s %"PRIu64", %d) "
3816 "returned %d, expected %d",
3817 oldpath, oldsize, newpath,
3818 newsize, replacing, error, expected_error);
3819 }
3820 out:
3821 mutex_exit(&ztest_vdev_lock);
3822
3823 umem_free(oldpath, MAXPATHLEN);
3824 umem_free(newpath, MAXPATHLEN);
3825 }
3826
3827 void
3828 ztest_device_removal(ztest_ds_t *zd, uint64_t id)
3829 {
3830 (void) zd, (void) id;
3831 spa_t *spa = ztest_spa;
3832 vdev_t *vd;
3833 uint64_t guid;
3834 int error;
3835
3836 mutex_enter(&ztest_vdev_lock);
3837
3838 if (ztest_device_removal_active) {
3839 mutex_exit(&ztest_vdev_lock);
3840 return;
3841 }
3842
3843 /*
3844 * Remove a random top-level vdev and wait for removal to finish.
3845 */
3846 spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);
3847 vd = vdev_lookup_top(spa, ztest_random_vdev_top(spa, B_FALSE));
3848 guid = vd->vdev_guid;
3849 spa_config_exit(spa, SCL_VDEV, FTAG);
3850
3851 error = spa_vdev_remove(spa, guid, B_FALSE);
3852 if (error == 0) {
3853 ztest_device_removal_active = B_TRUE;
3854 mutex_exit(&ztest_vdev_lock);
3855
3856 /*
3857 * spa->spa_vdev_removal is created in a sync task that
3858 * is initiated via dsl_sync_task_nowait(). Since the
3859 * task may not run before spa_vdev_remove() returns, we
3860 * must wait at least 1 txg to ensure that the removal
3861 * struct has been created.
3862 */
3863 txg_wait_synced(spa_get_dsl(spa), 0);
3864
3865 while (spa->spa_removing_phys.sr_state == DSS_SCANNING)
3866 txg_wait_synced(spa_get_dsl(spa), 0);
3867 } else {
3868 mutex_exit(&ztest_vdev_lock);
3869 return;
3870 }
3871
3872 /*
3873 * The pool needs to be scrubbed after completing device removal.
3874 * Failure to do so may result in checksum errors due to the
3875 * strategy employed by ztest_fault_inject() when selecting which
3876 * offset are redundant and can be damaged.
3877 */
3878 error = spa_scan(spa, POOL_SCAN_SCRUB);
3879 if (error == 0) {
3880 while (dsl_scan_scrubbing(spa_get_dsl(spa)))
3881 txg_wait_synced(spa_get_dsl(spa), 0);
3882 }
3883
3884 mutex_enter(&ztest_vdev_lock);
3885 ztest_device_removal_active = B_FALSE;
3886 mutex_exit(&ztest_vdev_lock);
3887 }
3888
3889 /*
3890 * Callback function which expands the physical size of the vdev.
3891 */
3892 static vdev_t *
3893 grow_vdev(vdev_t *vd, void *arg)
3894 {
3895 spa_t *spa __maybe_unused = vd->vdev_spa;
3896 size_t *newsize = arg;
3897 size_t fsize;
3898 int fd;
3899
3900 ASSERT3S(spa_config_held(spa, SCL_STATE, RW_READER), ==, SCL_STATE);
3901 ASSERT(vd->vdev_ops->vdev_op_leaf);
3902
3903 if ((fd = open(vd->vdev_path, O_RDWR)) == -1)
3904 return (vd);
3905
3906 fsize = lseek(fd, 0, SEEK_END);
3907 VERIFY0(ftruncate(fd, *newsize));
3908
3909 if (ztest_opts.zo_verbose >= 6) {
3910 (void) printf("%s grew from %lu to %lu bytes\n",
3911 vd->vdev_path, (ulong_t)fsize, (ulong_t)*newsize);
3912 }
3913 (void) close(fd);
3914 return (NULL);
3915 }
3916
3917 /*
3918 * Callback function which expands a given vdev by calling vdev_online().
3919 */
3920 static vdev_t *
3921 online_vdev(vdev_t *vd, void *arg)
3922 {
3923 (void) arg;
3924 spa_t *spa = vd->vdev_spa;
3925 vdev_t *tvd = vd->vdev_top;
3926 uint64_t guid = vd->vdev_guid;
3927 uint64_t generation = spa->spa_config_generation + 1;
3928 vdev_state_t newstate = VDEV_STATE_UNKNOWN;
3929 int error;
3930
3931 ASSERT3S(spa_config_held(spa, SCL_STATE, RW_READER), ==, SCL_STATE);
3932 ASSERT(vd->vdev_ops->vdev_op_leaf);
3933
3934 /* Calling vdev_online will initialize the new metaslabs */
3935 spa_config_exit(spa, SCL_STATE, spa);
3936 error = vdev_online(spa, guid, ZFS_ONLINE_EXPAND, &newstate);
3937 spa_config_enter(spa, SCL_STATE, spa, RW_READER);
3938
3939 /*
3940 * If vdev_online returned an error or the underlying vdev_open
3941 * failed then we abort the expand. The only way to know that
3942 * vdev_open fails is by checking the returned newstate.
3943 */
3944 if (error || newstate != VDEV_STATE_HEALTHY) {
3945 if (ztest_opts.zo_verbose >= 5) {
3946 (void) printf("Unable to expand vdev, state %u, "
3947 "error %d\n", newstate, error);
3948 }
3949 return (vd);
3950 }
3951 ASSERT3U(newstate, ==, VDEV_STATE_HEALTHY);
3952
3953 /*
3954 * Since we dropped the lock we need to ensure that we're
3955 * still talking to the original vdev. It's possible this
3956 * vdev may have been detached/replaced while we were
3957 * trying to online it.
3958 */
3959 if (generation != spa->spa_config_generation) {
3960 if (ztest_opts.zo_verbose >= 5) {
3961 (void) printf("vdev configuration has changed, "
3962 "guid %"PRIu64", state %"PRIu64", "
3963 "expected gen %"PRIu64", got gen %"PRIu64"\n",
3964 guid,
3965 tvd->vdev_state,
3966 generation,
3967 spa->spa_config_generation);
3968 }
3969 return (vd);
3970 }
3971 return (NULL);
3972 }
3973
3974 /*
3975 * Traverse the vdev tree calling the supplied function.
3976 * We continue to walk the tree until we either have walked all
3977 * children or we receive a non-NULL return from the callback.
3978 * If a NULL callback is passed, then we just return back the first
3979 * leaf vdev we encounter.
3980 */
3981 static vdev_t *
3982 vdev_walk_tree(vdev_t *vd, vdev_t *(*func)(vdev_t *, void *), void *arg)
3983 {
3984 uint_t c;
3985
3986 if (vd->vdev_ops->vdev_op_leaf) {
3987 if (func == NULL)
3988 return (vd);
3989 else
3990 return (func(vd, arg));
3991 }
3992
3993 for (c = 0; c < vd->vdev_children; c++) {
3994 vdev_t *cvd = vd->vdev_child[c];
3995 if ((cvd = vdev_walk_tree(cvd, func, arg)) != NULL)
3996 return (cvd);
3997 }
3998 return (NULL);
3999 }
4000
4001 /*
4002 * Verify that dynamic LUN growth works as expected.
4003 */
4004 void
4005 ztest_vdev_LUN_growth(ztest_ds_t *zd, uint64_t id)
4006 {
4007 (void) zd, (void) id;
4008 spa_t *spa = ztest_spa;
4009 vdev_t *vd, *tvd;
4010 metaslab_class_t *mc;
4011 metaslab_group_t *mg;
4012 size_t psize, newsize;
4013 uint64_t top;
4014 uint64_t old_class_space, new_class_space, old_ms_count, new_ms_count;
4015
4016 mutex_enter(&ztest_checkpoint_lock);
4017 mutex_enter(&ztest_vdev_lock);
4018 spa_config_enter(spa, SCL_STATE, spa, RW_READER);
4019
4020 /*
4021 * If there is a vdev removal in progress, it could complete while
4022 * we are running, in which case we would not be able to verify
4023 * that the metaslab_class space increased (because it decreases
4024 * when the device removal completes).
4025 */
4026 if (ztest_device_removal_active) {
4027 spa_config_exit(spa, SCL_STATE, spa);
4028 mutex_exit(&ztest_vdev_lock);
4029 mutex_exit(&ztest_checkpoint_lock);
4030 return;
4031 }
4032
4033 top = ztest_random_vdev_top(spa, B_TRUE);
4034
4035 tvd = spa->spa_root_vdev->vdev_child[top];
4036 mg = tvd->vdev_mg;
4037 mc = mg->mg_class;
4038 old_ms_count = tvd->vdev_ms_count;
4039 old_class_space = metaslab_class_get_space(mc);
4040
4041 /*
4042 * Determine the size of the first leaf vdev associated with
4043 * our top-level device.
4044 */
4045 vd = vdev_walk_tree(tvd, NULL, NULL);
4046 ASSERT3P(vd, !=, NULL);
4047 ASSERT(vd->vdev_ops->vdev_op_leaf);
4048
4049 psize = vd->vdev_psize;
4050
4051 /*
4052 * We only try to expand the vdev if it's healthy, less than 4x its
4053 * original size, and it has a valid psize.
4054 */
4055 if (tvd->vdev_state != VDEV_STATE_HEALTHY ||
4056 psize == 0 || psize >= 4 * ztest_opts.zo_vdev_size) {
4057 spa_config_exit(spa, SCL_STATE, spa);
4058 mutex_exit(&ztest_vdev_lock);
4059 mutex_exit(&ztest_checkpoint_lock);
4060 return;
4061 }
4062 ASSERT3U(psize, >, 0);
4063 newsize = psize + MAX(psize / 8, SPA_MAXBLOCKSIZE);
4064 ASSERT3U(newsize, >, psize);
4065
4066 if (ztest_opts.zo_verbose >= 6) {
4067 (void) printf("Expanding LUN %s from %lu to %lu\n",
4068 vd->vdev_path, (ulong_t)psize, (ulong_t)newsize);
4069 }
4070
4071 /*
4072 * Growing the vdev is a two step process:
4073 * 1). expand the physical size (i.e. relabel)
4074 * 2). online the vdev to create the new metaslabs
4075 */
4076 if (vdev_walk_tree(tvd, grow_vdev, &newsize) != NULL ||
4077 vdev_walk_tree(tvd, online_vdev, NULL) != NULL ||
4078 tvd->vdev_state != VDEV_STATE_HEALTHY) {
4079 if (ztest_opts.zo_verbose >= 5) {
4080 (void) printf("Could not expand LUN because "
4081 "the vdev configuration changed.\n");
4082 }
4083 spa_config_exit(spa, SCL_STATE, spa);
4084 mutex_exit(&ztest_vdev_lock);
4085 mutex_exit(&ztest_checkpoint_lock);
4086 return;
4087 }
4088
4089 spa_config_exit(spa, SCL_STATE, spa);
4090
4091 /*
4092 * Expanding the LUN will update the config asynchronously,
4093 * thus we must wait for the async thread to complete any
4094 * pending tasks before proceeding.
4095 */
4096 for (;;) {
4097 boolean_t done;
4098 mutex_enter(&spa->spa_async_lock);
4099 done = (spa->spa_async_thread == NULL && !spa->spa_async_tasks);
4100 mutex_exit(&spa->spa_async_lock);
4101 if (done)
4102 break;
4103 txg_wait_synced(spa_get_dsl(spa), 0);
4104 (void) poll(NULL, 0, 100);
4105 }
4106
4107 spa_config_enter(spa, SCL_STATE, spa, RW_READER);
4108
4109 tvd = spa->spa_root_vdev->vdev_child[top];
4110 new_ms_count = tvd->vdev_ms_count;
4111 new_class_space = metaslab_class_get_space(mc);
4112
4113 if (tvd->vdev_mg != mg || mg->mg_class != mc) {
4114 if (ztest_opts.zo_verbose >= 5) {
4115 (void) printf("Could not verify LUN expansion due to "
4116 "intervening vdev offline or remove.\n");
4117 }
4118 spa_config_exit(spa, SCL_STATE, spa);
4119 mutex_exit(&ztest_vdev_lock);
4120 mutex_exit(&ztest_checkpoint_lock);
4121 return;
4122 }
4123
4124 /*
4125 * Make sure we were able to grow the vdev.
4126 */
4127 if (new_ms_count <= old_ms_count) {
4128 fatal(B_FALSE,
4129 "LUN expansion failed: ms_count %"PRIu64" < %"PRIu64"\n",
4130 old_ms_count, new_ms_count);
4131 }
4132
4133 /*
4134 * Make sure we were able to grow the pool.
4135 */
4136 if (new_class_space <= old_class_space) {
4137 fatal(B_FALSE,
4138 "LUN expansion failed: class_space %"PRIu64" < %"PRIu64"\n",
4139 old_class_space, new_class_space);
4140 }
4141
4142 if (ztest_opts.zo_verbose >= 5) {
4143 char oldnumbuf[NN_NUMBUF_SZ], newnumbuf[NN_NUMBUF_SZ];
4144
4145 nicenum(old_class_space, oldnumbuf, sizeof (oldnumbuf));
4146 nicenum(new_class_space, newnumbuf, sizeof (newnumbuf));
4147 (void) printf("%s grew from %s to %s\n",
4148 spa->spa_name, oldnumbuf, newnumbuf);
4149 }
4150
4151 spa_config_exit(spa, SCL_STATE, spa);
4152 mutex_exit(&ztest_vdev_lock);
4153 mutex_exit(&ztest_checkpoint_lock);
4154 }
4155
4156 /*
4157 * Verify that dmu_objset_{create,destroy,open,close} work as expected.
4158 */
4159 static void
4160 ztest_objset_create_cb(objset_t *os, void *arg, cred_t *cr, dmu_tx_t *tx)
4161 {
4162 (void) arg, (void) cr;
4163
4164 /*
4165 * Create the objects common to all ztest datasets.
4166 */
4167 VERIFY0(zap_create_claim(os, ZTEST_DIROBJ,
4168 DMU_OT_ZAP_OTHER, DMU_OT_NONE, 0, tx));
4169 }
4170
4171 static int
4172 ztest_dataset_create(char *dsname)
4173 {
4174 int err;
4175 uint64_t rand;
4176 dsl_crypto_params_t *dcp = NULL;
4177
4178 /*
4179 * 50% of the time, we create encrypted datasets
4180 * using a random cipher suite and a hard-coded
4181 * wrapping key.
4182 */
4183 rand = ztest_random(2);
4184 if (rand != 0) {
4185 nvlist_t *crypto_args = fnvlist_alloc();
4186 nvlist_t *props = fnvlist_alloc();
4187
4188 /* slight bias towards the default cipher suite */
4189 rand = ztest_random(ZIO_CRYPT_FUNCTIONS);
4190 if (rand < ZIO_CRYPT_AES_128_CCM)
4191 rand = ZIO_CRYPT_ON;
4192
4193 fnvlist_add_uint64(props,
4194 zfs_prop_to_name(ZFS_PROP_ENCRYPTION), rand);
4195 fnvlist_add_uint8_array(crypto_args, "wkeydata",
4196 (uint8_t *)ztest_wkeydata, WRAPPING_KEY_LEN);
4197
4198 /*
4199 * These parameters aren't really used by the kernel. They
4200 * are simply stored so that userspace knows how to load
4201 * the wrapping key.
4202 */
4203 fnvlist_add_uint64(props,
4204 zfs_prop_to_name(ZFS_PROP_KEYFORMAT), ZFS_KEYFORMAT_RAW);
4205 fnvlist_add_string(props,
4206 zfs_prop_to_name(ZFS_PROP_KEYLOCATION), "prompt");
4207 fnvlist_add_uint64(props,
4208 zfs_prop_to_name(ZFS_PROP_PBKDF2_SALT), 0ULL);
4209 fnvlist_add_uint64(props,
4210 zfs_prop_to_name(ZFS_PROP_PBKDF2_ITERS), 0ULL);
4211
4212 VERIFY0(dsl_crypto_params_create_nvlist(DCP_CMD_NONE, props,
4213 crypto_args, &dcp));
4214
4215 /*
4216 * Cycle through all available encryption implementations
4217 * to verify interoperability.
4218 */
4219 VERIFY0(gcm_impl_set("cycle"));
4220 VERIFY0(aes_impl_set("cycle"));
4221
4222 fnvlist_free(crypto_args);
4223 fnvlist_free(props);
4224 }
4225
4226 err = dmu_objset_create(dsname, DMU_OST_OTHER, 0, dcp,
4227 ztest_objset_create_cb, NULL);
4228 dsl_crypto_params_free(dcp, !!err);
4229
4230 rand = ztest_random(100);
4231 if (err || rand < 80)
4232 return (err);
4233
4234 if (ztest_opts.zo_verbose >= 5)
4235 (void) printf("Setting dataset %s to sync always\n", dsname);
4236 return (ztest_dsl_prop_set_uint64(dsname, ZFS_PROP_SYNC,
4237 ZFS_SYNC_ALWAYS, B_FALSE));
4238 }
4239
4240 static int
4241 ztest_objset_destroy_cb(const char *name, void *arg)
4242 {
4243 (void) arg;
4244 objset_t *os;
4245 dmu_object_info_t doi;
4246 int error;
4247
4248 /*
4249 * Verify that the dataset contains a directory object.
4250 */
4251 VERIFY0(ztest_dmu_objset_own(name, DMU_OST_OTHER, B_TRUE,
4252 B_TRUE, FTAG, &os));
4253 error = dmu_object_info(os, ZTEST_DIROBJ, &doi);
4254 if (error != ENOENT) {
4255 /* We could have crashed in the middle of destroying it */
4256 ASSERT0(error);
4257 ASSERT3U(doi.doi_type, ==, DMU_OT_ZAP_OTHER);
4258 ASSERT3S(doi.doi_physical_blocks_512, >=, 0);
4259 }
4260 dmu_objset_disown(os, B_TRUE, FTAG);
4261
4262 /*
4263 * Destroy the dataset.
4264 */
4265 if (strchr(name, '@') != NULL) {
4266 error = dsl_destroy_snapshot(name, B_TRUE);
4267 if (error != ECHRNG) {
4268 /*
4269 * The program was executed, but encountered a runtime
4270 * error, such as insufficient slop, or a hold on the
4271 * dataset.
4272 */
4273 ASSERT0(error);
4274 }
4275 } else {
4276 error = dsl_destroy_head(name);
4277 if (error == ENOSPC) {
4278 /* There could be checkpoint or insufficient slop */
4279 ztest_record_enospc(FTAG);
4280 } else if (error != EBUSY) {
4281 /* There could be a hold on this dataset */
4282 ASSERT0(error);
4283 }
4284 }
4285 return (0);
4286 }
4287
4288 static boolean_t
4289 ztest_snapshot_create(char *osname, uint64_t id)
4290 {
4291 char snapname[ZFS_MAX_DATASET_NAME_LEN];
4292 int error;
4293
4294 (void) snprintf(snapname, sizeof (snapname), "%"PRIu64"", id);
4295
4296 error = dmu_objset_snapshot_one(osname, snapname);
4297 if (error == ENOSPC) {
4298 ztest_record_enospc(FTAG);
4299 return (B_FALSE);
4300 }
4301 if (error != 0 && error != EEXIST) {
4302 fatal(B_FALSE, "ztest_snapshot_create(%s@%s) = %d", osname,
4303 snapname, error);
4304 }
4305 return (B_TRUE);
4306 }
4307
4308 static boolean_t
4309 ztest_snapshot_destroy(char *osname, uint64_t id)
4310 {
4311 char snapname[ZFS_MAX_DATASET_NAME_LEN];
4312 int error;
4313
4314 (void) snprintf(snapname, sizeof (snapname), "%s@%"PRIu64"",
4315 osname, id);
4316
4317 error = dsl_destroy_snapshot(snapname, B_FALSE);
4318 if (error != 0 && error != ENOENT)
4319 fatal(B_FALSE, "ztest_snapshot_destroy(%s) = %d",
4320 snapname, error);
4321 return (B_TRUE);
4322 }
4323
4324 void
4325 ztest_dmu_objset_create_destroy(ztest_ds_t *zd, uint64_t id)
4326 {
4327 (void) zd;
4328 ztest_ds_t *zdtmp;
4329 int iters;
4330 int error;
4331 objset_t *os, *os2;
4332 char name[ZFS_MAX_DATASET_NAME_LEN];
4333 zilog_t *zilog;
4334 int i;
4335
4336 zdtmp = umem_alloc(sizeof (ztest_ds_t), UMEM_NOFAIL);
4337
4338 (void) pthread_rwlock_rdlock(&ztest_name_lock);
4339
4340 (void) snprintf(name, sizeof (name), "%s/temp_%"PRIu64"",
4341 ztest_opts.zo_pool, id);
4342
4343 /*
4344 * If this dataset exists from a previous run, process its replay log
4345 * half of the time. If we don't replay it, then dsl_destroy_head()
4346 * (invoked from ztest_objset_destroy_cb()) should just throw it away.
4347 */
4348 if (ztest_random(2) == 0 &&
4349 ztest_dmu_objset_own(name, DMU_OST_OTHER, B_FALSE,
4350 B_TRUE, FTAG, &os) == 0) {
4351 ztest_zd_init(zdtmp, NULL, os);
4352 zil_replay(os, zdtmp, ztest_replay_vector);
4353 ztest_zd_fini(zdtmp);
4354 dmu_objset_disown(os, B_TRUE, FTAG);
4355 }
4356
4357 /*
4358 * There may be an old instance of the dataset we're about to
4359 * create lying around from a previous run. If so, destroy it
4360 * and all of its snapshots.
4361 */
4362 (void) dmu_objset_find(name, ztest_objset_destroy_cb, NULL,
4363 DS_FIND_CHILDREN | DS_FIND_SNAPSHOTS);
4364
4365 /*
4366 * Verify that the destroyed dataset is no longer in the namespace.
4367 */
4368 VERIFY3U(ENOENT, ==, ztest_dmu_objset_own(name, DMU_OST_OTHER, B_TRUE,
4369 B_TRUE, FTAG, &os));
4370
4371 /*
4372 * Verify that we can create a new dataset.
4373 */
4374 error = ztest_dataset_create(name);
4375 if (error) {
4376 if (error == ENOSPC) {
4377 ztest_record_enospc(FTAG);
4378 goto out;
4379 }
4380 fatal(B_FALSE, "dmu_objset_create(%s) = %d", name, error);
4381 }
4382
4383 VERIFY0(ztest_dmu_objset_own(name, DMU_OST_OTHER, B_FALSE, B_TRUE,
4384 FTAG, &os));
4385
4386 ztest_zd_init(zdtmp, NULL, os);
4387
4388 /*
4389 * Open the intent log for it.
4390 */
4391 zilog = zil_open(os, ztest_get_data, NULL);
4392
4393 /*
4394 * Put some objects in there, do a little I/O to them,
4395 * and randomly take a couple of snapshots along the way.
4396 */
4397 iters = ztest_random(5);
4398 for (i = 0; i < iters; i++) {
4399 ztest_dmu_object_alloc_free(zdtmp, id);
4400 if (ztest_random(iters) == 0)
4401 (void) ztest_snapshot_create(name, i);
4402 }
4403
4404 /*
4405 * Verify that we cannot create an existing dataset.
4406 */
4407 VERIFY3U(EEXIST, ==,
4408 dmu_objset_create(name, DMU_OST_OTHER, 0, NULL, NULL, NULL));
4409
4410 /*
4411 * Verify that we can hold an objset that is also owned.
4412 */
4413 VERIFY0(dmu_objset_hold(name, FTAG, &os2));
4414 dmu_objset_rele(os2, FTAG);
4415
4416 /*
4417 * Verify that we cannot own an objset that is already owned.
4418 */
4419 VERIFY3U(EBUSY, ==, ztest_dmu_objset_own(name, DMU_OST_OTHER,
4420 B_FALSE, B_TRUE, FTAG, &os2));
4421
4422 zil_close(zilog);
4423 dmu_objset_disown(os, B_TRUE, FTAG);
4424 ztest_zd_fini(zdtmp);
4425 out:
4426 (void) pthread_rwlock_unlock(&ztest_name_lock);
4427
4428 umem_free(zdtmp, sizeof (ztest_ds_t));
4429 }
4430
4431 /*
4432 * Verify that dmu_snapshot_{create,destroy,open,close} work as expected.
4433 */
4434 void
4435 ztest_dmu_snapshot_create_destroy(ztest_ds_t *zd, uint64_t id)
4436 {
4437 (void) pthread_rwlock_rdlock(&ztest_name_lock);
4438 (void) ztest_snapshot_destroy(zd->zd_name, id);
4439 (void) ztest_snapshot_create(zd->zd_name, id);
4440 (void) pthread_rwlock_unlock(&ztest_name_lock);
4441 }
4442
4443 /*
4444 * Cleanup non-standard snapshots and clones.
4445 */
4446 static void
4447 ztest_dsl_dataset_cleanup(char *osname, uint64_t id)
4448 {
4449 char *snap1name;
4450 char *clone1name;
4451 char *snap2name;
4452 char *clone2name;
4453 char *snap3name;
4454 int error;
4455
4456 snap1name = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL);
4457 clone1name = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL);
4458 snap2name = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL);
4459 clone2name = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL);
4460 snap3name = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL);
4461
4462 (void) snprintf(snap1name, ZFS_MAX_DATASET_NAME_LEN, "%s@s1_%"PRIu64"",
4463 osname, id);
4464 (void) snprintf(clone1name, ZFS_MAX_DATASET_NAME_LEN, "%s/c1_%"PRIu64"",
4465 osname, id);
4466 (void) snprintf(snap2name, ZFS_MAX_DATASET_NAME_LEN, "%s@s2_%"PRIu64"",
4467 clone1name, id);
4468 (void) snprintf(clone2name, ZFS_MAX_DATASET_NAME_LEN, "%s/c2_%"PRIu64"",
4469 osname, id);
4470 (void) snprintf(snap3name, ZFS_MAX_DATASET_NAME_LEN, "%s@s3_%"PRIu64"",
4471 clone1name, id);
4472
4473 error = dsl_destroy_head(clone2name);
4474 if (error && error != ENOENT)
4475 fatal(B_FALSE, "dsl_destroy_head(%s) = %d", clone2name, error);
4476 error = dsl_destroy_snapshot(snap3name, B_FALSE);
4477 if (error && error != ENOENT)
4478 fatal(B_FALSE, "dsl_destroy_snapshot(%s) = %d",
4479 snap3name, error);
4480 error = dsl_destroy_snapshot(snap2name, B_FALSE);
4481 if (error && error != ENOENT)
4482 fatal(B_FALSE, "dsl_destroy_snapshot(%s) = %d",
4483 snap2name, error);
4484 error = dsl_destroy_head(clone1name);
4485 if (error && error != ENOENT)
4486 fatal(B_FALSE, "dsl_destroy_head(%s) = %d", clone1name, error);
4487 error = dsl_destroy_snapshot(snap1name, B_FALSE);
4488 if (error && error != ENOENT)
4489 fatal(B_FALSE, "dsl_destroy_snapshot(%s) = %d",
4490 snap1name, error);
4491
4492 umem_free(snap1name, ZFS_MAX_DATASET_NAME_LEN);
4493 umem_free(clone1name, ZFS_MAX_DATASET_NAME_LEN);
4494 umem_free(snap2name, ZFS_MAX_DATASET_NAME_LEN);
4495 umem_free(clone2name, ZFS_MAX_DATASET_NAME_LEN);
4496 umem_free(snap3name, ZFS_MAX_DATASET_NAME_LEN);
4497 }
4498
4499 /*
4500 * Verify dsl_dataset_promote handles EBUSY
4501 */
4502 void
4503 ztest_dsl_dataset_promote_busy(ztest_ds_t *zd, uint64_t id)
4504 {
4505 objset_t *os;
4506 char *snap1name;
4507 char *clone1name;
4508 char *snap2name;
4509 char *clone2name;
4510 char *snap3name;
4511 char *osname = zd->zd_name;
4512 int error;
4513
4514 snap1name = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL);
4515 clone1name = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL);
4516 snap2name = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL);
4517 clone2name = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL);
4518 snap3name = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL);
4519
4520 (void) pthread_rwlock_rdlock(&ztest_name_lock);
4521
4522 ztest_dsl_dataset_cleanup(osname, id);
4523
4524 (void) snprintf(snap1name, ZFS_MAX_DATASET_NAME_LEN, "%s@s1_%"PRIu64"",
4525 osname, id);
4526 (void) snprintf(clone1name, ZFS_MAX_DATASET_NAME_LEN, "%s/c1_%"PRIu64"",
4527 osname, id);
4528 (void) snprintf(snap2name, ZFS_MAX_DATASET_NAME_LEN, "%s@s2_%"PRIu64"",
4529 clone1name, id);
4530 (void) snprintf(clone2name, ZFS_MAX_DATASET_NAME_LEN, "%s/c2_%"PRIu64"",
4531 osname, id);
4532 (void) snprintf(snap3name, ZFS_MAX_DATASET_NAME_LEN, "%s@s3_%"PRIu64"",
4533 clone1name, id);
4534
4535 error = dmu_objset_snapshot_one(osname, strchr(snap1name, '@') + 1);
4536 if (error && error != EEXIST) {
4537 if (error == ENOSPC) {
4538 ztest_record_enospc(FTAG);
4539 goto out;
4540 }
4541 fatal(B_FALSE, "dmu_take_snapshot(%s) = %d", snap1name, error);
4542 }
4543
4544 error = dmu_objset_clone(clone1name, snap1name);
4545 if (error) {
4546 if (error == ENOSPC) {
4547 ztest_record_enospc(FTAG);
4548 goto out;
4549 }
4550 fatal(B_FALSE, "dmu_objset_create(%s) = %d", clone1name, error);
4551 }
4552
4553 error = dmu_objset_snapshot_one(clone1name, strchr(snap2name, '@') + 1);
4554 if (error && error != EEXIST) {
4555 if (error == ENOSPC) {
4556 ztest_record_enospc(FTAG);
4557 goto out;
4558 }
4559 fatal(B_FALSE, "dmu_open_snapshot(%s) = %d", snap2name, error);
4560 }
4561
4562 error = dmu_objset_snapshot_one(clone1name, strchr(snap3name, '@') + 1);
4563 if (error && error != EEXIST) {
4564 if (error == ENOSPC) {
4565 ztest_record_enospc(FTAG);
4566 goto out;
4567 }
4568 fatal(B_FALSE, "dmu_open_snapshot(%s) = %d", snap3name, error);
4569 }
4570
4571 error = dmu_objset_clone(clone2name, snap3name);
4572 if (error) {
4573 if (error == ENOSPC) {
4574 ztest_record_enospc(FTAG);
4575 goto out;
4576 }
4577 fatal(B_FALSE, "dmu_objset_create(%s) = %d", clone2name, error);
4578 }
4579
4580 error = ztest_dmu_objset_own(snap2name, DMU_OST_ANY, B_TRUE, B_TRUE,
4581 FTAG, &os);
4582 if (error)
4583 fatal(B_FALSE, "dmu_objset_own(%s) = %d", snap2name, error);
4584 error = dsl_dataset_promote(clone2name, NULL);
4585 if (error == ENOSPC) {
4586 dmu_objset_disown(os, B_TRUE, FTAG);
4587 ztest_record_enospc(FTAG);
4588 goto out;
4589 }
4590 if (error != EBUSY)
4591 fatal(B_FALSE, "dsl_dataset_promote(%s), %d, not EBUSY",
4592 clone2name, error);
4593 dmu_objset_disown(os, B_TRUE, FTAG);
4594
4595 out:
4596 ztest_dsl_dataset_cleanup(osname, id);
4597
4598 (void) pthread_rwlock_unlock(&ztest_name_lock);
4599
4600 umem_free(snap1name, ZFS_MAX_DATASET_NAME_LEN);
4601 umem_free(clone1name, ZFS_MAX_DATASET_NAME_LEN);
4602 umem_free(snap2name, ZFS_MAX_DATASET_NAME_LEN);
4603 umem_free(clone2name, ZFS_MAX_DATASET_NAME_LEN);
4604 umem_free(snap3name, ZFS_MAX_DATASET_NAME_LEN);
4605 }
4606
4607 #undef OD_ARRAY_SIZE
4608 #define OD_ARRAY_SIZE 4
4609
4610 /*
4611 * Verify that dmu_object_{alloc,free} work as expected.
4612 */
4613 void
4614 ztest_dmu_object_alloc_free(ztest_ds_t *zd, uint64_t id)
4615 {
4616 ztest_od_t *od;
4617 int batchsize;
4618 int size;
4619 int b;
4620
4621 size = sizeof (ztest_od_t) * OD_ARRAY_SIZE;
4622 od = umem_alloc(size, UMEM_NOFAIL);
4623 batchsize = OD_ARRAY_SIZE;
4624
4625 for (b = 0; b < batchsize; b++)
4626 ztest_od_init(od + b, id, FTAG, b, DMU_OT_UINT64_OTHER,
4627 0, 0, 0);
4628
4629 /*
4630 * Destroy the previous batch of objects, create a new batch,
4631 * and do some I/O on the new objects.
4632 */
4633 if (ztest_object_init(zd, od, size, B_TRUE) != 0)
4634 return;
4635
4636 while (ztest_random(4 * batchsize) != 0)
4637 ztest_io(zd, od[ztest_random(batchsize)].od_object,
4638 ztest_random(ZTEST_RANGE_LOCKS) << SPA_MAXBLOCKSHIFT);
4639
4640 umem_free(od, size);
4641 }
4642
4643 /*
4644 * Rewind the global allocator to verify object allocation backfilling.
4645 */
4646 void
4647 ztest_dmu_object_next_chunk(ztest_ds_t *zd, uint64_t id)
4648 {
4649 (void) id;
4650 objset_t *os = zd->zd_os;
4651 uint_t dnodes_per_chunk = 1 << dmu_object_alloc_chunk_shift;
4652 uint64_t object;
4653
4654 /*
4655 * Rewind the global allocator randomly back to a lower object number
4656 * to force backfilling and reclamation of recently freed dnodes.
4657 */
4658 mutex_enter(&os->os_obj_lock);
4659 object = ztest_random(os->os_obj_next_chunk);
4660 os->os_obj_next_chunk = P2ALIGN(object, dnodes_per_chunk);
4661 mutex_exit(&os->os_obj_lock);
4662 }
4663
4664 #undef OD_ARRAY_SIZE
4665 #define OD_ARRAY_SIZE 2
4666
4667 /*
4668 * Verify that dmu_{read,write} work as expected.
4669 */
4670 void
4671 ztest_dmu_read_write(ztest_ds_t *zd, uint64_t id)
4672 {
4673 int size;
4674 ztest_od_t *od;
4675
4676 objset_t *os = zd->zd_os;
4677 size = sizeof (ztest_od_t) * OD_ARRAY_SIZE;
4678 od = umem_alloc(size, UMEM_NOFAIL);
4679 dmu_tx_t *tx;
4680 int freeit, error;
4681 uint64_t i, n, s, txg;
4682 bufwad_t *packbuf, *bigbuf, *pack, *bigH, *bigT;
4683 uint64_t packobj, packoff, packsize, bigobj, bigoff, bigsize;
4684 uint64_t chunksize = (1000 + ztest_random(1000)) * sizeof (uint64_t);
4685 uint64_t regions = 997;
4686 uint64_t stride = 123456789ULL;
4687 uint64_t width = 40;
4688 int free_percent = 5;
4689
4690 /*
4691 * This test uses two objects, packobj and bigobj, that are always
4692 * updated together (i.e. in the same tx) so that their contents are
4693 * in sync and can be compared. Their contents relate to each other
4694 * in a simple way: packobj is a dense array of 'bufwad' structures,
4695 * while bigobj is a sparse array of the same bufwads. Specifically,
4696 * for any index n, there are three bufwads that should be identical:
4697 *
4698 * packobj, at offset n * sizeof (bufwad_t)
4699 * bigobj, at the head of the nth chunk
4700 * bigobj, at the tail of the nth chunk
4701 *
4702 * The chunk size is arbitrary. It doesn't have to be a power of two,
4703 * and it doesn't have any relation to the object blocksize.
4704 * The only requirement is that it can hold at least two bufwads.
4705 *
4706 * Normally, we write the bufwad to each of these locations.
4707 * However, free_percent of the time we instead write zeroes to
4708 * packobj and perform a dmu_free_range() on bigobj. By comparing
4709 * bigobj to packobj, we can verify that the DMU is correctly
4710 * tracking which parts of an object are allocated and free,
4711 * and that the contents of the allocated blocks are correct.
4712 */
4713
4714 /*
4715 * Read the directory info. If it's the first time, set things up.
4716 */
4717 ztest_od_init(od, id, FTAG, 0, DMU_OT_UINT64_OTHER, 0, 0, chunksize);
4718 ztest_od_init(od + 1, id, FTAG, 1, DMU_OT_UINT64_OTHER, 0, 0,
4719 chunksize);
4720
4721 if (ztest_object_init(zd, od, size, B_FALSE) != 0) {
4722 umem_free(od, size);
4723 return;
4724 }
4725
4726 bigobj = od[0].od_object;
4727 packobj = od[1].od_object;
4728 chunksize = od[0].od_gen;
4729 ASSERT3U(chunksize, ==, od[1].od_gen);
4730
4731 /*
4732 * Prefetch a random chunk of the big object.
4733 * Our aim here is to get some async reads in flight
4734 * for blocks that we may free below; the DMU should
4735 * handle this race correctly.
4736 */
4737 n = ztest_random(regions) * stride + ztest_random(width);
4738 s = 1 + ztest_random(2 * width - 1);
4739 dmu_prefetch(os, bigobj, 0, n * chunksize, s * chunksize,
4740 ZIO_PRIORITY_SYNC_READ);
4741
4742 /*
4743 * Pick a random index and compute the offsets into packobj and bigobj.
4744 */
4745 n = ztest_random(regions) * stride + ztest_random(width);
4746 s = 1 + ztest_random(width - 1);
4747
4748 packoff = n * sizeof (bufwad_t);
4749 packsize = s * sizeof (bufwad_t);
4750
4751 bigoff = n * chunksize;
4752 bigsize = s * chunksize;
4753
4754 packbuf = umem_alloc(packsize, UMEM_NOFAIL);
4755 bigbuf = umem_alloc(bigsize, UMEM_NOFAIL);
4756
4757 /*
4758 * free_percent of the time, free a range of bigobj rather than
4759 * overwriting it.
4760 */
4761 freeit = (ztest_random(100) < free_percent);
4762
4763 /*
4764 * Read the current contents of our objects.
4765 */
4766 error = dmu_read(os, packobj, packoff, packsize, packbuf,
4767 DMU_READ_PREFETCH);
4768 ASSERT0(error);
4769 error = dmu_read(os, bigobj, bigoff, bigsize, bigbuf,
4770 DMU_READ_PREFETCH);
4771 ASSERT0(error);
4772
4773 /*
4774 * Get a tx for the mods to both packobj and bigobj.
4775 */
4776 tx = dmu_tx_create(os);
4777
4778 dmu_tx_hold_write(tx, packobj, packoff, packsize);
4779
4780 if (freeit)
4781 dmu_tx_hold_free(tx, bigobj, bigoff, bigsize);
4782 else
4783 dmu_tx_hold_write(tx, bigobj, bigoff, bigsize);
4784
4785 /* This accounts for setting the checksum/compression. */
4786 dmu_tx_hold_bonus(tx, bigobj);
4787
4788 txg = ztest_tx_assign(tx, TXG_MIGHTWAIT, FTAG);
4789 if (txg == 0) {
4790 umem_free(packbuf, packsize);
4791 umem_free(bigbuf, bigsize);
4792 umem_free(od, size);
4793 return;
4794 }
4795
4796 enum zio_checksum cksum;
4797 do {
4798 cksum = (enum zio_checksum)
4799 ztest_random_dsl_prop(ZFS_PROP_CHECKSUM);
4800 } while (cksum >= ZIO_CHECKSUM_LEGACY_FUNCTIONS);
4801 dmu_object_set_checksum(os, bigobj, cksum, tx);
4802
4803 enum zio_compress comp;
4804 do {
4805 comp = (enum zio_compress)
4806 ztest_random_dsl_prop(ZFS_PROP_COMPRESSION);
4807 } while (comp >= ZIO_COMPRESS_LEGACY_FUNCTIONS);
4808 dmu_object_set_compress(os, bigobj, comp, tx);
4809
4810 /*
4811 * For each index from n to n + s, verify that the existing bufwad
4812 * in packobj matches the bufwads at the head and tail of the
4813 * corresponding chunk in bigobj. Then update all three bufwads
4814 * with the new values we want to write out.
4815 */
4816 for (i = 0; i < s; i++) {
4817 /* LINTED */
4818 pack = (bufwad_t *)((char *)packbuf + i * sizeof (bufwad_t));
4819 /* LINTED */
4820 bigH = (bufwad_t *)((char *)bigbuf + i * chunksize);
4821 /* LINTED */
4822 bigT = (bufwad_t *)((char *)bigH + chunksize) - 1;
4823
4824 ASSERT3U((uintptr_t)bigH - (uintptr_t)bigbuf, <, bigsize);
4825 ASSERT3U((uintptr_t)bigT - (uintptr_t)bigbuf, <, bigsize);
4826
4827 if (pack->bw_txg > txg)
4828 fatal(B_FALSE,
4829 "future leak: got %"PRIx64", open txg is %"PRIx64"",
4830 pack->bw_txg, txg);
4831
4832 if (pack->bw_data != 0 && pack->bw_index != n + i)
4833 fatal(B_FALSE, "wrong index: "
4834 "got %"PRIx64", wanted %"PRIx64"+%"PRIx64"",
4835 pack->bw_index, n, i);
4836
4837 if (memcmp(pack, bigH, sizeof (bufwad_t)) != 0)
4838 fatal(B_FALSE, "pack/bigH mismatch in %p/%p",
4839 pack, bigH);
4840
4841 if (memcmp(pack, bigT, sizeof (bufwad_t)) != 0)
4842 fatal(B_FALSE, "pack/bigT mismatch in %p/%p",
4843 pack, bigT);
4844
4845 if (freeit) {
4846 memset(pack, 0, sizeof (bufwad_t));
4847 } else {
4848 pack->bw_index = n + i;
4849 pack->bw_txg = txg;
4850 pack->bw_data = 1 + ztest_random(-2ULL);
4851 }
4852 *bigH = *pack;
4853 *bigT = *pack;
4854 }
4855
4856 /*
4857 * We've verified all the old bufwads, and made new ones.
4858 * Now write them out.
4859 */
4860 dmu_write(os, packobj, packoff, packsize, packbuf, tx);
4861
4862 if (freeit) {
4863 if (ztest_opts.zo_verbose >= 7) {
4864 (void) printf("freeing offset %"PRIx64" size %"PRIx64""
4865 " txg %"PRIx64"\n",
4866 bigoff, bigsize, txg);
4867 }
4868 VERIFY0(dmu_free_range(os, bigobj, bigoff, bigsize, tx));
4869 } else {
4870 if (ztest_opts.zo_verbose >= 7) {
4871 (void) printf("writing offset %"PRIx64" size %"PRIx64""
4872 " txg %"PRIx64"\n",
4873 bigoff, bigsize, txg);
4874 }
4875 dmu_write(os, bigobj, bigoff, bigsize, bigbuf, tx);
4876 }
4877
4878 dmu_tx_commit(tx);
4879
4880 /*
4881 * Sanity check the stuff we just wrote.
4882 */
4883 {
4884 void *packcheck = umem_alloc(packsize, UMEM_NOFAIL);
4885 void *bigcheck = umem_alloc(bigsize, UMEM_NOFAIL);
4886
4887 VERIFY0(dmu_read(os, packobj, packoff,
4888 packsize, packcheck, DMU_READ_PREFETCH));
4889 VERIFY0(dmu_read(os, bigobj, bigoff,
4890 bigsize, bigcheck, DMU_READ_PREFETCH));
4891
4892 ASSERT0(memcmp(packbuf, packcheck, packsize));
4893 ASSERT0(memcmp(bigbuf, bigcheck, bigsize));
4894
4895 umem_free(packcheck, packsize);
4896 umem_free(bigcheck, bigsize);
4897 }
4898
4899 umem_free(packbuf, packsize);
4900 umem_free(bigbuf, bigsize);
4901 umem_free(od, size);
4902 }
4903
4904 static void
4905 compare_and_update_pbbufs(uint64_t s, bufwad_t *packbuf, bufwad_t *bigbuf,
4906 uint64_t bigsize, uint64_t n, uint64_t chunksize, uint64_t txg)
4907 {
4908 uint64_t i;
4909 bufwad_t *pack;
4910 bufwad_t *bigH;
4911 bufwad_t *bigT;
4912
4913 /*
4914 * For each index from n to n + s, verify that the existing bufwad
4915 * in packobj matches the bufwads at the head and tail of the
4916 * corresponding chunk in bigobj. Then update all three bufwads
4917 * with the new values we want to write out.
4918 */
4919 for (i = 0; i < s; i++) {
4920 /* LINTED */
4921 pack = (bufwad_t *)((char *)packbuf + i * sizeof (bufwad_t));
4922 /* LINTED */
4923 bigH = (bufwad_t *)((char *)bigbuf + i * chunksize);
4924 /* LINTED */
4925 bigT = (bufwad_t *)((char *)bigH + chunksize) - 1;
4926
4927 ASSERT3U((uintptr_t)bigH - (uintptr_t)bigbuf, <, bigsize);
4928 ASSERT3U((uintptr_t)bigT - (uintptr_t)bigbuf, <, bigsize);
4929
4930 if (pack->bw_txg > txg)
4931 fatal(B_FALSE,
4932 "future leak: got %"PRIx64", open txg is %"PRIx64"",
4933 pack->bw_txg, txg);
4934
4935 if (pack->bw_data != 0 && pack->bw_index != n + i)
4936 fatal(B_FALSE, "wrong index: "
4937 "got %"PRIx64", wanted %"PRIx64"+%"PRIx64"",
4938 pack->bw_index, n, i);
4939
4940 if (memcmp(pack, bigH, sizeof (bufwad_t)) != 0)
4941 fatal(B_FALSE, "pack/bigH mismatch in %p/%p",
4942 pack, bigH);
4943
4944 if (memcmp(pack, bigT, sizeof (bufwad_t)) != 0)
4945 fatal(B_FALSE, "pack/bigT mismatch in %p/%p",
4946 pack, bigT);
4947
4948 pack->bw_index = n + i;
4949 pack->bw_txg = txg;
4950 pack->bw_data = 1 + ztest_random(-2ULL);
4951
4952 *bigH = *pack;
4953 *bigT = *pack;
4954 }
4955 }
4956
4957 #undef OD_ARRAY_SIZE
4958 #define OD_ARRAY_SIZE 2
4959
4960 void
4961 ztest_dmu_read_write_zcopy(ztest_ds_t *zd, uint64_t id)
4962 {
4963 objset_t *os = zd->zd_os;
4964 ztest_od_t *od;
4965 dmu_tx_t *tx;
4966 uint64_t i;
4967 int error;
4968 int size;
4969 uint64_t n, s, txg;
4970 bufwad_t *packbuf, *bigbuf;
4971 uint64_t packobj, packoff, packsize, bigobj, bigoff, bigsize;
4972 uint64_t blocksize = ztest_random_blocksize();
4973 uint64_t chunksize = blocksize;
4974 uint64_t regions = 997;
4975 uint64_t stride = 123456789ULL;
4976 uint64_t width = 9;
4977 dmu_buf_t *bonus_db;
4978 arc_buf_t **bigbuf_arcbufs;
4979 dmu_object_info_t doi;
4980
4981 size = sizeof (ztest_od_t) * OD_ARRAY_SIZE;
4982 od = umem_alloc(size, UMEM_NOFAIL);
4983
4984 /*
4985 * This test uses two objects, packobj and bigobj, that are always
4986 * updated together (i.e. in the same tx) so that their contents are
4987 * in sync and can be compared. Their contents relate to each other
4988 * in a simple way: packobj is a dense array of 'bufwad' structures,
4989 * while bigobj is a sparse array of the same bufwads. Specifically,
4990 * for any index n, there are three bufwads that should be identical:
4991 *
4992 * packobj, at offset n * sizeof (bufwad_t)
4993 * bigobj, at the head of the nth chunk
4994 * bigobj, at the tail of the nth chunk
4995 *
4996 * The chunk size is set equal to bigobj block size so that
4997 * dmu_assign_arcbuf_by_dbuf() can be tested for object updates.
4998 */
4999
5000 /*
5001 * Read the directory info. If it's the first time, set things up.
5002 */
5003 ztest_od_init(od, id, FTAG, 0, DMU_OT_UINT64_OTHER, blocksize, 0, 0);
5004 ztest_od_init(od + 1, id, FTAG, 1, DMU_OT_UINT64_OTHER, 0, 0,
5005 chunksize);
5006
5007
5008 if (ztest_object_init(zd, od, size, B_FALSE) != 0) {
5009 umem_free(od, size);
5010 return;
5011 }
5012
5013 bigobj = od[0].od_object;
5014 packobj = od[1].od_object;
5015 blocksize = od[0].od_blocksize;
5016 chunksize = blocksize;
5017 ASSERT3U(chunksize, ==, od[1].od_gen);
5018
5019 VERIFY0(dmu_object_info(os, bigobj, &doi));
5020 VERIFY(ISP2(doi.doi_data_block_size));
5021 VERIFY3U(chunksize, ==, doi.doi_data_block_size);
5022 VERIFY3U(chunksize, >=, 2 * sizeof (bufwad_t));
5023
5024 /*
5025 * Pick a random index and compute the offsets into packobj and bigobj.
5026 */
5027 n = ztest_random(regions) * stride + ztest_random(width);
5028 s = 1 + ztest_random(width - 1);
5029
5030 packoff = n * sizeof (bufwad_t);
5031 packsize = s * sizeof (bufwad_t);
5032
5033 bigoff = n * chunksize;
5034 bigsize = s * chunksize;
5035
5036 packbuf = umem_zalloc(packsize, UMEM_NOFAIL);
5037 bigbuf = umem_zalloc(bigsize, UMEM_NOFAIL);
5038
5039 VERIFY0(dmu_bonus_hold(os, bigobj, FTAG, &bonus_db));
5040
5041 bigbuf_arcbufs = umem_zalloc(2 * s * sizeof (arc_buf_t *), UMEM_NOFAIL);
5042
5043 /*
5044 * Iteration 0 test zcopy for DB_UNCACHED dbufs.
5045 * Iteration 1 test zcopy to already referenced dbufs.
5046 * Iteration 2 test zcopy to dirty dbuf in the same txg.
5047 * Iteration 3 test zcopy to dbuf dirty in previous txg.
5048 * Iteration 4 test zcopy when dbuf is no longer dirty.
5049 * Iteration 5 test zcopy when it can't be done.
5050 * Iteration 6 one more zcopy write.
5051 */
5052 for (i = 0; i < 7; i++) {
5053 uint64_t j;
5054 uint64_t off;
5055
5056 /*
5057 * In iteration 5 (i == 5) use arcbufs
5058 * that don't match bigobj blksz to test
5059 * dmu_assign_arcbuf_by_dbuf() when it can't directly
5060 * assign an arcbuf to a dbuf.
5061 */
5062 for (j = 0; j < s; j++) {
5063 if (i != 5 || chunksize < (SPA_MINBLOCKSIZE * 2)) {
5064 bigbuf_arcbufs[j] =
5065 dmu_request_arcbuf(bonus_db, chunksize);
5066 } else {
5067 bigbuf_arcbufs[2 * j] =
5068 dmu_request_arcbuf(bonus_db, chunksize / 2);
5069 bigbuf_arcbufs[2 * j + 1] =
5070 dmu_request_arcbuf(bonus_db, chunksize / 2);
5071 }
5072 }
5073
5074 /*
5075 * Get a tx for the mods to both packobj and bigobj.
5076 */
5077 tx = dmu_tx_create(os);
5078
5079 dmu_tx_hold_write(tx, packobj, packoff, packsize);
5080 dmu_tx_hold_write(tx, bigobj, bigoff, bigsize);
5081
5082 txg = ztest_tx_assign(tx, TXG_MIGHTWAIT, FTAG);
5083 if (txg == 0) {
5084 umem_free(packbuf, packsize);
5085 umem_free(bigbuf, bigsize);
5086 for (j = 0; j < s; j++) {
5087 if (i != 5 ||
5088 chunksize < (SPA_MINBLOCKSIZE * 2)) {
5089 dmu_return_arcbuf(bigbuf_arcbufs[j]);
5090 } else {
5091 dmu_return_arcbuf(
5092 bigbuf_arcbufs[2 * j]);
5093 dmu_return_arcbuf(
5094 bigbuf_arcbufs[2 * j + 1]);
5095 }
5096 }
5097 umem_free(bigbuf_arcbufs, 2 * s * sizeof (arc_buf_t *));
5098 umem_free(od, size);
5099 dmu_buf_rele(bonus_db, FTAG);
5100 return;
5101 }
5102
5103 /*
5104 * 50% of the time don't read objects in the 1st iteration to
5105 * test dmu_assign_arcbuf_by_dbuf() for the case when there are
5106 * no existing dbufs for the specified offsets.
5107 */
5108 if (i != 0 || ztest_random(2) != 0) {
5109 error = dmu_read(os, packobj, packoff,
5110 packsize, packbuf, DMU_READ_PREFETCH);
5111 ASSERT0(error);
5112 error = dmu_read(os, bigobj, bigoff, bigsize,
5113 bigbuf, DMU_READ_PREFETCH);
5114 ASSERT0(error);
5115 }
5116 compare_and_update_pbbufs(s, packbuf, bigbuf, bigsize,
5117 n, chunksize, txg);
5118
5119 /*
5120 * We've verified all the old bufwads, and made new ones.
5121 * Now write them out.
5122 */
5123 dmu_write(os, packobj, packoff, packsize, packbuf, tx);
5124 if (ztest_opts.zo_verbose >= 7) {
5125 (void) printf("writing offset %"PRIx64" size %"PRIx64""
5126 " txg %"PRIx64"\n",
5127 bigoff, bigsize, txg);
5128 }
5129 for (off = bigoff, j = 0; j < s; j++, off += chunksize) {
5130 dmu_buf_t *dbt;
5131 if (i != 5 || chunksize < (SPA_MINBLOCKSIZE * 2)) {
5132 memcpy(bigbuf_arcbufs[j]->b_data,
5133 (caddr_t)bigbuf + (off - bigoff),
5134 chunksize);
5135 } else {
5136 memcpy(bigbuf_arcbufs[2 * j]->b_data,
5137 (caddr_t)bigbuf + (off - bigoff),
5138 chunksize / 2);
5139 memcpy(bigbuf_arcbufs[2 * j + 1]->b_data,
5140 (caddr_t)bigbuf + (off - bigoff) +
5141 chunksize / 2,
5142 chunksize / 2);
5143 }
5144
5145 if (i == 1) {
5146 VERIFY(dmu_buf_hold(os, bigobj, off,
5147 FTAG, &dbt, DMU_READ_NO_PREFETCH) == 0);
5148 }
5149 if (i != 5 || chunksize < (SPA_MINBLOCKSIZE * 2)) {
5150 VERIFY0(dmu_assign_arcbuf_by_dbuf(bonus_db,
5151 off, bigbuf_arcbufs[j], tx));
5152 } else {
5153 VERIFY0(dmu_assign_arcbuf_by_dbuf(bonus_db,
5154 off, bigbuf_arcbufs[2 * j], tx));
5155 VERIFY0(dmu_assign_arcbuf_by_dbuf(bonus_db,
5156 off + chunksize / 2,
5157 bigbuf_arcbufs[2 * j + 1], tx));
5158 }
5159 if (i == 1) {
5160 dmu_buf_rele(dbt, FTAG);
5161 }
5162 }
5163 dmu_tx_commit(tx);
5164
5165 /*
5166 * Sanity check the stuff we just wrote.
5167 */
5168 {
5169 void *packcheck = umem_alloc(packsize, UMEM_NOFAIL);
5170 void *bigcheck = umem_alloc(bigsize, UMEM_NOFAIL);
5171
5172 VERIFY0(dmu_read(os, packobj, packoff,
5173 packsize, packcheck, DMU_READ_PREFETCH));
5174 VERIFY0(dmu_read(os, bigobj, bigoff,
5175 bigsize, bigcheck, DMU_READ_PREFETCH));
5176
5177 ASSERT0(memcmp(packbuf, packcheck, packsize));
5178 ASSERT0(memcmp(bigbuf, bigcheck, bigsize));
5179
5180 umem_free(packcheck, packsize);
5181 umem_free(bigcheck, bigsize);
5182 }
5183 if (i == 2) {
5184 txg_wait_open(dmu_objset_pool(os), 0, B_TRUE);
5185 } else if (i == 3) {
5186 txg_wait_synced(dmu_objset_pool(os), 0);
5187 }
5188 }
5189
5190 dmu_buf_rele(bonus_db, FTAG);
5191 umem_free(packbuf, packsize);
5192 umem_free(bigbuf, bigsize);
5193 umem_free(bigbuf_arcbufs, 2 * s * sizeof (arc_buf_t *));
5194 umem_free(od, size);
5195 }
5196
5197 void
5198 ztest_dmu_write_parallel(ztest_ds_t *zd, uint64_t id)
5199 {
5200 (void) id;
5201 ztest_od_t *od;
5202
5203 od = umem_alloc(sizeof (ztest_od_t), UMEM_NOFAIL);
5204 uint64_t offset = (1ULL << (ztest_random(20) + 43)) +
5205 (ztest_random(ZTEST_RANGE_LOCKS) << SPA_MAXBLOCKSHIFT);
5206
5207 /*
5208 * Have multiple threads write to large offsets in an object
5209 * to verify that parallel writes to an object -- even to the
5210 * same blocks within the object -- doesn't cause any trouble.
5211 */
5212 ztest_od_init(od, ID_PARALLEL, FTAG, 0, DMU_OT_UINT64_OTHER, 0, 0, 0);
5213
5214 if (ztest_object_init(zd, od, sizeof (ztest_od_t), B_FALSE) != 0)
5215 return;
5216
5217 while (ztest_random(10) != 0)
5218 ztest_io(zd, od->od_object, offset);
5219
5220 umem_free(od, sizeof (ztest_od_t));
5221 }
5222
5223 void
5224 ztest_dmu_prealloc(ztest_ds_t *zd, uint64_t id)
5225 {
5226 ztest_od_t *od;
5227 uint64_t offset = (1ULL << (ztest_random(4) + SPA_MAXBLOCKSHIFT)) +
5228 (ztest_random(ZTEST_RANGE_LOCKS) << SPA_MAXBLOCKSHIFT);
5229 uint64_t count = ztest_random(20) + 1;
5230 uint64_t blocksize = ztest_random_blocksize();
5231 void *data;
5232
5233 od = umem_alloc(sizeof (ztest_od_t), UMEM_NOFAIL);
5234
5235 ztest_od_init(od, id, FTAG, 0, DMU_OT_UINT64_OTHER, blocksize, 0, 0);
5236
5237 if (ztest_object_init(zd, od, sizeof (ztest_od_t),
5238 !ztest_random(2)) != 0) {
5239 umem_free(od, sizeof (ztest_od_t));
5240 return;
5241 }
5242
5243 if (ztest_truncate(zd, od->od_object, offset, count * blocksize) != 0) {
5244 umem_free(od, sizeof (ztest_od_t));
5245 return;
5246 }
5247
5248 ztest_prealloc(zd, od->od_object, offset, count * blocksize);
5249
5250 data = umem_zalloc(blocksize, UMEM_NOFAIL);
5251
5252 while (ztest_random(count) != 0) {
5253 uint64_t randoff = offset + (ztest_random(count) * blocksize);
5254 if (ztest_write(zd, od->od_object, randoff, blocksize,
5255 data) != 0)
5256 break;
5257 while (ztest_random(4) != 0)
5258 ztest_io(zd, od->od_object, randoff);
5259 }
5260
5261 umem_free(data, blocksize);
5262 umem_free(od, sizeof (ztest_od_t));
5263 }
5264
5265 /*
5266 * Verify that zap_{create,destroy,add,remove,update} work as expected.
5267 */
5268 #define ZTEST_ZAP_MIN_INTS 1
5269 #define ZTEST_ZAP_MAX_INTS 4
5270 #define ZTEST_ZAP_MAX_PROPS 1000
5271
5272 void
5273 ztest_zap(ztest_ds_t *zd, uint64_t id)
5274 {
5275 objset_t *os = zd->zd_os;
5276 ztest_od_t *od;
5277 uint64_t object;
5278 uint64_t txg, last_txg;
5279 uint64_t value[ZTEST_ZAP_MAX_INTS];
5280 uint64_t zl_ints, zl_intsize, prop;
5281 int i, ints;
5282 dmu_tx_t *tx;
5283 char propname[100], txgname[100];
5284 int error;
5285 const char *const hc[2] = { "s.acl.h", ".s.open.h.hyLZlg" };
5286
5287 od = umem_alloc(sizeof (ztest_od_t), UMEM_NOFAIL);
5288 ztest_od_init(od, id, FTAG, 0, DMU_OT_ZAP_OTHER, 0, 0, 0);
5289
5290 if (ztest_object_init(zd, od, sizeof (ztest_od_t),
5291 !ztest_random(2)) != 0)
5292 goto out;
5293
5294 object = od->od_object;
5295
5296 /*
5297 * Generate a known hash collision, and verify that
5298 * we can lookup and remove both entries.
5299 */
5300 tx = dmu_tx_create(os);
5301 dmu_tx_hold_zap(tx, object, B_TRUE, NULL);
5302 txg = ztest_tx_assign(tx, TXG_MIGHTWAIT, FTAG);
5303 if (txg == 0)
5304 goto out;
5305 for (i = 0; i < 2; i++) {
5306 value[i] = i;
5307 VERIFY0(zap_add(os, object, hc[i], sizeof (uint64_t),
5308 1, &value[i], tx));
5309 }
5310 for (i = 0; i < 2; i++) {
5311 VERIFY3U(EEXIST, ==, zap_add(os, object, hc[i],
5312 sizeof (uint64_t), 1, &value[i], tx));
5313 VERIFY0(
5314 zap_length(os, object, hc[i], &zl_intsize, &zl_ints));
5315 ASSERT3U(zl_intsize, ==, sizeof (uint64_t));
5316 ASSERT3U(zl_ints, ==, 1);
5317 }
5318 for (i = 0; i < 2; i++) {
5319 VERIFY0(zap_remove(os, object, hc[i], tx));
5320 }
5321 dmu_tx_commit(tx);
5322
5323 /*
5324 * Generate a bunch of random entries.
5325 */
5326 ints = MAX(ZTEST_ZAP_MIN_INTS, object % ZTEST_ZAP_MAX_INTS);
5327
5328 prop = ztest_random(ZTEST_ZAP_MAX_PROPS);
5329 (void) sprintf(propname, "prop_%"PRIu64"", prop);
5330 (void) sprintf(txgname, "txg_%"PRIu64"", prop);
5331 memset(value, 0, sizeof (value));
5332 last_txg = 0;
5333
5334 /*
5335 * If these zap entries already exist, validate their contents.
5336 */
5337 error = zap_length(os, object, txgname, &zl_intsize, &zl_ints);
5338 if (error == 0) {
5339 ASSERT3U(zl_intsize, ==, sizeof (uint64_t));
5340 ASSERT3U(zl_ints, ==, 1);
5341
5342 VERIFY0(zap_lookup(os, object, txgname, zl_intsize,
5343 zl_ints, &last_txg));
5344
5345 VERIFY0(zap_length(os, object, propname, &zl_intsize,
5346 &zl_ints));
5347
5348 ASSERT3U(zl_intsize, ==, sizeof (uint64_t));
5349 ASSERT3U(zl_ints, ==, ints);
5350
5351 VERIFY0(zap_lookup(os, object, propname, zl_intsize,
5352 zl_ints, value));
5353
5354 for (i = 0; i < ints; i++) {
5355 ASSERT3U(value[i], ==, last_txg + object + i);
5356 }
5357 } else {
5358 ASSERT3U(error, ==, ENOENT);
5359 }
5360
5361 /*
5362 * Atomically update two entries in our zap object.
5363 * The first is named txg_%llu, and contains the txg
5364 * in which the property was last updated. The second
5365 * is named prop_%llu, and the nth element of its value
5366 * should be txg + object + n.
5367 */
5368 tx = dmu_tx_create(os);
5369 dmu_tx_hold_zap(tx, object, B_TRUE, NULL);
5370 txg = ztest_tx_assign(tx, TXG_MIGHTWAIT, FTAG);
5371 if (txg == 0)
5372 goto out;
5373
5374 if (last_txg > txg)
5375 fatal(B_FALSE, "zap future leak: old %"PRIu64" new %"PRIu64"",
5376 last_txg, txg);
5377
5378 for (i = 0; i < ints; i++)
5379 value[i] = txg + object + i;
5380
5381 VERIFY0(zap_update(os, object, txgname, sizeof (uint64_t),
5382 1, &txg, tx));
5383 VERIFY0(zap_update(os, object, propname, sizeof (uint64_t),
5384 ints, value, tx));
5385
5386 dmu_tx_commit(tx);
5387
5388 /*
5389 * Remove a random pair of entries.
5390 */
5391 prop = ztest_random(ZTEST_ZAP_MAX_PROPS);
5392 (void) sprintf(propname, "prop_%"PRIu64"", prop);
5393 (void) sprintf(txgname, "txg_%"PRIu64"", prop);
5394
5395 error = zap_length(os, object, txgname, &zl_intsize, &zl_ints);
5396
5397 if (error == ENOENT)
5398 goto out;
5399
5400 ASSERT0(error);
5401
5402 tx = dmu_tx_create(os);
5403 dmu_tx_hold_zap(tx, object, B_TRUE, NULL);
5404 txg = ztest_tx_assign(tx, TXG_MIGHTWAIT, FTAG);
5405 if (txg == 0)
5406 goto out;
5407 VERIFY0(zap_remove(os, object, txgname, tx));
5408 VERIFY0(zap_remove(os, object, propname, tx));
5409 dmu_tx_commit(tx);
5410 out:
5411 umem_free(od, sizeof (ztest_od_t));
5412 }
5413
5414 /*
5415 * Test case to test the upgrading of a microzap to fatzap.
5416 */
5417 void
5418 ztest_fzap(ztest_ds_t *zd, uint64_t id)
5419 {
5420 objset_t *os = zd->zd_os;
5421 ztest_od_t *od;
5422 uint64_t object, txg, value;
5423
5424 od = umem_alloc(sizeof (ztest_od_t), UMEM_NOFAIL);
5425 ztest_od_init(od, id, FTAG, 0, DMU_OT_ZAP_OTHER, 0, 0, 0);
5426
5427 if (ztest_object_init(zd, od, sizeof (ztest_od_t),
5428 !ztest_random(2)) != 0)
5429 goto out;
5430 object = od->od_object;
5431
5432 /*
5433 * Add entries to this ZAP and make sure it spills over
5434 * and gets upgraded to a fatzap. Also, since we are adding
5435 * 2050 entries we should see ptrtbl growth and leaf-block split.
5436 */
5437 for (value = 0; value < 2050; value++) {
5438 char name[ZFS_MAX_DATASET_NAME_LEN];
5439 dmu_tx_t *tx;
5440 int error;
5441
5442 (void) snprintf(name, sizeof (name), "fzap-%"PRIu64"-%"PRIu64"",
5443 id, value);
5444
5445 tx = dmu_tx_create(os);
5446 dmu_tx_hold_zap(tx, object, B_TRUE, name);
5447 txg = ztest_tx_assign(tx, TXG_MIGHTWAIT, FTAG);
5448 if (txg == 0)
5449 goto out;
5450 error = zap_add(os, object, name, sizeof (uint64_t), 1,
5451 &value, tx);
5452 ASSERT(error == 0 || error == EEXIST);
5453 dmu_tx_commit(tx);
5454 }
5455 out:
5456 umem_free(od, sizeof (ztest_od_t));
5457 }
5458
5459 void
5460 ztest_zap_parallel(ztest_ds_t *zd, uint64_t id)
5461 {
5462 (void) id;
5463 objset_t *os = zd->zd_os;
5464 ztest_od_t *od;
5465 uint64_t txg, object, count, wsize, wc, zl_wsize, zl_wc;
5466 dmu_tx_t *tx;
5467 int i, namelen, error;
5468 int micro = ztest_random(2);
5469 char name[20], string_value[20];
5470 void *data;
5471
5472 od = umem_alloc(sizeof (ztest_od_t), UMEM_NOFAIL);
5473 ztest_od_init(od, ID_PARALLEL, FTAG, micro, DMU_OT_ZAP_OTHER, 0, 0, 0);
5474
5475 if (ztest_object_init(zd, od, sizeof (ztest_od_t), B_FALSE) != 0) {
5476 umem_free(od, sizeof (ztest_od_t));
5477 return;
5478 }
5479
5480 object = od->od_object;
5481
5482 /*
5483 * Generate a random name of the form 'xxx.....' where each
5484 * x is a random printable character and the dots are dots.
5485 * There are 94 such characters, and the name length goes from
5486 * 6 to 20, so there are 94^3 * 15 = 12,458,760 possible names.
5487 */
5488 namelen = ztest_random(sizeof (name) - 5) + 5 + 1;
5489
5490 for (i = 0; i < 3; i++)
5491 name[i] = '!' + ztest_random('~' - '!' + 1);
5492 for (; i < namelen - 1; i++)
5493 name[i] = '.';
5494 name[i] = '\0';
5495
5496 if ((namelen & 1) || micro) {
5497 wsize = sizeof (txg);
5498 wc = 1;
5499 data = &txg;
5500 } else {
5501 wsize = 1;
5502 wc = namelen;
5503 data = string_value;
5504 }
5505
5506 count = -1ULL;
5507 VERIFY0(zap_count(os, object, &count));
5508 ASSERT3S(count, !=, -1ULL);
5509
5510 /*
5511 * Select an operation: length, lookup, add, update, remove.
5512 */
5513 i = ztest_random(5);
5514
5515 if (i >= 2) {
5516 tx = dmu_tx_create(os);
5517 dmu_tx_hold_zap(tx, object, B_TRUE, NULL);
5518 txg = ztest_tx_assign(tx, TXG_MIGHTWAIT, FTAG);
5519 if (txg == 0) {
5520 umem_free(od, sizeof (ztest_od_t));
5521 return;
5522 }
5523 memcpy(string_value, name, namelen);
5524 } else {
5525 tx = NULL;
5526 txg = 0;
5527 memset(string_value, 0, namelen);
5528 }
5529
5530 switch (i) {
5531
5532 case 0:
5533 error = zap_length(os, object, name, &zl_wsize, &zl_wc);
5534 if (error == 0) {
5535 ASSERT3U(wsize, ==, zl_wsize);
5536 ASSERT3U(wc, ==, zl_wc);
5537 } else {
5538 ASSERT3U(error, ==, ENOENT);
5539 }
5540 break;
5541
5542 case 1:
5543 error = zap_lookup(os, object, name, wsize, wc, data);
5544 if (error == 0) {
5545 if (data == string_value &&
5546 memcmp(name, data, namelen) != 0)
5547 fatal(B_FALSE, "name '%s' != val '%s' len %d",
5548 name, (char *)data, namelen);
5549 } else {
5550 ASSERT3U(error, ==, ENOENT);
5551 }
5552 break;
5553
5554 case 2:
5555 error = zap_add(os, object, name, wsize, wc, data, tx);
5556 ASSERT(error == 0 || error == EEXIST);
5557 break;
5558
5559 case 3:
5560 VERIFY0(zap_update(os, object, name, wsize, wc, data, tx));
5561 break;
5562
5563 case 4:
5564 error = zap_remove(os, object, name, tx);
5565 ASSERT(error == 0 || error == ENOENT);
5566 break;
5567 }
5568
5569 if (tx != NULL)
5570 dmu_tx_commit(tx);
5571
5572 umem_free(od, sizeof (ztest_od_t));
5573 }
5574
5575 /*
5576 * Commit callback data.
5577 */
5578 typedef struct ztest_cb_data {
5579 list_node_t zcd_node;
5580 uint64_t zcd_txg;
5581 int zcd_expected_err;
5582 boolean_t zcd_added;
5583 boolean_t zcd_called;
5584 spa_t *zcd_spa;
5585 } ztest_cb_data_t;
5586
5587 /* This is the actual commit callback function */
5588 static void
5589 ztest_commit_callback(void *arg, int error)
5590 {
5591 ztest_cb_data_t *data = arg;
5592 uint64_t synced_txg;
5593
5594 VERIFY3P(data, !=, NULL);
5595 VERIFY3S(data->zcd_expected_err, ==, error);
5596 VERIFY(!data->zcd_called);
5597
5598 synced_txg = spa_last_synced_txg(data->zcd_spa);
5599 if (data->zcd_txg > synced_txg)
5600 fatal(B_FALSE,
5601 "commit callback of txg %"PRIu64" called prematurely, "
5602 "last synced txg = %"PRIu64"\n",
5603 data->zcd_txg, synced_txg);
5604
5605 data->zcd_called = B_TRUE;
5606
5607 if (error == ECANCELED) {
5608 ASSERT0(data->zcd_txg);
5609 ASSERT(!data->zcd_added);
5610
5611 /*
5612 * The private callback data should be destroyed here, but
5613 * since we are going to check the zcd_called field after
5614 * dmu_tx_abort(), we will destroy it there.
5615 */
5616 return;
5617 }
5618
5619 ASSERT(data->zcd_added);
5620 ASSERT3U(data->zcd_txg, !=, 0);
5621
5622 (void) mutex_enter(&zcl.zcl_callbacks_lock);
5623
5624 /* See if this cb was called more quickly */
5625 if ((synced_txg - data->zcd_txg) < zc_min_txg_delay)
5626 zc_min_txg_delay = synced_txg - data->zcd_txg;
5627
5628 /* Remove our callback from the list */
5629 list_remove(&zcl.zcl_callbacks, data);
5630
5631 (void) mutex_exit(&zcl.zcl_callbacks_lock);
5632
5633 umem_free(data, sizeof (ztest_cb_data_t));
5634 }
5635
5636 /* Allocate and initialize callback data structure */
5637 static ztest_cb_data_t *
5638 ztest_create_cb_data(objset_t *os, uint64_t txg)
5639 {
5640 ztest_cb_data_t *cb_data;
5641
5642 cb_data = umem_zalloc(sizeof (ztest_cb_data_t), UMEM_NOFAIL);
5643
5644 cb_data->zcd_txg = txg;
5645 cb_data->zcd_spa = dmu_objset_spa(os);
5646 list_link_init(&cb_data->zcd_node);
5647
5648 return (cb_data);
5649 }
5650
5651 /*
5652 * Commit callback test.
5653 */
5654 void
5655 ztest_dmu_commit_callbacks(ztest_ds_t *zd, uint64_t id)
5656 {
5657 objset_t *os = zd->zd_os;
5658 ztest_od_t *od;
5659 dmu_tx_t *tx;
5660 ztest_cb_data_t *cb_data[3], *tmp_cb;
5661 uint64_t old_txg, txg;
5662 int i, error = 0;
5663
5664 od = umem_alloc(sizeof (ztest_od_t), UMEM_NOFAIL);
5665 ztest_od_init(od, id, FTAG, 0, DMU_OT_UINT64_OTHER, 0, 0, 0);
5666
5667 if (ztest_object_init(zd, od, sizeof (ztest_od_t), B_FALSE) != 0) {
5668 umem_free(od, sizeof (ztest_od_t));
5669 return;
5670 }
5671
5672 tx = dmu_tx_create(os);
5673
5674 cb_data[0] = ztest_create_cb_data(os, 0);
5675 dmu_tx_callback_register(tx, ztest_commit_callback, cb_data[0]);
5676
5677 dmu_tx_hold_write(tx, od->od_object, 0, sizeof (uint64_t));
5678
5679 /* Every once in a while, abort the transaction on purpose */
5680 if (ztest_random(100) == 0)
5681 error = -1;
5682
5683 if (!error)
5684 error = dmu_tx_assign(tx, TXG_NOWAIT);
5685
5686 txg = error ? 0 : dmu_tx_get_txg(tx);
5687
5688 cb_data[0]->zcd_txg = txg;
5689 cb_data[1] = ztest_create_cb_data(os, txg);
5690 dmu_tx_callback_register(tx, ztest_commit_callback, cb_data[1]);
5691
5692 if (error) {
5693 /*
5694 * It's not a strict requirement to call the registered
5695 * callbacks from inside dmu_tx_abort(), but that's what
5696 * it's supposed to happen in the current implementation
5697 * so we will check for that.
5698 */
5699 for (i = 0; i < 2; i++) {
5700 cb_data[i]->zcd_expected_err = ECANCELED;
5701 VERIFY(!cb_data[i]->zcd_called);
5702 }
5703
5704 dmu_tx_abort(tx);
5705
5706 for (i = 0; i < 2; i++) {
5707 VERIFY(cb_data[i]->zcd_called);
5708 umem_free(cb_data[i], sizeof (ztest_cb_data_t));
5709 }
5710
5711 umem_free(od, sizeof (ztest_od_t));
5712 return;
5713 }
5714
5715 cb_data[2] = ztest_create_cb_data(os, txg);
5716 dmu_tx_callback_register(tx, ztest_commit_callback, cb_data[2]);
5717
5718 /*
5719 * Read existing data to make sure there isn't a future leak.
5720 */
5721 VERIFY0(dmu_read(os, od->od_object, 0, sizeof (uint64_t),
5722 &old_txg, DMU_READ_PREFETCH));
5723
5724 if (old_txg > txg)
5725 fatal(B_FALSE,
5726 "future leak: got %"PRIu64", open txg is %"PRIu64"",
5727 old_txg, txg);
5728
5729 dmu_write(os, od->od_object, 0, sizeof (uint64_t), &txg, tx);
5730
5731 (void) mutex_enter(&zcl.zcl_callbacks_lock);
5732
5733 /*
5734 * Since commit callbacks don't have any ordering requirement and since
5735 * it is theoretically possible for a commit callback to be called
5736 * after an arbitrary amount of time has elapsed since its txg has been
5737 * synced, it is difficult to reliably determine whether a commit
5738 * callback hasn't been called due to high load or due to a flawed
5739 * implementation.
5740 *
5741 * In practice, we will assume that if after a certain number of txgs a
5742 * commit callback hasn't been called, then most likely there's an
5743 * implementation bug..
5744 */
5745 tmp_cb = list_head(&zcl.zcl_callbacks);
5746 if (tmp_cb != NULL &&
5747 tmp_cb->zcd_txg + ZTEST_COMMIT_CB_THRESH < txg) {
5748 fatal(B_FALSE,
5749 "Commit callback threshold exceeded, "
5750 "oldest txg: %"PRIu64", open txg: %"PRIu64"\n",
5751 tmp_cb->zcd_txg, txg);
5752 }
5753
5754 /*
5755 * Let's find the place to insert our callbacks.
5756 *
5757 * Even though the list is ordered by txg, it is possible for the
5758 * insertion point to not be the end because our txg may already be
5759 * quiescing at this point and other callbacks in the open txg
5760 * (from other objsets) may have sneaked in.
5761 */
5762 tmp_cb = list_tail(&zcl.zcl_callbacks);
5763 while (tmp_cb != NULL && tmp_cb->zcd_txg > txg)
5764 tmp_cb = list_prev(&zcl.zcl_callbacks, tmp_cb);
5765
5766 /* Add the 3 callbacks to the list */
5767 for (i = 0; i < 3; i++) {
5768 if (tmp_cb == NULL)
5769 list_insert_head(&zcl.zcl_callbacks, cb_data[i]);
5770 else
5771 list_insert_after(&zcl.zcl_callbacks, tmp_cb,
5772 cb_data[i]);
5773
5774 cb_data[i]->zcd_added = B_TRUE;
5775 VERIFY(!cb_data[i]->zcd_called);
5776
5777 tmp_cb = cb_data[i];
5778 }
5779
5780 zc_cb_counter += 3;
5781
5782 (void) mutex_exit(&zcl.zcl_callbacks_lock);
5783
5784 dmu_tx_commit(tx);
5785
5786 umem_free(od, sizeof (ztest_od_t));
5787 }
5788
5789 /*
5790 * Visit each object in the dataset. Verify that its properties
5791 * are consistent what was stored in the block tag when it was created,
5792 * and that its unused bonus buffer space has not been overwritten.
5793 */
5794 void
5795 ztest_verify_dnode_bt(ztest_ds_t *zd, uint64_t id)
5796 {
5797 (void) id;
5798 objset_t *os = zd->zd_os;
5799 uint64_t obj;
5800 int err = 0;
5801
5802 for (obj = 0; err == 0; err = dmu_object_next(os, &obj, FALSE, 0)) {
5803 ztest_block_tag_t *bt = NULL;
5804 dmu_object_info_t doi;
5805 dmu_buf_t *db;
5806
5807 ztest_object_lock(zd, obj, RL_READER);
5808 if (dmu_bonus_hold(os, obj, FTAG, &db) != 0) {
5809 ztest_object_unlock(zd, obj);
5810 continue;
5811 }
5812
5813 dmu_object_info_from_db(db, &doi);
5814 if (doi.doi_bonus_size >= sizeof (*bt))
5815 bt = ztest_bt_bonus(db);
5816
5817 if (bt && bt->bt_magic == BT_MAGIC) {
5818 ztest_bt_verify(bt, os, obj, doi.doi_dnodesize,
5819 bt->bt_offset, bt->bt_gen, bt->bt_txg,
5820 bt->bt_crtxg);
5821 ztest_verify_unused_bonus(db, bt, obj, os, bt->bt_gen);
5822 }
5823
5824 dmu_buf_rele(db, FTAG);
5825 ztest_object_unlock(zd, obj);
5826 }
5827 }
5828
5829 void
5830 ztest_dsl_prop_get_set(ztest_ds_t *zd, uint64_t id)
5831 {
5832 (void) id;
5833 zfs_prop_t proplist[] = {
5834 ZFS_PROP_CHECKSUM,
5835 ZFS_PROP_COMPRESSION,
5836 ZFS_PROP_COPIES,
5837 ZFS_PROP_DEDUP
5838 };
5839
5840 (void) pthread_rwlock_rdlock(&ztest_name_lock);
5841
5842 for (int p = 0; p < sizeof (proplist) / sizeof (proplist[0]); p++)
5843 (void) ztest_dsl_prop_set_uint64(zd->zd_name, proplist[p],
5844 ztest_random_dsl_prop(proplist[p]), (int)ztest_random(2));
5845
5846 VERIFY0(ztest_dsl_prop_set_uint64(zd->zd_name, ZFS_PROP_RECORDSIZE,
5847 ztest_random_blocksize(), (int)ztest_random(2)));
5848
5849 (void) pthread_rwlock_unlock(&ztest_name_lock);
5850 }
5851
5852 void
5853 ztest_spa_prop_get_set(ztest_ds_t *zd, uint64_t id)
5854 {
5855 (void) zd, (void) id;
5856 nvlist_t *props = NULL;
5857
5858 (void) pthread_rwlock_rdlock(&ztest_name_lock);
5859
5860 (void) ztest_spa_prop_set_uint64(ZPOOL_PROP_AUTOTRIM, ztest_random(2));
5861
5862 VERIFY0(spa_prop_get(ztest_spa, &props));
5863
5864 if (ztest_opts.zo_verbose >= 6)
5865 dump_nvlist(props, 4);
5866
5867 fnvlist_free(props);
5868
5869 (void) pthread_rwlock_unlock(&ztest_name_lock);
5870 }
5871
5872 static int
5873 user_release_one(const char *snapname, const char *holdname)
5874 {
5875 nvlist_t *snaps, *holds;
5876 int error;
5877
5878 snaps = fnvlist_alloc();
5879 holds = fnvlist_alloc();
5880 fnvlist_add_boolean(holds, holdname);
5881 fnvlist_add_nvlist(snaps, snapname, holds);
5882 fnvlist_free(holds);
5883 error = dsl_dataset_user_release(snaps, NULL);
5884 fnvlist_free(snaps);
5885 return (error);
5886 }
5887
5888 /*
5889 * Test snapshot hold/release and deferred destroy.
5890 */
5891 void
5892 ztest_dmu_snapshot_hold(ztest_ds_t *zd, uint64_t id)
5893 {
5894 int error;
5895 objset_t *os = zd->zd_os;
5896 objset_t *origin;
5897 char snapname[100];
5898 char fullname[100];
5899 char clonename[100];
5900 char tag[100];
5901 char osname[ZFS_MAX_DATASET_NAME_LEN];
5902 nvlist_t *holds;
5903
5904 (void) pthread_rwlock_rdlock(&ztest_name_lock);
5905
5906 dmu_objset_name(os, osname);
5907
5908 (void) snprintf(snapname, sizeof (snapname), "sh1_%"PRIu64"", id);
5909 (void) snprintf(fullname, sizeof (fullname), "%s@%s", osname, snapname);
5910 (void) snprintf(clonename, sizeof (clonename), "%s/ch1_%"PRIu64"",
5911 osname, id);
5912 (void) snprintf(tag, sizeof (tag), "tag_%"PRIu64"", id);
5913
5914 /*
5915 * Clean up from any previous run.
5916 */
5917 error = dsl_destroy_head(clonename);
5918 if (error != ENOENT)
5919 ASSERT0(error);
5920 error = user_release_one(fullname, tag);
5921 if (error != ESRCH && error != ENOENT)
5922 ASSERT0(error);
5923 error = dsl_destroy_snapshot(fullname, B_FALSE);
5924 if (error != ENOENT)
5925 ASSERT0(error);
5926
5927 /*
5928 * Create snapshot, clone it, mark snap for deferred destroy,
5929 * destroy clone, verify snap was also destroyed.
5930 */
5931 error = dmu_objset_snapshot_one(osname, snapname);
5932 if (error) {
5933 if (error == ENOSPC) {
5934 ztest_record_enospc("dmu_objset_snapshot");
5935 goto out;
5936 }
5937 fatal(B_FALSE, "dmu_objset_snapshot(%s) = %d", fullname, error);
5938 }
5939
5940 error = dmu_objset_clone(clonename, fullname);
5941 if (error) {
5942 if (error == ENOSPC) {
5943 ztest_record_enospc("dmu_objset_clone");
5944 goto out;
5945 }
5946 fatal(B_FALSE, "dmu_objset_clone(%s) = %d", clonename, error);
5947 }
5948
5949 error = dsl_destroy_snapshot(fullname, B_TRUE);
5950 if (error) {
5951 fatal(B_FALSE, "dsl_destroy_snapshot(%s, B_TRUE) = %d",
5952 fullname, error);
5953 }
5954
5955 error = dsl_destroy_head(clonename);
5956 if (error)
5957 fatal(B_FALSE, "dsl_destroy_head(%s) = %d", clonename, error);
5958
5959 error = dmu_objset_hold(fullname, FTAG, &origin);
5960 if (error != ENOENT)
5961 fatal(B_FALSE, "dmu_objset_hold(%s) = %d", fullname, error);
5962
5963 /*
5964 * Create snapshot, add temporary hold, verify that we can't
5965 * destroy a held snapshot, mark for deferred destroy,
5966 * release hold, verify snapshot was destroyed.
5967 */
5968 error = dmu_objset_snapshot_one(osname, snapname);
5969 if (error) {
5970 if (error == ENOSPC) {
5971 ztest_record_enospc("dmu_objset_snapshot");
5972 goto out;
5973 }
5974 fatal(B_FALSE, "dmu_objset_snapshot(%s) = %d", fullname, error);
5975 }
5976
5977 holds = fnvlist_alloc();
5978 fnvlist_add_string(holds, fullname, tag);
5979 error = dsl_dataset_user_hold(holds, 0, NULL);
5980 fnvlist_free(holds);
5981
5982 if (error == ENOSPC) {
5983 ztest_record_enospc("dsl_dataset_user_hold");
5984 goto out;
5985 } else if (error) {
5986 fatal(B_FALSE, "dsl_dataset_user_hold(%s, %s) = %u",
5987 fullname, tag, error);
5988 }
5989
5990 error = dsl_destroy_snapshot(fullname, B_FALSE);
5991 if (error != EBUSY) {
5992 fatal(B_FALSE, "dsl_destroy_snapshot(%s, B_FALSE) = %d",
5993 fullname, error);
5994 }
5995
5996 error = dsl_destroy_snapshot(fullname, B_TRUE);
5997 if (error) {
5998 fatal(B_FALSE, "dsl_destroy_snapshot(%s, B_TRUE) = %d",
5999 fullname, error);
6000 }
6001
6002 error = user_release_one(fullname, tag);
6003 if (error)
6004 fatal(B_FALSE, "user_release_one(%s, %s) = %d",
6005 fullname, tag, error);
6006
6007 VERIFY3U(dmu_objset_hold(fullname, FTAG, &origin), ==, ENOENT);
6008
6009 out:
6010 (void) pthread_rwlock_unlock(&ztest_name_lock);
6011 }
6012
6013 /*
6014 * Inject random faults into the on-disk data.
6015 */
6016 void
6017 ztest_fault_inject(ztest_ds_t *zd, uint64_t id)
6018 {
6019 (void) zd, (void) id;
6020 ztest_shared_t *zs = ztest_shared;
6021 spa_t *spa = ztest_spa;
6022 int fd;
6023 uint64_t offset;
6024 uint64_t leaves;
6025 uint64_t bad = 0x1990c0ffeedecadeull;
6026 uint64_t top, leaf;
6027 char *path0;
6028 char *pathrand;
6029 size_t fsize;
6030 int bshift = SPA_MAXBLOCKSHIFT + 2;
6031 int iters = 1000;
6032 int maxfaults;
6033 int mirror_save;
6034 vdev_t *vd0 = NULL;
6035 uint64_t guid0 = 0;
6036 boolean_t islog = B_FALSE;
6037
6038 path0 = umem_alloc(MAXPATHLEN, UMEM_NOFAIL);
6039 pathrand = umem_alloc(MAXPATHLEN, UMEM_NOFAIL);
6040
6041 mutex_enter(&ztest_vdev_lock);
6042
6043 /*
6044 * Device removal is in progress, fault injection must be disabled
6045 * until it completes and the pool is scrubbed. The fault injection
6046 * strategy for damaging blocks does not take in to account evacuated
6047 * blocks which may have already been damaged.
6048 */
6049 if (ztest_device_removal_active) {
6050 mutex_exit(&ztest_vdev_lock);
6051 goto out;
6052 }
6053
6054 maxfaults = MAXFAULTS(zs);
6055 leaves = MAX(zs->zs_mirrors, 1) * ztest_opts.zo_raid_children;
6056 mirror_save = zs->zs_mirrors;
6057 mutex_exit(&ztest_vdev_lock);
6058
6059 ASSERT3U(leaves, >=, 1);
6060
6061 /*
6062 * While ztest is running the number of leaves will not change. This
6063 * is critical for the fault injection logic as it determines where
6064 * errors can be safely injected such that they are always repairable.
6065 *
6066 * When restarting ztest a different number of leaves may be requested
6067 * which will shift the regions to be damaged. This is fine as long
6068 * as the pool has been scrubbed prior to using the new mapping.
6069 * Failure to do can result in non-repairable damage being injected.
6070 */
6071 if (ztest_pool_scrubbed == B_FALSE)
6072 goto out;
6073
6074 /*
6075 * Grab the name lock as reader. There are some operations
6076 * which don't like to have their vdevs changed while
6077 * they are in progress (i.e. spa_change_guid). Those
6078 * operations will have grabbed the name lock as writer.
6079 */
6080 (void) pthread_rwlock_rdlock(&ztest_name_lock);
6081
6082 /*
6083 * We need SCL_STATE here because we're going to look at vd0->vdev_tsd.
6084 */
6085 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER);
6086
6087 if (ztest_random(2) == 0) {
6088 /*
6089 * Inject errors on a normal data device or slog device.
6090 */
6091 top = ztest_random_vdev_top(spa, B_TRUE);
6092 leaf = ztest_random(leaves) + zs->zs_splits;
6093
6094 /*
6095 * Generate paths to the first leaf in this top-level vdev,
6096 * and to the random leaf we selected. We'll induce transient
6097 * write failures and random online/offline activity on leaf 0,
6098 * and we'll write random garbage to the randomly chosen leaf.
6099 */
6100 (void) snprintf(path0, MAXPATHLEN, ztest_dev_template,
6101 ztest_opts.zo_dir, ztest_opts.zo_pool,
6102 top * leaves + zs->zs_splits);
6103 (void) snprintf(pathrand, MAXPATHLEN, ztest_dev_template,
6104 ztest_opts.zo_dir, ztest_opts.zo_pool,
6105 top * leaves + leaf);
6106
6107 vd0 = vdev_lookup_by_path(spa->spa_root_vdev, path0);
6108 if (vd0 != NULL && vd0->vdev_top->vdev_islog)
6109 islog = B_TRUE;
6110
6111 /*
6112 * If the top-level vdev needs to be resilvered
6113 * then we only allow faults on the device that is
6114 * resilvering.
6115 */
6116 if (vd0 != NULL && maxfaults != 1 &&
6117 (!vdev_resilver_needed(vd0->vdev_top, NULL, NULL) ||
6118 vd0->vdev_resilver_txg != 0)) {
6119 /*
6120 * Make vd0 explicitly claim to be unreadable,
6121 * or unwritable, or reach behind its back
6122 * and close the underlying fd. We can do this if
6123 * maxfaults == 0 because we'll fail and reexecute,
6124 * and we can do it if maxfaults >= 2 because we'll
6125 * have enough redundancy. If maxfaults == 1, the
6126 * combination of this with injection of random data
6127 * corruption below exceeds the pool's fault tolerance.
6128 */
6129 vdev_file_t *vf = vd0->vdev_tsd;
6130
6131 zfs_dbgmsg("injecting fault to vdev %llu; maxfaults=%d",
6132 (long long)vd0->vdev_id, (int)maxfaults);
6133
6134 if (vf != NULL && ztest_random(3) == 0) {
6135 (void) close(vf->vf_file->f_fd);
6136 vf->vf_file->f_fd = -1;
6137 } else if (ztest_random(2) == 0) {
6138 vd0->vdev_cant_read = B_TRUE;
6139 } else {
6140 vd0->vdev_cant_write = B_TRUE;
6141 }
6142 guid0 = vd0->vdev_guid;
6143 }
6144 } else {
6145 /*
6146 * Inject errors on an l2cache device.
6147 */
6148 spa_aux_vdev_t *sav = &spa->spa_l2cache;
6149
6150 if (sav->sav_count == 0) {
6151 spa_config_exit(spa, SCL_STATE, FTAG);
6152 (void) pthread_rwlock_unlock(&ztest_name_lock);
6153 goto out;
6154 }
6155 vd0 = sav->sav_vdevs[ztest_random(sav->sav_count)];
6156 guid0 = vd0->vdev_guid;
6157 (void) strlcpy(path0, vd0->vdev_path, MAXPATHLEN);
6158 (void) strlcpy(pathrand, vd0->vdev_path, MAXPATHLEN);
6159
6160 leaf = 0;
6161 leaves = 1;
6162 maxfaults = INT_MAX; /* no limit on cache devices */
6163 }
6164
6165 spa_config_exit(spa, SCL_STATE, FTAG);
6166 (void) pthread_rwlock_unlock(&ztest_name_lock);
6167
6168 /*
6169 * If we can tolerate two or more faults, or we're dealing
6170 * with a slog, randomly online/offline vd0.
6171 */
6172 if ((maxfaults >= 2 || islog) && guid0 != 0) {
6173 if (ztest_random(10) < 6) {
6174 int flags = (ztest_random(2) == 0 ?
6175 ZFS_OFFLINE_TEMPORARY : 0);
6176
6177 /*
6178 * We have to grab the zs_name_lock as writer to
6179 * prevent a race between offlining a slog and
6180 * destroying a dataset. Offlining the slog will
6181 * grab a reference on the dataset which may cause
6182 * dsl_destroy_head() to fail with EBUSY thus
6183 * leaving the dataset in an inconsistent state.
6184 */
6185 if (islog)
6186 (void) pthread_rwlock_wrlock(&ztest_name_lock);
6187
6188 VERIFY3U(vdev_offline(spa, guid0, flags), !=, EBUSY);
6189
6190 if (islog)
6191 (void) pthread_rwlock_unlock(&ztest_name_lock);
6192 } else {
6193 /*
6194 * Ideally we would like to be able to randomly
6195 * call vdev_[on|off]line without holding locks
6196 * to force unpredictable failures but the side
6197 * effects of vdev_[on|off]line prevent us from
6198 * doing so. We grab the ztest_vdev_lock here to
6199 * prevent a race between injection testing and
6200 * aux_vdev removal.
6201 */
6202 mutex_enter(&ztest_vdev_lock);
6203 (void) vdev_online(spa, guid0, 0, NULL);
6204 mutex_exit(&ztest_vdev_lock);
6205 }
6206 }
6207
6208 if (maxfaults == 0)
6209 goto out;
6210
6211 /*
6212 * We have at least single-fault tolerance, so inject data corruption.
6213 */
6214 fd = open(pathrand, O_RDWR);
6215
6216 if (fd == -1) /* we hit a gap in the device namespace */
6217 goto out;
6218
6219 fsize = lseek(fd, 0, SEEK_END);
6220
6221 while (--iters != 0) {
6222 /*
6223 * The offset must be chosen carefully to ensure that
6224 * we do not inject a given logical block with errors
6225 * on two different leaf devices, because ZFS can not
6226 * tolerate that (if maxfaults==1).
6227 *
6228 * To achieve this we divide each leaf device into
6229 * chunks of size (# leaves * SPA_MAXBLOCKSIZE * 4).
6230 * Each chunk is further divided into error-injection
6231 * ranges (can accept errors) and clear ranges (we do
6232 * not inject errors in those). Each error-injection
6233 * range can accept errors only for a single leaf vdev.
6234 * Error-injection ranges are separated by clear ranges.
6235 *
6236 * For example, with 3 leaves, each chunk looks like:
6237 * 0 to 32M: injection range for leaf 0
6238 * 32M to 64M: clear range - no injection allowed
6239 * 64M to 96M: injection range for leaf 1
6240 * 96M to 128M: clear range - no injection allowed
6241 * 128M to 160M: injection range for leaf 2
6242 * 160M to 192M: clear range - no injection allowed
6243 *
6244 * Each clear range must be large enough such that a
6245 * single block cannot straddle it. This way a block
6246 * can't be a target in two different injection ranges
6247 * (on different leaf vdevs).
6248 */
6249 offset = ztest_random(fsize / (leaves << bshift)) *
6250 (leaves << bshift) + (leaf << bshift) +
6251 (ztest_random(1ULL << (bshift - 1)) & -8ULL);
6252
6253 /*
6254 * Only allow damage to the labels at one end of the vdev.
6255 *
6256 * If all labels are damaged, the device will be totally
6257 * inaccessible, which will result in loss of data,
6258 * because we also damage (parts of) the other side of
6259 * the mirror/raidz.
6260 *
6261 * Additionally, we will always have both an even and an
6262 * odd label, so that we can handle crashes in the
6263 * middle of vdev_config_sync().
6264 */
6265 if ((leaf & 1) == 0 && offset < VDEV_LABEL_START_SIZE)
6266 continue;
6267
6268 /*
6269 * The two end labels are stored at the "end" of the disk, but
6270 * the end of the disk (vdev_psize) is aligned to
6271 * sizeof (vdev_label_t).
6272 */
6273 uint64_t psize = P2ALIGN(fsize, sizeof (vdev_label_t));
6274 if ((leaf & 1) == 1 &&
6275 offset + sizeof (bad) > psize - VDEV_LABEL_END_SIZE)
6276 continue;
6277
6278 mutex_enter(&ztest_vdev_lock);
6279 if (mirror_save != zs->zs_mirrors) {
6280 mutex_exit(&ztest_vdev_lock);
6281 (void) close(fd);
6282 goto out;
6283 }
6284
6285 if (pwrite(fd, &bad, sizeof (bad), offset) != sizeof (bad))
6286 fatal(B_TRUE,
6287 "can't inject bad word at 0x%"PRIx64" in %s",
6288 offset, pathrand);
6289
6290 mutex_exit(&ztest_vdev_lock);
6291
6292 if (ztest_opts.zo_verbose >= 7)
6293 (void) printf("injected bad word into %s,"
6294 " offset 0x%"PRIx64"\n", pathrand, offset);
6295 }
6296
6297 (void) close(fd);
6298 out:
6299 umem_free(path0, MAXPATHLEN);
6300 umem_free(pathrand, MAXPATHLEN);
6301 }
6302
6303 /*
6304 * By design ztest will never inject uncorrectable damage in to the pool.
6305 * Issue a scrub, wait for it to complete, and verify there is never any
6306 * persistent damage.
6307 *
6308 * Only after a full scrub has been completed is it safe to start injecting
6309 * data corruption. See the comment in zfs_fault_inject().
6310 */
6311 static int
6312 ztest_scrub_impl(spa_t *spa)
6313 {
6314 int error = spa_scan(spa, POOL_SCAN_SCRUB);
6315 if (error)
6316 return (error);
6317
6318 while (dsl_scan_scrubbing(spa_get_dsl(spa)))
6319 txg_wait_synced(spa_get_dsl(spa), 0);
6320
6321 if (spa_approx_errlog_size(spa) > 0)
6322 return (ECKSUM);
6323
6324 ztest_pool_scrubbed = B_TRUE;
6325
6326 return (0);
6327 }
6328
6329 /*
6330 * Scrub the pool.
6331 */
6332 void
6333 ztest_scrub(ztest_ds_t *zd, uint64_t id)
6334 {
6335 (void) zd, (void) id;
6336 spa_t *spa = ztest_spa;
6337 int error;
6338
6339 /*
6340 * Scrub in progress by device removal.
6341 */
6342 if (ztest_device_removal_active)
6343 return;
6344
6345 /*
6346 * Start a scrub, wait a moment, then force a restart.
6347 */
6348 (void) spa_scan(spa, POOL_SCAN_SCRUB);
6349 (void) poll(NULL, 0, 100);
6350
6351 error = ztest_scrub_impl(spa);
6352 if (error == EBUSY)
6353 error = 0;
6354 ASSERT0(error);
6355 }
6356
6357 /*
6358 * Change the guid for the pool.
6359 */
6360 void
6361 ztest_reguid(ztest_ds_t *zd, uint64_t id)
6362 {
6363 (void) zd, (void) id;
6364 spa_t *spa = ztest_spa;
6365 uint64_t orig, load;
6366 int error;
6367
6368 if (ztest_opts.zo_mmp_test)
6369 return;
6370
6371 orig = spa_guid(spa);
6372 load = spa_load_guid(spa);
6373
6374 (void) pthread_rwlock_wrlock(&ztest_name_lock);
6375 error = spa_change_guid(spa);
6376 (void) pthread_rwlock_unlock(&ztest_name_lock);
6377
6378 if (error != 0)
6379 return;
6380
6381 if (ztest_opts.zo_verbose >= 4) {
6382 (void) printf("Changed guid old %"PRIu64" -> %"PRIu64"\n",
6383 orig, spa_guid(spa));
6384 }
6385
6386 VERIFY3U(orig, !=, spa_guid(spa));
6387 VERIFY3U(load, ==, spa_load_guid(spa));
6388 }
6389
6390 void
6391 ztest_blake3(ztest_ds_t *zd, uint64_t id)
6392 {
6393 (void) zd, (void) id;
6394 hrtime_t end = gethrtime() + NANOSEC;
6395 zio_cksum_salt_t salt;
6396 void *salt_ptr = &salt.zcs_bytes;
6397 struct abd *abd_data, *abd_meta;
6398 void *buf, *templ;
6399 int i, *ptr;
6400 uint32_t size;
6401 BLAKE3_CTX ctx;
6402
6403 size = ztest_random_blocksize();
6404 buf = umem_alloc(size, UMEM_NOFAIL);
6405 abd_data = abd_alloc(size, B_FALSE);
6406 abd_meta = abd_alloc(size, B_TRUE);
6407
6408 for (i = 0, ptr = buf; i < size / sizeof (*ptr); i++, ptr++)
6409 *ptr = ztest_random(UINT_MAX);
6410 memset(salt_ptr, 'A', 32);
6411
6412 abd_copy_from_buf_off(abd_data, buf, 0, size);
6413 abd_copy_from_buf_off(abd_meta, buf, 0, size);
6414
6415 while (gethrtime() <= end) {
6416 int run_count = 100;
6417 zio_cksum_t zc_ref1, zc_ref2;
6418 zio_cksum_t zc_res1, zc_res2;
6419
6420 void *ref1 = &zc_ref1;
6421 void *ref2 = &zc_ref2;
6422 void *res1 = &zc_res1;
6423 void *res2 = &zc_res2;
6424
6425 /* BLAKE3_KEY_LEN = 32 */
6426 VERIFY0(blake3_impl_setname("generic"));
6427 templ = abd_checksum_blake3_tmpl_init(&salt);
6428 Blake3_InitKeyed(&ctx, salt_ptr);
6429 Blake3_Update(&ctx, buf, size);
6430 Blake3_Final(&ctx, ref1);
6431 zc_ref2 = zc_ref1;
6432 ZIO_CHECKSUM_BSWAP(&zc_ref2);
6433 abd_checksum_blake3_tmpl_free(templ);
6434
6435 VERIFY0(blake3_impl_setname("cycle"));
6436 while (run_count-- > 0) {
6437
6438 /* Test current implementation */
6439 Blake3_InitKeyed(&ctx, salt_ptr);
6440 Blake3_Update(&ctx, buf, size);
6441 Blake3_Final(&ctx, res1);
6442 zc_res2 = zc_res1;
6443 ZIO_CHECKSUM_BSWAP(&zc_res2);
6444
6445 VERIFY0(memcmp(ref1, res1, 32));
6446 VERIFY0(memcmp(ref2, res2, 32));
6447
6448 /* Test ABD - data */
6449 templ = abd_checksum_blake3_tmpl_init(&salt);
6450 abd_checksum_blake3_native(abd_data, size,
6451 templ, &zc_res1);
6452 abd_checksum_blake3_byteswap(abd_data, size,
6453 templ, &zc_res2);
6454
6455 VERIFY0(memcmp(ref1, res1, 32));
6456 VERIFY0(memcmp(ref2, res2, 32));
6457
6458 /* Test ABD - metadata */
6459 abd_checksum_blake3_native(abd_meta, size,
6460 templ, &zc_res1);
6461 abd_checksum_blake3_byteswap(abd_meta, size,
6462 templ, &zc_res2);
6463 abd_checksum_blake3_tmpl_free(templ);
6464
6465 VERIFY0(memcmp(ref1, res1, 32));
6466 VERIFY0(memcmp(ref2, res2, 32));
6467
6468 }
6469 }
6470
6471 abd_free(abd_data);
6472 abd_free(abd_meta);
6473 umem_free(buf, size);
6474 }
6475
6476 void
6477 ztest_fletcher(ztest_ds_t *zd, uint64_t id)
6478 {
6479 (void) zd, (void) id;
6480 hrtime_t end = gethrtime() + NANOSEC;
6481
6482 while (gethrtime() <= end) {
6483 int run_count = 100;
6484 void *buf;
6485 struct abd *abd_data, *abd_meta;
6486 uint32_t size;
6487 int *ptr;
6488 int i;
6489 zio_cksum_t zc_ref;
6490 zio_cksum_t zc_ref_byteswap;
6491
6492 size = ztest_random_blocksize();
6493
6494 buf = umem_alloc(size, UMEM_NOFAIL);
6495 abd_data = abd_alloc(size, B_FALSE);
6496 abd_meta = abd_alloc(size, B_TRUE);
6497
6498 for (i = 0, ptr = buf; i < size / sizeof (*ptr); i++, ptr++)
6499 *ptr = ztest_random(UINT_MAX);
6500
6501 abd_copy_from_buf_off(abd_data, buf, 0, size);
6502 abd_copy_from_buf_off(abd_meta, buf, 0, size);
6503
6504 VERIFY0(fletcher_4_impl_set("scalar"));
6505 fletcher_4_native(buf, size, NULL, &zc_ref);
6506 fletcher_4_byteswap(buf, size, NULL, &zc_ref_byteswap);
6507
6508 VERIFY0(fletcher_4_impl_set("cycle"));
6509 while (run_count-- > 0) {
6510 zio_cksum_t zc;
6511 zio_cksum_t zc_byteswap;
6512
6513 fletcher_4_byteswap(buf, size, NULL, &zc_byteswap);
6514 fletcher_4_native(buf, size, NULL, &zc);
6515
6516 VERIFY0(memcmp(&zc, &zc_ref, sizeof (zc)));
6517 VERIFY0(memcmp(&zc_byteswap, &zc_ref_byteswap,
6518 sizeof (zc_byteswap)));
6519
6520 /* Test ABD - data */
6521 abd_fletcher_4_byteswap(abd_data, size, NULL,
6522 &zc_byteswap);
6523 abd_fletcher_4_native(abd_data, size, NULL, &zc);
6524
6525 VERIFY0(memcmp(&zc, &zc_ref, sizeof (zc)));
6526 VERIFY0(memcmp(&zc_byteswap, &zc_ref_byteswap,
6527 sizeof (zc_byteswap)));
6528
6529 /* Test ABD - metadata */
6530 abd_fletcher_4_byteswap(abd_meta, size, NULL,
6531 &zc_byteswap);
6532 abd_fletcher_4_native(abd_meta, size, NULL, &zc);
6533
6534 VERIFY0(memcmp(&zc, &zc_ref, sizeof (zc)));
6535 VERIFY0(memcmp(&zc_byteswap, &zc_ref_byteswap,
6536 sizeof (zc_byteswap)));
6537
6538 }
6539
6540 umem_free(buf, size);
6541 abd_free(abd_data);
6542 abd_free(abd_meta);
6543 }
6544 }
6545
6546 void
6547 ztest_fletcher_incr(ztest_ds_t *zd, uint64_t id)
6548 {
6549 (void) zd, (void) id;
6550 void *buf;
6551 size_t size;
6552 int *ptr;
6553 int i;
6554 zio_cksum_t zc_ref;
6555 zio_cksum_t zc_ref_bswap;
6556
6557 hrtime_t end = gethrtime() + NANOSEC;
6558
6559 while (gethrtime() <= end) {
6560 int run_count = 100;
6561
6562 size = ztest_random_blocksize();
6563 buf = umem_alloc(size, UMEM_NOFAIL);
6564
6565 for (i = 0, ptr = buf; i < size / sizeof (*ptr); i++, ptr++)
6566 *ptr = ztest_random(UINT_MAX);
6567
6568 VERIFY0(fletcher_4_impl_set("scalar"));
6569 fletcher_4_native(buf, size, NULL, &zc_ref);
6570 fletcher_4_byteswap(buf, size, NULL, &zc_ref_bswap);
6571
6572 VERIFY0(fletcher_4_impl_set("cycle"));
6573
6574 while (run_count-- > 0) {
6575 zio_cksum_t zc;
6576 zio_cksum_t zc_bswap;
6577 size_t pos = 0;
6578
6579 ZIO_SET_CHECKSUM(&zc, 0, 0, 0, 0);
6580 ZIO_SET_CHECKSUM(&zc_bswap, 0, 0, 0, 0);
6581
6582 while (pos < size) {
6583 size_t inc = 64 * ztest_random(size / 67);
6584 /* sometimes add few bytes to test non-simd */
6585 if (ztest_random(100) < 10)
6586 inc += P2ALIGN(ztest_random(64),
6587 sizeof (uint32_t));
6588
6589 if (inc > (size - pos))
6590 inc = size - pos;
6591
6592 fletcher_4_incremental_native(buf + pos, inc,
6593 &zc);
6594 fletcher_4_incremental_byteswap(buf + pos, inc,
6595 &zc_bswap);
6596
6597 pos += inc;
6598 }
6599
6600 VERIFY3U(pos, ==, size);
6601
6602 VERIFY(ZIO_CHECKSUM_EQUAL(zc, zc_ref));
6603 VERIFY(ZIO_CHECKSUM_EQUAL(zc_bswap, zc_ref_bswap));
6604
6605 /*
6606 * verify if incremental on the whole buffer is
6607 * equivalent to non-incremental version
6608 */
6609 ZIO_SET_CHECKSUM(&zc, 0, 0, 0, 0);
6610 ZIO_SET_CHECKSUM(&zc_bswap, 0, 0, 0, 0);
6611
6612 fletcher_4_incremental_native(buf, size, &zc);
6613 fletcher_4_incremental_byteswap(buf, size, &zc_bswap);
6614
6615 VERIFY(ZIO_CHECKSUM_EQUAL(zc, zc_ref));
6616 VERIFY(ZIO_CHECKSUM_EQUAL(zc_bswap, zc_ref_bswap));
6617 }
6618
6619 umem_free(buf, size);
6620 }
6621 }
6622
6623 static int
6624 ztest_set_global_vars(void)
6625 {
6626 for (size_t i = 0; i < ztest_opts.zo_gvars_count; i++) {
6627 char *kv = ztest_opts.zo_gvars[i];
6628 VERIFY3U(strlen(kv), <=, ZO_GVARS_MAX_ARGLEN);
6629 VERIFY3U(strlen(kv), >, 0);
6630 int err = set_global_var(kv);
6631 if (ztest_opts.zo_verbose > 0) {
6632 (void) printf("setting global var %s ... %s\n", kv,
6633 err ? "failed" : "ok");
6634 }
6635 if (err != 0) {
6636 (void) fprintf(stderr,
6637 "failed to set global var '%s'\n", kv);
6638 return (err);
6639 }
6640 }
6641 return (0);
6642 }
6643
6644 static char **
6645 ztest_global_vars_to_zdb_args(void)
6646 {
6647 char **args = calloc(2*ztest_opts.zo_gvars_count + 1, sizeof (char *));
6648 char **cur = args;
6649 if (args == NULL)
6650 return (NULL);
6651 for (size_t i = 0; i < ztest_opts.zo_gvars_count; i++) {
6652 *cur++ = (char *)"-o";
6653 *cur++ = ztest_opts.zo_gvars[i];
6654 }
6655 ASSERT3P(cur, ==, &args[2*ztest_opts.zo_gvars_count]);
6656 *cur = NULL;
6657 return (args);
6658 }
6659
6660 /* The end of strings is indicated by a NULL element */
6661 static char *
6662 join_strings(char **strings, const char *sep)
6663 {
6664 size_t totallen = 0;
6665 for (char **sp = strings; *sp != NULL; sp++) {
6666 totallen += strlen(*sp);
6667 totallen += strlen(sep);
6668 }
6669 if (totallen > 0) {
6670 ASSERT(totallen >= strlen(sep));
6671 totallen -= strlen(sep);
6672 }
6673
6674 size_t buflen = totallen + 1;
6675 char *o = umem_alloc(buflen, UMEM_NOFAIL); /* trailing 0 byte */
6676 o[0] = '\0';
6677 for (char **sp = strings; *sp != NULL; sp++) {
6678 size_t would;
6679 would = strlcat(o, *sp, buflen);
6680 VERIFY3U(would, <, buflen);
6681 if (*(sp+1) == NULL) {
6682 break;
6683 }
6684 would = strlcat(o, sep, buflen);
6685 VERIFY3U(would, <, buflen);
6686 }
6687 ASSERT3S(strlen(o), ==, totallen);
6688 return (o);
6689 }
6690
6691 static int
6692 ztest_check_path(char *path)
6693 {
6694 struct stat s;
6695 /* return true on success */
6696 return (!stat(path, &s));
6697 }
6698
6699 static void
6700 ztest_get_zdb_bin(char *bin, int len)
6701 {
6702 char *zdb_path;
6703 /*
6704 * Try to use $ZDB and in-tree zdb path. If not successful, just
6705 * let popen to search through PATH.
6706 */
6707 if ((zdb_path = getenv("ZDB"))) {
6708 strlcpy(bin, zdb_path, len); /* In env */
6709 if (!ztest_check_path(bin)) {
6710 ztest_dump_core = 0;
6711 fatal(B_TRUE, "invalid ZDB '%s'", bin);
6712 }
6713 return;
6714 }
6715
6716 VERIFY3P(realpath(getexecname(), bin), !=, NULL);
6717 if (strstr(bin, ".libs/ztest")) {
6718 strstr(bin, ".libs/ztest")[0] = '\0'; /* In-tree */
6719 strcat(bin, "zdb");
6720 if (ztest_check_path(bin))
6721 return;
6722 }
6723 strcpy(bin, "zdb");
6724 }
6725
6726 static vdev_t *
6727 ztest_random_concrete_vdev_leaf(vdev_t *vd)
6728 {
6729 if (vd == NULL)
6730 return (NULL);
6731
6732 if (vd->vdev_children == 0)
6733 return (vd);
6734
6735 vdev_t *eligible[vd->vdev_children];
6736 int eligible_idx = 0, i;
6737 for (i = 0; i < vd->vdev_children; i++) {
6738 vdev_t *cvd = vd->vdev_child[i];
6739 if (cvd->vdev_top->vdev_removing)
6740 continue;
6741 if (cvd->vdev_children > 0 ||
6742 (vdev_is_concrete(cvd) && !cvd->vdev_detached)) {
6743 eligible[eligible_idx++] = cvd;
6744 }
6745 }
6746 VERIFY3S(eligible_idx, >, 0);
6747
6748 uint64_t child_no = ztest_random(eligible_idx);
6749 return (ztest_random_concrete_vdev_leaf(eligible[child_no]));
6750 }
6751
6752 void
6753 ztest_initialize(ztest_ds_t *zd, uint64_t id)
6754 {
6755 (void) zd, (void) id;
6756 spa_t *spa = ztest_spa;
6757 int error = 0;
6758
6759 mutex_enter(&ztest_vdev_lock);
6760
6761 spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);
6762
6763 /* Random leaf vdev */
6764 vdev_t *rand_vd = ztest_random_concrete_vdev_leaf(spa->spa_root_vdev);
6765 if (rand_vd == NULL) {
6766 spa_config_exit(spa, SCL_VDEV, FTAG);
6767 mutex_exit(&ztest_vdev_lock);
6768 return;
6769 }
6770
6771 /*
6772 * The random vdev we've selected may change as soon as we
6773 * drop the spa_config_lock. We create local copies of things
6774 * we're interested in.
6775 */
6776 uint64_t guid = rand_vd->vdev_guid;
6777 char *path = strdup(rand_vd->vdev_path);
6778 boolean_t active = rand_vd->vdev_initialize_thread != NULL;
6779
6780 zfs_dbgmsg("vd %px, guid %llu", rand_vd, (u_longlong_t)guid);
6781 spa_config_exit(spa, SCL_VDEV, FTAG);
6782
6783 uint64_t cmd = ztest_random(POOL_INITIALIZE_FUNCS);
6784
6785 nvlist_t *vdev_guids = fnvlist_alloc();
6786 nvlist_t *vdev_errlist = fnvlist_alloc();
6787 fnvlist_add_uint64(vdev_guids, path, guid);
6788 error = spa_vdev_initialize(spa, vdev_guids, cmd, vdev_errlist);
6789 fnvlist_free(vdev_guids);
6790 fnvlist_free(vdev_errlist);
6791
6792 switch (cmd) {
6793 case POOL_INITIALIZE_CANCEL:
6794 if (ztest_opts.zo_verbose >= 4) {
6795 (void) printf("Cancel initialize %s", path);
6796 if (!active)
6797 (void) printf(" failed (no initialize active)");
6798 (void) printf("\n");
6799 }
6800 break;
6801 case POOL_INITIALIZE_START:
6802 if (ztest_opts.zo_verbose >= 4) {
6803 (void) printf("Start initialize %s", path);
6804 if (active && error == 0)
6805 (void) printf(" failed (already active)");
6806 else if (error != 0)
6807 (void) printf(" failed (error %d)", error);
6808 (void) printf("\n");
6809 }
6810 break;
6811 case POOL_INITIALIZE_SUSPEND:
6812 if (ztest_opts.zo_verbose >= 4) {
6813 (void) printf("Suspend initialize %s", path);
6814 if (!active)
6815 (void) printf(" failed (no initialize active)");
6816 (void) printf("\n");
6817 }
6818 break;
6819 }
6820 free(path);
6821 mutex_exit(&ztest_vdev_lock);
6822 }
6823
6824 void
6825 ztest_trim(ztest_ds_t *zd, uint64_t id)
6826 {
6827 (void) zd, (void) id;
6828 spa_t *spa = ztest_spa;
6829 int error = 0;
6830
6831 mutex_enter(&ztest_vdev_lock);
6832
6833 spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);
6834
6835 /* Random leaf vdev */
6836 vdev_t *rand_vd = ztest_random_concrete_vdev_leaf(spa->spa_root_vdev);
6837 if (rand_vd == NULL) {
6838 spa_config_exit(spa, SCL_VDEV, FTAG);
6839 mutex_exit(&ztest_vdev_lock);
6840 return;
6841 }
6842
6843 /*
6844 * The random vdev we've selected may change as soon as we
6845 * drop the spa_config_lock. We create local copies of things
6846 * we're interested in.
6847 */
6848 uint64_t guid = rand_vd->vdev_guid;
6849 char *path = strdup(rand_vd->vdev_path);
6850 boolean_t active = rand_vd->vdev_trim_thread != NULL;
6851
6852 zfs_dbgmsg("vd %p, guid %llu", rand_vd, (u_longlong_t)guid);
6853 spa_config_exit(spa, SCL_VDEV, FTAG);
6854
6855 uint64_t cmd = ztest_random(POOL_TRIM_FUNCS);
6856 uint64_t rate = 1 << ztest_random(30);
6857 boolean_t partial = (ztest_random(5) > 0);
6858 boolean_t secure = (ztest_random(5) > 0);
6859
6860 nvlist_t *vdev_guids = fnvlist_alloc();
6861 nvlist_t *vdev_errlist = fnvlist_alloc();
6862 fnvlist_add_uint64(vdev_guids, path, guid);
6863 error = spa_vdev_trim(spa, vdev_guids, cmd, rate, partial,
6864 secure, vdev_errlist);
6865 fnvlist_free(vdev_guids);
6866 fnvlist_free(vdev_errlist);
6867
6868 switch (cmd) {
6869 case POOL_TRIM_CANCEL:
6870 if (ztest_opts.zo_verbose >= 4) {
6871 (void) printf("Cancel TRIM %s", path);
6872 if (!active)
6873 (void) printf(" failed (no TRIM active)");
6874 (void) printf("\n");
6875 }
6876 break;
6877 case POOL_TRIM_START:
6878 if (ztest_opts.zo_verbose >= 4) {
6879 (void) printf("Start TRIM %s", path);
6880 if (active && error == 0)
6881 (void) printf(" failed (already active)");
6882 else if (error != 0)
6883 (void) printf(" failed (error %d)", error);
6884 (void) printf("\n");
6885 }
6886 break;
6887 case POOL_TRIM_SUSPEND:
6888 if (ztest_opts.zo_verbose >= 4) {
6889 (void) printf("Suspend TRIM %s", path);
6890 if (!active)
6891 (void) printf(" failed (no TRIM active)");
6892 (void) printf("\n");
6893 }
6894 break;
6895 }
6896 free(path);
6897 mutex_exit(&ztest_vdev_lock);
6898 }
6899
6900 /*
6901 * Verify pool integrity by running zdb.
6902 */
6903 static void
6904 ztest_run_zdb(const char *pool)
6905 {
6906 int status;
6907 char *bin;
6908 char *zdb;
6909 char *zbuf;
6910 const int len = MAXPATHLEN + MAXNAMELEN + 20;
6911 FILE *fp;
6912
6913 bin = umem_alloc(len, UMEM_NOFAIL);
6914 zdb = umem_alloc(len, UMEM_NOFAIL);
6915 zbuf = umem_alloc(1024, UMEM_NOFAIL);
6916
6917 ztest_get_zdb_bin(bin, len);
6918
6919 char **set_gvars_args = ztest_global_vars_to_zdb_args();
6920 if (set_gvars_args == NULL) {
6921 fatal(B_FALSE, "Failed to allocate memory in "
6922 "ztest_global_vars_to_zdb_args(). Cannot run zdb.\n");
6923 }
6924 char *set_gvars_args_joined = join_strings(set_gvars_args, " ");
6925 free(set_gvars_args);
6926
6927 size_t would = snprintf(zdb, len,
6928 "%s -bcc%s%s -G -d -Y -e -y %s -p %s %s",
6929 bin,
6930 ztest_opts.zo_verbose >= 3 ? "s" : "",
6931 ztest_opts.zo_verbose >= 4 ? "v" : "",
6932 set_gvars_args_joined,
6933 ztest_opts.zo_dir,
6934 pool);
6935 ASSERT3U(would, <, len);
6936
6937 umem_free(set_gvars_args_joined, strlen(set_gvars_args_joined) + 1);
6938
6939 if (ztest_opts.zo_verbose >= 5)
6940 (void) printf("Executing %s\n", zdb);
6941
6942 fp = popen(zdb, "r");
6943
6944 while (fgets(zbuf, 1024, fp) != NULL)
6945 if (ztest_opts.zo_verbose >= 3)
6946 (void) printf("%s", zbuf);
6947
6948 status = pclose(fp);
6949
6950 if (status == 0)
6951 goto out;
6952
6953 ztest_dump_core = 0;
6954 if (WIFEXITED(status))
6955 fatal(B_FALSE, "'%s' exit code %d", zdb, WEXITSTATUS(status));
6956 else
6957 fatal(B_FALSE, "'%s' died with signal %d",
6958 zdb, WTERMSIG(status));
6959 out:
6960 umem_free(bin, len);
6961 umem_free(zdb, len);
6962 umem_free(zbuf, 1024);
6963 }
6964
6965 static void
6966 ztest_walk_pool_directory(const char *header)
6967 {
6968 spa_t *spa = NULL;
6969
6970 if (ztest_opts.zo_verbose >= 6)
6971 (void) puts(header);
6972
6973 mutex_enter(&spa_namespace_lock);
6974 while ((spa = spa_next(spa)) != NULL)
6975 if (ztest_opts.zo_verbose >= 6)
6976 (void) printf("\t%s\n", spa_name(spa));
6977 mutex_exit(&spa_namespace_lock);
6978 }
6979
6980 static void
6981 ztest_spa_import_export(char *oldname, char *newname)
6982 {
6983 nvlist_t *config, *newconfig;
6984 uint64_t pool_guid;
6985 spa_t *spa;
6986 int error;
6987
6988 if (ztest_opts.zo_verbose >= 4) {
6989 (void) printf("import/export: old = %s, new = %s\n",
6990 oldname, newname);
6991 }
6992
6993 /*
6994 * Clean up from previous runs.
6995 */
6996 (void) spa_destroy(newname);
6997
6998 /*
6999 * Get the pool's configuration and guid.
7000 */
7001 VERIFY0(spa_open(oldname, &spa, FTAG));
7002
7003 /*
7004 * Kick off a scrub to tickle scrub/export races.
7005 */
7006 if (ztest_random(2) == 0)
7007 (void) spa_scan(spa, POOL_SCAN_SCRUB);
7008
7009 pool_guid = spa_guid(spa);
7010 spa_close(spa, FTAG);
7011
7012 ztest_walk_pool_directory("pools before export");
7013
7014 /*
7015 * Export it.
7016 */
7017 VERIFY0(spa_export(oldname, &config, B_FALSE, B_FALSE));
7018
7019 ztest_walk_pool_directory("pools after export");
7020
7021 /*
7022 * Try to import it.
7023 */
7024 newconfig = spa_tryimport(config);
7025 ASSERT3P(newconfig, !=, NULL);
7026 fnvlist_free(newconfig);
7027
7028 /*
7029 * Import it under the new name.
7030 */
7031 error = spa_import(newname, config, NULL, 0);
7032 if (error != 0) {
7033 dump_nvlist(config, 0);
7034 fatal(B_FALSE, "couldn't import pool %s as %s: error %u",
7035 oldname, newname, error);
7036 }
7037
7038 ztest_walk_pool_directory("pools after import");
7039
7040 /*
7041 * Try to import it again -- should fail with EEXIST.
7042 */
7043 VERIFY3U(EEXIST, ==, spa_import(newname, config, NULL, 0));
7044
7045 /*
7046 * Try to import it under a different name -- should fail with EEXIST.
7047 */
7048 VERIFY3U(EEXIST, ==, spa_import(oldname, config, NULL, 0));
7049
7050 /*
7051 * Verify that the pool is no longer visible under the old name.
7052 */
7053 VERIFY3U(ENOENT, ==, spa_open(oldname, &spa, FTAG));
7054
7055 /*
7056 * Verify that we can open and close the pool using the new name.
7057 */
7058 VERIFY0(spa_open(newname, &spa, FTAG));
7059 ASSERT3U(pool_guid, ==, spa_guid(spa));
7060 spa_close(spa, FTAG);
7061
7062 fnvlist_free(config);
7063 }
7064
7065 static void
7066 ztest_resume(spa_t *spa)
7067 {
7068 if (spa_suspended(spa) && ztest_opts.zo_verbose >= 6)
7069 (void) printf("resuming from suspended state\n");
7070 spa_vdev_state_enter(spa, SCL_NONE);
7071 vdev_clear(spa, NULL);
7072 (void) spa_vdev_state_exit(spa, NULL, 0);
7073 (void) zio_resume(spa);
7074 }
7075
7076 static __attribute__((noreturn)) void
7077 ztest_resume_thread(void *arg)
7078 {
7079 spa_t *spa = arg;
7080
7081 while (!ztest_exiting) {
7082 if (spa_suspended(spa))
7083 ztest_resume(spa);
7084 (void) poll(NULL, 0, 100);
7085
7086 /*
7087 * Periodically change the zfs_compressed_arc_enabled setting.
7088 */
7089 if (ztest_random(10) == 0)
7090 zfs_compressed_arc_enabled = ztest_random(2);
7091
7092 /*
7093 * Periodically change the zfs_abd_scatter_enabled setting.
7094 */
7095 if (ztest_random(10) == 0)
7096 zfs_abd_scatter_enabled = ztest_random(2);
7097 }
7098
7099 thread_exit();
7100 }
7101
7102 static __attribute__((noreturn)) void
7103 ztest_deadman_thread(void *arg)
7104 {
7105 ztest_shared_t *zs = arg;
7106 spa_t *spa = ztest_spa;
7107 hrtime_t delay, overdue, last_run = gethrtime();
7108
7109 delay = (zs->zs_thread_stop - zs->zs_thread_start) +
7110 MSEC2NSEC(zfs_deadman_synctime_ms);
7111
7112 while (!ztest_exiting) {
7113 /*
7114 * Wait for the delay timer while checking occasionally
7115 * if we should stop.
7116 */
7117 if (gethrtime() < last_run + delay) {
7118 (void) poll(NULL, 0, 1000);
7119 continue;
7120 }
7121
7122 /*
7123 * If the pool is suspended then fail immediately. Otherwise,
7124 * check to see if the pool is making any progress. If
7125 * vdev_deadman() discovers that there hasn't been any recent
7126 * I/Os then it will end up aborting the tests.
7127 */
7128 if (spa_suspended(spa) || spa->spa_root_vdev == NULL) {
7129 fatal(B_FALSE,
7130 "aborting test after %llu seconds because "
7131 "pool has transitioned to a suspended state.",
7132 (u_longlong_t)zfs_deadman_synctime_ms / 1000);
7133 }
7134 vdev_deadman(spa->spa_root_vdev, FTAG);
7135
7136 /*
7137 * If the process doesn't complete within a grace period of
7138 * zfs_deadman_synctime_ms over the expected finish time,
7139 * then it may be hung and is terminated.
7140 */
7141 overdue = zs->zs_proc_stop + MSEC2NSEC(zfs_deadman_synctime_ms);
7142 if (gethrtime() > overdue) {
7143 fatal(B_FALSE,
7144 "aborting test after %llu seconds because "
7145 "the process is overdue for termination.",
7146 (gethrtime() - zs->zs_proc_start) / NANOSEC);
7147 }
7148
7149 (void) printf("ztest has been running for %lld seconds\n",
7150 (gethrtime() - zs->zs_proc_start) / NANOSEC);
7151
7152 last_run = gethrtime();
7153 delay = MSEC2NSEC(zfs_deadman_checktime_ms);
7154 }
7155
7156 thread_exit();
7157 }
7158
7159 static void
7160 ztest_execute(int test, ztest_info_t *zi, uint64_t id)
7161 {
7162 ztest_ds_t *zd = &ztest_ds[id % ztest_opts.zo_datasets];
7163 ztest_shared_callstate_t *zc = ZTEST_GET_SHARED_CALLSTATE(test);
7164 hrtime_t functime = gethrtime();
7165 int i;
7166
7167 for (i = 0; i < zi->zi_iters; i++)
7168 zi->zi_func(zd, id);
7169
7170 functime = gethrtime() - functime;
7171
7172 atomic_add_64(&zc->zc_count, 1);
7173 atomic_add_64(&zc->zc_time, functime);
7174
7175 if (ztest_opts.zo_verbose >= 4)
7176 (void) printf("%6.2f sec in %s\n",
7177 (double)functime / NANOSEC, zi->zi_funcname);
7178 }
7179
7180 static __attribute__((noreturn)) void
7181 ztest_thread(void *arg)
7182 {
7183 int rand;
7184 uint64_t id = (uintptr_t)arg;
7185 ztest_shared_t *zs = ztest_shared;
7186 uint64_t call_next;
7187 hrtime_t now;
7188 ztest_info_t *zi;
7189 ztest_shared_callstate_t *zc;
7190
7191 while ((now = gethrtime()) < zs->zs_thread_stop) {
7192 /*
7193 * See if it's time to force a crash.
7194 */
7195 if (now > zs->zs_thread_kill)
7196 ztest_kill(zs);
7197
7198 /*
7199 * If we're getting ENOSPC with some regularity, stop.
7200 */
7201 if (zs->zs_enospc_count > 10)
7202 break;
7203
7204 /*
7205 * Pick a random function to execute.
7206 */
7207 rand = ztest_random(ZTEST_FUNCS);
7208 zi = &ztest_info[rand];
7209 zc = ZTEST_GET_SHARED_CALLSTATE(rand);
7210 call_next = zc->zc_next;
7211
7212 if (now >= call_next &&
7213 atomic_cas_64(&zc->zc_next, call_next, call_next +
7214 ztest_random(2 * zi->zi_interval[0] + 1)) == call_next) {
7215 ztest_execute(rand, zi, id);
7216 }
7217 }
7218
7219 thread_exit();
7220 }
7221
7222 static void
7223 ztest_dataset_name(char *dsname, const char *pool, int d)
7224 {
7225 (void) snprintf(dsname, ZFS_MAX_DATASET_NAME_LEN, "%s/ds_%d", pool, d);
7226 }
7227
7228 static void
7229 ztest_dataset_destroy(int d)
7230 {
7231 char name[ZFS_MAX_DATASET_NAME_LEN];
7232 int t;
7233
7234 ztest_dataset_name(name, ztest_opts.zo_pool, d);
7235
7236 if (ztest_opts.zo_verbose >= 3)
7237 (void) printf("Destroying %s to free up space\n", name);
7238
7239 /*
7240 * Cleanup any non-standard clones and snapshots. In general,
7241 * ztest thread t operates on dataset (t % zopt_datasets),
7242 * so there may be more than one thing to clean up.
7243 */
7244 for (t = d; t < ztest_opts.zo_threads;
7245 t += ztest_opts.zo_datasets)
7246 ztest_dsl_dataset_cleanup(name, t);
7247
7248 (void) dmu_objset_find(name, ztest_objset_destroy_cb, NULL,
7249 DS_FIND_SNAPSHOTS | DS_FIND_CHILDREN);
7250 }
7251
7252 static void
7253 ztest_dataset_dirobj_verify(ztest_ds_t *zd)
7254 {
7255 uint64_t usedobjs, dirobjs, scratch;
7256
7257 /*
7258 * ZTEST_DIROBJ is the object directory for the entire dataset.
7259 * Therefore, the number of objects in use should equal the
7260 * number of ZTEST_DIROBJ entries, +1 for ZTEST_DIROBJ itself.
7261 * If not, we have an object leak.
7262 *
7263 * Note that we can only check this in ztest_dataset_open(),
7264 * when the open-context and syncing-context values agree.
7265 * That's because zap_count() returns the open-context value,
7266 * while dmu_objset_space() returns the rootbp fill count.
7267 */
7268 VERIFY0(zap_count(zd->zd_os, ZTEST_DIROBJ, &dirobjs));
7269 dmu_objset_space(zd->zd_os, &scratch, &scratch, &usedobjs, &scratch);
7270 ASSERT3U(dirobjs + 1, ==, usedobjs);
7271 }
7272
7273 static int
7274 ztest_dataset_open(int d)
7275 {
7276 ztest_ds_t *zd = &ztest_ds[d];
7277 uint64_t committed_seq = ZTEST_GET_SHARED_DS(d)->zd_seq;
7278 objset_t *os;
7279 zilog_t *zilog;
7280 char name[ZFS_MAX_DATASET_NAME_LEN];
7281 int error;
7282
7283 ztest_dataset_name(name, ztest_opts.zo_pool, d);
7284
7285 (void) pthread_rwlock_rdlock(&ztest_name_lock);
7286
7287 error = ztest_dataset_create(name);
7288 if (error == ENOSPC) {
7289 (void) pthread_rwlock_unlock(&ztest_name_lock);
7290 ztest_record_enospc(FTAG);
7291 return (error);
7292 }
7293 ASSERT(error == 0 || error == EEXIST);
7294
7295 VERIFY0(ztest_dmu_objset_own(name, DMU_OST_OTHER, B_FALSE,
7296 B_TRUE, zd, &os));
7297 (void) pthread_rwlock_unlock(&ztest_name_lock);
7298
7299 ztest_zd_init(zd, ZTEST_GET_SHARED_DS(d), os);
7300
7301 zilog = zd->zd_zilog;
7302
7303 if (zilog->zl_header->zh_claim_lr_seq != 0 &&
7304 zilog->zl_header->zh_claim_lr_seq < committed_seq)
7305 fatal(B_FALSE, "missing log records: "
7306 "claimed %"PRIu64" < committed %"PRIu64"",
7307 zilog->zl_header->zh_claim_lr_seq, committed_seq);
7308
7309 ztest_dataset_dirobj_verify(zd);
7310
7311 zil_replay(os, zd, ztest_replay_vector);
7312
7313 ztest_dataset_dirobj_verify(zd);
7314
7315 if (ztest_opts.zo_verbose >= 6)
7316 (void) printf("%s replay %"PRIu64" blocks, "
7317 "%"PRIu64" records, seq %"PRIu64"\n",
7318 zd->zd_name,
7319 zilog->zl_parse_blk_count,
7320 zilog->zl_parse_lr_count,
7321 zilog->zl_replaying_seq);
7322
7323 zilog = zil_open(os, ztest_get_data, NULL);
7324
7325 if (zilog->zl_replaying_seq != 0 &&
7326 zilog->zl_replaying_seq < committed_seq)
7327 fatal(B_FALSE, "missing log records: "
7328 "replayed %"PRIu64" < committed %"PRIu64"",
7329 zilog->zl_replaying_seq, committed_seq);
7330
7331 return (0);
7332 }
7333
7334 static void
7335 ztest_dataset_close(int d)
7336 {
7337 ztest_ds_t *zd = &ztest_ds[d];
7338
7339 zil_close(zd->zd_zilog);
7340 dmu_objset_disown(zd->zd_os, B_TRUE, zd);
7341
7342 ztest_zd_fini(zd);
7343 }
7344
7345 static int
7346 ztest_replay_zil_cb(const char *name, void *arg)
7347 {
7348 (void) arg;
7349 objset_t *os;
7350 ztest_ds_t *zdtmp;
7351
7352 VERIFY0(ztest_dmu_objset_own(name, DMU_OST_ANY, B_TRUE,
7353 B_TRUE, FTAG, &os));
7354
7355 zdtmp = umem_alloc(sizeof (ztest_ds_t), UMEM_NOFAIL);
7356
7357 ztest_zd_init(zdtmp, NULL, os);
7358 zil_replay(os, zdtmp, ztest_replay_vector);
7359 ztest_zd_fini(zdtmp);
7360
7361 if (dmu_objset_zil(os)->zl_parse_lr_count != 0 &&
7362 ztest_opts.zo_verbose >= 6) {
7363 zilog_t *zilog = dmu_objset_zil(os);
7364
7365 (void) printf("%s replay %"PRIu64" blocks, "
7366 "%"PRIu64" records, seq %"PRIu64"\n",
7367 name,
7368 zilog->zl_parse_blk_count,
7369 zilog->zl_parse_lr_count,
7370 zilog->zl_replaying_seq);
7371 }
7372
7373 umem_free(zdtmp, sizeof (ztest_ds_t));
7374
7375 dmu_objset_disown(os, B_TRUE, FTAG);
7376 return (0);
7377 }
7378
7379 static void
7380 ztest_freeze(void)
7381 {
7382 ztest_ds_t *zd = &ztest_ds[0];
7383 spa_t *spa;
7384 int numloops = 0;
7385
7386 if (ztest_opts.zo_verbose >= 3)
7387 (void) printf("testing spa_freeze()...\n");
7388
7389 kernel_init(SPA_MODE_READ | SPA_MODE_WRITE);
7390 VERIFY0(spa_open(ztest_opts.zo_pool, &spa, FTAG));
7391 VERIFY0(ztest_dataset_open(0));
7392 ztest_spa = spa;
7393
7394 /*
7395 * Force the first log block to be transactionally allocated.
7396 * We have to do this before we freeze the pool -- otherwise
7397 * the log chain won't be anchored.
7398 */
7399 while (BP_IS_HOLE(&zd->zd_zilog->zl_header->zh_log)) {
7400 ztest_dmu_object_alloc_free(zd, 0);
7401 zil_commit(zd->zd_zilog, 0);
7402 }
7403
7404 txg_wait_synced(spa_get_dsl(spa), 0);
7405
7406 /*
7407 * Freeze the pool. This stops spa_sync() from doing anything,
7408 * so that the only way to record changes from now on is the ZIL.
7409 */
7410 spa_freeze(spa);
7411
7412 /*
7413 * Because it is hard to predict how much space a write will actually
7414 * require beforehand, we leave ourselves some fudge space to write over
7415 * capacity.
7416 */
7417 uint64_t capacity = metaslab_class_get_space(spa_normal_class(spa)) / 2;
7418
7419 /*
7420 * Run tests that generate log records but don't alter the pool config
7421 * or depend on DSL sync tasks (snapshots, objset create/destroy, etc).
7422 * We do a txg_wait_synced() after each iteration to force the txg
7423 * to increase well beyond the last synced value in the uberblock.
7424 * The ZIL should be OK with that.
7425 *
7426 * Run a random number of times less than zo_maxloops and ensure we do
7427 * not run out of space on the pool.
7428 */
7429 while (ztest_random(10) != 0 &&
7430 numloops++ < ztest_opts.zo_maxloops &&
7431 metaslab_class_get_alloc(spa_normal_class(spa)) < capacity) {
7432 ztest_od_t od;
7433 ztest_od_init(&od, 0, FTAG, 0, DMU_OT_UINT64_OTHER, 0, 0, 0);
7434 VERIFY0(ztest_object_init(zd, &od, sizeof (od), B_FALSE));
7435 ztest_io(zd, od.od_object,
7436 ztest_random(ZTEST_RANGE_LOCKS) << SPA_MAXBLOCKSHIFT);
7437 txg_wait_synced(spa_get_dsl(spa), 0);
7438 }
7439
7440 /*
7441 * Commit all of the changes we just generated.
7442 */
7443 zil_commit(zd->zd_zilog, 0);
7444 txg_wait_synced(spa_get_dsl(spa), 0);
7445
7446 /*
7447 * Close our dataset and close the pool.
7448 */
7449 ztest_dataset_close(0);
7450 spa_close(spa, FTAG);
7451 kernel_fini();
7452
7453 /*
7454 * Open and close the pool and dataset to induce log replay.
7455 */
7456 kernel_init(SPA_MODE_READ | SPA_MODE_WRITE);
7457 VERIFY0(spa_open(ztest_opts.zo_pool, &spa, FTAG));
7458 ASSERT3U(spa_freeze_txg(spa), ==, UINT64_MAX);
7459 VERIFY0(ztest_dataset_open(0));
7460 ztest_spa = spa;
7461 txg_wait_synced(spa_get_dsl(spa), 0);
7462 ztest_dataset_close(0);
7463 ztest_reguid(NULL, 0);
7464
7465 spa_close(spa, FTAG);
7466 kernel_fini();
7467 }
7468
7469 static void
7470 ztest_import_impl(void)
7471 {
7472 importargs_t args = { 0 };
7473 nvlist_t *cfg = NULL;
7474 int nsearch = 1;
7475 char *searchdirs[nsearch];
7476 int flags = ZFS_IMPORT_MISSING_LOG;
7477
7478 searchdirs[0] = ztest_opts.zo_dir;
7479 args.paths = nsearch;
7480 args.path = searchdirs;
7481 args.can_be_active = B_FALSE;
7482
7483 libpc_handle_t lpch = {
7484 .lpc_lib_handle = NULL,
7485 .lpc_ops = &libzpool_config_ops,
7486 .lpc_printerr = B_TRUE
7487 };
7488 VERIFY0(zpool_find_config(&lpch, ztest_opts.zo_pool, &cfg, &args));
7489 VERIFY0(spa_import(ztest_opts.zo_pool, cfg, NULL, flags));
7490 fnvlist_free(cfg);
7491 }
7492
7493 /*
7494 * Import a storage pool with the given name.
7495 */
7496 static void
7497 ztest_import(ztest_shared_t *zs)
7498 {
7499 spa_t *spa;
7500
7501 mutex_init(&ztest_vdev_lock, NULL, MUTEX_DEFAULT, NULL);
7502 mutex_init(&ztest_checkpoint_lock, NULL, MUTEX_DEFAULT, NULL);
7503 VERIFY0(pthread_rwlock_init(&ztest_name_lock, NULL));
7504
7505 kernel_init(SPA_MODE_READ | SPA_MODE_WRITE);
7506
7507 ztest_import_impl();
7508
7509 VERIFY0(spa_open(ztest_opts.zo_pool, &spa, FTAG));
7510 zs->zs_metaslab_sz =
7511 1ULL << spa->spa_root_vdev->vdev_child[0]->vdev_ms_shift;
7512 spa_close(spa, FTAG);
7513
7514 kernel_fini();
7515
7516 if (!ztest_opts.zo_mmp_test) {
7517 ztest_run_zdb(ztest_opts.zo_pool);
7518 ztest_freeze();
7519 ztest_run_zdb(ztest_opts.zo_pool);
7520 }
7521
7522 (void) pthread_rwlock_destroy(&ztest_name_lock);
7523 mutex_destroy(&ztest_vdev_lock);
7524 mutex_destroy(&ztest_checkpoint_lock);
7525 }
7526
7527 /*
7528 * Kick off threads to run tests on all datasets in parallel.
7529 */
7530 static void
7531 ztest_run(ztest_shared_t *zs)
7532 {
7533 spa_t *spa;
7534 objset_t *os;
7535 kthread_t *resume_thread, *deadman_thread;
7536 kthread_t **run_threads;
7537 uint64_t object;
7538 int error;
7539 int t, d;
7540
7541 ztest_exiting = B_FALSE;
7542
7543 /*
7544 * Initialize parent/child shared state.
7545 */
7546 mutex_init(&ztest_vdev_lock, NULL, MUTEX_DEFAULT, NULL);
7547 mutex_init(&ztest_checkpoint_lock, NULL, MUTEX_DEFAULT, NULL);
7548 VERIFY0(pthread_rwlock_init(&ztest_name_lock, NULL));
7549
7550 zs->zs_thread_start = gethrtime();
7551 zs->zs_thread_stop =
7552 zs->zs_thread_start + ztest_opts.zo_passtime * NANOSEC;
7553 zs->zs_thread_stop = MIN(zs->zs_thread_stop, zs->zs_proc_stop);
7554 zs->zs_thread_kill = zs->zs_thread_stop;
7555 if (ztest_random(100) < ztest_opts.zo_killrate) {
7556 zs->zs_thread_kill -=
7557 ztest_random(ztest_opts.zo_passtime * NANOSEC);
7558 }
7559
7560 mutex_init(&zcl.zcl_callbacks_lock, NULL, MUTEX_DEFAULT, NULL);
7561
7562 list_create(&zcl.zcl_callbacks, sizeof (ztest_cb_data_t),
7563 offsetof(ztest_cb_data_t, zcd_node));
7564
7565 /*
7566 * Open our pool. It may need to be imported first depending on
7567 * what tests were running when the previous pass was terminated.
7568 */
7569 kernel_init(SPA_MODE_READ | SPA_MODE_WRITE);
7570 error = spa_open(ztest_opts.zo_pool, &spa, FTAG);
7571 if (error) {
7572 VERIFY3S(error, ==, ENOENT);
7573 ztest_import_impl();
7574 VERIFY0(spa_open(ztest_opts.zo_pool, &spa, FTAG));
7575 zs->zs_metaslab_sz =
7576 1ULL << spa->spa_root_vdev->vdev_child[0]->vdev_ms_shift;
7577 }
7578
7579 metaslab_preload_limit = ztest_random(20) + 1;
7580 ztest_spa = spa;
7581
7582 VERIFY0(vdev_raidz_impl_set("cycle"));
7583
7584 dmu_objset_stats_t dds;
7585 VERIFY0(ztest_dmu_objset_own(ztest_opts.zo_pool,
7586 DMU_OST_ANY, B_TRUE, B_TRUE, FTAG, &os));
7587 dsl_pool_config_enter(dmu_objset_pool(os), FTAG);
7588 dmu_objset_fast_stat(os, &dds);
7589 dsl_pool_config_exit(dmu_objset_pool(os), FTAG);
7590 zs->zs_guid = dds.dds_guid;
7591 dmu_objset_disown(os, B_TRUE, FTAG);
7592
7593 /*
7594 * Create a thread to periodically resume suspended I/O.
7595 */
7596 resume_thread = thread_create(NULL, 0, ztest_resume_thread,
7597 spa, 0, NULL, TS_RUN | TS_JOINABLE, defclsyspri);
7598
7599 /*
7600 * Create a deadman thread and set to panic if we hang.
7601 */
7602 deadman_thread = thread_create(NULL, 0, ztest_deadman_thread,
7603 zs, 0, NULL, TS_RUN | TS_JOINABLE, defclsyspri);
7604
7605 spa->spa_deadman_failmode = ZIO_FAILURE_MODE_PANIC;
7606
7607 /*
7608 * Verify that we can safely inquire about any object,
7609 * whether it's allocated or not. To make it interesting,
7610 * we probe a 5-wide window around each power of two.
7611 * This hits all edge cases, including zero and the max.
7612 */
7613 for (t = 0; t < 64; t++) {
7614 for (d = -5; d <= 5; d++) {
7615 error = dmu_object_info(spa->spa_meta_objset,
7616 (1ULL << t) + d, NULL);
7617 ASSERT(error == 0 || error == ENOENT ||
7618 error == EINVAL);
7619 }
7620 }
7621
7622 /*
7623 * If we got any ENOSPC errors on the previous run, destroy something.
7624 */
7625 if (zs->zs_enospc_count != 0) {
7626 int d = ztest_random(ztest_opts.zo_datasets);
7627 ztest_dataset_destroy(d);
7628 }
7629 zs->zs_enospc_count = 0;
7630
7631 /*
7632 * If we were in the middle of ztest_device_removal() and were killed
7633 * we need to ensure the removal and scrub complete before running
7634 * any tests that check ztest_device_removal_active. The removal will
7635 * be restarted automatically when the spa is opened, but we need to
7636 * initiate the scrub manually if it is not already in progress. Note
7637 * that we always run the scrub whenever an indirect vdev exists
7638 * because we have no way of knowing for sure if ztest_device_removal()
7639 * fully completed its scrub before the pool was reimported.
7640 */
7641 if (spa->spa_removing_phys.sr_state == DSS_SCANNING ||
7642 spa->spa_removing_phys.sr_prev_indirect_vdev != -1) {
7643 while (spa->spa_removing_phys.sr_state == DSS_SCANNING)
7644 txg_wait_synced(spa_get_dsl(spa), 0);
7645
7646 error = ztest_scrub_impl(spa);
7647 if (error == EBUSY)
7648 error = 0;
7649 ASSERT0(error);
7650 }
7651
7652 run_threads = umem_zalloc(ztest_opts.zo_threads * sizeof (kthread_t *),
7653 UMEM_NOFAIL);
7654
7655 if (ztest_opts.zo_verbose >= 4)
7656 (void) printf("starting main threads...\n");
7657
7658 /*
7659 * Replay all logs of all datasets in the pool. This is primarily for
7660 * temporary datasets which wouldn't otherwise get replayed, which
7661 * can trigger failures when attempting to offline a SLOG in
7662 * ztest_fault_inject().
7663 */
7664 (void) dmu_objset_find(ztest_opts.zo_pool, ztest_replay_zil_cb,
7665 NULL, DS_FIND_CHILDREN);
7666
7667 /*
7668 * Kick off all the tests that run in parallel.
7669 */
7670 for (t = 0; t < ztest_opts.zo_threads; t++) {
7671 if (t < ztest_opts.zo_datasets && ztest_dataset_open(t) != 0) {
7672 umem_free(run_threads, ztest_opts.zo_threads *
7673 sizeof (kthread_t *));
7674 return;
7675 }
7676
7677 run_threads[t] = thread_create(NULL, 0, ztest_thread,
7678 (void *)(uintptr_t)t, 0, NULL, TS_RUN | TS_JOINABLE,
7679 defclsyspri);
7680 }
7681
7682 /*
7683 * Wait for all of the tests to complete.
7684 */
7685 for (t = 0; t < ztest_opts.zo_threads; t++)
7686 VERIFY0(thread_join(run_threads[t]));
7687
7688 /*
7689 * Close all datasets. This must be done after all the threads
7690 * are joined so we can be sure none of the datasets are in-use
7691 * by any of the threads.
7692 */
7693 for (t = 0; t < ztest_opts.zo_threads; t++) {
7694 if (t < ztest_opts.zo_datasets)
7695 ztest_dataset_close(t);
7696 }
7697
7698 txg_wait_synced(spa_get_dsl(spa), 0);
7699
7700 zs->zs_alloc = metaslab_class_get_alloc(spa_normal_class(spa));
7701 zs->zs_space = metaslab_class_get_space(spa_normal_class(spa));
7702
7703 umem_free(run_threads, ztest_opts.zo_threads * sizeof (kthread_t *));
7704
7705 /* Kill the resume and deadman threads */
7706 ztest_exiting = B_TRUE;
7707 VERIFY0(thread_join(resume_thread));
7708 VERIFY0(thread_join(deadman_thread));
7709 ztest_resume(spa);
7710
7711 /*
7712 * Right before closing the pool, kick off a bunch of async I/O;
7713 * spa_close() should wait for it to complete.
7714 */
7715 for (object = 1; object < 50; object++) {
7716 dmu_prefetch(spa->spa_meta_objset, object, 0, 0, 1ULL << 20,
7717 ZIO_PRIORITY_SYNC_READ);
7718 }
7719
7720 /* Verify that at least one commit cb was called in a timely fashion */
7721 if (zc_cb_counter >= ZTEST_COMMIT_CB_MIN_REG)
7722 VERIFY0(zc_min_txg_delay);
7723
7724 spa_close(spa, FTAG);
7725
7726 /*
7727 * Verify that we can loop over all pools.
7728 */
7729 mutex_enter(&spa_namespace_lock);
7730 for (spa = spa_next(NULL); spa != NULL; spa = spa_next(spa))
7731 if (ztest_opts.zo_verbose > 3)
7732 (void) printf("spa_next: found %s\n", spa_name(spa));
7733 mutex_exit(&spa_namespace_lock);
7734
7735 /*
7736 * Verify that we can export the pool and reimport it under a
7737 * different name.
7738 */
7739 if ((ztest_random(2) == 0) && !ztest_opts.zo_mmp_test) {
7740 char name[ZFS_MAX_DATASET_NAME_LEN];
7741 (void) snprintf(name, sizeof (name), "%s_import",
7742 ztest_opts.zo_pool);
7743 ztest_spa_import_export(ztest_opts.zo_pool, name);
7744 ztest_spa_import_export(name, ztest_opts.zo_pool);
7745 }
7746
7747 kernel_fini();
7748
7749 list_destroy(&zcl.zcl_callbacks);
7750 mutex_destroy(&zcl.zcl_callbacks_lock);
7751 (void) pthread_rwlock_destroy(&ztest_name_lock);
7752 mutex_destroy(&ztest_vdev_lock);
7753 mutex_destroy(&ztest_checkpoint_lock);
7754 }
7755
7756 static void
7757 print_time(hrtime_t t, char *timebuf)
7758 {
7759 hrtime_t s = t / NANOSEC;
7760 hrtime_t m = s / 60;
7761 hrtime_t h = m / 60;
7762 hrtime_t d = h / 24;
7763
7764 s -= m * 60;
7765 m -= h * 60;
7766 h -= d * 24;
7767
7768 timebuf[0] = '\0';
7769
7770 if (d)
7771 (void) sprintf(timebuf,
7772 "%llud%02lluh%02llum%02llus", d, h, m, s);
7773 else if (h)
7774 (void) sprintf(timebuf, "%lluh%02llum%02llus", h, m, s);
7775 else if (m)
7776 (void) sprintf(timebuf, "%llum%02llus", m, s);
7777 else
7778 (void) sprintf(timebuf, "%llus", s);
7779 }
7780
7781 static nvlist_t *
7782 make_random_props(void)
7783 {
7784 nvlist_t *props;
7785
7786 props = fnvlist_alloc();
7787
7788 if (ztest_random(2) == 0)
7789 return (props);
7790
7791 fnvlist_add_uint64(props,
7792 zpool_prop_to_name(ZPOOL_PROP_AUTOREPLACE), 1);
7793
7794 return (props);
7795 }
7796
7797 /*
7798 * Create a storage pool with the given name and initial vdev size.
7799 * Then test spa_freeze() functionality.
7800 */
7801 static void
7802 ztest_init(ztest_shared_t *zs)
7803 {
7804 spa_t *spa;
7805 nvlist_t *nvroot, *props;
7806 int i;
7807
7808 mutex_init(&ztest_vdev_lock, NULL, MUTEX_DEFAULT, NULL);
7809 mutex_init(&ztest_checkpoint_lock, NULL, MUTEX_DEFAULT, NULL);
7810 VERIFY0(pthread_rwlock_init(&ztest_name_lock, NULL));
7811
7812 kernel_init(SPA_MODE_READ | SPA_MODE_WRITE);
7813
7814 /*
7815 * Create the storage pool.
7816 */
7817 (void) spa_destroy(ztest_opts.zo_pool);
7818 ztest_shared->zs_vdev_next_leaf = 0;
7819 zs->zs_splits = 0;
7820 zs->zs_mirrors = ztest_opts.zo_mirrors;
7821 nvroot = make_vdev_root(NULL, NULL, NULL, ztest_opts.zo_vdev_size, 0,
7822 NULL, ztest_opts.zo_raid_children, zs->zs_mirrors, 1);
7823 props = make_random_props();
7824
7825 /*
7826 * We don't expect the pool to suspend unless maxfaults == 0,
7827 * in which case ztest_fault_inject() temporarily takes away
7828 * the only valid replica.
7829 */
7830 fnvlist_add_uint64(props,
7831 zpool_prop_to_name(ZPOOL_PROP_FAILUREMODE),
7832 MAXFAULTS(zs) ? ZIO_FAILURE_MODE_PANIC : ZIO_FAILURE_MODE_WAIT);
7833
7834 for (i = 0; i < SPA_FEATURES; i++) {
7835 char *buf;
7836
7837 if (!spa_feature_table[i].fi_zfs_mod_supported)
7838 continue;
7839
7840 /*
7841 * 75% chance of using the log space map feature. We want ztest
7842 * to exercise both the code paths that use the log space map
7843 * feature and the ones that don't.
7844 */
7845 if (i == SPA_FEATURE_LOG_SPACEMAP && ztest_random(4) == 0)
7846 continue;
7847
7848 VERIFY3S(-1, !=, asprintf(&buf, "feature@%s",
7849 spa_feature_table[i].fi_uname));
7850 fnvlist_add_uint64(props, buf, 0);
7851 free(buf);
7852 }
7853
7854 VERIFY0(spa_create(ztest_opts.zo_pool, nvroot, props, NULL, NULL));
7855 fnvlist_free(nvroot);
7856 fnvlist_free(props);
7857
7858 VERIFY0(spa_open(ztest_opts.zo_pool, &spa, FTAG));
7859 zs->zs_metaslab_sz =
7860 1ULL << spa->spa_root_vdev->vdev_child[0]->vdev_ms_shift;
7861 spa_close(spa, FTAG);
7862
7863 kernel_fini();
7864
7865 if (!ztest_opts.zo_mmp_test) {
7866 ztest_run_zdb(ztest_opts.zo_pool);
7867 ztest_freeze();
7868 ztest_run_zdb(ztest_opts.zo_pool);
7869 }
7870
7871 (void) pthread_rwlock_destroy(&ztest_name_lock);
7872 mutex_destroy(&ztest_vdev_lock);
7873 mutex_destroy(&ztest_checkpoint_lock);
7874 }
7875
7876 static void
7877 setup_data_fd(void)
7878 {
7879 static char ztest_name_data[] = "/tmp/ztest.data.XXXXXX";
7880
7881 ztest_fd_data = mkstemp(ztest_name_data);
7882 ASSERT3S(ztest_fd_data, >=, 0);
7883 (void) unlink(ztest_name_data);
7884 }
7885
7886 static int
7887 shared_data_size(ztest_shared_hdr_t *hdr)
7888 {
7889 int size;
7890
7891 size = hdr->zh_hdr_size;
7892 size += hdr->zh_opts_size;
7893 size += hdr->zh_size;
7894 size += hdr->zh_stats_size * hdr->zh_stats_count;
7895 size += hdr->zh_ds_size * hdr->zh_ds_count;
7896
7897 return (size);
7898 }
7899
7900 static void
7901 setup_hdr(void)
7902 {
7903 int size;
7904 ztest_shared_hdr_t *hdr;
7905
7906 hdr = (void *)mmap(0, P2ROUNDUP(sizeof (*hdr), getpagesize()),
7907 PROT_READ | PROT_WRITE, MAP_SHARED, ztest_fd_data, 0);
7908 ASSERT3P(hdr, !=, MAP_FAILED);
7909
7910 VERIFY0(ftruncate(ztest_fd_data, sizeof (ztest_shared_hdr_t)));
7911
7912 hdr->zh_hdr_size = sizeof (ztest_shared_hdr_t);
7913 hdr->zh_opts_size = sizeof (ztest_shared_opts_t);
7914 hdr->zh_size = sizeof (ztest_shared_t);
7915 hdr->zh_stats_size = sizeof (ztest_shared_callstate_t);
7916 hdr->zh_stats_count = ZTEST_FUNCS;
7917 hdr->zh_ds_size = sizeof (ztest_shared_ds_t);
7918 hdr->zh_ds_count = ztest_opts.zo_datasets;
7919
7920 size = shared_data_size(hdr);
7921 VERIFY0(ftruncate(ztest_fd_data, size));
7922
7923 (void) munmap((caddr_t)hdr, P2ROUNDUP(sizeof (*hdr), getpagesize()));
7924 }
7925
7926 static void
7927 setup_data(void)
7928 {
7929 int size, offset;
7930 ztest_shared_hdr_t *hdr;
7931 uint8_t *buf;
7932
7933 hdr = (void *)mmap(0, P2ROUNDUP(sizeof (*hdr), getpagesize()),
7934 PROT_READ, MAP_SHARED, ztest_fd_data, 0);
7935 ASSERT3P(hdr, !=, MAP_FAILED);
7936
7937 size = shared_data_size(hdr);
7938
7939 (void) munmap((caddr_t)hdr, P2ROUNDUP(sizeof (*hdr), getpagesize()));
7940 hdr = ztest_shared_hdr = (void *)mmap(0, P2ROUNDUP(size, getpagesize()),
7941 PROT_READ | PROT_WRITE, MAP_SHARED, ztest_fd_data, 0);
7942 ASSERT3P(hdr, !=, MAP_FAILED);
7943 buf = (uint8_t *)hdr;
7944
7945 offset = hdr->zh_hdr_size;
7946 ztest_shared_opts = (void *)&buf[offset];
7947 offset += hdr->zh_opts_size;
7948 ztest_shared = (void *)&buf[offset];
7949 offset += hdr->zh_size;
7950 ztest_shared_callstate = (void *)&buf[offset];
7951 offset += hdr->zh_stats_size * hdr->zh_stats_count;
7952 ztest_shared_ds = (void *)&buf[offset];
7953 }
7954
7955 static boolean_t
7956 exec_child(char *cmd, char *libpath, boolean_t ignorekill, int *statusp)
7957 {
7958 pid_t pid;
7959 int status;
7960 char *cmdbuf = NULL;
7961
7962 pid = fork();
7963
7964 if (cmd == NULL) {
7965 cmdbuf = umem_alloc(MAXPATHLEN, UMEM_NOFAIL);
7966 (void) strlcpy(cmdbuf, getexecname(), MAXPATHLEN);
7967 cmd = cmdbuf;
7968 }
7969
7970 if (pid == -1)
7971 fatal(B_TRUE, "fork failed");
7972
7973 if (pid == 0) { /* child */
7974 char fd_data_str[12];
7975
7976 VERIFY3S(11, >=,
7977 snprintf(fd_data_str, 12, "%d", ztest_fd_data));
7978 VERIFY0(setenv("ZTEST_FD_DATA", fd_data_str, 1));
7979
7980 if (libpath != NULL) {
7981 const char *curlp = getenv("LD_LIBRARY_PATH");
7982 if (curlp == NULL)
7983 VERIFY0(setenv("LD_LIBRARY_PATH", libpath, 1));
7984 else {
7985 char *newlp = NULL;
7986 VERIFY3S(-1, !=,
7987 asprintf(&newlp, "%s:%s", libpath, curlp));
7988 VERIFY0(setenv("LD_LIBRARY_PATH", newlp, 1));
7989 free(newlp);
7990 }
7991 }
7992 (void) execl(cmd, cmd, (char *)NULL);
7993 ztest_dump_core = B_FALSE;
7994 fatal(B_TRUE, "exec failed: %s", cmd);
7995 }
7996
7997 if (cmdbuf != NULL) {
7998 umem_free(cmdbuf, MAXPATHLEN);
7999 cmd = NULL;
8000 }
8001
8002 while (waitpid(pid, &status, 0) != pid)
8003 continue;
8004 if (statusp != NULL)
8005 *statusp = status;
8006
8007 if (WIFEXITED(status)) {
8008 if (WEXITSTATUS(status) != 0) {
8009 (void) fprintf(stderr, "child exited with code %d\n",
8010 WEXITSTATUS(status));
8011 exit(2);
8012 }
8013 return (B_FALSE);
8014 } else if (WIFSIGNALED(status)) {
8015 if (!ignorekill || WTERMSIG(status) != SIGKILL) {
8016 (void) fprintf(stderr, "child died with signal %d\n",
8017 WTERMSIG(status));
8018 exit(3);
8019 }
8020 return (B_TRUE);
8021 } else {
8022 (void) fprintf(stderr, "something strange happened to child\n");
8023 exit(4);
8024 }
8025 }
8026
8027 static void
8028 ztest_run_init(void)
8029 {
8030 int i;
8031
8032 ztest_shared_t *zs = ztest_shared;
8033
8034 /*
8035 * Blow away any existing copy of zpool.cache
8036 */
8037 (void) remove(spa_config_path);
8038
8039 if (ztest_opts.zo_init == 0) {
8040 if (ztest_opts.zo_verbose >= 1)
8041 (void) printf("Importing pool %s\n",
8042 ztest_opts.zo_pool);
8043 ztest_import(zs);
8044 return;
8045 }
8046
8047 /*
8048 * Create and initialize our storage pool.
8049 */
8050 for (i = 1; i <= ztest_opts.zo_init; i++) {
8051 memset(zs, 0, sizeof (*zs));
8052 if (ztest_opts.zo_verbose >= 3 &&
8053 ztest_opts.zo_init != 1) {
8054 (void) printf("ztest_init(), pass %d\n", i);
8055 }
8056 ztest_init(zs);
8057 }
8058 }
8059
8060 int
8061 main(int argc, char **argv)
8062 {
8063 int kills = 0;
8064 int iters = 0;
8065 int older = 0;
8066 int newer = 0;
8067 ztest_shared_t *zs;
8068 ztest_info_t *zi;
8069 ztest_shared_callstate_t *zc;
8070 char timebuf[100];
8071 char numbuf[NN_NUMBUF_SZ];
8072 char *cmd;
8073 boolean_t hasalt;
8074 int f, err;
8075 char *fd_data_str = getenv("ZTEST_FD_DATA");
8076 struct sigaction action;
8077
8078 (void) setvbuf(stdout, NULL, _IOLBF, 0);
8079
8080 dprintf_setup(&argc, argv);
8081 zfs_deadman_synctime_ms = 300000;
8082 zfs_deadman_checktime_ms = 30000;
8083 /*
8084 * As two-word space map entries may not come up often (especially
8085 * if pool and vdev sizes are small) we want to force at least some
8086 * of them so the feature get tested.
8087 */
8088 zfs_force_some_double_word_sm_entries = B_TRUE;
8089
8090 /*
8091 * Verify that even extensively damaged split blocks with many
8092 * segments can be reconstructed in a reasonable amount of time
8093 * when reconstruction is known to be possible.
8094 *
8095 * Note: the lower this value is, the more damage we inflict, and
8096 * the more time ztest spends in recovering that damage. We chose
8097 * to induce damage 1/100th of the time so recovery is tested but
8098 * not so frequently that ztest doesn't get to test other code paths.
8099 */
8100 zfs_reconstruct_indirect_damage_fraction = 100;
8101
8102 action.sa_handler = sig_handler;
8103 sigemptyset(&action.sa_mask);
8104 action.sa_flags = 0;
8105
8106 if (sigaction(SIGSEGV, &action, NULL) < 0) {
8107 (void) fprintf(stderr, "ztest: cannot catch SIGSEGV: %s.\n",
8108 strerror(errno));
8109 exit(EXIT_FAILURE);
8110 }
8111
8112 if (sigaction(SIGABRT, &action, NULL) < 0) {
8113 (void) fprintf(stderr, "ztest: cannot catch SIGABRT: %s.\n",
8114 strerror(errno));
8115 exit(EXIT_FAILURE);
8116 }
8117
8118 /*
8119 * Force random_get_bytes() to use /dev/urandom in order to prevent
8120 * ztest from needlessly depleting the system entropy pool.
8121 */
8122 random_path = "/dev/urandom";
8123 ztest_fd_rand = open(random_path, O_RDONLY | O_CLOEXEC);
8124 ASSERT3S(ztest_fd_rand, >=, 0);
8125
8126 if (!fd_data_str) {
8127 process_options(argc, argv);
8128
8129 setup_data_fd();
8130 setup_hdr();
8131 setup_data();
8132 memcpy(ztest_shared_opts, &ztest_opts,
8133 sizeof (*ztest_shared_opts));
8134 } else {
8135 ztest_fd_data = atoi(fd_data_str);
8136 setup_data();
8137 memcpy(&ztest_opts, ztest_shared_opts, sizeof (ztest_opts));
8138 }
8139 ASSERT3U(ztest_opts.zo_datasets, ==, ztest_shared_hdr->zh_ds_count);
8140
8141 err = ztest_set_global_vars();
8142 if (err != 0 && !fd_data_str) {
8143 /* error message done by ztest_set_global_vars */
8144 exit(EXIT_FAILURE);
8145 } else {
8146 /* children should not be spawned if setting gvars fails */
8147 VERIFY3S(err, ==, 0);
8148 }
8149
8150 /* Override location of zpool.cache */
8151 VERIFY3S(asprintf((char **)&spa_config_path, "%s/zpool.cache",
8152 ztest_opts.zo_dir), !=, -1);
8153
8154 ztest_ds = umem_alloc(ztest_opts.zo_datasets * sizeof (ztest_ds_t),
8155 UMEM_NOFAIL);
8156 zs = ztest_shared;
8157
8158 if (fd_data_str) {
8159 metaslab_force_ganging = ztest_opts.zo_metaslab_force_ganging;
8160 metaslab_df_alloc_threshold =
8161 zs->zs_metaslab_df_alloc_threshold;
8162
8163 if (zs->zs_do_init)
8164 ztest_run_init();
8165 else
8166 ztest_run(zs);
8167 exit(0);
8168 }
8169
8170 hasalt = (strlen(ztest_opts.zo_alt_ztest) != 0);
8171
8172 if (ztest_opts.zo_verbose >= 1) {
8173 (void) printf("%"PRIu64" vdevs, %d datasets, %d threads,"
8174 "%d %s disks, %"PRIu64" seconds...\n\n",
8175 ztest_opts.zo_vdevs,
8176 ztest_opts.zo_datasets,
8177 ztest_opts.zo_threads,
8178 ztest_opts.zo_raid_children,
8179 ztest_opts.zo_raid_type,
8180 ztest_opts.zo_time);
8181 }
8182
8183 cmd = umem_alloc(MAXNAMELEN, UMEM_NOFAIL);
8184 (void) strlcpy(cmd, getexecname(), MAXNAMELEN);
8185
8186 zs->zs_do_init = B_TRUE;
8187 if (strlen(ztest_opts.zo_alt_ztest) != 0) {
8188 if (ztest_opts.zo_verbose >= 1) {
8189 (void) printf("Executing older ztest for "
8190 "initialization: %s\n", ztest_opts.zo_alt_ztest);
8191 }
8192 VERIFY(!exec_child(ztest_opts.zo_alt_ztest,
8193 ztest_opts.zo_alt_libpath, B_FALSE, NULL));
8194 } else {
8195 VERIFY(!exec_child(NULL, NULL, B_FALSE, NULL));
8196 }
8197 zs->zs_do_init = B_FALSE;
8198
8199 zs->zs_proc_start = gethrtime();
8200 zs->zs_proc_stop = zs->zs_proc_start + ztest_opts.zo_time * NANOSEC;
8201
8202 for (f = 0; f < ZTEST_FUNCS; f++) {
8203 zi = &ztest_info[f];
8204 zc = ZTEST_GET_SHARED_CALLSTATE(f);
8205 if (zs->zs_proc_start + zi->zi_interval[0] > zs->zs_proc_stop)
8206 zc->zc_next = UINT64_MAX;
8207 else
8208 zc->zc_next = zs->zs_proc_start +
8209 ztest_random(2 * zi->zi_interval[0] + 1);
8210 }
8211
8212 /*
8213 * Run the tests in a loop. These tests include fault injection
8214 * to verify that self-healing data works, and forced crashes
8215 * to verify that we never lose on-disk consistency.
8216 */
8217 while (gethrtime() < zs->zs_proc_stop) {
8218 int status;
8219 boolean_t killed;
8220
8221 /*
8222 * Initialize the workload counters for each function.
8223 */
8224 for (f = 0; f < ZTEST_FUNCS; f++) {
8225 zc = ZTEST_GET_SHARED_CALLSTATE(f);
8226 zc->zc_count = 0;
8227 zc->zc_time = 0;
8228 }
8229
8230 /* Set the allocation switch size */
8231 zs->zs_metaslab_df_alloc_threshold =
8232 ztest_random(zs->zs_metaslab_sz / 4) + 1;
8233
8234 if (!hasalt || ztest_random(2) == 0) {
8235 if (hasalt && ztest_opts.zo_verbose >= 1) {
8236 (void) printf("Executing newer ztest: %s\n",
8237 cmd);
8238 }
8239 newer++;
8240 killed = exec_child(cmd, NULL, B_TRUE, &status);
8241 } else {
8242 if (hasalt && ztest_opts.zo_verbose >= 1) {
8243 (void) printf("Executing older ztest: %s\n",
8244 ztest_opts.zo_alt_ztest);
8245 }
8246 older++;
8247 killed = exec_child(ztest_opts.zo_alt_ztest,
8248 ztest_opts.zo_alt_libpath, B_TRUE, &status);
8249 }
8250
8251 if (killed)
8252 kills++;
8253 iters++;
8254
8255 if (ztest_opts.zo_verbose >= 1) {
8256 hrtime_t now = gethrtime();
8257
8258 now = MIN(now, zs->zs_proc_stop);
8259 print_time(zs->zs_proc_stop - now, timebuf);
8260 nicenum(zs->zs_space, numbuf, sizeof (numbuf));
8261
8262 (void) printf("Pass %3d, %8s, %3"PRIu64" ENOSPC, "
8263 "%4.1f%% of %5s used, %3.0f%% done, %8s to go\n",
8264 iters,
8265 WIFEXITED(status) ? "Complete" : "SIGKILL",
8266 zs->zs_enospc_count,
8267 100.0 * zs->zs_alloc / zs->zs_space,
8268 numbuf,
8269 100.0 * (now - zs->zs_proc_start) /
8270 (ztest_opts.zo_time * NANOSEC), timebuf);
8271 }
8272
8273 if (ztest_opts.zo_verbose >= 2) {
8274 (void) printf("\nWorkload summary:\n\n");
8275 (void) printf("%7s %9s %s\n",
8276 "Calls", "Time", "Function");
8277 (void) printf("%7s %9s %s\n",
8278 "-----", "----", "--------");
8279 for (f = 0; f < ZTEST_FUNCS; f++) {
8280 zi = &ztest_info[f];
8281 zc = ZTEST_GET_SHARED_CALLSTATE(f);
8282 print_time(zc->zc_time, timebuf);
8283 (void) printf("%7"PRIu64" %9s %s\n",
8284 zc->zc_count, timebuf,
8285 zi->zi_funcname);
8286 }
8287 (void) printf("\n");
8288 }
8289
8290 if (!ztest_opts.zo_mmp_test)
8291 ztest_run_zdb(ztest_opts.zo_pool);
8292 }
8293
8294 if (ztest_opts.zo_verbose >= 1) {
8295 if (hasalt) {
8296 (void) printf("%d runs of older ztest: %s\n", older,
8297 ztest_opts.zo_alt_ztest);
8298 (void) printf("%d runs of newer ztest: %s\n", newer,
8299 cmd);
8300 }
8301 (void) printf("%d killed, %d completed, %.0f%% kill rate\n",
8302 kills, iters - kills, (100.0 * kills) / MAX(1, iters));
8303 }
8304
8305 umem_free(cmd, MAXNAMELEN);
8306
8307 return (0);
8308 }
OpenZFS on Linux and FreeBSD