?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
OpenZFS 7003 - zap_lockdir() should tag hold
[zfs.git] / module / zfs / dbuf.c
blobd89d030d34dfd845a1383e69d2ea997cd4217c61
1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
24 * Copyright (c) 2012, 2015 by Delphix. All rights reserved.
25 * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
26 * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
27 */
28
29 #include <sys/zfs_context.h>
30 #include <sys/arc.h>
31 #include <sys/dmu.h>
32 #include <sys/dmu_send.h>
33 #include <sys/dmu_impl.h>
34 #include <sys/dbuf.h>
35 #include <sys/dmu_objset.h>
36 #include <sys/dsl_dataset.h>
37 #include <sys/dsl_dir.h>
38 #include <sys/dmu_tx.h>
39 #include <sys/spa.h>
40 #include <sys/zio.h>
41 #include <sys/dmu_zfetch.h>
42 #include <sys/sa.h>
43 #include <sys/sa_impl.h>
44 #include <sys/zfeature.h>
45 #include <sys/blkptr.h>
46 #include <sys/range_tree.h>
47 #include <sys/trace_dbuf.h>
48
49 struct dbuf_hold_impl_data {
50 /* Function arguments */
51 dnode_t *dh_dn;
52 uint8_t dh_level;
53 uint64_t dh_blkid;
54 boolean_t dh_fail_sparse;
55 boolean_t dh_fail_uncached;
56 void *dh_tag;
57 dmu_buf_impl_t **dh_dbp;
58 /* Local variables */
59 dmu_buf_impl_t *dh_db;
60 dmu_buf_impl_t *dh_parent;
61 blkptr_t *dh_bp;
62 int dh_err;
63 dbuf_dirty_record_t *dh_dr;
64 arc_buf_contents_t dh_type;
65 int dh_depth;
66 };
67
68 static void __dbuf_hold_impl_init(struct dbuf_hold_impl_data *dh,
69 dnode_t *dn, uint8_t level, uint64_t blkid, boolean_t fail_sparse,
70 boolean_t fail_uncached,
71 void *tag, dmu_buf_impl_t **dbp, int depth);
72 static int __dbuf_hold_impl(struct dbuf_hold_impl_data *dh);
73
74 /*
75 * Number of times that zfs_free_range() took the slow path while doing
76 * a zfs receive. A nonzero value indicates a potential performance problem.
77 */
78 uint64_t zfs_free_range_recv_miss;
79
80 static void dbuf_destroy(dmu_buf_impl_t *db);
81 static boolean_t dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx);
82 static void dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx);
83
84 #ifndef __lint
85 extern inline void dmu_buf_init_user(dmu_buf_user_t *dbu,
86 dmu_buf_evict_func_t *evict_func, dmu_buf_t **clear_on_evict_dbufp);
87 #endif /* ! __lint */
88
89 /*
90 * Global data structures and functions for the dbuf cache.
91 */
92 static kmem_cache_t *dbuf_cache;
93 static taskq_t *dbu_evict_taskq;
94
95 /* ARGSUSED */
96 static int
97 dbuf_cons(void *vdb, void *unused, int kmflag)
98 {
99 dmu_buf_impl_t *db = vdb;
100 bzero(db, sizeof (dmu_buf_impl_t));
101
102 mutex_init(&db->db_mtx, NULL, MUTEX_DEFAULT, NULL);
103 cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL);
104 refcount_create(&db->db_holds);
105
106 return (0);
107 }
108
109 /* ARGSUSED */
110 static void
111 dbuf_dest(void *vdb, void *unused)
112 {
113 dmu_buf_impl_t *db = vdb;
114 mutex_destroy(&db->db_mtx);
115 cv_destroy(&db->db_changed);
116 refcount_destroy(&db->db_holds);
117 }
118
119 /*
120 * dbuf hash table routines
121 */
122 static dbuf_hash_table_t dbuf_hash_table;
123
124 static uint64_t dbuf_hash_count;
125
126 static uint64_t
127 dbuf_hash(void *os, uint64_t obj, uint8_t lvl, uint64_t blkid)
128 {
129 uintptr_t osv = (uintptr_t)os;
130 uint64_t crc = -1ULL;
131
132 ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY);
133 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (lvl)) & 0xFF];
134 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (osv >> 6)) & 0xFF];
135 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 0)) & 0xFF];
136 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 8)) & 0xFF];
137 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 0)) & 0xFF];
138 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 8)) & 0xFF];
139
140 crc ^= (osv>>14) ^ (obj>>16) ^ (blkid>>16);
141
142 return (crc);
143 }
144
145 #define DBUF_HASH(os, obj, level, blkid) dbuf_hash(os, obj, level, blkid);
146
147 #define DBUF_EQUAL(dbuf, os, obj, level, blkid) \
148 ((dbuf)->db.db_object == (obj) && \
149 (dbuf)->db_objset == (os) && \
150 (dbuf)->db_level == (level) && \
151 (dbuf)->db_blkid == (blkid))
152
153 dmu_buf_impl_t *
154 dbuf_find(objset_t *os, uint64_t obj, uint8_t level, uint64_t blkid)
155 {
156 dbuf_hash_table_t *h = &dbuf_hash_table;
157 uint64_t hv;
158 uint64_t idx;
159 dmu_buf_impl_t *db;
160
161 hv = DBUF_HASH(os, obj, level, blkid);
162 idx = hv & h->hash_table_mask;
163
164 mutex_enter(DBUF_HASH_MUTEX(h, idx));
165 for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) {
166 if (DBUF_EQUAL(db, os, obj, level, blkid)) {
167 mutex_enter(&db->db_mtx);
168 if (db->db_state != DB_EVICTING) {
169 mutex_exit(DBUF_HASH_MUTEX(h, idx));
170 return (db);
171 }
172 mutex_exit(&db->db_mtx);
173 }
174 }
175 mutex_exit(DBUF_HASH_MUTEX(h, idx));
176 return (NULL);
177 }
178
179 static dmu_buf_impl_t *
180 dbuf_find_bonus(objset_t *os, uint64_t object)
181 {
182 dnode_t *dn;
183 dmu_buf_impl_t *db = NULL;
184
185 if (dnode_hold(os, object, FTAG, &dn) == 0) {
186 rw_enter(&dn->dn_struct_rwlock, RW_READER);
187 if (dn->dn_bonus != NULL) {
188 db = dn->dn_bonus;
189 mutex_enter(&db->db_mtx);
190 }
191 rw_exit(&dn->dn_struct_rwlock);
192 dnode_rele(dn, FTAG);
193 }
194 return (db);
195 }
196
197 /*
198 * Insert an entry into the hash table. If there is already an element
199 * equal to elem in the hash table, then the already existing element
200 * will be returned and the new element will not be inserted.
201 * Otherwise returns NULL.
202 */
203 static dmu_buf_impl_t *
204 dbuf_hash_insert(dmu_buf_impl_t *db)
205 {
206 dbuf_hash_table_t *h = &dbuf_hash_table;
207 objset_t *os = db->db_objset;
208 uint64_t obj = db->db.db_object;
209 int level = db->db_level;
210 uint64_t blkid, hv, idx;
211 dmu_buf_impl_t *dbf;
212
213 blkid = db->db_blkid;
214 hv = DBUF_HASH(os, obj, level, blkid);
215 idx = hv & h->hash_table_mask;
216
217 mutex_enter(DBUF_HASH_MUTEX(h, idx));
218 for (dbf = h->hash_table[idx]; dbf != NULL; dbf = dbf->db_hash_next) {
219 if (DBUF_EQUAL(dbf, os, obj, level, blkid)) {
220 mutex_enter(&dbf->db_mtx);
221 if (dbf->db_state != DB_EVICTING) {
222 mutex_exit(DBUF_HASH_MUTEX(h, idx));
223 return (dbf);
224 }
225 mutex_exit(&dbf->db_mtx);
226 }
227 }
228
229 mutex_enter(&db->db_mtx);
230 db->db_hash_next = h->hash_table[idx];
231 h->hash_table[idx] = db;
232 mutex_exit(DBUF_HASH_MUTEX(h, idx));
233 atomic_inc_64(&dbuf_hash_count);
234
235 return (NULL);
236 }
237
238 /*
239 * Remove an entry from the hash table. It must be in the EVICTING state.
240 */
241 static void
242 dbuf_hash_remove(dmu_buf_impl_t *db)
243 {
244 dbuf_hash_table_t *h = &dbuf_hash_table;
245 uint64_t hv, idx;
246 dmu_buf_impl_t *dbf, **dbp;
247
248 hv = DBUF_HASH(db->db_objset, db->db.db_object,
249 db->db_level, db->db_blkid);
250 idx = hv & h->hash_table_mask;
251
252 /*
253 * We musn't hold db_mtx to maintain lock ordering:
254 * DBUF_HASH_MUTEX > db_mtx.
255 */
256 ASSERT(refcount_is_zero(&db->db_holds));
257 ASSERT(db->db_state == DB_EVICTING);
258 ASSERT(!MUTEX_HELD(&db->db_mtx));
259
260 mutex_enter(DBUF_HASH_MUTEX(h, idx));
261 dbp = &h->hash_table[idx];
262 while ((dbf = *dbp) != db) {
263 dbp = &dbf->db_hash_next;
264 ASSERT(dbf != NULL);
265 }
266 *dbp = db->db_hash_next;
267 db->db_hash_next = NULL;
268 mutex_exit(DBUF_HASH_MUTEX(h, idx));
269 atomic_dec_64(&dbuf_hash_count);
270 }
271
272 static arc_evict_func_t dbuf_do_evict;
273
274 typedef enum {
275 DBVU_EVICTING,
276 DBVU_NOT_EVICTING
277 } dbvu_verify_type_t;
278
279 static void
280 dbuf_verify_user(dmu_buf_impl_t *db, dbvu_verify_type_t verify_type)
281 {
282 #ifdef ZFS_DEBUG
283 int64_t holds;
284
285 if (db->db_user == NULL)
286 return;
287
288 /* Only data blocks support the attachment of user data. */
289 ASSERT(db->db_level == 0);
290
291 /* Clients must resolve a dbuf before attaching user data. */
292 ASSERT(db->db.db_data != NULL);
293 ASSERT3U(db->db_state, ==, DB_CACHED);
294
295 holds = refcount_count(&db->db_holds);
296 if (verify_type == DBVU_EVICTING) {
297 /*
298 * Immediate eviction occurs when holds == dirtycnt.
299 * For normal eviction buffers, holds is zero on
300 * eviction, except when dbuf_fix_old_data() calls
301 * dbuf_clear_data(). However, the hold count can grow
302 * during eviction even though db_mtx is held (see
303 * dmu_bonus_hold() for an example), so we can only
304 * test the generic invariant that holds >= dirtycnt.
305 */
306 ASSERT3U(holds, >=, db->db_dirtycnt);
307 } else {
308 if (db->db_user_immediate_evict == TRUE)
309 ASSERT3U(holds, >=, db->db_dirtycnt);
310 else
311 ASSERT3U(holds, >, 0);
312 }
313 #endif
314 }
315
316 static void
317 dbuf_evict_user(dmu_buf_impl_t *db)
318 {
319 dmu_buf_user_t *dbu = db->db_user;
320
321 ASSERT(MUTEX_HELD(&db->db_mtx));
322
323 if (dbu == NULL)
324 return;
325
326 dbuf_verify_user(db, DBVU_EVICTING);
327 db->db_user = NULL;
328
329 #ifdef ZFS_DEBUG
330 if (dbu->dbu_clear_on_evict_dbufp != NULL)
331 *dbu->dbu_clear_on_evict_dbufp = NULL;
332 #endif
333
334 /*
335 * Invoke the callback from a taskq to avoid lock order reversals
336 * and limit stack depth.
337 */
338 taskq_dispatch_ent(dbu_evict_taskq, dbu->dbu_evict_func, dbu, 0,
339 &dbu->dbu_tqent);
340 }
341
342 boolean_t
343 dbuf_is_metadata(dmu_buf_impl_t *db)
344 {
345 /*
346 * Consider indirect blocks and spill blocks to be meta data.
347 */
348 if (db->db_level > 0 || db->db_blkid == DMU_SPILL_BLKID) {
349 return (B_TRUE);
350 } else {
351 boolean_t is_metadata;
352
353 DB_DNODE_ENTER(db);
354 is_metadata = DMU_OT_IS_METADATA(DB_DNODE(db)->dn_type);
355 DB_DNODE_EXIT(db);
356
357 return (is_metadata);
358 }
359 }
360
361 void
362 dbuf_evict(dmu_buf_impl_t *db)
363 {
364 ASSERT(MUTEX_HELD(&db->db_mtx));
365 ASSERT(db->db_buf == NULL);
366 ASSERT(db->db_data_pending == NULL);
367
368 dbuf_clear(db);
369 dbuf_destroy(db);
370 }
371
372 void
373 dbuf_init(void)
374 {
375 uint64_t hsize = 1ULL << 16;
376 dbuf_hash_table_t *h = &dbuf_hash_table;
377 int i;
378
379 /*
380 * The hash table is big enough to fill all of physical memory
381 * with an average block size of zfs_arc_average_blocksize (default 8K).
382 * By default, the table will take up
383 * totalmem * sizeof(void*) / 8K (1MB per GB with 8-byte pointers).
384 */
385 while (hsize * zfs_arc_average_blocksize < physmem * PAGESIZE)
386 hsize <<= 1;
387
388 retry:
389 h->hash_table_mask = hsize - 1;
390 #if defined(_KERNEL) && defined(HAVE_SPL)
391 /*
392 * Large allocations which do not require contiguous pages
393 * should be using vmem_alloc() in the linux kernel
394 */
395 h->hash_table = vmem_zalloc(hsize * sizeof (void *), KM_SLEEP);
396 #else
397 h->hash_table = kmem_zalloc(hsize * sizeof (void *), KM_NOSLEEP);
398 #endif
399 if (h->hash_table == NULL) {
400 /* XXX - we should really return an error instead of assert */
401 ASSERT(hsize > (1ULL << 10));
402 hsize >>= 1;
403 goto retry;
404 }
405
406 dbuf_cache = kmem_cache_create("dmu_buf_impl_t",
407 sizeof (dmu_buf_impl_t),
408 0, dbuf_cons, dbuf_dest, NULL, NULL, NULL, 0);
409
410 for (i = 0; i < DBUF_MUTEXES; i++)
411 mutex_init(&h->hash_mutexes[i], NULL, MUTEX_DEFAULT, NULL);
412
413 dbuf_stats_init(h);
414
415 /*
416 * All entries are queued via taskq_dispatch_ent(), so min/maxalloc
417 * configuration is not required.
418 */
419 dbu_evict_taskq = taskq_create("dbu_evict", 1, defclsyspri, 0, 0, 0);
420 }
421
422 void
423 dbuf_fini(void)
424 {
425 dbuf_hash_table_t *h = &dbuf_hash_table;
426 int i;
427
428 dbuf_stats_destroy();
429
430 for (i = 0; i < DBUF_MUTEXES; i++)
431 mutex_destroy(&h->hash_mutexes[i]);
432 #if defined(_KERNEL) && defined(HAVE_SPL)
433 /*
434 * Large allocations which do not require contiguous pages
435 * should be using vmem_free() in the linux kernel
436 */
437 vmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *));
438 #else
439 kmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *));
440 #endif
441 kmem_cache_destroy(dbuf_cache);
442 taskq_destroy(dbu_evict_taskq);
443 }
444
445 /*
446 * Other stuff.
447 */
448
449 #ifdef ZFS_DEBUG
450 static void
451 dbuf_verify(dmu_buf_impl_t *db)
452 {
453 dnode_t *dn;
454 dbuf_dirty_record_t *dr;
455
456 ASSERT(MUTEX_HELD(&db->db_mtx));
457
458 if (!(zfs_flags & ZFS_DEBUG_DBUF_VERIFY))
459 return;
460
461 ASSERT(db->db_objset != NULL);
462 DB_DNODE_ENTER(db);
463 dn = DB_DNODE(db);
464 if (dn == NULL) {
465 ASSERT(db->db_parent == NULL);
466 ASSERT(db->db_blkptr == NULL);
467 } else {
468 ASSERT3U(db->db.db_object, ==, dn->dn_object);
469 ASSERT3P(db->db_objset, ==, dn->dn_objset);
470 ASSERT3U(db->db_level, <, dn->dn_nlevels);
471 ASSERT(db->db_blkid == DMU_BONUS_BLKID ||
472 db->db_blkid == DMU_SPILL_BLKID ||
473 !avl_is_empty(&dn->dn_dbufs));
474 }
475 if (db->db_blkid == DMU_BONUS_BLKID) {
476 ASSERT(dn != NULL);
477 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen);
478 ASSERT3U(db->db.db_offset, ==, DMU_BONUS_BLKID);
479 } else if (db->db_blkid == DMU_SPILL_BLKID) {
480 ASSERT(dn != NULL);
481 ASSERT0(db->db.db_offset);
482 } else {
483 ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size);
484 }
485
486 for (dr = db->db_data_pending; dr != NULL; dr = dr->dr_next)
487 ASSERT(dr->dr_dbuf == db);
488
489 for (dr = db->db_last_dirty; dr != NULL; dr = dr->dr_next)
490 ASSERT(dr->dr_dbuf == db);
491
492 /*
493 * We can't assert that db_size matches dn_datablksz because it
494 * can be momentarily different when another thread is doing
495 * dnode_set_blksz().
496 */
497 if (db->db_level == 0 && db->db.db_object == DMU_META_DNODE_OBJECT) {
498 dr = db->db_data_pending;
499 /*
500 * It should only be modified in syncing context, so
501 * make sure we only have one copy of the data.
502 */
503 ASSERT(dr == NULL || dr->dt.dl.dr_data == db->db_buf);
504 }
505
506 /* verify db->db_blkptr */
507 if (db->db_blkptr) {
508 if (db->db_parent == dn->dn_dbuf) {
509 /* db is pointed to by the dnode */
510 /* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */
511 if (DMU_OBJECT_IS_SPECIAL(db->db.db_object))
512 ASSERT(db->db_parent == NULL);
513 else
514 ASSERT(db->db_parent != NULL);
515 if (db->db_blkid != DMU_SPILL_BLKID)
516 ASSERT3P(db->db_blkptr, ==,
517 &dn->dn_phys->dn_blkptr[db->db_blkid]);
518 } else {
519 /* db is pointed to by an indirect block */
520 ASSERTV(int epb = db->db_parent->db.db_size >>
521 SPA_BLKPTRSHIFT);
522 ASSERT3U(db->db_parent->db_level, ==, db->db_level+1);
523 ASSERT3U(db->db_parent->db.db_object, ==,
524 db->db.db_object);
525 /*
526 * dnode_grow_indblksz() can make this fail if we don't
527 * have the struct_rwlock. XXX indblksz no longer
528 * grows. safe to do this now?
529 */
530 if (RW_WRITE_HELD(&dn->dn_struct_rwlock)) {
531 ASSERT3P(db->db_blkptr, ==,
532 ((blkptr_t *)db->db_parent->db.db_data +
533 db->db_blkid % epb));
534 }
535 }
536 }
537 if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) &&
538 (db->db_buf == NULL || db->db_buf->b_data) &&
539 db->db.db_data && db->db_blkid != DMU_BONUS_BLKID &&
540 db->db_state != DB_FILL && !dn->dn_free_txg) {
541 /*
542 * If the blkptr isn't set but they have nonzero data,
543 * it had better be dirty, otherwise we'll lose that
544 * data when we evict this buffer.
545 *
546 * There is an exception to this rule for indirect blocks; in
547 * this case, if the indirect block is a hole, we fill in a few
548 * fields on each of the child blocks (importantly, birth time)
549 * to prevent hole birth times from being lost when you
550 * partially fill in a hole.
551 */
552 if (db->db_dirtycnt == 0) {
553 if (db->db_level == 0) {
554 uint64_t *buf = db->db.db_data;
555 int i;
556
557 for (i = 0; i < db->db.db_size >> 3; i++) {
558 ASSERT(buf[i] == 0);
559 }
560 } else {
561 int i;
562 blkptr_t *bps = db->db.db_data;
563 ASSERT3U(1 << DB_DNODE(db)->dn_indblkshift, ==,
564 db->db.db_size);
565 /*
566 * We want to verify that all the blkptrs in the
567 * indirect block are holes, but we may have
568 * automatically set up a few fields for them.
569 * We iterate through each blkptr and verify
570 * they only have those fields set.
571 */
572 for (i = 0;
573 i < db->db.db_size / sizeof (blkptr_t);
574 i++) {
575 blkptr_t *bp = &bps[i];
576 ASSERT(ZIO_CHECKSUM_IS_ZERO(
577 &bp->blk_cksum));
578 ASSERT(
579 DVA_IS_EMPTY(&bp->blk_dva[0]) &&
580 DVA_IS_EMPTY(&bp->blk_dva[1]) &&
581 DVA_IS_EMPTY(&bp->blk_dva[2]));
582 ASSERT0(bp->blk_fill);
583 ASSERT0(bp->blk_pad[0]);
584 ASSERT0(bp->blk_pad[1]);
585 ASSERT(!BP_IS_EMBEDDED(bp));
586 ASSERT(BP_IS_HOLE(bp));
587 ASSERT0(bp->blk_phys_birth);
588 }
589 }
590 }
591 }
592 DB_DNODE_EXIT(db);
593 }
594 #endif
595
596 static void
597 dbuf_clear_data(dmu_buf_impl_t *db)
598 {
599 ASSERT(MUTEX_HELD(&db->db_mtx));
600 dbuf_evict_user(db);
601 db->db_buf = NULL;
602 db->db.db_data = NULL;
603 if (db->db_state != DB_NOFILL)
604 db->db_state = DB_UNCACHED;
605 }
606
607 static void
608 dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf)
609 {
610 ASSERT(MUTEX_HELD(&db->db_mtx));
611 ASSERT(buf != NULL);
612
613 db->db_buf = buf;
614 ASSERT(buf->b_data != NULL);
615 db->db.db_data = buf->b_data;
616 if (!arc_released(buf))
617 arc_set_callback(buf, dbuf_do_evict, db);
618 }
619
620 /*
621 * Loan out an arc_buf for read. Return the loaned arc_buf.
622 */
623 arc_buf_t *
624 dbuf_loan_arcbuf(dmu_buf_impl_t *db)
625 {
626 arc_buf_t *abuf;
627
628 mutex_enter(&db->db_mtx);
629 if (arc_released(db->db_buf) || refcount_count(&db->db_holds) > 1) {
630 int blksz = db->db.db_size;
631 spa_t *spa = db->db_objset->os_spa;
632
633 mutex_exit(&db->db_mtx);
634 abuf = arc_loan_buf(spa, blksz);
635 bcopy(db->db.db_data, abuf->b_data, blksz);
636 } else {
637 abuf = db->db_buf;
638 arc_loan_inuse_buf(abuf, db);
639 dbuf_clear_data(db);
640 mutex_exit(&db->db_mtx);
641 }
642 return (abuf);
643 }
644
645 /*
646 * Calculate which level n block references the data at the level 0 offset
647 * provided.
648 */
649 uint64_t
650 dbuf_whichblock(dnode_t *dn, int64_t level, uint64_t offset)
651 {
652 if (dn->dn_datablkshift != 0 && dn->dn_indblkshift != 0) {
653 /*
654 * The level n blkid is equal to the level 0 blkid divided by
655 * the number of level 0s in a level n block.
656 *
657 * The level 0 blkid is offset >> datablkshift =
658 * offset / 2^datablkshift.
659 *
660 * The number of level 0s in a level n is the number of block
661 * pointers in an indirect block, raised to the power of level.
662 * This is 2^(indblkshift - SPA_BLKPTRSHIFT)^level =
663 * 2^(level*(indblkshift - SPA_BLKPTRSHIFT)).
664 *
665 * Thus, the level n blkid is: offset /
666 * ((2^datablkshift)*(2^(level*(indblkshift - SPA_BLKPTRSHIFT)))
667 * = offset / 2^(datablkshift + level *
668 * (indblkshift - SPA_BLKPTRSHIFT))
669 * = offset >> (datablkshift + level *
670 * (indblkshift - SPA_BLKPTRSHIFT))
671 */
672 return (offset >> (dn->dn_datablkshift + level *
673 (dn->dn_indblkshift - SPA_BLKPTRSHIFT)));
674 } else {
675 ASSERT3U(offset, <, dn->dn_datablksz);
676 return (0);
677 }
678 }
679
680 static void
681 dbuf_read_done(zio_t *zio, arc_buf_t *buf, void *vdb)
682 {
683 dmu_buf_impl_t *db = vdb;
684
685 mutex_enter(&db->db_mtx);
686 ASSERT3U(db->db_state, ==, DB_READ);
687 /*
688 * All reads are synchronous, so we must have a hold on the dbuf
689 */
690 ASSERT(refcount_count(&db->db_holds) > 0);
691 ASSERT(db->db_buf == NULL);
692 ASSERT(db->db.db_data == NULL);
693 if (db->db_level == 0 && db->db_freed_in_flight) {
694 /* we were freed in flight; disregard any error */
695 arc_release(buf, db);
696 bzero(buf->b_data, db->db.db_size);
697 arc_buf_freeze(buf);
698 db->db_freed_in_flight = FALSE;
699 dbuf_set_data(db, buf);
700 db->db_state = DB_CACHED;
701 } else if (zio == NULL || zio->io_error == 0) {
702 dbuf_set_data(db, buf);
703 db->db_state = DB_CACHED;
704 } else {
705 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
706 ASSERT3P(db->db_buf, ==, NULL);
707 VERIFY(arc_buf_remove_ref(buf, db));
708 db->db_state = DB_UNCACHED;
709 }
710 cv_broadcast(&db->db_changed);
711 dbuf_rele_and_unlock(db, NULL);
712 }
713
714 static int
715 dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags)
716 {
717 dnode_t *dn;
718 zbookmark_phys_t zb;
719 uint32_t aflags = ARC_FLAG_NOWAIT;
720 int err;
721
722 DB_DNODE_ENTER(db);
723 dn = DB_DNODE(db);
724 ASSERT(!refcount_is_zero(&db->db_holds));
725 /* We need the struct_rwlock to prevent db_blkptr from changing. */
726 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
727 ASSERT(MUTEX_HELD(&db->db_mtx));
728 ASSERT(db->db_state == DB_UNCACHED);
729 ASSERT(db->db_buf == NULL);
730
731 if (db->db_blkid == DMU_BONUS_BLKID) {
732 /*
733 * The bonus length stored in the dnode may be less than
734 * the maximum available space in the bonus buffer.
735 */
736 int bonuslen = MIN(dn->dn_bonuslen, dn->dn_phys->dn_bonuslen);
737 int max_bonuslen = DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots);
738
739 ASSERT3U(bonuslen, <=, db->db.db_size);
740 db->db.db_data = zio_buf_alloc(max_bonuslen);
741 arc_space_consume(max_bonuslen, ARC_SPACE_BONUS);
742 if (bonuslen < max_bonuslen)
743 bzero(db->db.db_data, max_bonuslen);
744 if (bonuslen)
745 bcopy(DN_BONUS(dn->dn_phys), db->db.db_data, bonuslen);
746 DB_DNODE_EXIT(db);
747 db->db_state = DB_CACHED;
748 mutex_exit(&db->db_mtx);
749 return (0);
750 }
751
752 /*
753 * Recheck BP_IS_HOLE() after dnode_block_freed() in case dnode_sync()
754 * processes the delete record and clears the bp while we are waiting
755 * for the dn_mtx (resulting in a "no" from block_freed).
756 */
757 if (db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr) ||
758 (db->db_level == 0 && (dnode_block_freed(dn, db->db_blkid) ||
759 BP_IS_HOLE(db->db_blkptr)))) {
760 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
761
762 dbuf_set_data(db, arc_buf_alloc(db->db_objset->os_spa,
763 db->db.db_size, db, type));
764 bzero(db->db.db_data, db->db.db_size);
765
766 if (db->db_blkptr != NULL && db->db_level > 0 &&
767 BP_IS_HOLE(db->db_blkptr) &&
768 db->db_blkptr->blk_birth != 0) {
769 blkptr_t *bps = db->db.db_data;
770 int i;
771 for (i = 0; i < ((1 <<
772 DB_DNODE(db)->dn_indblkshift) / sizeof (blkptr_t));
773 i++) {
774 blkptr_t *bp = &bps[i];
775 ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==,
776 1 << dn->dn_indblkshift);
777 BP_SET_LSIZE(bp,
778 BP_GET_LEVEL(db->db_blkptr) == 1 ?
779 dn->dn_datablksz :
780 BP_GET_LSIZE(db->db_blkptr));
781 BP_SET_TYPE(bp, BP_GET_TYPE(db->db_blkptr));
782 BP_SET_LEVEL(bp,
783 BP_GET_LEVEL(db->db_blkptr) - 1);
784 BP_SET_BIRTH(bp, db->db_blkptr->blk_birth, 0);
785 }
786 }
787 DB_DNODE_EXIT(db);
788 db->db_state = DB_CACHED;
789 mutex_exit(&db->db_mtx);
790 return (0);
791 }
792
793 DB_DNODE_EXIT(db);
794
795 db->db_state = DB_READ;
796 mutex_exit(&db->db_mtx);
797
798 if (DBUF_IS_L2CACHEABLE(db))
799 aflags |= ARC_FLAG_L2CACHE;
800 if (DBUF_IS_L2COMPRESSIBLE(db))
801 aflags |= ARC_FLAG_L2COMPRESS;
802
803 SET_BOOKMARK(&zb, db->db_objset->os_dsl_dataset ?
804 db->db_objset->os_dsl_dataset->ds_object : DMU_META_OBJSET,
805 db->db.db_object, db->db_level, db->db_blkid);
806
807 dbuf_add_ref(db, NULL);
808
809 err = arc_read(zio, db->db_objset->os_spa, db->db_blkptr,
810 dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ,
811 (flags & DB_RF_CANFAIL) ? ZIO_FLAG_CANFAIL : ZIO_FLAG_MUSTSUCCEED,
812 &aflags, &zb);
813
814 return (SET_ERROR(err));
815 }
816
817 int
818 dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags)
819 {
820 int err = 0;
821 boolean_t havepzio = (zio != NULL);
822 boolean_t prefetch;
823 dnode_t *dn;
824
825 /*
826 * We don't have to hold the mutex to check db_state because it
827 * can't be freed while we have a hold on the buffer.
828 */
829 ASSERT(!refcount_is_zero(&db->db_holds));
830
831 if (db->db_state == DB_NOFILL)
832 return (SET_ERROR(EIO));
833
834 DB_DNODE_ENTER(db);
835 dn = DB_DNODE(db);
836 if ((flags & DB_RF_HAVESTRUCT) == 0)
837 rw_enter(&dn->dn_struct_rwlock, RW_READER);
838
839 prefetch = db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID &&
840 (flags & DB_RF_NOPREFETCH) == 0 && dn != NULL &&
841 DBUF_IS_CACHEABLE(db);
842
843 mutex_enter(&db->db_mtx);
844 if (db->db_state == DB_CACHED) {
845 mutex_exit(&db->db_mtx);
846 if (prefetch)
847 dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1);
848 if ((flags & DB_RF_HAVESTRUCT) == 0)
849 rw_exit(&dn->dn_struct_rwlock);
850 DB_DNODE_EXIT(db);
851 } else if (db->db_state == DB_UNCACHED) {
852 spa_t *spa = dn->dn_objset->os_spa;
853
854 if (zio == NULL)
855 zio = zio_root(spa, NULL, NULL, ZIO_FLAG_CANFAIL);
856
857 err = dbuf_read_impl(db, zio, flags);
858
859 /* dbuf_read_impl has dropped db_mtx for us */
860
861 if (!err && prefetch)
862 dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1);
863
864 if ((flags & DB_RF_HAVESTRUCT) == 0)
865 rw_exit(&dn->dn_struct_rwlock);
866 DB_DNODE_EXIT(db);
867
868 if (!err && !havepzio)
869 err = zio_wait(zio);
870 } else {
871 /*
872 * Another reader came in while the dbuf was in flight
873 * between UNCACHED and CACHED. Either a writer will finish
874 * writing the buffer (sending the dbuf to CACHED) or the
875 * first reader's request will reach the read_done callback
876 * and send the dbuf to CACHED. Otherwise, a failure
877 * occurred and the dbuf went to UNCACHED.
878 */
879 mutex_exit(&db->db_mtx);
880 if (prefetch)
881 dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1);
882 if ((flags & DB_RF_HAVESTRUCT) == 0)
883 rw_exit(&dn->dn_struct_rwlock);
884 DB_DNODE_EXIT(db);
885
886 /* Skip the wait per the caller's request. */
887 mutex_enter(&db->db_mtx);
888 if ((flags & DB_RF_NEVERWAIT) == 0) {
889 while (db->db_state == DB_READ ||
890 db->db_state == DB_FILL) {
891 ASSERT(db->db_state == DB_READ ||
892 (flags & DB_RF_HAVESTRUCT) == 0);
893 DTRACE_PROBE2(blocked__read, dmu_buf_impl_t *,
894 db, zio_t *, zio);
895 cv_wait(&db->db_changed, &db->db_mtx);
896 }
897 if (db->db_state == DB_UNCACHED)
898 err = SET_ERROR(EIO);
899 }
900 mutex_exit(&db->db_mtx);
901 }
902
903 ASSERT(err || havepzio || db->db_state == DB_CACHED);
904 return (err);
905 }
906
907 static void
908 dbuf_noread(dmu_buf_impl_t *db)
909 {
910 ASSERT(!refcount_is_zero(&db->db_holds));
911 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
912 mutex_enter(&db->db_mtx);
913 while (db->db_state == DB_READ || db->db_state == DB_FILL)
914 cv_wait(&db->db_changed, &db->db_mtx);
915 if (db->db_state == DB_UNCACHED) {
916 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
917 spa_t *spa = db->db_objset->os_spa;
918
919 ASSERT(db->db_buf == NULL);
920 ASSERT(db->db.db_data == NULL);
921 dbuf_set_data(db, arc_buf_alloc(spa, db->db.db_size, db, type));
922 db->db_state = DB_FILL;
923 } else if (db->db_state == DB_NOFILL) {
924 dbuf_clear_data(db);
925 } else {
926 ASSERT3U(db->db_state, ==, DB_CACHED);
927 }
928 mutex_exit(&db->db_mtx);
929 }
930
931 /*
932 * This is our just-in-time copy function. It makes a copy of
933 * buffers, that have been modified in a previous transaction
934 * group, before we modify them in the current active group.
935 *
936 * This function is used in two places: when we are dirtying a
937 * buffer for the first time in a txg, and when we are freeing
938 * a range in a dnode that includes this buffer.
939 *
940 * Note that when we are called from dbuf_free_range() we do
941 * not put a hold on the buffer, we just traverse the active
942 * dbuf list for the dnode.
943 */
944 static void
945 dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg)
946 {
947 dbuf_dirty_record_t *dr = db->db_last_dirty;
948
949 ASSERT(MUTEX_HELD(&db->db_mtx));
950 ASSERT(db->db.db_data != NULL);
951 ASSERT(db->db_level == 0);
952 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT);
953
954 if (dr == NULL ||
955 (dr->dt.dl.dr_data !=
956 ((db->db_blkid == DMU_BONUS_BLKID) ? db->db.db_data : db->db_buf)))
957 return;
958
959 /*
960 * If the last dirty record for this dbuf has not yet synced
961 * and its referencing the dbuf data, either:
962 * reset the reference to point to a new copy,
963 * or (if there a no active holders)
964 * just null out the current db_data pointer.
965 */
966 ASSERT(dr->dr_txg >= txg - 2);
967 if (db->db_blkid == DMU_BONUS_BLKID) {
968 /* Note that the data bufs here are zio_bufs */
969 dnode_t *dn = DB_DNODE(db);
970 int bonuslen = DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots);
971 dr->dt.dl.dr_data = zio_buf_alloc(bonuslen);
972 arc_space_consume(bonuslen, ARC_SPACE_BONUS);
973 bcopy(db->db.db_data, dr->dt.dl.dr_data, bonuslen);
974 } else if (refcount_count(&db->db_holds) > db->db_dirtycnt) {
975 int size = db->db.db_size;
976 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
977 spa_t *spa = db->db_objset->os_spa;
978
979 dr->dt.dl.dr_data = arc_buf_alloc(spa, size, db, type);
980 bcopy(db->db.db_data, dr->dt.dl.dr_data->b_data, size);
981 } else {
982 dbuf_clear_data(db);
983 }
984 }
985
986 void
987 dbuf_unoverride(dbuf_dirty_record_t *dr)
988 {
989 dmu_buf_impl_t *db = dr->dr_dbuf;
990 blkptr_t *bp = &dr->dt.dl.dr_overridden_by;
991 uint64_t txg = dr->dr_txg;
992
993 ASSERT(MUTEX_HELD(&db->db_mtx));
994 ASSERT(dr->dt.dl.dr_override_state != DR_IN_DMU_SYNC);
995 ASSERT(db->db_level == 0);
996
997 if (db->db_blkid == DMU_BONUS_BLKID ||
998 dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN)
999 return;
1000
1001 ASSERT(db->db_data_pending != dr);
1002
1003 /* free this block */
1004 if (!BP_IS_HOLE(bp) && !dr->dt.dl.dr_nopwrite)
1005 zio_free(db->db_objset->os_spa, txg, bp);
1006
1007 dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN;
1008 dr->dt.dl.dr_nopwrite = B_FALSE;
1009
1010 /*
1011 * Release the already-written buffer, so we leave it in
1012 * a consistent dirty state. Note that all callers are
1013 * modifying the buffer, so they will immediately do
1014 * another (redundant) arc_release(). Therefore, leave
1015 * the buf thawed to save the effort of freezing &
1016 * immediately re-thawing it.
1017 */
1018 arc_release(dr->dt.dl.dr_data, db);
1019 }
1020
1021 /*
1022 * Evict (if its unreferenced) or clear (if its referenced) any level-0
1023 * data blocks in the free range, so that any future readers will find
1024 * empty blocks.
1025 *
1026 * This is a no-op if the dataset is in the middle of an incremental
1027 * receive; see comment below for details.
1028 */
1029 void
1030 dbuf_free_range(dnode_t *dn, uint64_t start_blkid, uint64_t end_blkid,
1031 dmu_tx_t *tx)
1032 {
1033 dmu_buf_impl_t *db_search;
1034 dmu_buf_impl_t *db, *db_next;
1035 uint64_t txg = tx->tx_txg;
1036 avl_index_t where;
1037 boolean_t freespill =
1038 (start_blkid == DMU_SPILL_BLKID || end_blkid == DMU_SPILL_BLKID);
1039
1040 if (end_blkid > dn->dn_maxblkid && !freespill)
1041 end_blkid = dn->dn_maxblkid;
1042 dprintf_dnode(dn, "start=%llu end=%llu\n", start_blkid, end_blkid);
1043
1044 db_search = kmem_alloc(sizeof (dmu_buf_impl_t), KM_SLEEP);
1045 db_search->db_level = 0;
1046 db_search->db_blkid = start_blkid;
1047 db_search->db_state = DB_SEARCH;
1048
1049 mutex_enter(&dn->dn_dbufs_mtx);
1050 if (start_blkid >= dn->dn_unlisted_l0_blkid && !freespill) {
1051 /* There can't be any dbufs in this range; no need to search. */
1052 #ifdef DEBUG
1053 db = avl_find(&dn->dn_dbufs, db_search, &where);
1054 ASSERT3P(db, ==, NULL);
1055 db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER);
1056 ASSERT(db == NULL || db->db_level > 0);
1057 #endif
1058 goto out;
1059 } else if (dmu_objset_is_receiving(dn->dn_objset)) {
1060 /*
1061 * If we are receiving, we expect there to be no dbufs in
1062 * the range to be freed, because receive modifies each
1063 * block at most once, and in offset order. If this is
1064 * not the case, it can lead to performance problems,
1065 * so note that we unexpectedly took the slow path.
1066 */
1067 atomic_inc_64(&zfs_free_range_recv_miss);
1068 }
1069
1070 db = avl_find(&dn->dn_dbufs, db_search, &where);
1071 ASSERT3P(db, ==, NULL);
1072 db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER);
1073
1074 for (; db != NULL; db = db_next) {
1075 db_next = AVL_NEXT(&dn->dn_dbufs, db);
1076 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1077
1078 if (db->db_level != 0 || db->db_blkid > end_blkid) {
1079 break;
1080 }
1081 ASSERT3U(db->db_blkid, >=, start_blkid);
1082
1083 /* found a level 0 buffer in the range */
1084 mutex_enter(&db->db_mtx);
1085 if (dbuf_undirty(db, tx)) {
1086 /* mutex has been dropped and dbuf destroyed */
1087 continue;
1088 }
1089
1090 if (db->db_state == DB_UNCACHED ||
1091 db->db_state == DB_NOFILL ||
1092 db->db_state == DB_EVICTING) {
1093 ASSERT(db->db.db_data == NULL);
1094 mutex_exit(&db->db_mtx);
1095 continue;
1096 }
1097 if (db->db_state == DB_READ || db->db_state == DB_FILL) {
1098 /* will be handled in dbuf_read_done or dbuf_rele */
1099 db->db_freed_in_flight = TRUE;
1100 mutex_exit(&db->db_mtx);
1101 continue;
1102 }
1103 if (refcount_count(&db->db_holds) == 0) {
1104 ASSERT(db->db_buf);
1105 dbuf_clear(db);
1106 continue;
1107 }
1108 /* The dbuf is referenced */
1109
1110 if (db->db_last_dirty != NULL) {
1111 dbuf_dirty_record_t *dr = db->db_last_dirty;
1112
1113 if (dr->dr_txg == txg) {
1114 /*
1115 * This buffer is "in-use", re-adjust the file
1116 * size to reflect that this buffer may
1117 * contain new data when we sync.
1118 */
1119 if (db->db_blkid != DMU_SPILL_BLKID &&
1120 db->db_blkid > dn->dn_maxblkid)
1121 dn->dn_maxblkid = db->db_blkid;
1122 dbuf_unoverride(dr);
1123 } else {
1124 /*
1125 * This dbuf is not dirty in the open context.
1126 * Either uncache it (if its not referenced in
1127 * the open context) or reset its contents to
1128 * empty.
1129 */
1130 dbuf_fix_old_data(db, txg);
1131 }
1132 }
1133 /* clear the contents if its cached */
1134 if (db->db_state == DB_CACHED) {
1135 ASSERT(db->db.db_data != NULL);
1136 arc_release(db->db_buf, db);
1137 bzero(db->db.db_data, db->db.db_size);
1138 arc_buf_freeze(db->db_buf);
1139 }
1140
1141 mutex_exit(&db->db_mtx);
1142 }
1143
1144 out:
1145 kmem_free(db_search, sizeof (dmu_buf_impl_t));
1146 mutex_exit(&dn->dn_dbufs_mtx);
1147 }
1148
1149 static int
1150 dbuf_block_freeable(dmu_buf_impl_t *db)
1151 {
1152 dsl_dataset_t *ds = db->db_objset->os_dsl_dataset;
1153 uint64_t birth_txg = 0;
1154
1155 /*
1156 * We don't need any locking to protect db_blkptr:
1157 * If it's syncing, then db_last_dirty will be set
1158 * so we'll ignore db_blkptr.
1159 *
1160 * This logic ensures that only block births for
1161 * filled blocks are considered.
1162 */
1163 ASSERT(MUTEX_HELD(&db->db_mtx));
1164 if (db->db_last_dirty && (db->db_blkptr == NULL ||
1165 !BP_IS_HOLE(db->db_blkptr))) {
1166 birth_txg = db->db_last_dirty->dr_txg;
1167 } else if (db->db_blkptr != NULL && !BP_IS_HOLE(db->db_blkptr)) {
1168 birth_txg = db->db_blkptr->blk_birth;
1169 }
1170
1171 /*
1172 * If this block don't exist or is in a snapshot, it can't be freed.
1173 * Don't pass the bp to dsl_dataset_block_freeable() since we
1174 * are holding the db_mtx lock and might deadlock if we are
1175 * prefetching a dedup-ed block.
1176 */
1177 if (birth_txg != 0)
1178 return (ds == NULL ||
1179 dsl_dataset_block_freeable(ds, NULL, birth_txg));
1180 else
1181 return (B_FALSE);
1182 }
1183
1184 void
1185 dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx)
1186 {
1187 arc_buf_t *buf, *obuf;
1188 int osize = db->db.db_size;
1189 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
1190 dnode_t *dn;
1191
1192 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1193
1194 DB_DNODE_ENTER(db);
1195 dn = DB_DNODE(db);
1196
1197 /* XXX does *this* func really need the lock? */
1198 ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock));
1199
1200 /*
1201 * This call to dmu_buf_will_dirty() with the dn_struct_rwlock held
1202 * is OK, because there can be no other references to the db
1203 * when we are changing its size, so no concurrent DB_FILL can
1204 * be happening.
1205 */
1206 /*
1207 * XXX we should be doing a dbuf_read, checking the return
1208 * value and returning that up to our callers
1209 */
1210 dmu_buf_will_dirty(&db->db, tx);
1211
1212 /* create the data buffer for the new block */
1213 buf = arc_buf_alloc(dn->dn_objset->os_spa, size, db, type);
1214
1215 /* copy old block data to the new block */
1216 obuf = db->db_buf;
1217 bcopy(obuf->b_data, buf->b_data, MIN(osize, size));
1218 /* zero the remainder */
1219 if (size > osize)
1220 bzero((uint8_t *)buf->b_data + osize, size - osize);
1221
1222 mutex_enter(&db->db_mtx);
1223 dbuf_set_data(db, buf);
1224 VERIFY(arc_buf_remove_ref(obuf, db));
1225 db->db.db_size = size;
1226
1227 if (db->db_level == 0) {
1228 ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg);
1229 db->db_last_dirty->dt.dl.dr_data = buf;
1230 }
1231 mutex_exit(&db->db_mtx);
1232
1233 dnode_willuse_space(dn, size-osize, tx);
1234 DB_DNODE_EXIT(db);
1235 }
1236
1237 void
1238 dbuf_release_bp(dmu_buf_impl_t *db)
1239 {
1240 ASSERTV(objset_t *os = db->db_objset);
1241
1242 ASSERT(dsl_pool_sync_context(dmu_objset_pool(os)));
1243 ASSERT(arc_released(os->os_phys_buf) ||
1244 list_link_active(&os->os_dsl_dataset->ds_synced_link));
1245 ASSERT(db->db_parent == NULL || arc_released(db->db_parent->db_buf));
1246
1247 (void) arc_release(db->db_buf, db);
1248 }
1249
1250 /*
1251 * We already have a dirty record for this TXG, and we are being
1252 * dirtied again.
1253 */
1254 static void
1255 dbuf_redirty(dbuf_dirty_record_t *dr)
1256 {
1257 dmu_buf_impl_t *db = dr->dr_dbuf;
1258
1259 ASSERT(MUTEX_HELD(&db->db_mtx));
1260
1261 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID) {
1262 /*
1263 * If this buffer has already been written out,
1264 * we now need to reset its state.
1265 */
1266 dbuf_unoverride(dr);
1267 if (db->db.db_object != DMU_META_DNODE_OBJECT &&
1268 db->db_state != DB_NOFILL) {
1269 /* Already released on initial dirty, so just thaw. */
1270 ASSERT(arc_released(db->db_buf));
1271 arc_buf_thaw(db->db_buf);
1272 }
1273 }
1274 }
1275
1276 dbuf_dirty_record_t *
1277 dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
1278 {
1279 dnode_t *dn;
1280 objset_t *os;
1281 dbuf_dirty_record_t **drp, *dr;
1282 int drop_struct_lock = FALSE;
1283 boolean_t do_free_accounting = B_FALSE;
1284 int txgoff = tx->tx_txg & TXG_MASK;
1285
1286 ASSERT(tx->tx_txg != 0);
1287 ASSERT(!refcount_is_zero(&db->db_holds));
1288 DMU_TX_DIRTY_BUF(tx, db);
1289
1290 DB_DNODE_ENTER(db);
1291 dn = DB_DNODE(db);
1292 /*
1293 * Shouldn't dirty a regular buffer in syncing context. Private
1294 * objects may be dirtied in syncing context, but only if they
1295 * were already pre-dirtied in open context.
1296 */
1297 ASSERT(!dmu_tx_is_syncing(tx) ||
1298 BP_IS_HOLE(dn->dn_objset->os_rootbp) ||
1299 DMU_OBJECT_IS_SPECIAL(dn->dn_object) ||
1300 dn->dn_objset->os_dsl_dataset == NULL);
1301 /*
1302 * We make this assert for private objects as well, but after we
1303 * check if we're already dirty. They are allowed to re-dirty
1304 * in syncing context.
1305 */
1306 ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT ||
1307 dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx ==
1308 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN));
1309
1310 mutex_enter(&db->db_mtx);
1311 /*
1312 * XXX make this true for indirects too? The problem is that
1313 * transactions created with dmu_tx_create_assigned() from
1314 * syncing context don't bother holding ahead.
1315 */
1316 ASSERT(db->db_level != 0 ||
1317 db->db_state == DB_CACHED || db->db_state == DB_FILL ||
1318 db->db_state == DB_NOFILL);
1319
1320 mutex_enter(&dn->dn_mtx);
1321 /*
1322 * Don't set dirtyctx to SYNC if we're just modifying this as we
1323 * initialize the objset.
1324 */
1325 if (dn->dn_dirtyctx == DN_UNDIRTIED &&
1326 !BP_IS_HOLE(dn->dn_objset->os_rootbp)) {
1327 dn->dn_dirtyctx =
1328 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN);
1329 ASSERT(dn->dn_dirtyctx_firstset == NULL);
1330 dn->dn_dirtyctx_firstset = kmem_alloc(1, KM_SLEEP);
1331 }
1332 mutex_exit(&dn->dn_mtx);
1333
1334 if (db->db_blkid == DMU_SPILL_BLKID)
1335 dn->dn_have_spill = B_TRUE;
1336
1337 /*
1338 * If this buffer is already dirty, we're done.
1339 */
1340 drp = &db->db_last_dirty;
1341 ASSERT(*drp == NULL || (*drp)->dr_txg <= tx->tx_txg ||
1342 db->db.db_object == DMU_META_DNODE_OBJECT);
1343 while ((dr = *drp) != NULL && dr->dr_txg > tx->tx_txg)
1344 drp = &dr->dr_next;
1345 if (dr && dr->dr_txg == tx->tx_txg) {
1346 DB_DNODE_EXIT(db);
1347
1348 dbuf_redirty(dr);
1349 mutex_exit(&db->db_mtx);
1350 return (dr);
1351 }
1352
1353 /*
1354 * Only valid if not already dirty.
1355 */
1356 ASSERT(dn->dn_object == 0 ||
1357 dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx ==
1358 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN));
1359
1360 ASSERT3U(dn->dn_nlevels, >, db->db_level);
1361 ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) ||
1362 dn->dn_phys->dn_nlevels > db->db_level ||
1363 dn->dn_next_nlevels[txgoff] > db->db_level ||
1364 dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level ||
1365 dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level);
1366
1367 /*
1368 * We should only be dirtying in syncing context if it's the
1369 * mos or we're initializing the os or it's a special object.
1370 * However, we are allowed to dirty in syncing context provided
1371 * we already dirtied it in open context. Hence we must make
1372 * this assertion only if we're not already dirty.
1373 */
1374 os = dn->dn_objset;
1375 ASSERT(!dmu_tx_is_syncing(tx) || DMU_OBJECT_IS_SPECIAL(dn->dn_object) ||
1376 os->os_dsl_dataset == NULL || BP_IS_HOLE(os->os_rootbp));
1377 ASSERT(db->db.db_size != 0);
1378
1379 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size);
1380
1381 if (db->db_blkid != DMU_BONUS_BLKID) {
1382 /*
1383 * Update the accounting.
1384 * Note: we delay "free accounting" until after we drop
1385 * the db_mtx. This keeps us from grabbing other locks
1386 * (and possibly deadlocking) in bp_get_dsize() while
1387 * also holding the db_mtx.
1388 */
1389 dnode_willuse_space(dn, db->db.db_size, tx);
1390 do_free_accounting = dbuf_block_freeable(db);
1391 }
1392
1393 /*
1394 * If this buffer is dirty in an old transaction group we need
1395 * to make a copy of it so that the changes we make in this
1396 * transaction group won't leak out when we sync the older txg.
1397 */
1398 dr = kmem_zalloc(sizeof (dbuf_dirty_record_t), KM_SLEEP);
1399 list_link_init(&dr->dr_dirty_node);
1400 if (db->db_level == 0) {
1401 void *data_old = db->db_buf;
1402
1403 if (db->db_state != DB_NOFILL) {
1404 if (db->db_blkid == DMU_BONUS_BLKID) {
1405 dbuf_fix_old_data(db, tx->tx_txg);
1406 data_old = db->db.db_data;
1407 } else if (db->db.db_object != DMU_META_DNODE_OBJECT) {
1408 /*
1409 * Release the data buffer from the cache so
1410 * that we can modify it without impacting
1411 * possible other users of this cached data
1412 * block. Note that indirect blocks and
1413 * private objects are not released until the
1414 * syncing state (since they are only modified
1415 * then).
1416 */
1417 arc_release(db->db_buf, db);
1418 dbuf_fix_old_data(db, tx->tx_txg);
1419 data_old = db->db_buf;
1420 }
1421 ASSERT(data_old != NULL);
1422 }
1423 dr->dt.dl.dr_data = data_old;
1424 } else {
1425 mutex_init(&dr->dt.di.dr_mtx, NULL, MUTEX_NOLOCKDEP, NULL);
1426 list_create(&dr->dt.di.dr_children,
1427 sizeof (dbuf_dirty_record_t),
1428 offsetof(dbuf_dirty_record_t, dr_dirty_node));
1429 }
1430 if (db->db_blkid != DMU_BONUS_BLKID && os->os_dsl_dataset != NULL)
1431 dr->dr_accounted = db->db.db_size;
1432 dr->dr_dbuf = db;
1433 dr->dr_txg = tx->tx_txg;
1434 dr->dr_next = *drp;
1435 *drp = dr;
1436
1437 /*
1438 * We could have been freed_in_flight between the dbuf_noread
1439 * and dbuf_dirty. We win, as though the dbuf_noread() had
1440 * happened after the free.
1441 */
1442 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID &&
1443 db->db_blkid != DMU_SPILL_BLKID) {
1444 mutex_enter(&dn->dn_mtx);
1445 if (dn->dn_free_ranges[txgoff] != NULL) {
1446 range_tree_clear(dn->dn_free_ranges[txgoff],
1447 db->db_blkid, 1);
1448 }
1449 mutex_exit(&dn->dn_mtx);
1450 db->db_freed_in_flight = FALSE;
1451 }
1452
1453 /*
1454 * This buffer is now part of this txg
1455 */
1456 dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg);
1457 db->db_dirtycnt += 1;
1458 ASSERT3U(db->db_dirtycnt, <=, 3);
1459
1460 mutex_exit(&db->db_mtx);
1461
1462 if (db->db_blkid == DMU_BONUS_BLKID ||
1463 db->db_blkid == DMU_SPILL_BLKID) {
1464 mutex_enter(&dn->dn_mtx);
1465 ASSERT(!list_link_active(&dr->dr_dirty_node));
1466 list_insert_tail(&dn->dn_dirty_records[txgoff], dr);
1467 mutex_exit(&dn->dn_mtx);
1468 dnode_setdirty(dn, tx);
1469 DB_DNODE_EXIT(db);
1470 return (dr);
1471 } else if (do_free_accounting) {
1472 blkptr_t *bp = db->db_blkptr;
1473 int64_t willfree = (bp && !BP_IS_HOLE(bp)) ?
1474 bp_get_dsize(os->os_spa, bp) : db->db.db_size;
1475 /*
1476 * This is only a guess -- if the dbuf is dirty
1477 * in a previous txg, we don't know how much
1478 * space it will use on disk yet. We should
1479 * really have the struct_rwlock to access
1480 * db_blkptr, but since this is just a guess,
1481 * it's OK if we get an odd answer.
1482 */
1483 ddt_prefetch(os->os_spa, bp);
1484 dnode_willuse_space(dn, -willfree, tx);
1485 }
1486
1487 if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) {
1488 rw_enter(&dn->dn_struct_rwlock, RW_READER);
1489 drop_struct_lock = TRUE;
1490 }
1491
1492 if (db->db_level == 0) {
1493 dnode_new_blkid(dn, db->db_blkid, tx, drop_struct_lock);
1494 ASSERT(dn->dn_maxblkid >= db->db_blkid);
1495 }
1496
1497 if (db->db_level+1 < dn->dn_nlevels) {
1498 dmu_buf_impl_t *parent = db->db_parent;
1499 dbuf_dirty_record_t *di;
1500 int parent_held = FALSE;
1501
1502 if (db->db_parent == NULL || db->db_parent == dn->dn_dbuf) {
1503 int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
1504
1505 parent = dbuf_hold_level(dn, db->db_level+1,
1506 db->db_blkid >> epbs, FTAG);
1507 ASSERT(parent != NULL);
1508 parent_held = TRUE;
1509 }
1510 if (drop_struct_lock)
1511 rw_exit(&dn->dn_struct_rwlock);
1512 ASSERT3U(db->db_level+1, ==, parent->db_level);
1513 di = dbuf_dirty(parent, tx);
1514 if (parent_held)
1515 dbuf_rele(parent, FTAG);
1516
1517 mutex_enter(&db->db_mtx);
1518 /*
1519 * Since we've dropped the mutex, it's possible that
1520 * dbuf_undirty() might have changed this out from under us.
1521 */
1522 if (db->db_last_dirty == dr ||
1523 dn->dn_object == DMU_META_DNODE_OBJECT) {
1524 mutex_enter(&di->dt.di.dr_mtx);
1525 ASSERT3U(di->dr_txg, ==, tx->tx_txg);
1526 ASSERT(!list_link_active(&dr->dr_dirty_node));
1527 list_insert_tail(&di->dt.di.dr_children, dr);
1528 mutex_exit(&di->dt.di.dr_mtx);
1529 dr->dr_parent = di;
1530 }
1531 mutex_exit(&db->db_mtx);
1532 } else {
1533 ASSERT(db->db_level+1 == dn->dn_nlevels);
1534 ASSERT(db->db_blkid < dn->dn_nblkptr);
1535 ASSERT(db->db_parent == NULL || db->db_parent == dn->dn_dbuf);
1536 mutex_enter(&dn->dn_mtx);
1537 ASSERT(!list_link_active(&dr->dr_dirty_node));
1538 list_insert_tail(&dn->dn_dirty_records[txgoff], dr);
1539 mutex_exit(&dn->dn_mtx);
1540 if (drop_struct_lock)
1541 rw_exit(&dn->dn_struct_rwlock);
1542 }
1543
1544 dnode_setdirty(dn, tx);
1545 DB_DNODE_EXIT(db);
1546 return (dr);
1547 }
1548
1549 /*
1550 * Undirty a buffer in the transaction group referenced by the given
1551 * transaction. Return whether this evicted the dbuf.
1552 */
1553 static boolean_t
1554 dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
1555 {
1556 dnode_t *dn;
1557 uint64_t txg = tx->tx_txg;
1558 dbuf_dirty_record_t *dr, **drp;
1559
1560 ASSERT(txg != 0);
1561
1562 /*
1563 * Due to our use of dn_nlevels below, this can only be called
1564 * in open context, unless we are operating on the MOS.
1565 * From syncing context, dn_nlevels may be different from the
1566 * dn_nlevels used when dbuf was dirtied.
1567 */
1568 ASSERT(db->db_objset ==
1569 dmu_objset_pool(db->db_objset)->dp_meta_objset ||
1570 txg != spa_syncing_txg(dmu_objset_spa(db->db_objset)));
1571 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1572 ASSERT0(db->db_level);
1573 ASSERT(MUTEX_HELD(&db->db_mtx));
1574
1575 /*
1576 * If this buffer is not dirty, we're done.
1577 */
1578 for (drp = &db->db_last_dirty; (dr = *drp) != NULL; drp = &dr->dr_next)
1579 if (dr->dr_txg <= txg)
1580 break;
1581 if (dr == NULL || dr->dr_txg < txg)
1582 return (B_FALSE);
1583 ASSERT(dr->dr_txg == txg);
1584 ASSERT(dr->dr_dbuf == db);
1585
1586 DB_DNODE_ENTER(db);
1587 dn = DB_DNODE(db);
1588
1589 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size);
1590
1591 ASSERT(db->db.db_size != 0);
1592
1593 dsl_pool_undirty_space(dmu_objset_pool(dn->dn_objset),
1594 dr->dr_accounted, txg);
1595
1596 *drp = dr->dr_next;
1597
1598 /*
1599 * Note that there are three places in dbuf_dirty()
1600 * where this dirty record may be put on a list.
1601 * Make sure to do a list_remove corresponding to
1602 * every one of those list_insert calls.
1603 */
1604 if (dr->dr_parent) {
1605 mutex_enter(&dr->dr_parent->dt.di.dr_mtx);
1606 list_remove(&dr->dr_parent->dt.di.dr_children, dr);
1607 mutex_exit(&dr->dr_parent->dt.di.dr_mtx);
1608 } else if (db->db_blkid == DMU_SPILL_BLKID ||
1609 db->db_level + 1 == dn->dn_nlevels) {
1610 ASSERT(db->db_blkptr == NULL || db->db_parent == dn->dn_dbuf);
1611 mutex_enter(&dn->dn_mtx);
1612 list_remove(&dn->dn_dirty_records[txg & TXG_MASK], dr);
1613 mutex_exit(&dn->dn_mtx);
1614 }
1615 DB_DNODE_EXIT(db);
1616
1617 if (db->db_state != DB_NOFILL) {
1618 dbuf_unoverride(dr);
1619
1620 ASSERT(db->db_buf != NULL);
1621 ASSERT(dr->dt.dl.dr_data != NULL);
1622 if (dr->dt.dl.dr_data != db->db_buf)
1623 VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, db));
1624 }
1625
1626 kmem_free(dr, sizeof (dbuf_dirty_record_t));
1627
1628 ASSERT(db->db_dirtycnt > 0);
1629 db->db_dirtycnt -= 1;
1630
1631 if (refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) {
1632 arc_buf_t *buf = db->db_buf;
1633
1634 ASSERT(db->db_state == DB_NOFILL || arc_released(buf));
1635 dbuf_clear_data(db);
1636 VERIFY(arc_buf_remove_ref(buf, db));
1637 dbuf_evict(db);
1638 return (B_TRUE);
1639 }
1640
1641 return (B_FALSE);
1642 }
1643
1644 void
1645 dmu_buf_will_dirty(dmu_buf_t *db_fake, dmu_tx_t *tx)
1646 {
1647 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
1648 int rf = DB_RF_MUST_SUCCEED | DB_RF_NOPREFETCH;
1649 dbuf_dirty_record_t *dr;
1650
1651 ASSERT(tx->tx_txg != 0);
1652 ASSERT(!refcount_is_zero(&db->db_holds));
1653
1654 /*
1655 * Quick check for dirtyness. For already dirty blocks, this
1656 * reduces runtime of this function by >90%, and overall performance
1657 * by 50% for some workloads (e.g. file deletion with indirect blocks
1658 * cached).
1659 */
1660 mutex_enter(&db->db_mtx);
1661
1662 for (dr = db->db_last_dirty;
1663 dr != NULL && dr->dr_txg >= tx->tx_txg; dr = dr->dr_next) {
1664 /*
1665 * It's possible that it is already dirty but not cached,
1666 * because there are some calls to dbuf_dirty() that don't
1667 * go through dmu_buf_will_dirty().
1668 */
1669 if (dr->dr_txg == tx->tx_txg && db->db_state == DB_CACHED) {
1670 /* This dbuf is already dirty and cached. */
1671 dbuf_redirty(dr);
1672 mutex_exit(&db->db_mtx);
1673 return;
1674 }
1675 }
1676 mutex_exit(&db->db_mtx);
1677
1678 DB_DNODE_ENTER(db);
1679 if (RW_WRITE_HELD(&DB_DNODE(db)->dn_struct_rwlock))
1680 rf |= DB_RF_HAVESTRUCT;
1681 DB_DNODE_EXIT(db);
1682 (void) dbuf_read(db, NULL, rf);
1683 (void) dbuf_dirty(db, tx);
1684 }
1685
1686 void
1687 dmu_buf_will_not_fill(dmu_buf_t *db_fake, dmu_tx_t *tx)
1688 {
1689 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
1690
1691 db->db_state = DB_NOFILL;
1692
1693 dmu_buf_will_fill(db_fake, tx);
1694 }
1695
1696 void
1697 dmu_buf_will_fill(dmu_buf_t *db_fake, dmu_tx_t *tx)
1698 {
1699 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
1700
1701 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1702 ASSERT(tx->tx_txg != 0);
1703 ASSERT(db->db_level == 0);
1704 ASSERT(!refcount_is_zero(&db->db_holds));
1705
1706 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT ||
1707 dmu_tx_private_ok(tx));
1708
1709 dbuf_noread(db);
1710 (void) dbuf_dirty(db, tx);
1711 }
1712
1713 #pragma weak dmu_buf_fill_done = dbuf_fill_done
1714 /* ARGSUSED */
1715 void
1716 dbuf_fill_done(dmu_buf_impl_t *db, dmu_tx_t *tx)
1717 {
1718 mutex_enter(&db->db_mtx);
1719 DBUF_VERIFY(db);
1720
1721 if (db->db_state == DB_FILL) {
1722 if (db->db_level == 0 && db->db_freed_in_flight) {
1723 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1724 /* we were freed while filling */
1725 /* XXX dbuf_undirty? */
1726 bzero(db->db.db_data, db->db.db_size);
1727 db->db_freed_in_flight = FALSE;
1728 }
1729 db->db_state = DB_CACHED;
1730 cv_broadcast(&db->db_changed);
1731 }
1732 mutex_exit(&db->db_mtx);
1733 }
1734
1735 void
1736 dmu_buf_write_embedded(dmu_buf_t *dbuf, void *data,
1737 bp_embedded_type_t etype, enum zio_compress comp,
1738 int uncompressed_size, int compressed_size, int byteorder,
1739 dmu_tx_t *tx)
1740 {
1741 dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf;
1742 struct dirty_leaf *dl;
1743 dmu_object_type_t type;
1744
1745 if (etype == BP_EMBEDDED_TYPE_DATA) {
1746 ASSERT(spa_feature_is_active(dmu_objset_spa(db->db_objset),
1747 SPA_FEATURE_EMBEDDED_DATA));
1748 }
1749
1750 DB_DNODE_ENTER(db);
1751 type = DB_DNODE(db)->dn_type;
1752 DB_DNODE_EXIT(db);
1753
1754 ASSERT0(db->db_level);
1755 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1756
1757 dmu_buf_will_not_fill(dbuf, tx);
1758
1759 ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg);
1760 dl = &db->db_last_dirty->dt.dl;
1761 encode_embedded_bp_compressed(&dl->dr_overridden_by,
1762 data, comp, uncompressed_size, compressed_size);
1763 BPE_SET_ETYPE(&dl->dr_overridden_by, etype);
1764 BP_SET_TYPE(&dl->dr_overridden_by, type);
1765 BP_SET_LEVEL(&dl->dr_overridden_by, 0);
1766 BP_SET_BYTEORDER(&dl->dr_overridden_by, byteorder);
1767
1768 dl->dr_override_state = DR_OVERRIDDEN;
1769 dl->dr_overridden_by.blk_birth = db->db_last_dirty->dr_txg;
1770 }
1771
1772 /*
1773 * Directly assign a provided arc buf to a given dbuf if it's not referenced
1774 * by anybody except our caller. Otherwise copy arcbuf's contents to dbuf.
1775 */
1776 void
1777 dbuf_assign_arcbuf(dmu_buf_impl_t *db, arc_buf_t *buf, dmu_tx_t *tx)
1778 {
1779 ASSERT(!refcount_is_zero(&db->db_holds));
1780 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1781 ASSERT(db->db_level == 0);
1782 ASSERT(DBUF_GET_BUFC_TYPE(db) == ARC_BUFC_DATA);
1783 ASSERT(buf != NULL);
1784 ASSERT(arc_buf_size(buf) == db->db.db_size);
1785 ASSERT(tx->tx_txg != 0);
1786
1787 arc_return_buf(buf, db);
1788 ASSERT(arc_released(buf));
1789
1790 mutex_enter(&db->db_mtx);
1791
1792 while (db->db_state == DB_READ || db->db_state == DB_FILL)
1793 cv_wait(&db->db_changed, &db->db_mtx);
1794
1795 ASSERT(db->db_state == DB_CACHED || db->db_state == DB_UNCACHED);
1796
1797 if (db->db_state == DB_CACHED &&
1798 refcount_count(&db->db_holds) - 1 > db->db_dirtycnt) {
1799 mutex_exit(&db->db_mtx);
1800 (void) dbuf_dirty(db, tx);
1801 bcopy(buf->b_data, db->db.db_data, db->db.db_size);
1802 VERIFY(arc_buf_remove_ref(buf, db));
1803 xuio_stat_wbuf_copied();
1804 return;
1805 }
1806
1807 xuio_stat_wbuf_nocopy();
1808 if (db->db_state == DB_CACHED) {
1809 dbuf_dirty_record_t *dr = db->db_last_dirty;
1810
1811 ASSERT(db->db_buf != NULL);
1812 if (dr != NULL && dr->dr_txg == tx->tx_txg) {
1813 ASSERT(dr->dt.dl.dr_data == db->db_buf);
1814 if (!arc_released(db->db_buf)) {
1815 ASSERT(dr->dt.dl.dr_override_state ==
1816 DR_OVERRIDDEN);
1817 arc_release(db->db_buf, db);
1818 }
1819 dr->dt.dl.dr_data = buf;
1820 VERIFY(arc_buf_remove_ref(db->db_buf, db));
1821 } else if (dr == NULL || dr->dt.dl.dr_data != db->db_buf) {
1822 arc_release(db->db_buf, db);
1823 VERIFY(arc_buf_remove_ref(db->db_buf, db));
1824 }
1825 db->db_buf = NULL;
1826 }
1827 ASSERT(db->db_buf == NULL);
1828 dbuf_set_data(db, buf);
1829 db->db_state = DB_FILL;
1830 mutex_exit(&db->db_mtx);
1831 (void) dbuf_dirty(db, tx);
1832 dmu_buf_fill_done(&db->db, tx);
1833 }
1834
1835 /*
1836 * "Clear" the contents of this dbuf. This will mark the dbuf
1837 * EVICTING and clear *most* of its references. Unfortunately,
1838 * when we are not holding the dn_dbufs_mtx, we can't clear the
1839 * entry in the dn_dbufs list. We have to wait until dbuf_destroy()
1840 * in this case. For callers from the DMU we will usually see:
1841 * dbuf_clear()->arc_clear_callback()->dbuf_do_evict()->dbuf_destroy()
1842 * For the arc callback, we will usually see:
1843 * dbuf_do_evict()->dbuf_clear();dbuf_destroy()
1844 * Sometimes, though, we will get a mix of these two:
1845 * DMU: dbuf_clear()->arc_clear_callback()
1846 * ARC: dbuf_do_evict()->dbuf_destroy()
1847 *
1848 * This routine will dissociate the dbuf from the arc, by calling
1849 * arc_clear_callback(), but will not evict the data from the ARC.
1850 */
1851 void
1852 dbuf_clear(dmu_buf_impl_t *db)
1853 {
1854 dnode_t *dn;
1855 dmu_buf_impl_t *parent = db->db_parent;
1856 dmu_buf_impl_t *dndb;
1857 boolean_t dbuf_gone = B_FALSE;
1858
1859 ASSERT(MUTEX_HELD(&db->db_mtx));
1860 ASSERT(refcount_is_zero(&db->db_holds));
1861
1862 dbuf_evict_user(db);
1863
1864 if (db->db_state == DB_CACHED) {
1865 ASSERT(db->db.db_data != NULL);
1866 if (db->db_blkid == DMU_BONUS_BLKID) {
1867 int slots = DB_DNODE(db)->dn_num_slots;
1868 int bonuslen = DN_SLOTS_TO_BONUSLEN(slots);
1869 zio_buf_free(db->db.db_data, bonuslen);
1870 arc_space_return(bonuslen, ARC_SPACE_BONUS);
1871 }
1872 db->db.db_data = NULL;
1873 db->db_state = DB_UNCACHED;
1874 }
1875
1876 ASSERT(db->db_state == DB_UNCACHED || db->db_state == DB_NOFILL);
1877 ASSERT(db->db_data_pending == NULL);
1878
1879 db->db_state = DB_EVICTING;
1880 db->db_blkptr = NULL;
1881
1882 DB_DNODE_ENTER(db);
1883 dn = DB_DNODE(db);
1884 dndb = dn->dn_dbuf;
1885 if (db->db_blkid != DMU_BONUS_BLKID && MUTEX_HELD(&dn->dn_dbufs_mtx)) {
1886 avl_remove(&dn->dn_dbufs, db);
1887 atomic_dec_32(&dn->dn_dbufs_count);
1888 membar_producer();
1889 DB_DNODE_EXIT(db);
1890 /*
1891 * Decrementing the dbuf count means that the hold corresponding
1892 * to the removed dbuf is no longer discounted in dnode_move(),
1893 * so the dnode cannot be moved until after we release the hold.
1894 * The membar_producer() ensures visibility of the decremented
1895 * value in dnode_move(), since DB_DNODE_EXIT doesn't actually
1896 * release any lock.
1897 */
1898 dnode_rele(dn, db);
1899 db->db_dnode_handle = NULL;
1900 } else {
1901 DB_DNODE_EXIT(db);
1902 }
1903
1904 if (db->db_buf)
1905 dbuf_gone = arc_clear_callback(db->db_buf);
1906
1907 if (!dbuf_gone)
1908 mutex_exit(&db->db_mtx);
1909
1910 /*
1911 * If this dbuf is referenced from an indirect dbuf,
1912 * decrement the ref count on the indirect dbuf.
1913 */
1914 if (parent && parent != dndb)
1915 dbuf_rele(parent, db);
1916 }
1917
1918 /*
1919 * Note: While bpp will always be updated if the function returns success,
1920 * parentp will not be updated if the dnode does not have dn_dbuf filled in;
1921 * this happens when the dnode is the meta-dnode, or a userused or groupused
1922 * object.
1923 */
1924 __attribute__((always_inline))
1925 static inline int
1926 dbuf_findbp(dnode_t *dn, int level, uint64_t blkid, int fail_sparse,
1927 dmu_buf_impl_t **parentp, blkptr_t **bpp, struct dbuf_hold_impl_data *dh)
1928 {
1929 int nlevels, epbs;
1930
1931 *parentp = NULL;
1932 *bpp = NULL;
1933
1934 ASSERT(blkid != DMU_BONUS_BLKID);
1935
1936 if (blkid == DMU_SPILL_BLKID) {
1937 mutex_enter(&dn->dn_mtx);
1938 if (dn->dn_have_spill &&
1939 (dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR))
1940 *bpp = DN_SPILL_BLKPTR(dn->dn_phys);
1941 else
1942 *bpp = NULL;
1943 dbuf_add_ref(dn->dn_dbuf, NULL);
1944 *parentp = dn->dn_dbuf;
1945 mutex_exit(&dn->dn_mtx);
1946 return (0);
1947 }
1948
1949 nlevels =
1950 (dn->dn_phys->dn_nlevels == 0) ? 1 : dn->dn_phys->dn_nlevels;
1951 epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
1952
1953 ASSERT3U(level * epbs, <, 64);
1954 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
1955 /*
1956 * This assertion shouldn't trip as long as the max indirect block size
1957 * is less than 1M. The reason for this is that up to that point,
1958 * the number of levels required to address an entire object with blocks
1959 * of size SPA_MINBLOCKSIZE satisfies nlevels * epbs + 1 <= 64. In
1960 * other words, if N * epbs + 1 > 64, then if (N-1) * epbs + 1 > 55
1961 * (i.e. we can address the entire object), objects will all use at most
1962 * N-1 levels and the assertion won't overflow. However, once epbs is
1963 * 13, 4 * 13 + 1 = 53, but 5 * 13 + 1 = 66. Then, 4 levels will not be
1964 * enough to address an entire object, so objects will have 5 levels,
1965 * but then this assertion will overflow.
1966 *
1967 * All this is to say that if we ever increase DN_MAX_INDBLKSHIFT, we
1968 * need to redo this logic to handle overflows.
1969 */
1970 ASSERT(level >= nlevels ||
1971 ((nlevels - level - 1) * epbs) +
1972 highbit64(dn->dn_phys->dn_nblkptr) <= 64);
1973 if (level >= nlevels ||
1974 blkid >= ((uint64_t)dn->dn_phys->dn_nblkptr <<
1975 ((nlevels - level - 1) * epbs)) ||
1976 (fail_sparse &&
1977 blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))) {
1978 /* the buffer has no parent yet */
1979 return (SET_ERROR(ENOENT));
1980 } else if (level < nlevels-1) {
1981 /* this block is referenced from an indirect block */
1982 int err;
1983 if (dh == NULL) {
1984 err = dbuf_hold_impl(dn, level+1,
1985 blkid >> epbs, fail_sparse, FALSE, NULL, parentp);
1986 } else {
1987 __dbuf_hold_impl_init(dh + 1, dn, dh->dh_level + 1,
1988 blkid >> epbs, fail_sparse, FALSE, NULL,
1989 parentp, dh->dh_depth + 1);
1990 err = __dbuf_hold_impl(dh + 1);
1991 }
1992 if (err)
1993 return (err);
1994 err = dbuf_read(*parentp, NULL,
1995 (DB_RF_HAVESTRUCT | DB_RF_NOPREFETCH | DB_RF_CANFAIL));
1996 if (err) {
1997 dbuf_rele(*parentp, NULL);
1998 *parentp = NULL;
1999 return (err);
2000 }
2001 *bpp = ((blkptr_t *)(*parentp)->db.db_data) +
2002 (blkid & ((1ULL << epbs) - 1));
2003 if (blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))
2004 ASSERT(BP_IS_HOLE(*bpp));
2005 return (0);
2006 } else {
2007 /* the block is referenced from the dnode */
2008 ASSERT3U(level, ==, nlevels-1);
2009 ASSERT(dn->dn_phys->dn_nblkptr == 0 ||
2010 blkid < dn->dn_phys->dn_nblkptr);
2011 if (dn->dn_dbuf) {
2012 dbuf_add_ref(dn->dn_dbuf, NULL);
2013 *parentp = dn->dn_dbuf;
2014 }
2015 *bpp = &dn->dn_phys->dn_blkptr[blkid];
2016 return (0);
2017 }
2018 }
2019
2020 static dmu_buf_impl_t *
2021 dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid,
2022 dmu_buf_impl_t *parent, blkptr_t *blkptr)
2023 {
2024 objset_t *os = dn->dn_objset;
2025 dmu_buf_impl_t *db, *odb;
2026
2027 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
2028 ASSERT(dn->dn_type != DMU_OT_NONE);
2029
2030 db = kmem_cache_alloc(dbuf_cache, KM_SLEEP);
2031
2032 db->db_objset = os;
2033 db->db.db_object = dn->dn_object;
2034 db->db_level = level;
2035 db->db_blkid = blkid;
2036 db->db_last_dirty = NULL;
2037 db->db_dirtycnt = 0;
2038 db->db_dnode_handle = dn->dn_handle;
2039 db->db_parent = parent;
2040 db->db_blkptr = blkptr;
2041
2042 db->db_user = NULL;
2043 db->db_user_immediate_evict = FALSE;
2044 db->db_freed_in_flight = FALSE;
2045 db->db_pending_evict = FALSE;
2046
2047 if (blkid == DMU_BONUS_BLKID) {
2048 ASSERT3P(parent, ==, dn->dn_dbuf);
2049 db->db.db_size = DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots) -
2050 (dn->dn_nblkptr-1) * sizeof (blkptr_t);
2051 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen);
2052 db->db.db_offset = DMU_BONUS_BLKID;
2053 db->db_state = DB_UNCACHED;
2054 /* the bonus dbuf is not placed in the hash table */
2055 arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_DBUF);
2056 return (db);
2057 } else if (blkid == DMU_SPILL_BLKID) {
2058 db->db.db_size = (blkptr != NULL) ?
2059 BP_GET_LSIZE(blkptr) : SPA_MINBLOCKSIZE;
2060 db->db.db_offset = 0;
2061 } else {
2062 int blocksize =
2063 db->db_level ? 1 << dn->dn_indblkshift : dn->dn_datablksz;
2064 db->db.db_size = blocksize;
2065 db->db.db_offset = db->db_blkid * blocksize;
2066 }
2067
2068 /*
2069 * Hold the dn_dbufs_mtx while we get the new dbuf
2070 * in the hash table *and* added to the dbufs list.
2071 * This prevents a possible deadlock with someone
2072 * trying to look up this dbuf before its added to the
2073 * dn_dbufs list.
2074 */
2075 mutex_enter(&dn->dn_dbufs_mtx);
2076 db->db_state = DB_EVICTING;
2077 if ((odb = dbuf_hash_insert(db)) != NULL) {
2078 /* someone else inserted it first */
2079 kmem_cache_free(dbuf_cache, db);
2080 mutex_exit(&dn->dn_dbufs_mtx);
2081 return (odb);
2082 }
2083 avl_add(&dn->dn_dbufs, db);
2084 if (db->db_level == 0 && db->db_blkid >=
2085 dn->dn_unlisted_l0_blkid)
2086 dn->dn_unlisted_l0_blkid = db->db_blkid + 1;
2087 db->db_state = DB_UNCACHED;
2088 mutex_exit(&dn->dn_dbufs_mtx);
2089 arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_DBUF);
2090
2091 if (parent && parent != dn->dn_dbuf)
2092 dbuf_add_ref(parent, db);
2093
2094 ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT ||
2095 refcount_count(&dn->dn_holds) > 0);
2096 (void) refcount_add(&dn->dn_holds, db);
2097 atomic_inc_32(&dn->dn_dbufs_count);
2098
2099 dprintf_dbuf(db, "db=%p\n", db);
2100
2101 return (db);
2102 }
2103
2104 static int
2105 dbuf_do_evict(void *private)
2106 {
2107 dmu_buf_impl_t *db = private;
2108
2109 if (!MUTEX_HELD(&db->db_mtx))
2110 mutex_enter(&db->db_mtx);
2111
2112 ASSERT(refcount_is_zero(&db->db_holds));
2113
2114 if (db->db_state != DB_EVICTING) {
2115 ASSERT(db->db_state == DB_CACHED);
2116 DBUF_VERIFY(db);
2117 db->db_buf = NULL;
2118 dbuf_evict(db);
2119 } else {
2120 mutex_exit(&db->db_mtx);
2121 dbuf_destroy(db);
2122 }
2123 return (0);
2124 }
2125
2126 static void
2127 dbuf_destroy(dmu_buf_impl_t *db)
2128 {
2129 ASSERT(refcount_is_zero(&db->db_holds));
2130
2131 if (db->db_blkid != DMU_BONUS_BLKID) {
2132 /*
2133 * If this dbuf is still on the dn_dbufs list,
2134 * remove it from that list.
2135 */
2136 if (db->db_dnode_handle != NULL) {
2137 dnode_t *dn;
2138
2139 DB_DNODE_ENTER(db);
2140 dn = DB_DNODE(db);
2141 mutex_enter(&dn->dn_dbufs_mtx);
2142 avl_remove(&dn->dn_dbufs, db);
2143 atomic_dec_32(&dn->dn_dbufs_count);
2144 mutex_exit(&dn->dn_dbufs_mtx);
2145 DB_DNODE_EXIT(db);
2146 /*
2147 * Decrementing the dbuf count means that the hold
2148 * corresponding to the removed dbuf is no longer
2149 * discounted in dnode_move(), so the dnode cannot be
2150 * moved until after we release the hold.
2151 */
2152 dnode_rele(dn, db);
2153 db->db_dnode_handle = NULL;
2154 }
2155 dbuf_hash_remove(db);
2156 }
2157 db->db_parent = NULL;
2158 db->db_buf = NULL;
2159
2160 ASSERT(db->db.db_data == NULL);
2161 ASSERT(db->db_hash_next == NULL);
2162 ASSERT(db->db_blkptr == NULL);
2163 ASSERT(db->db_data_pending == NULL);
2164
2165 kmem_cache_free(dbuf_cache, db);
2166 arc_space_return(sizeof (dmu_buf_impl_t), ARC_SPACE_DBUF);
2167 }
2168
2169 typedef struct dbuf_prefetch_arg {
2170 spa_t *dpa_spa; /* The spa to issue the prefetch in. */
2171 zbookmark_phys_t dpa_zb; /* The target block to prefetch. */
2172 int dpa_epbs; /* Entries (blkptr_t's) Per Block Shift. */
2173 int dpa_curlevel; /* The current level that we're reading */
2174 zio_priority_t dpa_prio; /* The priority I/Os should be issued at. */
2175 zio_t *dpa_zio; /* The parent zio_t for all prefetches. */
2176 arc_flags_t dpa_aflags; /* Flags to pass to the final prefetch. */
2177 } dbuf_prefetch_arg_t;
2178
2179 /*
2180 * Actually issue the prefetch read for the block given.
2181 */
2182 static void
2183 dbuf_issue_final_prefetch(dbuf_prefetch_arg_t *dpa, blkptr_t *bp)
2184 {
2185 arc_flags_t aflags;
2186 if (BP_IS_HOLE(bp) || BP_IS_EMBEDDED(bp))
2187 return;
2188
2189 aflags = dpa->dpa_aflags | ARC_FLAG_NOWAIT | ARC_FLAG_PREFETCH;
2190
2191 ASSERT3U(dpa->dpa_curlevel, ==, BP_GET_LEVEL(bp));
2192 ASSERT3U(dpa->dpa_curlevel, ==, dpa->dpa_zb.zb_level);
2193 ASSERT(dpa->dpa_zio != NULL);
2194 (void) arc_read(dpa->dpa_zio, dpa->dpa_spa, bp, NULL, NULL,
2195 dpa->dpa_prio, ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE,
2196 &aflags, &dpa->dpa_zb);
2197 }
2198
2199 /*
2200 * Called when an indirect block above our prefetch target is read in. This
2201 * will either read in the next indirect block down the tree or issue the actual
2202 * prefetch if the next block down is our target.
2203 */
2204 static void
2205 dbuf_prefetch_indirect_done(zio_t *zio, arc_buf_t *abuf, void *private)
2206 {
2207 dbuf_prefetch_arg_t *dpa = private;
2208 uint64_t nextblkid;
2209 blkptr_t *bp;
2210
2211 ASSERT3S(dpa->dpa_zb.zb_level, <, dpa->dpa_curlevel);
2212 ASSERT3S(dpa->dpa_curlevel, >, 0);
2213 if (zio != NULL) {
2214 ASSERT3S(BP_GET_LEVEL(zio->io_bp), ==, dpa->dpa_curlevel);
2215 ASSERT3U(BP_GET_LSIZE(zio->io_bp), ==, zio->io_size);
2216 ASSERT3P(zio->io_spa, ==, dpa->dpa_spa);
2217 }
2218
2219 dpa->dpa_curlevel--;
2220
2221 nextblkid = dpa->dpa_zb.zb_blkid >>
2222 (dpa->dpa_epbs * (dpa->dpa_curlevel - dpa->dpa_zb.zb_level));
2223 bp = ((blkptr_t *)abuf->b_data) +
2224 P2PHASE(nextblkid, 1ULL << dpa->dpa_epbs);
2225 if (BP_IS_HOLE(bp) || (zio != NULL && zio->io_error != 0)) {
2226 kmem_free(dpa, sizeof (*dpa));
2227 } else if (dpa->dpa_curlevel == dpa->dpa_zb.zb_level) {
2228 ASSERT3U(nextblkid, ==, dpa->dpa_zb.zb_blkid);
2229 dbuf_issue_final_prefetch(dpa, bp);
2230 kmem_free(dpa, sizeof (*dpa));
2231 } else {
2232 arc_flags_t iter_aflags = ARC_FLAG_NOWAIT;
2233 zbookmark_phys_t zb;
2234
2235 ASSERT3U(dpa->dpa_curlevel, ==, BP_GET_LEVEL(bp));
2236
2237 SET_BOOKMARK(&zb, dpa->dpa_zb.zb_objset,
2238 dpa->dpa_zb.zb_object, dpa->dpa_curlevel, nextblkid);
2239
2240 (void) arc_read(dpa->dpa_zio, dpa->dpa_spa,
2241 bp, dbuf_prefetch_indirect_done, dpa, dpa->dpa_prio,
2242 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE,
2243 &iter_aflags, &zb);
2244 }
2245 (void) arc_buf_remove_ref(abuf, private);
2246 }
2247
2248 /*
2249 * Issue prefetch reads for the given block on the given level. If the indirect
2250 * blocks above that block are not in memory, we will read them in
2251 * asynchronously. As a result, this call never blocks waiting for a read to
2252 * complete.
2253 */
2254 void
2255 dbuf_prefetch(dnode_t *dn, int64_t level, uint64_t blkid, zio_priority_t prio,
2256 arc_flags_t aflags)
2257 {
2258 blkptr_t bp;
2259 int epbs, nlevels, curlevel;
2260 uint64_t curblkid;
2261 dmu_buf_impl_t *db;
2262 zio_t *pio;
2263 dbuf_prefetch_arg_t *dpa;
2264 dsl_dataset_t *ds;
2265
2266 ASSERT(blkid != DMU_BONUS_BLKID);
2267 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
2268
2269 if (blkid > dn->dn_maxblkid)
2270 return;
2271
2272 if (dnode_block_freed(dn, blkid))
2273 return;
2274
2275 /*
2276 * This dnode hasn't been written to disk yet, so there's nothing to
2277 * prefetch.
2278 */
2279 nlevels = dn->dn_phys->dn_nlevels;
2280 if (level >= nlevels || dn->dn_phys->dn_nblkptr == 0)
2281 return;
2282
2283 epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT;
2284 if (dn->dn_phys->dn_maxblkid < blkid << (epbs * level))
2285 return;
2286
2287 db = dbuf_find(dn->dn_objset, dn->dn_object,
2288 level, blkid);
2289 if (db != NULL) {
2290 mutex_exit(&db->db_mtx);
2291 /*
2292 * This dbuf already exists. It is either CACHED, or
2293 * (we assume) about to be read or filled.
2294 */
2295 return;
2296 }
2297
2298 /*
2299 * Find the closest ancestor (indirect block) of the target block
2300 * that is present in the cache. In this indirect block, we will
2301 * find the bp that is at curlevel, curblkid.
2302 */
2303 curlevel = level;
2304 curblkid = blkid;
2305 while (curlevel < nlevels - 1) {
2306 int parent_level = curlevel + 1;
2307 uint64_t parent_blkid = curblkid >> epbs;
2308 dmu_buf_impl_t *db;
2309
2310 if (dbuf_hold_impl(dn, parent_level, parent_blkid,
2311 FALSE, TRUE, FTAG, &db) == 0) {
2312 blkptr_t *bpp = db->db_buf->b_data;
2313 bp = bpp[P2PHASE(curblkid, 1 << epbs)];
2314 dbuf_rele(db, FTAG);
2315 break;
2316 }
2317
2318 curlevel = parent_level;
2319 curblkid = parent_blkid;
2320 }
2321
2322 if (curlevel == nlevels - 1) {
2323 /* No cached indirect blocks found. */
2324 ASSERT3U(curblkid, <, dn->dn_phys->dn_nblkptr);
2325 bp = dn->dn_phys->dn_blkptr[curblkid];
2326 }
2327 if (BP_IS_HOLE(&bp))
2328 return;
2329
2330 ASSERT3U(curlevel, ==, BP_GET_LEVEL(&bp));
2331
2332 pio = zio_root(dmu_objset_spa(dn->dn_objset), NULL, NULL,
2333 ZIO_FLAG_CANFAIL);
2334
2335 dpa = kmem_zalloc(sizeof (*dpa), KM_SLEEP);
2336 ds = dn->dn_objset->os_dsl_dataset;
2337 SET_BOOKMARK(&dpa->dpa_zb, ds != NULL ? ds->ds_object : DMU_META_OBJSET,
2338 dn->dn_object, level, blkid);
2339 dpa->dpa_curlevel = curlevel;
2340 dpa->dpa_prio = prio;
2341 dpa->dpa_aflags = aflags;
2342 dpa->dpa_spa = dn->dn_objset->os_spa;
2343 dpa->dpa_epbs = epbs;
2344 dpa->dpa_zio = pio;
2345
2346 /*
2347 * If we have the indirect just above us, no need to do the asynchronous
2348 * prefetch chain; we'll just run the last step ourselves. If we're at
2349 * a higher level, though, we want to issue the prefetches for all the
2350 * indirect blocks asynchronously, so we can go on with whatever we were
2351 * doing.
2352 */
2353 if (curlevel == level) {
2354 ASSERT3U(curblkid, ==, blkid);
2355 dbuf_issue_final_prefetch(dpa, &bp);
2356 kmem_free(dpa, sizeof (*dpa));
2357 } else {
2358 arc_flags_t iter_aflags = ARC_FLAG_NOWAIT;
2359 zbookmark_phys_t zb;
2360
2361 SET_BOOKMARK(&zb, ds != NULL ? ds->ds_object : DMU_META_OBJSET,
2362 dn->dn_object, curlevel, curblkid);
2363 (void) arc_read(dpa->dpa_zio, dpa->dpa_spa,
2364 &bp, dbuf_prefetch_indirect_done, dpa, prio,
2365 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE,
2366 &iter_aflags, &zb);
2367 }
2368 /*
2369 * We use pio here instead of dpa_zio since it's possible that
2370 * dpa may have already been freed.
2371 */
2372 zio_nowait(pio);
2373 }
2374
2375 #define DBUF_HOLD_IMPL_MAX_DEPTH 20
2376
2377 /*
2378 * Returns with db_holds incremented, and db_mtx not held.
2379 * Note: dn_struct_rwlock must be held.
2380 */
2381 static int
2382 __dbuf_hold_impl(struct dbuf_hold_impl_data *dh)
2383 {
2384 ASSERT3S(dh->dh_depth, <, DBUF_HOLD_IMPL_MAX_DEPTH);
2385 dh->dh_parent = NULL;
2386
2387 ASSERT(dh->dh_blkid != DMU_BONUS_BLKID);
2388 ASSERT(RW_LOCK_HELD(&dh->dh_dn->dn_struct_rwlock));
2389 ASSERT3U(dh->dh_dn->dn_nlevels, >, dh->dh_level);
2390
2391 *(dh->dh_dbp) = NULL;
2392 top:
2393 /* dbuf_find() returns with db_mtx held */
2394 dh->dh_db = dbuf_find(dh->dh_dn->dn_objset, dh->dh_dn->dn_object,
2395 dh->dh_level, dh->dh_blkid);
2396
2397 if (dh->dh_db == NULL) {
2398 dh->dh_bp = NULL;
2399
2400 if (dh->dh_fail_uncached)
2401 return (SET_ERROR(ENOENT));
2402
2403 ASSERT3P(dh->dh_parent, ==, NULL);
2404 dh->dh_err = dbuf_findbp(dh->dh_dn, dh->dh_level, dh->dh_blkid,
2405 dh->dh_fail_sparse, &dh->dh_parent,
2406 &dh->dh_bp, dh);
2407 if (dh->dh_fail_sparse) {
2408 if (dh->dh_err == 0 &&
2409 dh->dh_bp && BP_IS_HOLE(dh->dh_bp))
2410 dh->dh_err = SET_ERROR(ENOENT);
2411 if (dh->dh_err) {
2412 if (dh->dh_parent)
2413 dbuf_rele(dh->dh_parent, NULL);
2414 return (dh->dh_err);
2415 }
2416 }
2417 if (dh->dh_err && dh->dh_err != ENOENT)
2418 return (dh->dh_err);
2419 dh->dh_db = dbuf_create(dh->dh_dn, dh->dh_level, dh->dh_blkid,
2420 dh->dh_parent, dh->dh_bp);
2421 }
2422
2423 if (dh->dh_fail_uncached && dh->dh_db->db_state != DB_CACHED) {
2424 mutex_exit(&dh->dh_db->db_mtx);
2425 return (SET_ERROR(ENOENT));
2426 }
2427
2428 if (dh->dh_db->db_buf && refcount_is_zero(&dh->dh_db->db_holds)) {
2429 arc_buf_add_ref(dh->dh_db->db_buf, dh->dh_db);
2430 if (dh->dh_db->db_buf->b_data == NULL) {
2431 dbuf_clear(dh->dh_db);
2432 if (dh->dh_parent) {
2433 dbuf_rele(dh->dh_parent, NULL);
2434 dh->dh_parent = NULL;
2435 }
2436 goto top;
2437 }
2438 ASSERT3P(dh->dh_db->db.db_data, ==, dh->dh_db->db_buf->b_data);
2439 }
2440
2441 ASSERT(dh->dh_db->db_buf == NULL || arc_referenced(dh->dh_db->db_buf));
2442
2443 /*
2444 * If this buffer is currently syncing out, and we are are
2445 * still referencing it from db_data, we need to make a copy
2446 * of it in case we decide we want to dirty it again in this txg.
2447 */
2448 if (dh->dh_db->db_level == 0 &&
2449 dh->dh_db->db_blkid != DMU_BONUS_BLKID &&
2450 dh->dh_dn->dn_object != DMU_META_DNODE_OBJECT &&
2451 dh->dh_db->db_state == DB_CACHED && dh->dh_db->db_data_pending) {
2452 dh->dh_dr = dh->dh_db->db_data_pending;
2453
2454 if (dh->dh_dr->dt.dl.dr_data == dh->dh_db->db_buf) {
2455 dh->dh_type = DBUF_GET_BUFC_TYPE(dh->dh_db);
2456
2457 dbuf_set_data(dh->dh_db,
2458 arc_buf_alloc(dh->dh_dn->dn_objset->os_spa,
2459 dh->dh_db->db.db_size, dh->dh_db, dh->dh_type));
2460 bcopy(dh->dh_dr->dt.dl.dr_data->b_data,
2461 dh->dh_db->db.db_data, dh->dh_db->db.db_size);
2462 }
2463 }
2464
2465 (void) refcount_add(&dh->dh_db->db_holds, dh->dh_tag);
2466 DBUF_VERIFY(dh->dh_db);
2467 mutex_exit(&dh->dh_db->db_mtx);
2468
2469 /* NOTE: we can't rele the parent until after we drop the db_mtx */
2470 if (dh->dh_parent)
2471 dbuf_rele(dh->dh_parent, NULL);
2472
2473 ASSERT3P(DB_DNODE(dh->dh_db), ==, dh->dh_dn);
2474 ASSERT3U(dh->dh_db->db_blkid, ==, dh->dh_blkid);
2475 ASSERT3U(dh->dh_db->db_level, ==, dh->dh_level);
2476 *(dh->dh_dbp) = dh->dh_db;
2477
2478 return (0);
2479 }
2480
2481 /*
2482 * The following code preserves the recursive function dbuf_hold_impl()
2483 * but moves the local variables AND function arguments to the heap to
2484 * minimize the stack frame size. Enough space is initially allocated
2485 * on the stack for 20 levels of recursion.
2486 */
2487 int
2488 dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid,
2489 boolean_t fail_sparse, boolean_t fail_uncached,
2490 void *tag, dmu_buf_impl_t **dbp)
2491 {
2492 struct dbuf_hold_impl_data *dh;
2493 int error;
2494
2495 dh = kmem_alloc(sizeof (struct dbuf_hold_impl_data) *
2496 DBUF_HOLD_IMPL_MAX_DEPTH, KM_SLEEP);
2497 __dbuf_hold_impl_init(dh, dn, level, blkid, fail_sparse,
2498 fail_uncached, tag, dbp, 0);
2499
2500 error = __dbuf_hold_impl(dh);
2501
2502 kmem_free(dh, sizeof (struct dbuf_hold_impl_data) *
2503 DBUF_HOLD_IMPL_MAX_DEPTH);
2504
2505 return (error);
2506 }
2507
2508 static void
2509 __dbuf_hold_impl_init(struct dbuf_hold_impl_data *dh,
2510 dnode_t *dn, uint8_t level, uint64_t blkid,
2511 boolean_t fail_sparse, boolean_t fail_uncached,
2512 void *tag, dmu_buf_impl_t **dbp, int depth)
2513 {
2514 dh->dh_dn = dn;
2515 dh->dh_level = level;
2516 dh->dh_blkid = blkid;
2517
2518 dh->dh_fail_sparse = fail_sparse;
2519 dh->dh_fail_uncached = fail_uncached;
2520
2521 dh->dh_tag = tag;
2522 dh->dh_dbp = dbp;
2523
2524 dh->dh_db = NULL;
2525 dh->dh_parent = NULL;
2526 dh->dh_bp = NULL;
2527 dh->dh_err = 0;
2528 dh->dh_dr = NULL;
2529 dh->dh_type = 0;
2530
2531 dh->dh_depth = depth;
2532 }
2533
2534 dmu_buf_impl_t *
2535 dbuf_hold(dnode_t *dn, uint64_t blkid, void *tag)
2536 {
2537 return (dbuf_hold_level(dn, 0, blkid, tag));
2538 }
2539
2540 dmu_buf_impl_t *
2541 dbuf_hold_level(dnode_t *dn, int level, uint64_t blkid, void *tag)
2542 {
2543 dmu_buf_impl_t *db;
2544 int err = dbuf_hold_impl(dn, level, blkid, FALSE, FALSE, tag, &db);
2545 return (err ? NULL : db);
2546 }
2547
2548 void
2549 dbuf_create_bonus(dnode_t *dn)
2550 {
2551 ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock));
2552
2553 ASSERT(dn->dn_bonus == NULL);
2554 dn->dn_bonus = dbuf_create(dn, 0, DMU_BONUS_BLKID, dn->dn_dbuf, NULL);
2555 }
2556
2557 int
2558 dbuf_spill_set_blksz(dmu_buf_t *db_fake, uint64_t blksz, dmu_tx_t *tx)
2559 {
2560 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2561 dnode_t *dn;
2562
2563 if (db->db_blkid != DMU_SPILL_BLKID)
2564 return (SET_ERROR(ENOTSUP));
2565 if (blksz == 0)
2566 blksz = SPA_MINBLOCKSIZE;
2567 ASSERT3U(blksz, <=, spa_maxblocksize(dmu_objset_spa(db->db_objset)));
2568 blksz = P2ROUNDUP(blksz, SPA_MINBLOCKSIZE);
2569
2570 DB_DNODE_ENTER(db);
2571 dn = DB_DNODE(db);
2572 rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
2573 dbuf_new_size(db, blksz, tx);
2574 rw_exit(&dn->dn_struct_rwlock);
2575 DB_DNODE_EXIT(db);
2576
2577 return (0);
2578 }
2579
2580 void
2581 dbuf_rm_spill(dnode_t *dn, dmu_tx_t *tx)
2582 {
2583 dbuf_free_range(dn, DMU_SPILL_BLKID, DMU_SPILL_BLKID, tx);
2584 }
2585
2586 #pragma weak dmu_buf_add_ref = dbuf_add_ref
2587 void
2588 dbuf_add_ref(dmu_buf_impl_t *db, void *tag)
2589 {
2590 VERIFY(refcount_add(&db->db_holds, tag) > 1);
2591 }
2592
2593 #pragma weak dmu_buf_try_add_ref = dbuf_try_add_ref
2594 boolean_t
2595 dbuf_try_add_ref(dmu_buf_t *db_fake, objset_t *os, uint64_t obj, uint64_t blkid,
2596 void *tag)
2597 {
2598 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2599 dmu_buf_impl_t *found_db;
2600 boolean_t result = B_FALSE;
2601
2602 if (blkid == DMU_BONUS_BLKID)
2603 found_db = dbuf_find_bonus(os, obj);
2604 else
2605 found_db = dbuf_find(os, obj, 0, blkid);
2606
2607 if (found_db != NULL) {
2608 if (db == found_db && dbuf_refcount(db) > db->db_dirtycnt) {
2609 (void) refcount_add(&db->db_holds, tag);
2610 result = B_TRUE;
2611 }
2612 mutex_exit(&found_db->db_mtx);
2613 }
2614 return (result);
2615 }
2616
2617 /*
2618 * If you call dbuf_rele() you had better not be referencing the dnode handle
2619 * unless you have some other direct or indirect hold on the dnode. (An indirect
2620 * hold is a hold on one of the dnode's dbufs, including the bonus buffer.)
2621 * Without that, the dbuf_rele() could lead to a dnode_rele() followed by the
2622 * dnode's parent dbuf evicting its dnode handles.
2623 */
2624 void
2625 dbuf_rele(dmu_buf_impl_t *db, void *tag)
2626 {
2627 mutex_enter(&db->db_mtx);
2628 dbuf_rele_and_unlock(db, tag);
2629 }
2630
2631 void
2632 dmu_buf_rele(dmu_buf_t *db, void *tag)
2633 {
2634 dbuf_rele((dmu_buf_impl_t *)db, tag);
2635 }
2636
2637 /*
2638 * dbuf_rele() for an already-locked dbuf. This is necessary to allow
2639 * db_dirtycnt and db_holds to be updated atomically.
2640 */
2641 void
2642 dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag)
2643 {
2644 int64_t holds;
2645
2646 ASSERT(MUTEX_HELD(&db->db_mtx));
2647 DBUF_VERIFY(db);
2648
2649 /*
2650 * Remove the reference to the dbuf before removing its hold on the
2651 * dnode so we can guarantee in dnode_move() that a referenced bonus
2652 * buffer has a corresponding dnode hold.
2653 */
2654 holds = refcount_remove(&db->db_holds, tag);
2655 ASSERT(holds >= 0);
2656
2657 /*
2658 * We can't freeze indirects if there is a possibility that they
2659 * may be modified in the current syncing context.
2660 */
2661 if (db->db_buf && holds == (db->db_level == 0 ? db->db_dirtycnt : 0))
2662 arc_buf_freeze(db->db_buf);
2663
2664 if (holds == db->db_dirtycnt &&
2665 db->db_level == 0 && db->db_user_immediate_evict)
2666 dbuf_evict_user(db);
2667
2668 if (holds == 0) {
2669 if (db->db_blkid == DMU_BONUS_BLKID) {
2670 dnode_t *dn;
2671 boolean_t evict_dbuf = db->db_pending_evict;
2672
2673 /*
2674 * If the dnode moves here, we cannot cross this
2675 * barrier until the move completes.
2676 */
2677 DB_DNODE_ENTER(db);
2678
2679 dn = DB_DNODE(db);
2680 atomic_dec_32(&dn->dn_dbufs_count);
2681
2682 /*
2683 * Decrementing the dbuf count means that the bonus
2684 * buffer's dnode hold is no longer discounted in
2685 * dnode_move(). The dnode cannot move until after
2686 * the dnode_rele() below.
2687 */
2688 DB_DNODE_EXIT(db);
2689
2690 /*
2691 * Do not reference db after its lock is dropped.
2692 * Another thread may evict it.
2693 */
2694 mutex_exit(&db->db_mtx);
2695
2696 if (evict_dbuf)
2697 dnode_evict_bonus(dn);
2698
2699 dnode_rele(dn, db);
2700 } else if (db->db_buf == NULL) {
2701 /*
2702 * This is a special case: we never associated this
2703 * dbuf with any data allocated from the ARC.
2704 */
2705 ASSERT(db->db_state == DB_UNCACHED ||
2706 db->db_state == DB_NOFILL);
2707 dbuf_evict(db);
2708 } else if (arc_released(db->db_buf)) {
2709 arc_buf_t *buf = db->db_buf;
2710 /*
2711 * This dbuf has anonymous data associated with it.
2712 */
2713 dbuf_clear_data(db);
2714 VERIFY(arc_buf_remove_ref(buf, db));
2715 dbuf_evict(db);
2716 } else {
2717 VERIFY(!arc_buf_remove_ref(db->db_buf, db));
2718
2719 /*
2720 * A dbuf will be eligible for eviction if either the
2721 * 'primarycache' property is set or a duplicate
2722 * copy of this buffer is already cached in the arc.
2723 *
2724 * In the case of the 'primarycache' a buffer
2725 * is considered for eviction if it matches the
2726 * criteria set in the property.
2727 *
2728 * To decide if our buffer is considered a
2729 * duplicate, we must call into the arc to determine
2730 * if multiple buffers are referencing the same
2731 * block on-disk. If so, then we simply evict
2732 * ourselves.
2733 */
2734 if (!DBUF_IS_CACHEABLE(db)) {
2735 if (db->db_blkptr != NULL &&
2736 !BP_IS_HOLE(db->db_blkptr) &&
2737 !BP_IS_EMBEDDED(db->db_blkptr)) {
2738 spa_t *spa =
2739 dmu_objset_spa(db->db_objset);
2740 blkptr_t bp = *db->db_blkptr;
2741 dbuf_clear(db);
2742 arc_freed(spa, &bp);
2743 } else {
2744 dbuf_clear(db);
2745 }
2746 } else if (db->db_pending_evict ||
2747 arc_buf_eviction_needed(db->db_buf)) {
2748 dbuf_clear(db);
2749 } else {
2750 mutex_exit(&db->db_mtx);
2751 }
2752 }
2753 } else {
2754 mutex_exit(&db->db_mtx);
2755 }
2756 }
2757
2758 #pragma weak dmu_buf_refcount = dbuf_refcount
2759 uint64_t
2760 dbuf_refcount(dmu_buf_impl_t *db)
2761 {
2762 return (refcount_count(&db->db_holds));
2763 }
2764
2765 void *
2766 dmu_buf_replace_user(dmu_buf_t *db_fake, dmu_buf_user_t *old_user,
2767 dmu_buf_user_t *new_user)
2768 {
2769 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2770
2771 mutex_enter(&db->db_mtx);
2772 dbuf_verify_user(db, DBVU_NOT_EVICTING);
2773 if (db->db_user == old_user)
2774 db->db_user = new_user;
2775 else
2776 old_user = db->db_user;
2777 dbuf_verify_user(db, DBVU_NOT_EVICTING);
2778 mutex_exit(&db->db_mtx);
2779
2780 return (old_user);
2781 }
2782
2783 void *
2784 dmu_buf_set_user(dmu_buf_t *db_fake, dmu_buf_user_t *user)
2785 {
2786 return (dmu_buf_replace_user(db_fake, NULL, user));
2787 }
2788
2789 void *
2790 dmu_buf_set_user_ie(dmu_buf_t *db_fake, dmu_buf_user_t *user)
2791 {
2792 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2793
2794 db->db_user_immediate_evict = TRUE;
2795 return (dmu_buf_set_user(db_fake, user));
2796 }
2797
2798 void *
2799 dmu_buf_remove_user(dmu_buf_t *db_fake, dmu_buf_user_t *user)
2800 {
2801 return (dmu_buf_replace_user(db_fake, user, NULL));
2802 }
2803
2804 void *
2805 dmu_buf_get_user(dmu_buf_t *db_fake)
2806 {
2807 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2808
2809 dbuf_verify_user(db, DBVU_NOT_EVICTING);
2810 return (db->db_user);
2811 }
2812
2813 void
2814 dmu_buf_user_evict_wait()
2815 {
2816 taskq_wait(dbu_evict_taskq);
2817 }
2818
2819 boolean_t
2820 dmu_buf_freeable(dmu_buf_t *dbuf)
2821 {
2822 boolean_t res = B_FALSE;
2823 dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf;
2824
2825 if (db->db_blkptr)
2826 res = dsl_dataset_block_freeable(db->db_objset->os_dsl_dataset,
2827 db->db_blkptr, db->db_blkptr->blk_birth);
2828
2829 return (res);
2830 }
2831
2832 blkptr_t *
2833 dmu_buf_get_blkptr(dmu_buf_t *db)
2834 {
2835 dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db;
2836 return (dbi->db_blkptr);
2837 }
2838
2839 objset_t *
2840 dmu_buf_get_objset(dmu_buf_t *db)
2841 {
2842 dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db;
2843 return (dbi->db_objset);
2844 }
2845
2846 static void
2847 dbuf_check_blkptr(dnode_t *dn, dmu_buf_impl_t *db)
2848 {
2849 /* ASSERT(dmu_tx_is_syncing(tx) */
2850 ASSERT(MUTEX_HELD(&db->db_mtx));
2851
2852 if (db->db_blkptr != NULL)
2853 return;
2854
2855 if (db->db_blkid == DMU_SPILL_BLKID) {
2856 db->db_blkptr = DN_SPILL_BLKPTR(dn->dn_phys);
2857 BP_ZERO(db->db_blkptr);
2858 return;
2859 }
2860 if (db->db_level == dn->dn_phys->dn_nlevels-1) {
2861 /*
2862 * This buffer was allocated at a time when there was
2863 * no available blkptrs from the dnode, or it was
2864 * inappropriate to hook it in (i.e., nlevels mis-match).
2865 */
2866 ASSERT(db->db_blkid < dn->dn_phys->dn_nblkptr);
2867 ASSERT(db->db_parent == NULL);
2868 db->db_parent = dn->dn_dbuf;
2869 db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid];
2870 DBUF_VERIFY(db);
2871 } else {
2872 dmu_buf_impl_t *parent = db->db_parent;
2873 int epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT;
2874
2875 ASSERT(dn->dn_phys->dn_nlevels > 1);
2876 if (parent == NULL) {
2877 mutex_exit(&db->db_mtx);
2878 rw_enter(&dn->dn_struct_rwlock, RW_READER);
2879 parent = dbuf_hold_level(dn, db->db_level + 1,
2880 db->db_blkid >> epbs, db);
2881 rw_exit(&dn->dn_struct_rwlock);
2882 mutex_enter(&db->db_mtx);
2883 db->db_parent = parent;
2884 }
2885 db->db_blkptr = (blkptr_t *)parent->db.db_data +
2886 (db->db_blkid & ((1ULL << epbs) - 1));
2887 DBUF_VERIFY(db);
2888 }
2889 }
2890
2891 /*
2892 * dbuf_sync_indirect() is called recursively from dbuf_sync_list() so it
2893 * is critical the we not allow the compiler to inline this function in to
2894 * dbuf_sync_list() thereby drastically bloating the stack usage.
2895 */
2896 noinline static void
2897 dbuf_sync_indirect(dbuf_dirty_record_t *dr, dmu_tx_t *tx)
2898 {
2899 dmu_buf_impl_t *db = dr->dr_dbuf;
2900 dnode_t *dn;
2901 zio_t *zio;
2902
2903 ASSERT(dmu_tx_is_syncing(tx));
2904
2905 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr);
2906
2907 mutex_enter(&db->db_mtx);
2908
2909 ASSERT(db->db_level > 0);
2910 DBUF_VERIFY(db);
2911
2912 /* Read the block if it hasn't been read yet. */
2913 if (db->db_buf == NULL) {
2914 mutex_exit(&db->db_mtx);
2915 (void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED);
2916 mutex_enter(&db->db_mtx);
2917 }
2918 ASSERT3U(db->db_state, ==, DB_CACHED);
2919 ASSERT(db->db_buf != NULL);
2920
2921 DB_DNODE_ENTER(db);
2922 dn = DB_DNODE(db);
2923 /* Indirect block size must match what the dnode thinks it is. */
2924 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift);
2925 dbuf_check_blkptr(dn, db);
2926 DB_DNODE_EXIT(db);
2927
2928 /* Provide the pending dirty record to child dbufs */
2929 db->db_data_pending = dr;
2930
2931 mutex_exit(&db->db_mtx);
2932 dbuf_write(dr, db->db_buf, tx);
2933
2934 zio = dr->dr_zio;
2935 mutex_enter(&dr->dt.di.dr_mtx);
2936 dbuf_sync_list(&dr->dt.di.dr_children, db->db_level - 1, tx);
2937 ASSERT(list_head(&dr->dt.di.dr_children) == NULL);
2938 mutex_exit(&dr->dt.di.dr_mtx);
2939 zio_nowait(zio);
2940 }
2941
2942 /*
2943 * dbuf_sync_leaf() is called recursively from dbuf_sync_list() so it is
2944 * critical the we not allow the compiler to inline this function in to
2945 * dbuf_sync_list() thereby drastically bloating the stack usage.
2946 */
2947 noinline static void
2948 dbuf_sync_leaf(dbuf_dirty_record_t *dr, dmu_tx_t *tx)
2949 {
2950 arc_buf_t **datap = &dr->dt.dl.dr_data;
2951 dmu_buf_impl_t *db = dr->dr_dbuf;
2952 dnode_t *dn;
2953 objset_t *os;
2954 uint64_t txg = tx->tx_txg;
2955
2956 ASSERT(dmu_tx_is_syncing(tx));
2957
2958 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr);
2959
2960 mutex_enter(&db->db_mtx);
2961 /*
2962 * To be synced, we must be dirtied. But we
2963 * might have been freed after the dirty.
2964 */
2965 if (db->db_state == DB_UNCACHED) {
2966 /* This buffer has been freed since it was dirtied */
2967 ASSERT(db->db.db_data == NULL);
2968 } else if (db->db_state == DB_FILL) {
2969 /* This buffer was freed and is now being re-filled */
2970 ASSERT(db->db.db_data != dr->dt.dl.dr_data);
2971 } else {
2972 ASSERT(db->db_state == DB_CACHED || db->db_state == DB_NOFILL);
2973 }
2974 DBUF_VERIFY(db);
2975
2976 DB_DNODE_ENTER(db);
2977 dn = DB_DNODE(db);
2978
2979 if (db->db_blkid == DMU_SPILL_BLKID) {
2980 mutex_enter(&dn->dn_mtx);
2981 if (!(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR)) {
2982 /*
2983 * In the previous transaction group, the bonus buffer
2984 * was entirely used to store the attributes for the
2985 * dnode which overrode the dn_spill field. However,
2986 * when adding more attributes to the file a spill
2987 * block was required to hold the extra attributes.
2988 *
2989 * Make sure to clear the garbage left in the dn_spill
2990 * field from the previous attributes in the bonus
2991 * buffer. Otherwise, after writing out the spill
2992 * block to the new allocated dva, it will free
2993 * the old block pointed to by the invalid dn_spill.
2994 */
2995 db->db_blkptr = NULL;
2996 }
2997 dn->dn_phys->dn_flags |= DNODE_FLAG_SPILL_BLKPTR;
2998 mutex_exit(&dn->dn_mtx);
2999 }
3000
3001 /*
3002 * If this is a bonus buffer, simply copy the bonus data into the
3003 * dnode. It will be written out when the dnode is synced (and it
3004 * will be synced, since it must have been dirty for dbuf_sync to
3005 * be called).
3006 */
3007 if (db->db_blkid == DMU_BONUS_BLKID) {
3008 dbuf_dirty_record_t **drp;
3009
3010 ASSERT(*datap != NULL);
3011 ASSERT0(db->db_level);
3012 ASSERT3U(dn->dn_phys->dn_bonuslen, <=,
3013 DN_SLOTS_TO_BONUSLEN(dn->dn_phys->dn_extra_slots + 1));
3014 bcopy(*datap, DN_BONUS(dn->dn_phys), dn->dn_phys->dn_bonuslen);
3015 DB_DNODE_EXIT(db);
3016
3017 if (*datap != db->db.db_data) {
3018 int slots = DB_DNODE(db)->dn_num_slots;
3019 int bonuslen = DN_SLOTS_TO_BONUSLEN(slots);
3020 zio_buf_free(*datap, bonuslen);
3021 arc_space_return(bonuslen, ARC_SPACE_BONUS);
3022 }
3023 db->db_data_pending = NULL;
3024 drp = &db->db_last_dirty;
3025 while (*drp != dr)
3026 drp = &(*drp)->dr_next;
3027 ASSERT(dr->dr_next == NULL);
3028 ASSERT(dr->dr_dbuf == db);
3029 *drp = dr->dr_next;
3030 if (dr->dr_dbuf->db_level != 0) {
3031 mutex_destroy(&dr->dt.di.dr_mtx);
3032 list_destroy(&dr->dt.di.dr_children);
3033 }
3034 kmem_free(dr, sizeof (dbuf_dirty_record_t));
3035 ASSERT(db->db_dirtycnt > 0);
3036 db->db_dirtycnt -= 1;
3037 dbuf_rele_and_unlock(db, (void *)(uintptr_t)txg);
3038 return;
3039 }
3040
3041 os = dn->dn_objset;
3042
3043 /*
3044 * This function may have dropped the db_mtx lock allowing a dmu_sync
3045 * operation to sneak in. As a result, we need to ensure that we
3046 * don't check the dr_override_state until we have returned from
3047 * dbuf_check_blkptr.
3048 */
3049 dbuf_check_blkptr(dn, db);
3050
3051 /*
3052 * If this buffer is in the middle of an immediate write,
3053 * wait for the synchronous IO to complete.
3054 */
3055 while (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC) {
3056 ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT);
3057 cv_wait(&db->db_changed, &db->db_mtx);
3058 ASSERT(dr->dt.dl.dr_override_state != DR_NOT_OVERRIDDEN);
3059 }
3060
3061 if (db->db_state != DB_NOFILL &&
3062 dn->dn_object != DMU_META_DNODE_OBJECT &&
3063 refcount_count(&db->db_holds) > 1 &&
3064 dr->dt.dl.dr_override_state != DR_OVERRIDDEN &&
3065 *datap == db->db_buf) {
3066 /*
3067 * If this buffer is currently "in use" (i.e., there
3068 * are active holds and db_data still references it),
3069 * then make a copy before we start the write so that
3070 * any modifications from the open txg will not leak
3071 * into this write.
3072 *
3073 * NOTE: this copy does not need to be made for
3074 * objects only modified in the syncing context (e.g.
3075 * DNONE_DNODE blocks).
3076 */
3077 int blksz = arc_buf_size(*datap);
3078 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
3079 *datap = arc_buf_alloc(os->os_spa, blksz, db, type);
3080 bcopy(db->db.db_data, (*datap)->b_data, blksz);
3081 }
3082 db->db_data_pending = dr;
3083
3084 mutex_exit(&db->db_mtx);
3085
3086 dbuf_write(dr, *datap, tx);
3087
3088 ASSERT(!list_link_active(&dr->dr_dirty_node));
3089 if (dn->dn_object == DMU_META_DNODE_OBJECT) {
3090 list_insert_tail(&dn->dn_dirty_records[txg&TXG_MASK], dr);
3091 DB_DNODE_EXIT(db);
3092 } else {
3093 /*
3094 * Although zio_nowait() does not "wait for an IO", it does
3095 * initiate the IO. If this is an empty write it seems plausible
3096 * that the IO could actually be completed before the nowait
3097 * returns. We need to DB_DNODE_EXIT() first in case
3098 * zio_nowait() invalidates the dbuf.
3099 */
3100 DB_DNODE_EXIT(db);
3101 zio_nowait(dr->dr_zio);
3102 }
3103 }
3104
3105 void
3106 dbuf_sync_list(list_t *list, int level, dmu_tx_t *tx)
3107 {
3108 dbuf_dirty_record_t *dr;
3109
3110 while ((dr = list_head(list))) {
3111 if (dr->dr_zio != NULL) {
3112 /*
3113 * If we find an already initialized zio then we
3114 * are processing the meta-dnode, and we have finished.
3115 * The dbufs for all dnodes are put back on the list
3116 * during processing, so that we can zio_wait()
3117 * these IOs after initiating all child IOs.
3118 */
3119 ASSERT3U(dr->dr_dbuf->db.db_object, ==,
3120 DMU_META_DNODE_OBJECT);
3121 break;
3122 }
3123 if (dr->dr_dbuf->db_blkid != DMU_BONUS_BLKID &&
3124 dr->dr_dbuf->db_blkid != DMU_SPILL_BLKID) {
3125 VERIFY3U(dr->dr_dbuf->db_level, ==, level);
3126 }
3127 list_remove(list, dr);
3128 if (dr->dr_dbuf->db_level > 0)
3129 dbuf_sync_indirect(dr, tx);
3130 else
3131 dbuf_sync_leaf(dr, tx);
3132 }
3133 }
3134
3135 /* ARGSUSED */
3136 static void
3137 dbuf_write_ready(zio_t *zio, arc_buf_t *buf, void *vdb)
3138 {
3139 dmu_buf_impl_t *db = vdb;
3140 dnode_t *dn;
3141 blkptr_t *bp = zio->io_bp;
3142 blkptr_t *bp_orig = &zio->io_bp_orig;
3143 spa_t *spa = zio->io_spa;
3144 int64_t delta;
3145 uint64_t fill = 0;
3146 int i;
3147
3148 ASSERT3P(db->db_blkptr, !=, NULL);
3149 ASSERT3P(&db->db_data_pending->dr_bp_copy, ==, bp);
3150
3151 DB_DNODE_ENTER(db);
3152 dn = DB_DNODE(db);
3153 delta = bp_get_dsize_sync(spa, bp) - bp_get_dsize_sync(spa, bp_orig);
3154 dnode_diduse_space(dn, delta - zio->io_prev_space_delta);
3155 zio->io_prev_space_delta = delta;
3156
3157 if (bp->blk_birth != 0) {
3158 ASSERT((db->db_blkid != DMU_SPILL_BLKID &&
3159 BP_GET_TYPE(bp) == dn->dn_type) ||
3160 (db->db_blkid == DMU_SPILL_BLKID &&
3161 BP_GET_TYPE(bp) == dn->dn_bonustype) ||
3162 BP_IS_EMBEDDED(bp));
3163 ASSERT(BP_GET_LEVEL(bp) == db->db_level);
3164 }
3165
3166 mutex_enter(&db->db_mtx);
3167
3168 #ifdef ZFS_DEBUG
3169 if (db->db_blkid == DMU_SPILL_BLKID) {
3170 ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR);
3171 ASSERT(!(BP_IS_HOLE(bp)) &&
3172 db->db_blkptr == DN_SPILL_BLKPTR(dn->dn_phys));
3173 }
3174 #endif
3175
3176 if (db->db_level == 0) {
3177 mutex_enter(&dn->dn_mtx);
3178 if (db->db_blkid > dn->dn_phys->dn_maxblkid &&
3179 db->db_blkid != DMU_SPILL_BLKID)
3180 dn->dn_phys->dn_maxblkid = db->db_blkid;
3181 mutex_exit(&dn->dn_mtx);
3182
3183 if (dn->dn_type == DMU_OT_DNODE) {
3184 i = 0;
3185 while (i < db->db.db_size) {
3186 dnode_phys_t *dnp = db->db.db_data + i;
3187
3188 i += DNODE_MIN_SIZE;
3189 if (dnp->dn_type != DMU_OT_NONE) {
3190 fill++;
3191 i += dnp->dn_extra_slots *
3192 DNODE_MIN_SIZE;
3193 }
3194 }
3195 } else {
3196 if (BP_IS_HOLE(bp)) {
3197 fill = 0;
3198 } else {
3199 fill = 1;
3200 }
3201 }
3202 } else {
3203 blkptr_t *ibp = db->db.db_data;
3204 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift);
3205 for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, ibp++) {
3206 if (BP_IS_HOLE(ibp))
3207 continue;
3208 fill += BP_GET_FILL(ibp);
3209 }
3210 }
3211 DB_DNODE_EXIT(db);
3212
3213 if (!BP_IS_EMBEDDED(bp))
3214 bp->blk_fill = fill;
3215
3216 mutex_exit(&db->db_mtx);
3217
3218 rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
3219 *db->db_blkptr = *bp;
3220 rw_exit(&dn->dn_struct_rwlock);
3221 }
3222
3223 /* ARGSUSED */
3224 /*
3225 * This function gets called just prior to running through the compression
3226 * stage of the zio pipeline. If we're an indirect block comprised of only
3227 * holes, then we want this indirect to be compressed away to a hole. In
3228 * order to do that we must zero out any information about the holes that
3229 * this indirect points to prior to before we try to compress it.
3230 */
3231 static void
3232 dbuf_write_children_ready(zio_t *zio, arc_buf_t *buf, void *vdb)
3233 {
3234 dmu_buf_impl_t *db = vdb;
3235 dnode_t *dn;
3236 blkptr_t *bp;
3237 uint64_t i;
3238 int epbs;
3239
3240 ASSERT3U(db->db_level, >, 0);
3241 DB_DNODE_ENTER(db);
3242 dn = DB_DNODE(db);
3243 epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT;
3244
3245 /* Determine if all our children are holes */
3246 for (i = 0, bp = db->db.db_data; i < 1 << epbs; i++, bp++) {
3247 if (!BP_IS_HOLE(bp))
3248 break;
3249 }
3250
3251 /*
3252 * If all the children are holes, then zero them all out so that
3253 * we may get compressed away.
3254 */
3255 if (i == 1 << epbs) {
3256 /* didn't find any non-holes */
3257 bzero(db->db.db_data, db->db.db_size);
3258 }
3259 DB_DNODE_EXIT(db);
3260 }
3261
3262 /*
3263 * The SPA will call this callback several times for each zio - once
3264 * for every physical child i/o (zio->io_phys_children times). This
3265 * allows the DMU to monitor the progress of each logical i/o. For example,
3266 * there may be 2 copies of an indirect block, or many fragments of a RAID-Z
3267 * block. There may be a long delay before all copies/fragments are completed,
3268 * so this callback allows us to retire dirty space gradually, as the physical
3269 * i/os complete.
3270 */
3271 /* ARGSUSED */
3272 static void
3273 dbuf_write_physdone(zio_t *zio, arc_buf_t *buf, void *arg)
3274 {
3275 dmu_buf_impl_t *db = arg;
3276 objset_t *os = db->db_objset;
3277 dsl_pool_t *dp = dmu_objset_pool(os);
3278 dbuf_dirty_record_t *dr;
3279 int delta = 0;
3280
3281 dr = db->db_data_pending;
3282 ASSERT3U(dr->dr_txg, ==, zio->io_txg);
3283
3284 /*
3285 * The callback will be called io_phys_children times. Retire one
3286 * portion of our dirty space each time we are called. Any rounding
3287 * error will be cleaned up by dsl_pool_sync()'s call to
3288 * dsl_pool_undirty_space().
3289 */
3290 delta = dr->dr_accounted / zio->io_phys_children;
3291 dsl_pool_undirty_space(dp, delta, zio->io_txg);
3292 }
3293
3294 /* ARGSUSED */
3295 static void
3296 dbuf_write_done(zio_t *zio, arc_buf_t *buf, void *vdb)
3297 {
3298 dmu_buf_impl_t *db = vdb;
3299 blkptr_t *bp_orig = &zio->io_bp_orig;
3300 blkptr_t *bp = db->db_blkptr;
3301 objset_t *os = db->db_objset;
3302 dmu_tx_t *tx = os->os_synctx;
3303 dbuf_dirty_record_t **drp, *dr;
3304
3305 ASSERT0(zio->io_error);
3306 ASSERT(db->db_blkptr == bp);
3307
3308 /*
3309 * For nopwrites and rewrites we ensure that the bp matches our
3310 * original and bypass all the accounting.
3311 */
3312 if (zio->io_flags & (ZIO_FLAG_IO_REWRITE | ZIO_FLAG_NOPWRITE)) {
3313 ASSERT(BP_EQUAL(bp, bp_orig));
3314 } else {
3315 dsl_dataset_t *ds = os->os_dsl_dataset;
3316 (void) dsl_dataset_block_kill(ds, bp_orig, tx, B_TRUE);
3317 dsl_dataset_block_born(ds, bp, tx);
3318 }
3319
3320 mutex_enter(&db->db_mtx);
3321
3322 DBUF_VERIFY(db);
3323
3324 drp = &db->db_last_dirty;
3325 while ((dr = *drp) != db->db_data_pending)
3326 drp = &dr->dr_next;
3327 ASSERT(!list_link_active(&dr->dr_dirty_node));
3328 ASSERT(dr->dr_dbuf == db);
3329 ASSERT(dr->dr_next == NULL);
3330 *drp = dr->dr_next;
3331
3332 #ifdef ZFS_DEBUG
3333 if (db->db_blkid == DMU_SPILL_BLKID) {
3334 dnode_t *dn;
3335
3336 DB_DNODE_ENTER(db);
3337 dn = DB_DNODE(db);
3338 ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR);
3339 ASSERT(!(BP_IS_HOLE(db->db_blkptr)) &&
3340 db->db_blkptr == DN_SPILL_BLKPTR(dn->dn_phys));
3341 DB_DNODE_EXIT(db);
3342 }
3343 #endif
3344
3345 if (db->db_level == 0) {
3346 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
3347 ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN);
3348 if (db->db_state != DB_NOFILL) {
3349 if (dr->dt.dl.dr_data != db->db_buf)
3350 VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data,
3351 db));
3352 else if (!arc_released(db->db_buf))
3353 arc_set_callback(db->db_buf, dbuf_do_evict, db);
3354 }
3355 } else {
3356 dnode_t *dn;
3357
3358 DB_DNODE_ENTER(db);
3359 dn = DB_DNODE(db);
3360 ASSERT(list_head(&dr->dt.di.dr_children) == NULL);
3361 ASSERT3U(db->db.db_size, ==, 1 << dn->dn_phys->dn_indblkshift);
3362 if (!BP_IS_HOLE(db->db_blkptr)) {
3363 ASSERTV(int epbs = dn->dn_phys->dn_indblkshift -
3364 SPA_BLKPTRSHIFT);
3365 ASSERT3U(db->db_blkid, <=,
3366 dn->dn_phys->dn_maxblkid >> (db->db_level * epbs));
3367 ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==,
3368 db->db.db_size);
3369 if (!arc_released(db->db_buf))
3370 arc_set_callback(db->db_buf, dbuf_do_evict, db);
3371 }
3372 DB_DNODE_EXIT(db);
3373 mutex_destroy(&dr->dt.di.dr_mtx);
3374 list_destroy(&dr->dt.di.dr_children);
3375 }
3376 kmem_free(dr, sizeof (dbuf_dirty_record_t));
3377
3378 cv_broadcast(&db->db_changed);
3379 ASSERT(db->db_dirtycnt > 0);
3380 db->db_dirtycnt -= 1;
3381 db->db_data_pending = NULL;
3382 dbuf_rele_and_unlock(db, (void *)(uintptr_t)tx->tx_txg);
3383 }
3384
3385 static void
3386 dbuf_write_nofill_ready(zio_t *zio)
3387 {
3388 dbuf_write_ready(zio, NULL, zio->io_private);
3389 }
3390
3391 static void
3392 dbuf_write_nofill_done(zio_t *zio)
3393 {
3394 dbuf_write_done(zio, NULL, zio->io_private);
3395 }
3396
3397 static void
3398 dbuf_write_override_ready(zio_t *zio)
3399 {
3400 dbuf_dirty_record_t *dr = zio->io_private;
3401 dmu_buf_impl_t *db = dr->dr_dbuf;
3402
3403 dbuf_write_ready(zio, NULL, db);
3404 }
3405
3406 static void
3407 dbuf_write_override_done(zio_t *zio)
3408 {
3409 dbuf_dirty_record_t *dr = zio->io_private;
3410 dmu_buf_impl_t *db = dr->dr_dbuf;
3411 blkptr_t *obp = &dr->dt.dl.dr_overridden_by;
3412
3413 mutex_enter(&db->db_mtx);
3414 if (!BP_EQUAL(zio->io_bp, obp)) {
3415 if (!BP_IS_HOLE(obp))
3416 dsl_free(spa_get_dsl(zio->io_spa), zio->io_txg, obp);
3417 arc_release(dr->dt.dl.dr_data, db);
3418 }
3419 mutex_exit(&db->db_mtx);
3420
3421 dbuf_write_done(zio, NULL, db);
3422 }
3423
3424 /* Issue I/O to commit a dirty buffer to disk. */
3425 static void
3426 dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx)
3427 {
3428 dmu_buf_impl_t *db = dr->dr_dbuf;
3429 dnode_t *dn;
3430 objset_t *os;
3431 dmu_buf_impl_t *parent = db->db_parent;
3432 uint64_t txg = tx->tx_txg;
3433 zbookmark_phys_t zb;
3434 zio_prop_t zp;
3435 zio_t *zio;
3436 int wp_flag = 0;
3437
3438 ASSERT(dmu_tx_is_syncing(tx));
3439
3440 DB_DNODE_ENTER(db);
3441 dn = DB_DNODE(db);
3442 os = dn->dn_objset;
3443
3444 if (db->db_state != DB_NOFILL) {
3445 if (db->db_level > 0 || dn->dn_type == DMU_OT_DNODE) {
3446 /*
3447 * Private object buffers are released here rather
3448 * than in dbuf_dirty() since they are only modified
3449 * in the syncing context and we don't want the
3450 * overhead of making multiple copies of the data.
3451 */
3452 if (BP_IS_HOLE(db->db_blkptr)) {
3453 arc_buf_thaw(data);
3454 } else {
3455 dbuf_release_bp(db);
3456 }
3457 }
3458 }
3459
3460 if (parent != dn->dn_dbuf) {
3461 /* Our parent is an indirect block. */
3462 /* We have a dirty parent that has been scheduled for write. */
3463 ASSERT(parent && parent->db_data_pending);
3464 /* Our parent's buffer is one level closer to the dnode. */
3465 ASSERT(db->db_level == parent->db_level-1);
3466 /*
3467 * We're about to modify our parent's db_data by modifying
3468 * our block pointer, so the parent must be released.
3469 */
3470 ASSERT(arc_released(parent->db_buf));
3471 zio = parent->db_data_pending->dr_zio;
3472 } else {
3473 /* Our parent is the dnode itself. */
3474 ASSERT((db->db_level == dn->dn_phys->dn_nlevels-1 &&
3475 db->db_blkid != DMU_SPILL_BLKID) ||
3476 (db->db_blkid == DMU_SPILL_BLKID && db->db_level == 0));
3477 if (db->db_blkid != DMU_SPILL_BLKID)
3478 ASSERT3P(db->db_blkptr, ==,
3479 &dn->dn_phys->dn_blkptr[db->db_blkid]);
3480 zio = dn->dn_zio;
3481 }
3482
3483 ASSERT(db->db_level == 0 || data == db->db_buf);
3484 ASSERT3U(db->db_blkptr->blk_birth, <=, txg);
3485 ASSERT(zio);
3486
3487 SET_BOOKMARK(&zb, os->os_dsl_dataset ?
3488 os->os_dsl_dataset->ds_object : DMU_META_OBJSET,
3489 db->db.db_object, db->db_level, db->db_blkid);
3490
3491 if (db->db_blkid == DMU_SPILL_BLKID)
3492 wp_flag = WP_SPILL;
3493 wp_flag |= (db->db_state == DB_NOFILL) ? WP_NOFILL : 0;
3494
3495 dmu_write_policy(os, dn, db->db_level, wp_flag, &zp);
3496 DB_DNODE_EXIT(db);
3497
3498 /*
3499 * We copy the blkptr now (rather than when we instantiate the dirty
3500 * record), because its value can change between open context and
3501 * syncing context. We do not need to hold dn_struct_rwlock to read
3502 * db_blkptr because we are in syncing context.
3503 */
3504 dr->dr_bp_copy = *db->db_blkptr;
3505
3506 if (db->db_level == 0 &&
3507 dr->dt.dl.dr_override_state == DR_OVERRIDDEN) {
3508 /*
3509 * The BP for this block has been provided by open context
3510 * (by dmu_sync() or dmu_buf_write_embedded()).
3511 */
3512 void *contents = (data != NULL) ? data->b_data : NULL;
3513
3514 dr->dr_zio = zio_write(zio, os->os_spa, txg,
3515 &dr->dr_bp_copy, contents, db->db.db_size, &zp,
3516 dbuf_write_override_ready, NULL, NULL,
3517 dbuf_write_override_done,
3518 dr, ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb);
3519 mutex_enter(&db->db_mtx);
3520 dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN;
3521 zio_write_override(dr->dr_zio, &dr->dt.dl.dr_overridden_by,
3522 dr->dt.dl.dr_copies, dr->dt.dl.dr_nopwrite);
3523 mutex_exit(&db->db_mtx);
3524 } else if (db->db_state == DB_NOFILL) {
3525 ASSERT(zp.zp_checksum == ZIO_CHECKSUM_OFF);
3526 dr->dr_zio = zio_write(zio, os->os_spa, txg,
3527 &dr->dr_bp_copy, NULL, db->db.db_size, &zp,
3528 dbuf_write_nofill_ready, NULL, NULL,
3529 dbuf_write_nofill_done, db,
3530 ZIO_PRIORITY_ASYNC_WRITE,
3531 ZIO_FLAG_MUSTSUCCEED | ZIO_FLAG_NODATA, &zb);
3532 } else {
3533 arc_done_func_t *children_ready_cb = NULL;
3534 ASSERT(arc_released(data));
3535
3536 /*
3537 * For indirect blocks, we want to setup the children
3538 * ready callback so that we can properly handle an indirect
3539 * block that only contains holes.
3540 */
3541 if (db->db_level != 0)
3542 children_ready_cb = dbuf_write_children_ready;
3543
3544 dr->dr_zio = arc_write(zio, os->os_spa, txg,
3545 &dr->dr_bp_copy, data, DBUF_IS_L2CACHEABLE(db),
3546 DBUF_IS_L2COMPRESSIBLE(db), &zp, dbuf_write_ready,
3547 children_ready_cb,
3548 dbuf_write_physdone, dbuf_write_done, db,
3549 ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb);
3550 }
3551 }
3552
3553 #if defined(_KERNEL) && defined(HAVE_SPL)
3554 EXPORT_SYMBOL(dbuf_find);
3555 EXPORT_SYMBOL(dbuf_is_metadata);
3556 EXPORT_SYMBOL(dbuf_evict);
3557 EXPORT_SYMBOL(dbuf_loan_arcbuf);
3558 EXPORT_SYMBOL(dbuf_whichblock);
3559 EXPORT_SYMBOL(dbuf_read);
3560 EXPORT_SYMBOL(dbuf_unoverride);
3561 EXPORT_SYMBOL(dbuf_free_range);
3562 EXPORT_SYMBOL(dbuf_new_size);
3563 EXPORT_SYMBOL(dbuf_release_bp);
3564 EXPORT_SYMBOL(dbuf_dirty);
3565 EXPORT_SYMBOL(dmu_buf_will_dirty);
3566 EXPORT_SYMBOL(dmu_buf_will_not_fill);
3567 EXPORT_SYMBOL(dmu_buf_will_fill);
3568 EXPORT_SYMBOL(dmu_buf_fill_done);
3569 EXPORT_SYMBOL(dmu_buf_rele);
3570 EXPORT_SYMBOL(dbuf_assign_arcbuf);
3571 EXPORT_SYMBOL(dbuf_clear);
3572 EXPORT_SYMBOL(dbuf_prefetch);
3573 EXPORT_SYMBOL(dbuf_hold_impl);
3574 EXPORT_SYMBOL(dbuf_hold);
3575 EXPORT_SYMBOL(dbuf_hold_level);
3576 EXPORT_SYMBOL(dbuf_create_bonus);
3577 EXPORT_SYMBOL(dbuf_spill_set_blksz);
3578 EXPORT_SYMBOL(dbuf_rm_spill);
3579 EXPORT_SYMBOL(dbuf_add_ref);
3580 EXPORT_SYMBOL(dbuf_rele);
3581 EXPORT_SYMBOL(dbuf_rele_and_unlock);
3582 EXPORT_SYMBOL(dbuf_refcount);
3583 EXPORT_SYMBOL(dbuf_sync_list);
3584 EXPORT_SYMBOL(dmu_buf_set_user);
3585 EXPORT_SYMBOL(dmu_buf_set_user_ie);
3586 EXPORT_SYMBOL(dmu_buf_get_user);
3587 EXPORT_SYMBOL(dmu_buf_freeable);
3588 EXPORT_SYMBOL(dmu_buf_get_blkptr);
3589 #endif
OpenZFS on Linux and FreeBSD