drivers/md/dm-vdo/indexer/io-factory.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright 2023 Red Hat
   4  */
   5
   6 #include "io-factory.h"
   7
   8 #include <linux/atomic.h>
   9 #include <linux/blkdev.h>
  10 #include <linux/err.h>
  11 #include <linux/mount.h>
  12
  13 #include "logger.h"
  14 #include "memory-alloc.h"
  15 #include "numeric.h"
  16
  17 /*
  18  * The I/O factory object manages access to index storage, which is a contiguous range of blocks on
  19  * a block device.
  20  *
  21  * The factory holds the open device and is responsible for closing it. The factory has methods to
  22  * make helper structures that can be used to access sections of the index.
  23  */
  24 struct io_factory {
  25         struct block_device *bdev;
  26         atomic_t ref_count;
  27 };
  28
  29 /* The buffered reader allows efficient I/O by reading page-sized segments into a buffer. */
  30 struct buffered_reader {
  31         struct io_factory *factory;
  32         struct dm_bufio_client *client;
  33         struct dm_buffer *buffer;
  34         sector_t limit;
  35         sector_t block_number;
  36         u8 *start;
  37         u8 *end;
  38 };
  39
  40 #define MAX_READ_AHEAD_BLOCKS 4
  41
  42 /*
  43  * The buffered writer allows efficient I/O by buffering writes and committing page-sized segments
  44  * to storage.
  45  */
  46 struct buffered_writer {
  47         struct io_factory *factory;
  48         struct dm_bufio_client *client;
  49         struct dm_buffer *buffer;
  50         sector_t limit;
  51         sector_t block_number;
  52         u8 *start;
  53         u8 *end;
  54         int error;
  55 };
  56
  57 static void uds_get_io_factory(struct io_factory *factory)
  58 {
  59         atomic_inc(&factory->ref_count);
  60 }
  61
  62 int uds_make_io_factory(struct block_device *bdev, struct io_factory **factory_ptr)
  63 {
  64         int result;
  65         struct io_factory *factory;
  66
  67         result = vdo_allocate(1, struct io_factory, __func__, &factory);
  68         if (result != VDO_SUCCESS)
  69                 return result;
  70
  71         factory->bdev = bdev;
  72         atomic_set_release(&factory->ref_count, 1);
  73
  74         *factory_ptr = factory;
  75         return UDS_SUCCESS;
  76 }
  77
  78 int uds_replace_storage(struct io_factory *factory, struct block_device *bdev)
  79 {
  80         factory->bdev = bdev;
  81         return UDS_SUCCESS;
  82 }
  83
  84 /* Free an I/O factory once all references have been released. */
  85 void uds_put_io_factory(struct io_factory *factory)
  86 {
  87         if (atomic_add_return(-1, &factory->ref_count) <= 0)
  88                 vdo_free(factory);
  89 }
  90
  91 size_t uds_get_writable_size(struct io_factory *factory)
  92 {
  93         return bdev_nr_bytes(factory->bdev);
  94 }
  95
  96 /* Create a struct dm_bufio_client for an index region starting at offset. */
  97 int uds_make_bufio(struct io_factory *factory, off_t block_offset, size_t block_size,
  98                    unsigned int reserved_buffers, struct dm_bufio_client **client_ptr)
  99 {
 100         struct dm_bufio_client *client;
 101
 102         client = dm_bufio_client_create(factory->bdev, block_size, reserved_buffers, 0,
 103                                         NULL, NULL, 0);
 104         if (IS_ERR(client))
 105                 return -PTR_ERR(client);
 106
 107         dm_bufio_set_sector_offset(client, block_offset * SECTORS_PER_BLOCK);
 108         *client_ptr = client;
 109         return UDS_SUCCESS;
 110 }
 111
 112 static void read_ahead(struct buffered_reader *reader, sector_t block_number)
 113 {
 114         if (block_number < reader->limit) {
 115                 sector_t read_ahead = min((sector_t) MAX_READ_AHEAD_BLOCKS,
 116                                           reader->limit - block_number);
 117
 118                 dm_bufio_prefetch(reader->client, block_number, read_ahead);
 119         }
 120 }
 121
 122 void uds_free_buffered_reader(struct buffered_reader *reader)
 123 {
 124         if (reader == NULL)
 125                 return;
 126
 127         if (reader->buffer != NULL)
 128                 dm_bufio_release(reader->buffer);
 129
 130         dm_bufio_client_destroy(reader->client);
 131         uds_put_io_factory(reader->factory);
 132         vdo_free(reader);
 133 }
 134
 135 /* Create a buffered reader for an index region starting at offset. */
 136 int uds_make_buffered_reader(struct io_factory *factory, off_t offset, u64 block_count,
 137                              struct buffered_reader **reader_ptr)
 138 {
 139         int result;
 140         struct dm_bufio_client *client = NULL;
 141         struct buffered_reader *reader = NULL;
 142
 143         result = uds_make_bufio(factory, offset, UDS_BLOCK_SIZE, 1, &client);
 144         if (result != UDS_SUCCESS)
 145                 return result;
 146
 147         result = vdo_allocate(1, struct buffered_reader, "buffered reader", &reader);
 148         if (result != VDO_SUCCESS) {
 149                 dm_bufio_client_destroy(client);
 150                 return result;
 151         }
 152
 153         *reader = (struct buffered_reader) {
 154                 .factory = factory,
 155                 .client = client,
 156                 .buffer = NULL,
 157                 .limit = block_count,
 158                 .block_number = 0,
 159                 .start = NULL,
 160                 .end = NULL,
 161         };
 162
 163         read_ahead(reader, 0);
 164         uds_get_io_factory(factory);
 165         *reader_ptr = reader;
 166         return UDS_SUCCESS;
 167 }
 168
 169 static int position_reader(struct buffered_reader *reader, sector_t block_number,
 170                            off_t offset)
 171 {
 172         struct dm_buffer *buffer = NULL;
 173         void *data;
 174
 175         if ((reader->end == NULL) || (block_number != reader->block_number)) {
 176                 if (block_number >= reader->limit)
 177                         return UDS_OUT_OF_RANGE;
 178
 179                 if (reader->buffer != NULL)
 180                         dm_bufio_release(vdo_forget(reader->buffer));
 181
 182                 data = dm_bufio_read(reader->client, block_number, &buffer);
 183                 if (IS_ERR(data))
 184                         return -PTR_ERR(data);
 185
 186                 reader->buffer = buffer;
 187                 reader->start = data;
 188                 if (block_number == reader->block_number + 1)
 189                         read_ahead(reader, block_number + 1);
 190         }
 191
 192         reader->block_number = block_number;
 193         reader->end = reader->start + offset;
 194         return UDS_SUCCESS;
 195 }
 196
 197 static size_t bytes_remaining_in_read_buffer(struct buffered_reader *reader)
 198 {
 199         return (reader->end == NULL) ? 0 : reader->start + UDS_BLOCK_SIZE - reader->end;
 200 }
 201
 202 static int reset_reader(struct buffered_reader *reader)
 203 {
 204         sector_t block_number;
 205
 206         if (bytes_remaining_in_read_buffer(reader) > 0)
 207                 return UDS_SUCCESS;
 208
 209         block_number = reader->block_number;
 210         if (reader->end != NULL)
 211                 block_number++;
 212
 213         return position_reader(reader, block_number, 0);
 214 }
 215
 216 int uds_read_from_buffered_reader(struct buffered_reader *reader, u8 *data,
 217                                   size_t length)
 218 {
 219         int result = UDS_SUCCESS;
 220         size_t chunk_size;
 221
 222         while (length > 0) {
 223                 result = reset_reader(reader);
 224                 if (result != UDS_SUCCESS)
 225                         return result;
 226
 227                 chunk_size = min(length, bytes_remaining_in_read_buffer(reader));
 228                 memcpy(data, reader->end, chunk_size);
 229                 length -= chunk_size;
 230                 data += chunk_size;
 231                 reader->end += chunk_size;
 232         }
 233
 234         return UDS_SUCCESS;
 235 }
 236
 237 /*
 238  * Verify that the next data on the reader matches the required value. If the value matches, the
 239  * matching contents are consumed. If the value does not match, the reader state is unchanged.
 240  */
 241 int uds_verify_buffered_data(struct buffered_reader *reader, const u8 *value,
 242                              size_t length)
 243 {
 244         int result = UDS_SUCCESS;
 245         size_t chunk_size;
 246         sector_t start_block_number = reader->block_number;
 247         int start_offset = reader->end - reader->start;
 248
 249         while (length > 0) {
 250                 result = reset_reader(reader);
 251                 if (result != UDS_SUCCESS) {
 252                         result = UDS_CORRUPT_DATA;
 253                         break;
 254                 }
 255
 256                 chunk_size = min(length, bytes_remaining_in_read_buffer(reader));
 257                 if (memcmp(value, reader->end, chunk_size) != 0) {
 258                         result = UDS_CORRUPT_DATA;
 259                         break;
 260                 }
 261
 262                 length -= chunk_size;
 263                 value += chunk_size;
 264                 reader->end += chunk_size;
 265         }
 266
 267         if (result != UDS_SUCCESS)
 268                 position_reader(reader, start_block_number, start_offset);
 269
 270         return result;
 271 }
 272
 273 /* Create a buffered writer for an index region starting at offset. */
 274 int uds_make_buffered_writer(struct io_factory *factory, off_t offset, u64 block_count,
 275                              struct buffered_writer **writer_ptr)
 276 {
 277         int result;
 278         struct dm_bufio_client *client = NULL;
 279         struct buffered_writer *writer;
 280
 281         result = uds_make_bufio(factory, offset, UDS_BLOCK_SIZE, 1, &client);
 282         if (result != UDS_SUCCESS)
 283                 return result;
 284
 285         result = vdo_allocate(1, struct buffered_writer, "buffered writer", &writer);
 286         if (result != VDO_SUCCESS) {
 287                 dm_bufio_client_destroy(client);
 288                 return result;
 289         }
 290
 291         *writer = (struct buffered_writer) {
 292                 .factory = factory,
 293                 .client = client,
 294                 .buffer = NULL,
 295                 .limit = block_count,
 296                 .start = NULL,
 297                 .end = NULL,
 298                 .block_number = 0,
 299                 .error = UDS_SUCCESS,
 300         };
 301
 302         uds_get_io_factory(factory);
 303         *writer_ptr = writer;
 304         return UDS_SUCCESS;
 305 }
 306
 307 static size_t get_remaining_write_space(struct buffered_writer *writer)
 308 {
 309         return writer->start + UDS_BLOCK_SIZE - writer->end;
 310 }
 311
 312 static int __must_check prepare_next_buffer(struct buffered_writer *writer)
 313 {
 314         struct dm_buffer *buffer = NULL;
 315         void *data;
 316
 317         if (writer->block_number >= writer->limit) {
 318                 writer->error = UDS_OUT_OF_RANGE;
 319                 return UDS_OUT_OF_RANGE;
 320         }
 321
 322         data = dm_bufio_new(writer->client, writer->block_number, &buffer);
 323         if (IS_ERR(data)) {
 324                 writer->error = -PTR_ERR(data);
 325                 return writer->error;
 326         }
 327
 328         writer->buffer = buffer;
 329         writer->start = data;
 330         writer->end = data;
 331         return UDS_SUCCESS;
 332 }
 333
 334 static int flush_previous_buffer(struct buffered_writer *writer)
 335 {
 336         size_t available;
 337
 338         if (writer->buffer == NULL)
 339                 return writer->error;
 340
 341         if (writer->error == UDS_SUCCESS) {
 342                 available = get_remaining_write_space(writer);
 343
 344                 if (available > 0)
 345                         memset(writer->end, 0, available);
 346
 347                 dm_bufio_mark_buffer_dirty(writer->buffer);
 348         }
 349
 350         dm_bufio_release(writer->buffer);
 351         writer->buffer = NULL;
 352         writer->start = NULL;
 353         writer->end = NULL;
 354         writer->block_number++;
 355         return writer->error;
 356 }
 357
 358 void uds_free_buffered_writer(struct buffered_writer *writer)
 359 {
 360         int result;
 361
 362         if (writer == NULL)
 363                 return;
 364
 365         flush_previous_buffer(writer);
 366         result = -dm_bufio_write_dirty_buffers(writer->client);
 367         if (result != UDS_SUCCESS)
 368                 vdo_log_warning_strerror(result, "%s: failed to sync storage", __func__);
 369
 370         dm_bufio_client_destroy(writer->client);
 371         uds_put_io_factory(writer->factory);
 372         vdo_free(writer);
 373 }
 374
 375 /*
 376  * Append data to the buffer, writing as needed. If no data is provided, zeros are written instead.
 377  * If a write error occurs, it is recorded and returned on every subsequent write attempt.
 378  */
 379 int uds_write_to_buffered_writer(struct buffered_writer *writer, const u8 *data,
 380                                  size_t length)
 381 {
 382         int result = writer->error;
 383         size_t chunk_size;
 384
 385         while ((length > 0) && (result == UDS_SUCCESS)) {
 386                 if (writer->buffer == NULL) {
 387                         result = prepare_next_buffer(writer);
 388                         continue;
 389                 }
 390
 391                 chunk_size = min(length, get_remaining_write_space(writer));
 392                 if (data == NULL) {
 393                         memset(writer->end, 0, chunk_size);
 394                 } else {
 395                         memcpy(writer->end, data, chunk_size);
 396                         data += chunk_size;
 397                 }
 398
 399                 length -= chunk_size;
 400                 writer->end += chunk_size;
 401
 402                 if (get_remaining_write_space(writer) == 0)
 403                         result = uds_flush_buffered_writer(writer);
 404         }
 405
 406         return result;
 407 }
 408
 409 int uds_flush_buffered_writer(struct buffered_writer *writer)
 410 {
 411         if (writer->error != UDS_SUCCESS)
 412                 return writer->error;
 413
 414         return flush_previous_buffer(writer);
 415 }