drivers/md/dm-vdo/logical-zone.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright 2023 Red Hat
   4  */
   5
   6 #include "logical-zone.h"
   7
   8 #include "logger.h"
   9 #include "memory-alloc.h"
  10 #include "permassert.h"
  11 #include "string-utils.h"
  12
  13 #include "action-manager.h"
  14 #include "admin-state.h"
  15 #include "block-map.h"
  16 #include "completion.h"
  17 #include "constants.h"
  18 #include "data-vio.h"
  19 #include "flush.h"
  20 #include "int-map.h"
  21 #include "physical-zone.h"
  22 #include "vdo.h"
  23
  24 #define ALLOCATIONS_PER_ZONE 128
  25
  26 /**
  27  * as_logical_zone() - Convert a generic vdo_completion to a logical_zone.
  28  * @completion: The completion to convert.
  29  *
  30  * Return: The completion as a logical_zone.
  31  */
  32 static struct logical_zone *as_logical_zone(struct vdo_completion *completion)
  33 {
  34         vdo_assert_completion_type(completion, VDO_GENERATION_FLUSHED_COMPLETION);
  35         return container_of(completion, struct logical_zone, completion);
  36 }
  37
  38 /* get_thread_id_for_zone() - Implements vdo_zone_thread_getter_fn. */
  39 static thread_id_t get_thread_id_for_zone(void *context, zone_count_t zone_number)
  40 {
  41         struct logical_zones *zones = context;
  42
  43         return zones->zones[zone_number].thread_id;
  44 }
  45
  46 /**
  47  * initialize_zone() - Initialize a logical zone.
  48  * @zones: The logical_zones to which this zone belongs.
  49  * @zone_number: The logical_zone's index.
  50  */
  51 static int initialize_zone(struct logical_zones *zones, zone_count_t zone_number)
  52 {
  53         int result;
  54         struct vdo *vdo = zones->vdo;
  55         struct logical_zone *zone = &zones->zones[zone_number];
  56         zone_count_t allocation_zone_number;
  57
  58         result = vdo_int_map_create(VDO_LOCK_MAP_CAPACITY, &zone->lbn_operations);
  59         if (result != VDO_SUCCESS)
  60                 return result;
  61
  62         if (zone_number < vdo->thread_config.logical_zone_count - 1)
  63                 zone->next = &zones->zones[zone_number + 1];
  64
  65         vdo_initialize_completion(&zone->completion, vdo,
  66                                   VDO_GENERATION_FLUSHED_COMPLETION);
  67         zone->zones = zones;
  68         zone->zone_number = zone_number;
  69         zone->thread_id = vdo->thread_config.logical_threads[zone_number];
  70         zone->block_map_zone = &vdo->block_map->zones[zone_number];
  71         INIT_LIST_HEAD(&zone->write_vios);
  72         vdo_set_admin_state_code(&zone->state, VDO_ADMIN_STATE_NORMAL_OPERATION);
  73
  74         allocation_zone_number = zone->thread_id % vdo->thread_config.physical_zone_count;
  75         zone->allocation_zone = &vdo->physical_zones->zones[allocation_zone_number];
  76
  77         return vdo_make_default_thread(vdo, zone->thread_id);
  78 }
  79
  80 /**
  81  * vdo_make_logical_zones() - Create a set of logical zones.
  82  * @vdo: The vdo to which the zones will belong.
  83  * @zones_ptr: A pointer to hold the new zones.
  84  *
  85  * Return: VDO_SUCCESS or an error code.
  86  */
  87 int vdo_make_logical_zones(struct vdo *vdo, struct logical_zones **zones_ptr)
  88 {
  89         struct logical_zones *zones;
  90         int result;
  91         zone_count_t zone;
  92         zone_count_t zone_count = vdo->thread_config.logical_zone_count;
  93
  94         if (zone_count == 0)
  95                 return VDO_SUCCESS;
  96
  97         result = vdo_allocate_extended(struct logical_zones, zone_count,
  98                                        struct logical_zone, __func__, &zones);
  99         if (result != VDO_SUCCESS)
 100                 return result;
 101
 102         zones->vdo = vdo;
 103         zones->zone_count = zone_count;
 104         for (zone = 0; zone < zone_count; zone++) {
 105                 result = initialize_zone(zones, zone);
 106                 if (result != VDO_SUCCESS) {
 107                         vdo_free_logical_zones(zones);
 108                         return result;
 109                 }
 110         }
 111
 112         result = vdo_make_action_manager(zones->zone_count, get_thread_id_for_zone,
 113                                          vdo->thread_config.admin_thread, zones, NULL,
 114                                          vdo, &zones->manager);
 115         if (result != VDO_SUCCESS) {
 116                 vdo_free_logical_zones(zones);
 117                 return result;
 118         }
 119
 120         *zones_ptr = zones;
 121         return VDO_SUCCESS;
 122 }
 123
 124 /**
 125  * vdo_free_logical_zones() - Free a set of logical zones.
 126  * @zones: The set of zones to free.
 127  */
 128 void vdo_free_logical_zones(struct logical_zones *zones)
 129 {
 130         zone_count_t index;
 131
 132         if (zones == NULL)
 133                 return;
 134
 135         vdo_free(vdo_forget(zones->manager));
 136
 137         for (index = 0; index < zones->zone_count; index++)
 138                 vdo_int_map_free(vdo_forget(zones->zones[index].lbn_operations));
 139
 140         vdo_free(zones);
 141 }
 142
 143 static inline void assert_on_zone_thread(struct logical_zone *zone, const char *what)
 144 {
 145         VDO_ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == zone->thread_id),
 146                             "%s() called on correct thread", what);
 147 }
 148
 149 /**
 150  * check_for_drain_complete() - Check whether this zone has drained.
 151  * @zone: The zone to check.
 152  */
 153 static void check_for_drain_complete(struct logical_zone *zone)
 154 {
 155         if (!vdo_is_state_draining(&zone->state) || zone->notifying ||
 156             !list_empty(&zone->write_vios))
 157                 return;
 158
 159         vdo_finish_draining(&zone->state);
 160 }
 161
 162 /**
 163  * initiate_drain() - Initiate a drain.
 164  *
 165  * Implements vdo_admin_initiator_fn.
 166  */
 167 static void initiate_drain(struct admin_state *state)
 168 {
 169         check_for_drain_complete(container_of(state, struct logical_zone, state));
 170 }
 171
 172 /**
 173  * drain_logical_zone() - Drain a logical zone.
 174  *
 175  * Implements vdo_zone_action_fn.
 176  */
 177 static void drain_logical_zone(void *context, zone_count_t zone_number,
 178                                struct vdo_completion *parent)
 179 {
 180         struct logical_zones *zones = context;
 181
 182         vdo_start_draining(&zones->zones[zone_number].state,
 183                            vdo_get_current_manager_operation(zones->manager), parent,
 184                            initiate_drain);
 185 }
 186
 187 void vdo_drain_logical_zones(struct logical_zones *zones,
 188                              const struct admin_state_code *operation,
 189                              struct vdo_completion *parent)
 190 {
 191         vdo_schedule_operation(zones->manager, operation, NULL, drain_logical_zone, NULL,
 192                                parent);
 193 }
 194
 195 /**
 196  * resume_logical_zone() - Resume a logical zone.
 197  *
 198  * Implements vdo_zone_action_fn.
 199  */
 200 static void resume_logical_zone(void *context, zone_count_t zone_number,
 201                                 struct vdo_completion *parent)
 202 {
 203         struct logical_zone *zone = &(((struct logical_zones *) context)->zones[zone_number]);
 204
 205         vdo_fail_completion(parent, vdo_resume_if_quiescent(&zone->state));
 206 }
 207
 208 /**
 209  * vdo_resume_logical_zones() - Resume a set of logical zones.
 210  * @zones: The logical zones to resume.
 211  * @parent: The object to notify when the zones have resumed.
 212  */
 213 void vdo_resume_logical_zones(struct logical_zones *zones, struct vdo_completion *parent)
 214 {
 215         vdo_schedule_operation(zones->manager, VDO_ADMIN_STATE_RESUMING, NULL,
 216                                resume_logical_zone, NULL, parent);
 217 }
 218
 219 /**
 220  * update_oldest_active_generation() - Update the oldest active generation.
 221  * @zone: The zone.
 222  *
 223  * Return: true if the oldest active generation has changed.
 224  */
 225 static bool update_oldest_active_generation(struct logical_zone *zone)
 226 {
 227         struct data_vio *data_vio =
 228                 list_first_entry_or_null(&zone->write_vios, struct data_vio,
 229                                          write_entry);
 230         sequence_number_t oldest =
 231                 (data_vio == NULL) ? zone->flush_generation : data_vio->flush_generation;
 232
 233         if (oldest == zone->oldest_active_generation)
 234                 return false;
 235
 236         WRITE_ONCE(zone->oldest_active_generation, oldest);
 237         return true;
 238 }
 239
 240 /**
 241  * vdo_increment_logical_zone_flush_generation() - Increment the flush generation in a logical
 242  *                                                 zone.
 243  * @zone: The logical zone.
 244  * @expected_generation: The expected value of the flush generation before the increment.
 245  */
 246 void vdo_increment_logical_zone_flush_generation(struct logical_zone *zone,
 247                                                  sequence_number_t expected_generation)
 248 {
 249         assert_on_zone_thread(zone, __func__);
 250         VDO_ASSERT_LOG_ONLY((zone->flush_generation == expected_generation),
 251                             "logical zone %u flush generation %llu should be %llu before increment",
 252                             zone->zone_number, (unsigned long long) zone->flush_generation,
 253                             (unsigned long long) expected_generation);
 254
 255         zone->flush_generation++;
 256         zone->ios_in_flush_generation = 0;
 257         update_oldest_active_generation(zone);
 258 }
 259
 260 /**
 261  * vdo_acquire_flush_generation_lock() - Acquire the shared lock on a flush generation by a write
 262  *                                       data_vio.
 263  * @data_vio: The data_vio.
 264  */
 265 void vdo_acquire_flush_generation_lock(struct data_vio *data_vio)
 266 {
 267         struct logical_zone *zone = data_vio->logical.zone;
 268
 269         assert_on_zone_thread(zone, __func__);
 270         VDO_ASSERT_LOG_ONLY(vdo_is_state_normal(&zone->state), "vdo state is normal");
 271
 272         data_vio->flush_generation = zone->flush_generation;
 273         list_add_tail(&data_vio->write_entry, &zone->write_vios);
 274         zone->ios_in_flush_generation++;
 275 }
 276
 277 static void attempt_generation_complete_notification(struct vdo_completion *completion);
 278
 279 /**
 280  * notify_flusher() - Notify the flush that at least one generation no longer has active VIOs.
 281  * @completion: The zone completion.
 282  *
 283  * This callback is registered in attempt_generation_complete_notification().
 284  */
 285 static void notify_flusher(struct vdo_completion *completion)
 286 {
 287         struct logical_zone *zone = as_logical_zone(completion);
 288
 289         vdo_complete_flushes(zone->zones->vdo->flusher);
 290         vdo_launch_completion_callback(completion,
 291                                        attempt_generation_complete_notification,
 292                                        zone->thread_id);
 293 }
 294
 295 /**
 296  * attempt_generation_complete_notification() - Notify the flusher if some generation no
 297  *                                              longer has active VIOs.
 298  * @completion: The zone completion.
 299  */
 300 static void attempt_generation_complete_notification(struct vdo_completion *completion)
 301 {
 302         struct logical_zone *zone = as_logical_zone(completion);
 303
 304         assert_on_zone_thread(zone, __func__);
 305         if (zone->oldest_active_generation <= zone->notification_generation) {
 306                 zone->notifying = false;
 307                 check_for_drain_complete(zone);
 308                 return;
 309         }
 310
 311         zone->notifying = true;
 312         zone->notification_generation = zone->oldest_active_generation;
 313         vdo_launch_completion_callback(&zone->completion, notify_flusher,
 314                                        vdo_get_flusher_thread_id(zone->zones->vdo->flusher));
 315 }
 316
 317 /**
 318  * vdo_release_flush_generation_lock() - Release the shared lock on a flush generation held by a
 319  *                                       write data_vio.
 320  * @data_vio: The data_vio whose lock is to be released.
 321  *
 322  * If there are pending flushes, and this data_vio completes the oldest generation active in this
 323  * zone, an attempt will be made to finish any flushes which may now be complete.
 324  */
 325 void vdo_release_flush_generation_lock(struct data_vio *data_vio)
 326 {
 327         struct logical_zone *zone = data_vio->logical.zone;
 328
 329         assert_on_zone_thread(zone, __func__);
 330
 331         if (!data_vio_has_flush_generation_lock(data_vio))
 332                 return;
 333
 334         list_del_init(&data_vio->write_entry);
 335         VDO_ASSERT_LOG_ONLY((zone->oldest_active_generation <= data_vio->flush_generation),
 336                             "data_vio releasing lock on generation %llu is not older than oldest active generation %llu",
 337                             (unsigned long long) data_vio->flush_generation,
 338                             (unsigned long long) zone->oldest_active_generation);
 339
 340         if (!update_oldest_active_generation(zone) || zone->notifying)
 341                 return;
 342
 343         attempt_generation_complete_notification(&zone->completion);
 344 }
 345
 346 struct physical_zone *vdo_get_next_allocation_zone(struct logical_zone *zone)
 347 {
 348         if (zone->allocation_count == ALLOCATIONS_PER_ZONE) {
 349                 zone->allocation_count = 0;
 350                 zone->allocation_zone = zone->allocation_zone->next;
 351         }
 352
 353         zone->allocation_count++;
 354         return zone->allocation_zone;
 355 }
 356
 357 /**
 358  * vdo_dump_logical_zone() - Dump information about a logical zone to the log for debugging.
 359  * @zone: The zone to dump
 360  *
 361  * Context: the information is dumped in a thread-unsafe fashion.
 362  *
 363  */
 364 void vdo_dump_logical_zone(const struct logical_zone *zone)
 365 {
 366         vdo_log_info("logical_zone %u", zone->zone_number);
 367         vdo_log_info("  flush_generation=%llu oldest_active_generation=%llu notification_generation=%llu notifying=%s ios_in_flush_generation=%llu",
 368                      (unsigned long long) READ_ONCE(zone->flush_generation),
 369                      (unsigned long long) READ_ONCE(zone->oldest_active_generation),
 370                      (unsigned long long) READ_ONCE(zone->notification_generation),
 371                      vdo_bool_to_string(READ_ONCE(zone->notifying)),
 372                      (unsigned long long) READ_ONCE(zone->ios_in_flush_generation));
 373 }