fs/netfs/read_retry.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Network filesystem read subrequest retrying.
   3  *
   4  * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved.
   5  * Written by David Howells (dhowells@redhat.com)
   6  */
   7
   8 #include <linux/fs.h>
   9 #include <linux/slab.h>
  10 #include "internal.h"
  11
  12 static void netfs_reissue_read(struct netfs_io_request *rreq,
  13                                struct netfs_io_subrequest *subreq)
  14 {
  15         struct iov_iter *io_iter = &subreq->io_iter;
  16
  17         if (iov_iter_is_folioq(io_iter)) {
  18                 subreq->curr_folioq = (struct folio_queue *)io_iter->folioq;
  19                 subreq->curr_folioq_slot = io_iter->folioq_slot;
  20                 subreq->curr_folio_order = subreq->curr_folioq->orders[subreq->curr_folioq_slot];
  21         }
  22
  23         atomic_inc(&rreq->nr_outstanding);
  24         __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
  25         netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit);
  26         subreq->rreq->netfs_ops->issue_read(subreq);
  27 }
  28
  29 /*
  30  * Go through the list of failed/short reads, retrying all retryable ones.  We
  31  * need to switch failed cache reads to network downloads.
  32  */
  33 static void netfs_retry_read_subrequests(struct netfs_io_request *rreq)
  34 {
  35         struct netfs_io_subrequest *subreq;
  36         struct netfs_io_stream *stream0 = &rreq->io_streams[0];
  37         LIST_HEAD(sublist);
  38         LIST_HEAD(queue);
  39
  40         _enter("R=%x", rreq->debug_id);
  41
  42         if (list_empty(&rreq->subrequests))
  43                 return;
  44
  45         if (rreq->netfs_ops->retry_request)
  46                 rreq->netfs_ops->retry_request(rreq, NULL);
  47
  48         /* If there's no renegotiation to do, just resend each retryable subreq
  49          * up to the first permanently failed one.
  50          */
  51         if (!rreq->netfs_ops->prepare_read &&
  52             !test_bit(NETFS_RREQ_COPY_TO_CACHE, &rreq->flags)) {
  53                 struct netfs_io_subrequest *subreq;
  54
  55                 list_for_each_entry(subreq, &rreq->subrequests, rreq_link) {
  56                         if (test_bit(NETFS_SREQ_FAILED, &subreq->flags))
  57                                 break;
  58                         if (__test_and_clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) {
  59                                 netfs_reset_iter(subreq);
  60                                 netfs_reissue_read(rreq, subreq);
  61                         }
  62                 }
  63                 return;
  64         }
  65
  66         /* Okay, we need to renegotiate all the download requests and flip any
  67          * failed cache reads over to being download requests and negotiate
  68          * those also.  All fully successful subreqs have been removed from the
  69          * list and any spare data from those has been donated.
  70          *
  71          * What we do is decant the list and rebuild it one subreq at a time so
  72          * that we don't end up with donations jumping over a gap we're busy
  73          * populating with smaller subrequests.  In the event that the subreq
  74          * we just launched finishes before we insert the next subreq, it'll
  75          * fill in rreq->prev_donated instead.
  76
  77          * Note: Alternatively, we could split the tail subrequest right before
  78          * we reissue it and fix up the donations under lock.
  79          */
  80         list_splice_init(&rreq->subrequests, &queue);
  81
  82         do {
  83                 struct netfs_io_subrequest *from;
  84                 struct iov_iter source;
  85                 unsigned long long start, len;
  86                 size_t part, deferred_next_donated = 0;
  87                 bool boundary = false;
  88
  89                 /* Go through the subreqs and find the next span of contiguous
  90                  * buffer that we then rejig (cifs, for example, needs the
  91                  * rsize renegotiating) and reissue.
  92                  */
  93                 from = list_first_entry(&queue, struct netfs_io_subrequest, rreq_link);
  94                 list_move_tail(&from->rreq_link, &sublist);
  95                 start = from->start + from->transferred;
  96                 len   = from->len   - from->transferred;
  97
  98                 _debug("from R=%08x[%x] s=%llx ctl=%zx/%zx/%zx",
  99                        rreq->debug_id, from->debug_index,
 100                        from->start, from->consumed, from->transferred, from->len);
 101
 102                 if (test_bit(NETFS_SREQ_FAILED, &from->flags) ||
 103                     !test_bit(NETFS_SREQ_NEED_RETRY, &from->flags))
 104                         goto abandon;
 105
 106                 deferred_next_donated = from->next_donated;
 107                 while ((subreq = list_first_entry_or_null(
 108                                 &queue, struct netfs_io_subrequest, rreq_link))) {
 109                         if (subreq->start != start + len ||
 110                             subreq->transferred > 0 ||
 111                             !test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags))
 112                                 break;
 113                         list_move_tail(&subreq->rreq_link, &sublist);
 114                         len += subreq->len;
 115                         deferred_next_donated = subreq->next_donated;
 116                         if (test_bit(NETFS_SREQ_BOUNDARY, &subreq->flags))
 117                                 break;
 118                 }
 119
 120                 _debug(" - range: %llx-%llx %llx", start, start + len - 1, len);
 121
 122                 /* Determine the set of buffers we're going to use.  Each
 123                  * subreq gets a subset of a single overall contiguous buffer.
 124                  */
 125                 netfs_reset_iter(from);
 126                 source = from->io_iter;
 127                 source.count = len;
 128
 129                 /* Work through the sublist. */
 130                 while ((subreq = list_first_entry_or_null(
 131                                 &sublist, struct netfs_io_subrequest, rreq_link))) {
 132                         list_del(&subreq->rreq_link);
 133
 134                         subreq->source  = NETFS_DOWNLOAD_FROM_SERVER;
 135                         subreq->start   = start - subreq->transferred;
 136                         subreq->len     = len   + subreq->transferred;
 137                         stream0->sreq_max_len = subreq->len;
 138
 139                         __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);
 140                         __set_bit(NETFS_SREQ_RETRYING, &subreq->flags);
 141
 142                         spin_lock_bh(&rreq->lock);
 143                         list_add_tail(&subreq->rreq_link, &rreq->subrequests);
 144                         subreq->prev_donated += rreq->prev_donated;
 145                         rreq->prev_donated = 0;
 146                         trace_netfs_sreq(subreq, netfs_sreq_trace_retry);
 147                         spin_unlock_bh(&rreq->lock);
 148
 149                         BUG_ON(!len);
 150
 151                         /* Renegotiate max_len (rsize) */
 152                         if (rreq->netfs_ops->prepare_read(subreq) < 0) {
 153                                 trace_netfs_sreq(subreq, netfs_sreq_trace_reprep_failed);
 154                                 __set_bit(NETFS_SREQ_FAILED, &subreq->flags);
 155                         }
 156
 157                         part = umin(len, stream0->sreq_max_len);
 158                         if (unlikely(rreq->io_streams[0].sreq_max_segs))
 159                                 part = netfs_limit_iter(&source, 0, part, stream0->sreq_max_segs);
 160                         subreq->len = subreq->transferred + part;
 161                         subreq->io_iter = source;
 162                         iov_iter_truncate(&subreq->io_iter, part);
 163                         iov_iter_advance(&source, part);
 164                         len -= part;
 165                         start += part;
 166                         if (!len) {
 167                                 if (boundary)
 168                                         __set_bit(NETFS_SREQ_BOUNDARY, &subreq->flags);
 169                                 subreq->next_donated = deferred_next_donated;
 170                         } else {
 171                                 __clear_bit(NETFS_SREQ_BOUNDARY, &subreq->flags);
 172                                 subreq->next_donated = 0;
 173                         }
 174
 175                         netfs_reissue_read(rreq, subreq);
 176                         if (!len)
 177                                 break;
 178
 179                         /* If we ran out of subrequests, allocate another. */
 180                         if (list_empty(&sublist)) {
 181                                 subreq = netfs_alloc_subrequest(rreq);
 182                                 if (!subreq)
 183                                         goto abandon;
 184                                 subreq->source = NETFS_DOWNLOAD_FROM_SERVER;
 185                                 subreq->start = start;
 186
 187                                 /* We get two refs, but need just one. */
 188                                 netfs_put_subrequest(subreq, false, netfs_sreq_trace_new);
 189                                 trace_netfs_sreq(subreq, netfs_sreq_trace_split);
 190                                 list_add_tail(&subreq->rreq_link, &sublist);
 191                         }
 192                 }
 193
 194                 /* If we managed to use fewer subreqs, we can discard the
 195                  * excess.
 196                  */
 197                 while ((subreq = list_first_entry_or_null(
 198                                 &sublist, struct netfs_io_subrequest, rreq_link))) {
 199                         trace_netfs_sreq(subreq, netfs_sreq_trace_discard);
 200                         list_del(&subreq->rreq_link);
 201                         netfs_put_subrequest(subreq, false, netfs_sreq_trace_put_done);
 202                 }
 203
 204         } while (!list_empty(&queue));
 205
 206         return;
 207
 208         /* If we hit ENOMEM, fail all remaining subrequests */
 209 abandon:
 210         list_splice_init(&sublist, &queue);
 211         list_for_each_entry(subreq, &queue, rreq_link) {
 212                 if (!subreq->error)
 213                         subreq->error = -ENOMEM;
 214                 __clear_bit(NETFS_SREQ_FAILED, &subreq->flags);
 215                 __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);
 216                 __clear_bit(NETFS_SREQ_RETRYING, &subreq->flags);
 217         }
 218         spin_lock_bh(&rreq->lock);
 219         list_splice_tail_init(&queue, &rreq->subrequests);
 220         spin_unlock_bh(&rreq->lock);
 221 }
 222
 223 /*
 224  * Retry reads.
 225  */
 226 void netfs_retry_reads(struct netfs_io_request *rreq)
 227 {
 228         trace_netfs_rreq(rreq, netfs_rreq_trace_resubmit);
 229
 230         atomic_inc(&rreq->nr_outstanding);
 231
 232         netfs_retry_read_subrequests(rreq);
 233
 234         if (atomic_dec_and_test(&rreq->nr_outstanding))
 235                 netfs_rreq_terminated(rreq, false);
 236 }
 237
 238 /*
 239  * Unlock any the pages that haven't been unlocked yet due to abandoned
 240  * subrequests.
 241  */
 242 void netfs_unlock_abandoned_read_pages(struct netfs_io_request *rreq)
 243 {
 244         struct folio_queue *p;
 245
 246         for (p = rreq->buffer; p; p = p->next) {
 247                 for (int slot = 0; slot < folioq_count(p); slot++) {
 248                         struct folio *folio = folioq_folio(p, slot);
 249
 250                         if (folio && !folioq_is_marked2(p, slot)) {
 251                                 trace_netfs_folio(folio, netfs_folio_trace_abandon);
 252                                 folio_unlock(folio);
 253                         }
 254                 }
 255         }
 256 }