fs/bcachefs/error.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include "bcachefs.h"
   3 #include "btree_iter.h"
   4 #include "error.h"
   5 #include "journal.h"
   6 #include "recovery_passes.h"
   7 #include "super.h"
   8 #include "thread_with_file.h"
   9
  10 #define FSCK_ERR_RATELIMIT_NR   10
  11
  12 bool bch2_inconsistent_error(struct bch_fs *c)
  13 {
  14         set_bit(BCH_FS_error, &c->flags);
  15
  16         switch (c->opts.errors) {
  17         case BCH_ON_ERROR_continue:
  18                 return false;
  19         case BCH_ON_ERROR_fix_safe:
  20         case BCH_ON_ERROR_ro:
  21                 if (bch2_fs_emergency_read_only(c))
  22                         bch_err(c, "inconsistency detected - emergency read only at journal seq %llu",
  23                                 journal_cur_seq(&c->journal));
  24                 return true;
  25         case BCH_ON_ERROR_panic:
  26                 panic(bch2_fmt(c, "panic after error"));
  27                 return true;
  28         default:
  29                 BUG();
  30         }
  31 }
  32
  33 int bch2_topology_error(struct bch_fs *c)
  34 {
  35         set_bit(BCH_FS_topology_error, &c->flags);
  36         if (!test_bit(BCH_FS_fsck_running, &c->flags)) {
  37                 bch2_inconsistent_error(c);
  38                 return -BCH_ERR_btree_need_topology_repair;
  39         } else {
  40                 return bch2_run_explicit_recovery_pass(c, BCH_RECOVERY_PASS_check_topology) ?:
  41                         -BCH_ERR_btree_node_read_validate_error;
  42         }
  43 }
  44
  45 void bch2_fatal_error(struct bch_fs *c)
  46 {
  47         if (bch2_fs_emergency_read_only(c))
  48                 bch_err(c, "fatal error - emergency read only");
  49 }
  50
  51 void bch2_io_error_work(struct work_struct *work)
  52 {
  53         struct bch_dev *ca = container_of(work, struct bch_dev, io_error_work);
  54         struct bch_fs *c = ca->fs;
  55         bool dev;
  56
  57         down_write(&c->state_lock);
  58         dev = bch2_dev_state_allowed(c, ca, BCH_MEMBER_STATE_ro,
  59                                     BCH_FORCE_IF_DEGRADED);
  60         if (dev
  61             ? __bch2_dev_set_state(c, ca, BCH_MEMBER_STATE_ro,
  62                                   BCH_FORCE_IF_DEGRADED)
  63             : bch2_fs_emergency_read_only(c))
  64                 bch_err(ca,
  65                         "too many IO errors, setting %s RO",
  66                         dev ? "device" : "filesystem");
  67         up_write(&c->state_lock);
  68 }
  69
  70 void bch2_io_error(struct bch_dev *ca, enum bch_member_error_type type)
  71 {
  72         atomic64_inc(&ca->errors[type]);
  73         //queue_work(system_long_wq, &ca->io_error_work);
  74 }
  75
  76 enum ask_yn {
  77         YN_NO,
  78         YN_YES,
  79         YN_ALLNO,
  80         YN_ALLYES,
  81 };
  82
  83 static enum ask_yn parse_yn_response(char *buf)
  84 {
  85         buf = strim(buf);
  86
  87         if (strlen(buf) == 1)
  88                 switch (buf[0]) {
  89                 case 'n':
  90                         return YN_NO;
  91                 case 'y':
  92                         return YN_YES;
  93                 case 'N':
  94                         return YN_ALLNO;
  95                 case 'Y':
  96                         return YN_ALLYES;
  97                 }
  98         return -1;
  99 }
 100
 101 #ifdef __KERNEL__
 102 static enum ask_yn bch2_fsck_ask_yn(struct bch_fs *c, struct btree_trans *trans)
 103 {
 104         struct stdio_redirect *stdio = c->stdio;
 105
 106         if (c->stdio_filter && c->stdio_filter != current)
 107                 stdio = NULL;
 108
 109         if (!stdio)
 110                 return YN_NO;
 111
 112         if (trans)
 113                 bch2_trans_unlock(trans);
 114
 115         unsigned long unlock_long_at = trans ? jiffies + HZ * 2 : 0;
 116         darray_char line = {};
 117         int ret;
 118
 119         do {
 120                 unsigned long t;
 121                 bch2_print(c, " (y,n, or Y,N for all errors of this type) ");
 122 rewait:
 123                 t = unlock_long_at
 124                         ? max_t(long, unlock_long_at - jiffies, 0)
 125                         : MAX_SCHEDULE_TIMEOUT;
 126
 127                 int r = bch2_stdio_redirect_readline_timeout(stdio, &line, t);
 128                 if (r == -ETIME) {
 129                         bch2_trans_unlock_long(trans);
 130                         unlock_long_at = 0;
 131                         goto rewait;
 132                 }
 133
 134                 if (r < 0) {
 135                         ret = YN_NO;
 136                         break;
 137                 }
 138
 139                 darray_last(line) = '\0';
 140         } while ((ret = parse_yn_response(line.data)) < 0);
 141
 142         darray_exit(&line);
 143         return ret;
 144 }
 145 #else
 146
 147 #include "tools-util.h"
 148
 149 static enum ask_yn bch2_fsck_ask_yn(struct bch_fs *c, struct btree_trans *trans)
 150 {
 151         char *buf = NULL;
 152         size_t buflen = 0;
 153         int ret;
 154
 155         do {
 156                 fputs(" (y,n, or Y,N for all errors of this type) ", stdout);
 157                 fflush(stdout);
 158
 159                 if (getline(&buf, &buflen, stdin) < 0)
 160                         die("error reading from standard input");
 161         } while ((ret = parse_yn_response(buf)) < 0);
 162
 163         free(buf);
 164         return ret;
 165 }
 166
 167 #endif
 168
 169 static struct fsck_err_state *fsck_err_get(struct bch_fs *c, const char *fmt)
 170 {
 171         struct fsck_err_state *s;
 172
 173         if (!test_bit(BCH_FS_fsck_running, &c->flags))
 174                 return NULL;
 175
 176         list_for_each_entry(s, &c->fsck_error_msgs, list)
 177                 if (s->fmt == fmt) {
 178                         /*
 179                          * move it to the head of the list: repeated fsck errors
 180                          * are common
 181                          */
 182                         list_move(&s->list, &c->fsck_error_msgs);
 183                         return s;
 184                 }
 185
 186         s = kzalloc(sizeof(*s), GFP_NOFS);
 187         if (!s) {
 188                 if (!c->fsck_alloc_msgs_err)
 189                         bch_err(c, "kmalloc err, cannot ratelimit fsck errs");
 190                 c->fsck_alloc_msgs_err = true;
 191                 return NULL;
 192         }
 193
 194         INIT_LIST_HEAD(&s->list);
 195         s->fmt = fmt;
 196         list_add(&s->list, &c->fsck_error_msgs);
 197         return s;
 198 }
 199
 200 /* s/fix?/fixing/ s/recreate?/recreating/ */
 201 static void prt_actioning(struct printbuf *out, const char *action)
 202 {
 203         unsigned len = strlen(action);
 204
 205         BUG_ON(action[len - 1] != '?');
 206         --len;
 207
 208         if (action[len - 1] == 'e')
 209                 --len;
 210
 211         prt_bytes(out, action, len);
 212         prt_str(out, "ing");
 213 }
 214
 215 static const u8 fsck_flags_extra[] = {
 216 #define x(t, n, flags)          [BCH_FSCK_ERR_##t] = flags,
 217         BCH_SB_ERRS()
 218 #undef x
 219 };
 220
 221 int __bch2_fsck_err(struct bch_fs *c,
 222                   struct btree_trans *trans,
 223                   enum bch_fsck_flags flags,
 224                   enum bch_sb_error_id err,
 225                   const char *fmt, ...)
 226 {
 227         struct fsck_err_state *s = NULL;
 228         va_list args;
 229         bool print = true, suppressing = false, inconsistent = false;
 230         struct printbuf buf = PRINTBUF, *out = &buf;
 231         int ret = -BCH_ERR_fsck_ignore;
 232         const char *action_orig = "fix?", *action = action_orig;
 233
 234         might_sleep();
 235
 236         if (!WARN_ON(err >= ARRAY_SIZE(fsck_flags_extra)))
 237                 flags |= fsck_flags_extra[err];
 238
 239         if (!c)
 240                 c = trans->c;
 241
 242         /*
 243          * Ugly: if there's a transaction in the current task it has to be
 244          * passed in to unlock if we prompt for user input.
 245          *
 246          * But, plumbing a transaction and transaction restarts into
 247          * bkey_validate() is problematic.
 248          *
 249          * So:
 250          * - make all bkey errors AUTOFIX, they're simple anyways (we just
 251          *   delete the key)
 252          * - and we don't need to warn if we're not prompting
 253          */
 254         WARN_ON((flags & FSCK_CAN_FIX) &&
 255                 !(flags & FSCK_AUTOFIX) &&
 256                 !trans &&
 257                 bch2_current_has_btree_trans(c));
 258
 259         if ((flags & FSCK_CAN_FIX) &&
 260             test_bit(err, c->sb.errors_silent))
 261                 return -BCH_ERR_fsck_fix;
 262
 263         bch2_sb_error_count(c, err);
 264
 265         va_start(args, fmt);
 266         prt_vprintf(out, fmt, args);
 267         va_end(args);
 268
 269         /* Custom fix/continue/recreate/etc.? */
 270         if (out->buf[out->pos - 1] == '?') {
 271                 const char *p = strrchr(out->buf, ',');
 272                 if (p) {
 273                         out->pos = p - out->buf;
 274                         action = kstrdup(p + 2, GFP_KERNEL);
 275                         if (!action) {
 276                                 ret = -ENOMEM;
 277                                 goto err;
 278                         }
 279                 }
 280         }
 281
 282         mutex_lock(&c->fsck_error_msgs_lock);
 283         s = fsck_err_get(c, fmt);
 284         if (s) {
 285                 /*
 286                  * We may be called multiple times for the same error on
 287                  * transaction restart - this memoizes instead of asking the user
 288                  * multiple times for the same error:
 289                  */
 290                 if (s->last_msg && !strcmp(buf.buf, s->last_msg)) {
 291                         ret = s->ret;
 292                         mutex_unlock(&c->fsck_error_msgs_lock);
 293                         goto err;
 294                 }
 295
 296                 kfree(s->last_msg);
 297                 s->last_msg = kstrdup(buf.buf, GFP_KERNEL);
 298                 if (!s->last_msg) {
 299                         mutex_unlock(&c->fsck_error_msgs_lock);
 300                         ret = -ENOMEM;
 301                         goto err;
 302                 }
 303
 304                 if (c->opts.ratelimit_errors &&
 305                     !(flags & FSCK_NO_RATELIMIT) &&
 306                     s->nr >= FSCK_ERR_RATELIMIT_NR) {
 307                         if (s->nr == FSCK_ERR_RATELIMIT_NR)
 308                                 suppressing = true;
 309                         else
 310                                 print = false;
 311                 }
 312
 313                 s->nr++;
 314         }
 315
 316 #ifdef BCACHEFS_LOG_PREFIX
 317         if (!strncmp(fmt, "bcachefs:", 9))
 318                 prt_printf(out, bch2_log_msg(c, ""));
 319 #endif
 320
 321         if ((flags & FSCK_CAN_FIX) &&
 322             (flags & FSCK_AUTOFIX) &&
 323             (c->opts.errors == BCH_ON_ERROR_continue ||
 324              c->opts.errors == BCH_ON_ERROR_fix_safe)) {
 325                 prt_str(out, ", ");
 326                 prt_actioning(out, action);
 327                 ret = -BCH_ERR_fsck_fix;
 328         } else if (!test_bit(BCH_FS_fsck_running, &c->flags)) {
 329                 if (c->opts.errors != BCH_ON_ERROR_continue ||
 330                     !(flags & (FSCK_CAN_FIX|FSCK_CAN_IGNORE))) {
 331                         prt_str(out, ", shutting down");
 332                         inconsistent = true;
 333                         ret = -BCH_ERR_fsck_errors_not_fixed;
 334                 } else if (flags & FSCK_CAN_FIX) {
 335                         prt_str(out, ", ");
 336                         prt_actioning(out, action);
 337                         ret = -BCH_ERR_fsck_fix;
 338                 } else {
 339                         prt_str(out, ", continuing");
 340                         ret = -BCH_ERR_fsck_ignore;
 341                 }
 342         } else if (c->opts.fix_errors == FSCK_FIX_exit) {
 343                 prt_str(out, ", exiting");
 344                 ret = -BCH_ERR_fsck_errors_not_fixed;
 345         } else if (flags & FSCK_CAN_FIX) {
 346                 int fix = s && s->fix
 347                         ? s->fix
 348                         : c->opts.fix_errors;
 349
 350                 if (fix == FSCK_FIX_ask) {
 351                         prt_str(out, ", ");
 352                         prt_str(out, action);
 353
 354                         if (bch2_fs_stdio_redirect(c))
 355                                 bch2_print(c, "%s", out->buf);
 356                         else
 357                                 bch2_print_string_as_lines(KERN_ERR, out->buf);
 358                         print = false;
 359
 360                         int ask = bch2_fsck_ask_yn(c, trans);
 361
 362                         if (trans) {
 363                                 ret = bch2_trans_relock(trans);
 364                                 if (ret) {
 365                                         mutex_unlock(&c->fsck_error_msgs_lock);
 366                                         goto err;
 367                                 }
 368                         }
 369
 370                         if (ask >= YN_ALLNO && s)
 371                                 s->fix = ask == YN_ALLNO
 372                                         ? FSCK_FIX_no
 373                                         : FSCK_FIX_yes;
 374
 375                         ret = ask & 1
 376                                 ? -BCH_ERR_fsck_fix
 377                                 : -BCH_ERR_fsck_ignore;
 378                 } else if (fix == FSCK_FIX_yes ||
 379                            (c->opts.nochanges &&
 380                             !(flags & FSCK_CAN_IGNORE))) {
 381                         prt_str(out, ", ");
 382                         prt_actioning(out, action);
 383                         ret = -BCH_ERR_fsck_fix;
 384                 } else {
 385                         prt_str(out, ", not ");
 386                         prt_actioning(out, action);
 387                 }
 388         } else if (flags & FSCK_NEED_FSCK) {
 389                 prt_str(out, " (run fsck to correct)");
 390         } else {
 391                 prt_str(out, " (repair unimplemented)");
 392         }
 393
 394         if (ret == -BCH_ERR_fsck_ignore &&
 395             (c->opts.fix_errors == FSCK_FIX_exit ||
 396              !(flags & FSCK_CAN_IGNORE)))
 397                 ret = -BCH_ERR_fsck_errors_not_fixed;
 398
 399         bool exiting =
 400                 test_bit(BCH_FS_fsck_running, &c->flags) &&
 401                 (ret != -BCH_ERR_fsck_fix &&
 402                  ret != -BCH_ERR_fsck_ignore);
 403
 404         if (exiting)
 405                 print = true;
 406
 407         if (print) {
 408                 if (bch2_fs_stdio_redirect(c))
 409                         bch2_print(c, "%s\n", out->buf);
 410                 else
 411                         bch2_print_string_as_lines(KERN_ERR, out->buf);
 412         }
 413
 414         if (exiting)
 415                 bch_err(c, "Unable to continue, halting");
 416         else if (suppressing)
 417                 bch_err(c, "Ratelimiting new instances of previous error");
 418
 419         if (s)
 420                 s->ret = ret;
 421
 422         mutex_unlock(&c->fsck_error_msgs_lock);
 423
 424         if (inconsistent)
 425                 bch2_inconsistent_error(c);
 426
 427         if (ret == -BCH_ERR_fsck_fix) {
 428                 set_bit(BCH_FS_errors_fixed, &c->flags);
 429         } else {
 430                 set_bit(BCH_FS_errors_not_fixed, &c->flags);
 431                 set_bit(BCH_FS_error, &c->flags);
 432         }
 433 err:
 434         if (action != action_orig)
 435                 kfree(action);
 436         printbuf_exit(&buf);
 437         return ret;
 438 }
 439
 440 int __bch2_bkey_fsck_err(struct bch_fs *c,
 441                          struct bkey_s_c k,
 442                          enum bch_validate_flags validate_flags,
 443                          enum bch_sb_error_id err,
 444                          const char *fmt, ...)
 445 {
 446         if (validate_flags & BCH_VALIDATE_silent)
 447                 return -BCH_ERR_fsck_delete_bkey;
 448
 449         unsigned fsck_flags = 0;
 450         if (!(validate_flags & (BCH_VALIDATE_write|BCH_VALIDATE_commit)))
 451                 fsck_flags |= FSCK_AUTOFIX|FSCK_CAN_FIX;
 452
 453         struct printbuf buf = PRINTBUF;
 454         va_list args;
 455
 456         prt_str(&buf, "invalid bkey ");
 457         bch2_bkey_val_to_text(&buf, c, k);
 458         prt_str(&buf, "\n  ");
 459         va_start(args, fmt);
 460         prt_vprintf(&buf, fmt, args);
 461         va_end(args);
 462         prt_str(&buf, ": delete?");
 463
 464         int ret = __bch2_fsck_err(c, NULL, fsck_flags, err, "%s", buf.buf);
 465         printbuf_exit(&buf);
 466         return ret;
 467 }
 468
 469 void bch2_flush_fsck_errs(struct bch_fs *c)
 470 {
 471         struct fsck_err_state *s, *n;
 472
 473         mutex_lock(&c->fsck_error_msgs_lock);
 474
 475         list_for_each_entry_safe(s, n, &c->fsck_error_msgs, list) {
 476                 if (s->ratelimited && s->last_msg)
 477                         bch_err(c, "Saw %llu errors like:\n    %s", s->nr, s->last_msg);
 478
 479                 list_del(&s->list);
 480                 kfree(s->last_msg);
 481                 kfree(s);
 482         }
 483
 484         mutex_unlock(&c->fsck_error_msgs_lock);
 485 }