Quelle  recovery_passes.c   Sprache: C

// SPDX-License-Identifier: GPL-2.0

#include "bcachefs.h"
#include "alloc_background.h"
#include "backpointers.h"
#include "btree_gc.h"
#include "btree_node_scan.h"
#include "disk_accounting.h"
#include "ec.h"
#include "fsck.h"
#include "inode.h"
#include "journal.h"
#include "lru.h"
#include "logged_ops.h"
#include "movinggc.h"
#include "rebalance.h"
#include "recovery.h"
#include "recovery_passes.h"
#include "snapshot.h"
#include "subvolume.h"
#include "super.h"
#include "super-io.h"

const char * const bch2_recovery_passes[] = {
#define x(_fn, ...) #_fn,
 BCH_RECOVERY_PASSES()
#undef x
 NULL
};

static const u8 passes_to_stable_map[] = {
#define x(n, id, ...) [BCH_RECOVERY_PASS_##n] = BCH_RECOVERY_PASS_STABLE_##n,
 BCH_RECOVERY_PASSES()
#undef x
};

static const u8 passes_from_stable_map[] = {
#define x(n, id, ...) [BCH_RECOVERY_PASS_STABLE_##n] = BCH_RECOVERY_PASS_##n,
 BCH_RECOVERY_PASSES()
#undef x
};

static enum bch_recovery_pass_stable bch2_recovery_pass_to_stable(enum bch_recovery_pass pass)
{
 return passes_to_stable_map[pass];
}

u64 bch2_recovery_passes_to_stable(u64 v)
{
 u64 ret = 0;
 for (unsigned i = 0; i < ARRAY_SIZE(passes_to_stable_map); i++)
  if (v & BIT_ULL(i))
   ret |= BIT_ULL(passes_to_stable_map[i]);
 return ret;
}

static enum bch_recovery_pass bch2_recovery_pass_from_stable(enum bch_recovery_pass_stable pass)
{
 return pass < ARRAY_SIZE(passes_from_stable_map)
  ? passes_from_stable_map[pass]
  : 0;
}

u64 bch2_recovery_passes_from_stable(u64 v)
{
 u64 ret = 0;
 for (unsigned i = 0; i < ARRAY_SIZE(passes_from_stable_map); i++)
  if (v & BIT_ULL(i))
   ret |= BIT_ULL(passes_from_stable_map[i]);
 return ret;
}

static int bch2_sb_recovery_passes_validate(struct bch_sb *sb, struct bch_sb_field *f,
         enum bch_validate_flags flags, struct printbuf *err)
{
 return 0;
}

static void bch2_sb_recovery_passes_to_text(struct printbuf *out,
         struct bch_sb *sb,
         struct bch_sb_field *f)
{
 struct bch_sb_field_recovery_passes *r =
  field_to_type(f, recovery_passes);
 unsigned nr = recovery_passes_nr_entries(r);

 if (out->nr_tabstops < 1)
  printbuf_tabstop_push(out, 32);
 if (out->nr_tabstops < 2)
  printbuf_tabstop_push(out, 16);

 prt_printf(out, "Pass\tLast run\tLast runtime\n");

 for (struct recovery_pass_entry *i = r->start; i < r->start + nr; i++) {
  if (!i->last_run)
   continue;

  unsigned idx = i - r->start;

  prt_printf(out, "%s\t", bch2_recovery_passes[bch2_recovery_pass_from_stable(idx)]);

  bch2_prt_datetime(out, le64_to_cpu(i->last_run));
  prt_tab(out);

  bch2_pr_time_units(out, le32_to_cpu(i->last_runtime) * NSEC_PER_SEC);

  if (BCH_RECOVERY_PASS_NO_RATELIMIT(i))
   prt_str(out, " (no ratelimit)");

  prt_newline(out);
 }
}

static struct recovery_pass_entry *bch2_sb_recovery_pass_entry(struct bch_fs *c,
              enum bch_recovery_pass pass)
{
 enum bch_recovery_pass_stable stable = bch2_recovery_pass_to_stable(pass);

 lockdep_assert_held(&c->sb_lock);

 struct bch_sb_field_recovery_passes *r =
  bch2_sb_field_get(c->disk_sb.sb, recovery_passes);

 if (stable >= recovery_passes_nr_entries(r)) {
  unsigned u64s = struct_size(r, start, stable + 1) / sizeof(u64);

  r = bch2_sb_field_resize(&c->disk_sb, recovery_passes, u64s);
  if (!r) {
   bch_err(c, "error creating recovery_passes sb section");
   return NULL;
  }
 }

 return r->start + stable;
}

static void bch2_sb_recovery_pass_complete(struct bch_fs *c,
        enum bch_recovery_pass pass,
        s64 start_time)
{
 guard(mutex)(&c->sb_lock);
 struct bch_sb_field_ext *ext = bch2_sb_field_get(c->disk_sb.sb, ext);
 __clear_bit_le64(bch2_recovery_pass_to_stable(pass),
    ext->recovery_passes_required);

 struct recovery_pass_entry *e = bch2_sb_recovery_pass_entry(c, pass);
 if (e) {
  s64 end_time = ktime_get_real_seconds();
  e->last_run = cpu_to_le64(end_time);
  e->last_runtime = cpu_to_le32(max(0, end_time - start_time));
  SET_BCH_RECOVERY_PASS_NO_RATELIMIT(e, false);
 }

 bch2_write_super(c);
}

void bch2_recovery_pass_set_no_ratelimit(struct bch_fs *c,
      enum bch_recovery_pass pass)
{
 guard(mutex)(&c->sb_lock);

 struct recovery_pass_entry *e = bch2_sb_recovery_pass_entry(c, pass);
 if (e && !BCH_RECOVERY_PASS_NO_RATELIMIT(e)) {
  SET_BCH_RECOVERY_PASS_NO_RATELIMIT(e, false);
  bch2_write_super(c);
 }
}

static bool bch2_recovery_pass_want_ratelimit(struct bch_fs *c, enum bch_recovery_pass pass)
{
 enum bch_recovery_pass_stable stable = bch2_recovery_pass_to_stable(pass);
 bool ret = false;

 lockdep_assert_held(&c->sb_lock);

 struct bch_sb_field_recovery_passes *r =
  bch2_sb_field_get(c->disk_sb.sb, recovery_passes);

 if (stable < recovery_passes_nr_entries(r)) {
  struct recovery_pass_entry *i = r->start + stable;

  /*
 * Ratelimit if the last runtime was more than 1% of the time
 * since we last ran
 */
  ret = (u64) le32_to_cpu(i->last_runtime) * 100 >
   ktime_get_real_seconds() - le64_to_cpu(i->last_run);

  if (BCH_RECOVERY_PASS_NO_RATELIMIT(i))
   ret = false;
 }

 return ret;
}

const struct bch_sb_field_ops bch_sb_field_ops_recovery_passes = {
 .validate = bch2_sb_recovery_passes_validate,
 .to_text = bch2_sb_recovery_passes_to_text
};

/* Fake recovery pass, so that scan_for_btree_nodes isn't 0: */
static int bch2_recovery_pass_empty(struct bch_fs *c)
{
 return 0;
}

static int bch2_set_may_go_rw(struct bch_fs *c)
{
 struct journal_keys *keys = &c->journal_keys;

 /*
 * After we go RW, the journal keys buffer can't be modified (except for
 * setting journal_key->overwritten: it will be accessed by multiple
 * threads
 */
 move_gap(keys, keys->nr);

 set_bit(BCH_FS_may_go_rw, &c->flags);

 if (go_rw_in_recovery(c)) {
  if (c->sb.features & BIT_ULL(BCH_FEATURE_no_alloc_info)) {
   bch_info(c, "mounting a filesystem with no alloc info read-write; will recreate");
   bch2_reconstruct_alloc(c);
  }

  return bch2_fs_read_write_early(c);
 }
 return 0;
}

/*
 * Make sure root inode is readable while we're still in recovery and can rewind
 * for repair:
 */
static int bch2_lookup_root_inode(struct bch_fs *c)
{
 subvol_inum inum = BCACHEFS_ROOT_SUBVOL_INUM;
 struct bch_inode_unpacked inode_u;
 struct bch_subvolume subvol;

 return bch2_trans_do(c,
  bch2_subvolume_get(trans, inum.subvol, true, &subvol) ?:
  bch2_inode_find_by_inum_trans(trans, inum, &inode_u));
}

struct recovery_pass_fn {
 int  (*fn)(struct bch_fs *);
 unsigned when;
};

static struct recovery_pass_fn recovery_pass_fns[] = {
#define x(_fn, _id, _when) { .fn = bch2_##_fn, .when = _when },
 BCH_RECOVERY_PASSES()
#undef x
};

static u64 bch2_recovery_passes_match(unsigned flags)
{
 u64 ret = 0;

 for (unsigned i = 0; i < ARRAY_SIZE(recovery_pass_fns); i++)
  if (recovery_pass_fns[i].when & flags)
   ret |= BIT_ULL(i);
 return ret;
}

u64 bch2_fsck_recovery_passes(void)
{
 return bch2_recovery_passes_match(PASS_FSCK);
}

static void bch2_run_async_recovery_passes(struct bch_fs *c)
{
 if (!down_trylock(&c->recovery.run_lock))
  return;

 if (!enumerated_ref_tryget(&c->writes, BCH_WRITE_REF_async_recovery_passes))
  goto unlock;

 if (queue_work(system_long_wq, &c->recovery.work))
  return;

 enumerated_ref_put(&c->writes, BCH_WRITE_REF_async_recovery_passes);
unlock:
 up(&c->recovery.run_lock);
}

static bool recovery_pass_needs_set(struct bch_fs *c,
        enum bch_recovery_pass pass,
        enum bch_run_recovery_pass_flags *flags)
{
 struct bch_fs_recovery *r = &c->recovery;

 /*
 * Never run scan_for_btree_nodes persistently: check_topology will run
 * it if required
 */
 if (pass == BCH_RECOVERY_PASS_scan_for_btree_nodes)
  *flags |= RUN_RECOVERY_PASS_nopersistent;

 if ((*flags & RUN_RECOVERY_PASS_ratelimit) &&
     !bch2_recovery_pass_want_ratelimit(c, pass))
  *flags &= ~RUN_RECOVERY_PASS_ratelimit;

 /*
 * If RUN_RECOVERY_PASS_nopersistent is set, we don't want to do
 * anything if the pass has already run: these mean we need a prior pass
 * to run before we continue to repair, we don't expect that pass to fix
 * the damage we encountered.
 *
 * Otherwise, we run run_explicit_recovery_pass when we find damage, so
 * it should run again even if it's already run:
 */
 bool in_recovery = test_bit(BCH_FS_in_recovery, &c->flags);
 bool persistent = !in_recovery || !(*flags & RUN_RECOVERY_PASS_nopersistent);
 bool rewind = in_recovery &&
  r->curr_pass > pass &&
  !(r->passes_complete & BIT_ULL(pass));

 if (persistent
     ? !(c->sb.recovery_passes_required & BIT_ULL(pass))
     : !((r->passes_to_run|r->passes_complete) & BIT_ULL(pass)))
  return true;

 if (!(*flags & RUN_RECOVERY_PASS_ratelimit) &&
     (r->passes_ratelimiting & BIT_ULL(pass)))
  return true;

 if (rewind)
  return true;

 return false;
}

/*
 * For when we need to rewind recovery passes and run a pass we skipped:
 */
int __bch2_run_explicit_recovery_pass(struct bch_fs *c,
          struct printbuf *out,
          enum bch_recovery_pass pass,
          enum bch_run_recovery_pass_flags flags)
{
 struct bch_fs_recovery *r = &c->recovery;
 int ret = 0;

 lockdep_assert_held(&c->sb_lock);

 bch2_printbuf_make_room(out, 1024);
 out->atomic++;

 unsigned long lockflags;
 spin_lock_irqsave(&r->lock, lockflags);

 if (!recovery_pass_needs_set(c, pass, &flags))
  goto out;

 bool in_recovery = test_bit(BCH_FS_in_recovery, &c->flags);
 bool rewind = in_recovery &&
  r->curr_pass > pass &&
  !(r->passes_complete & BIT_ULL(pass));
 bool ratelimit = flags & RUN_RECOVERY_PASS_ratelimit;

 if (!(flags & RUN_RECOVERY_PASS_nopersistent)) {
  struct bch_sb_field_ext *ext = bch2_sb_field_get(c->disk_sb.sb, ext);
  __set_bit_le64(bch2_recovery_pass_to_stable(pass), ext->recovery_passes_required);
 }

 if (pass < BCH_RECOVERY_PASS_set_may_go_rw &&
     (!in_recovery || r->curr_pass >= BCH_RECOVERY_PASS_set_may_go_rw)) {
  prt_printf(out, "need recovery pass %s (%u), but already rw\n",
      bch2_recovery_passes[pass], pass);
  ret = bch_err_throw(c, cannot_rewind_recovery);
  goto out;
 }

 if (ratelimit)
  r->passes_ratelimiting |= BIT_ULL(pass);
 else
  r->passes_ratelimiting &= ~BIT_ULL(pass);

 if (in_recovery && !ratelimit) {
  prt_printf(out, "running recovery pass %s (%u), currently at %s (%u)%s\n",
      bch2_recovery_passes[pass], pass,
      bch2_recovery_passes[r->curr_pass], r->curr_pass,
      rewind ? " - rewinding" : "");

  r->passes_to_run |= BIT_ULL(pass);

  if (rewind) {
   r->next_pass = pass;
   r->passes_complete &= (1ULL << pass) >> 1;
   ret = bch_err_throw(c, restart_recovery);
  }
 } else {
  prt_printf(out, "scheduling recovery pass %s (%u)%s\n",
      bch2_recovery_passes[pass], pass,
      ratelimit ? " - ratelimiting" : "");

  struct recovery_pass_fn *p = recovery_pass_fns + pass;
  if (p->when & PASS_ONLINE)
   bch2_run_async_recovery_passes(c);
 }
out:
 spin_unlock_irqrestore(&r->lock, lockflags);
 --out->atomic;
 return ret;
}

int bch2_run_explicit_recovery_pass(struct bch_fs *c,
        struct printbuf *out,
        enum bch_recovery_pass pass,
        enum bch_run_recovery_pass_flags flags)
{
 int ret = 0;

 if (recovery_pass_needs_set(c, pass, &flags)) {
  guard(mutex)(&c->sb_lock);
  ret = __bch2_run_explicit_recovery_pass(c, out, pass, flags);
  bch2_write_super(c);
 }

 return ret;
}

/*
 * Returns 0 if @pass has run recently, otherwise one of
 * -BCH_ERR_restart_recovery
 * -BCH_ERR_recovery_pass_will_run
 */
int bch2_require_recovery_pass(struct bch_fs *c,
          struct printbuf *out,
          enum bch_recovery_pass pass)
{
 if (test_bit(BCH_FS_in_recovery, &c->flags) &&
     c->recovery.passes_complete & BIT_ULL(pass))
  return 0;

 guard(mutex)(&c->sb_lock);

 if (bch2_recovery_pass_want_ratelimit(c, pass))
  return 0;

 enum bch_run_recovery_pass_flags flags = 0;
 int ret = 0;

 if (recovery_pass_needs_set(c, pass, &flags)) {
  ret = __bch2_run_explicit_recovery_pass(c, out, pass, flags);
  bch2_write_super(c);
 }

 return ret ?: bch_err_throw(c, recovery_pass_will_run);
}

int bch2_run_print_explicit_recovery_pass(struct bch_fs *c, enum bch_recovery_pass pass)
{
 enum bch_run_recovery_pass_flags flags = 0;

 if (!recovery_pass_needs_set(c, pass, &flags))
  return 0;

 struct printbuf buf = PRINTBUF;
 bch2_log_msg_start(c, &buf);

 mutex_lock(&c->sb_lock);
 int ret = __bch2_run_explicit_recovery_pass(c, &buf, pass,
      RUN_RECOVERY_PASS_nopersistent);
 mutex_unlock(&c->sb_lock);

 bch2_print_str(c, KERN_NOTICE, buf.buf);
 printbuf_exit(&buf);
 return ret;
}

static int bch2_run_recovery_pass(struct bch_fs *c, enum bch_recovery_pass pass)
{
 struct bch_fs_recovery *r = &c->recovery;
 struct recovery_pass_fn *p = recovery_pass_fns + pass;

 if (!(p->when & PASS_SILENT))
  bch2_print(c, KERN_INFO bch2_log_msg(c, "%s..."),
      bch2_recovery_passes[pass]);

 s64 start_time = ktime_get_real_seconds();
 int ret = p->fn(c);

 r->passes_to_run &= ~BIT_ULL(pass);

 if (ret) {
  r->passes_failing |= BIT_ULL(pass);
  return ret;
 }

 r->passes_failing = 0;

 if (!test_bit(BCH_FS_error, &c->flags))
  bch2_sb_recovery_pass_complete(c, pass, start_time);

 if (!(p->when & PASS_SILENT))
  bch2_print(c, KERN_CONT " done\n");

 return 0;
}

static int __bch2_run_recovery_passes(struct bch_fs *c, u64 orig_passes_to_run,
          bool online)
{
 struct bch_fs_recovery *r = &c->recovery;
 int ret = 0;

 spin_lock_irq(&r->lock);

 if (online)
  orig_passes_to_run &= bch2_recovery_passes_match(PASS_ONLINE);

 if (c->sb.features & BIT_ULL(BCH_FEATURE_no_alloc_info))
  orig_passes_to_run &= ~bch2_recovery_passes_match(PASS_ALLOC);

 /*
 * A failed recovery pass will be retried after another pass succeeds -
 * but not this iteration.
 *
 * This is because some passes depend on repair done by other passes: we
 * may want to retry, but we don't want to loop on failing passes.
 */

 orig_passes_to_run &= ~r->passes_failing;

 r->passes_to_run = orig_passes_to_run;

 while (r->passes_to_run) {
  unsigned prev_done = r->pass_done;
  unsigned pass = __ffs64(r->passes_to_run);
  r->curr_pass = pass;
  r->next_pass = r->curr_pass + 1;
  r->passes_to_run &= ~BIT_ULL(pass);

  spin_unlock_irq(&r->lock);

  int ret2 = bch2_run_recovery_pass(c, pass) ?:
   bch2_journal_flush(&c->journal);

  spin_lock_irq(&r->lock);

  if (r->next_pass < r->curr_pass) {
   /* Rewind: */
   r->passes_to_run |= orig_passes_to_run & (~0ULL << r->next_pass);
  } else if (!ret2) {
   r->pass_done = max(r->pass_done, pass);
   r->passes_complete |= BIT_ULL(pass);
  } else {
   ret = ret2;
  }

  if (ret && !online)
   break;

  if (prev_done <= BCH_RECOVERY_PASS_check_snapshots &&
      r->pass_done > BCH_RECOVERY_PASS_check_snapshots) {
   bch2_copygc_wakeup(c);
   bch2_rebalance_wakeup(c);
  }
 }

 clear_bit(BCH_FS_in_recovery, &c->flags);
 spin_unlock_irq(&r->lock);

 return ret;
}

static void bch2_async_recovery_passes_work(struct work_struct *work)
{
 struct bch_fs *c = container_of(work, struct bch_fs, recovery.work);
 struct bch_fs_recovery *r = &c->recovery;

 __bch2_run_recovery_passes(c,
  c->sb.recovery_passes_required & ~r->passes_ratelimiting,
  true);

 up(&r->run_lock);
 enumerated_ref_put(&c->writes, BCH_WRITE_REF_async_recovery_passes);
}

int bch2_run_online_recovery_passes(struct bch_fs *c, u64 passes)
{
 return __bch2_run_recovery_passes(c, c->sb.recovery_passes_required|passes, true);
}

int bch2_run_recovery_passes(struct bch_fs *c, enum bch_recovery_pass from)
{
 u64 passes =
  bch2_recovery_passes_match(PASS_ALWAYS) |
  (!c->sb.clean ? bch2_recovery_passes_match(PASS_UNCLEAN) : 0) |
  (c->opts.fsck ? bch2_recovery_passes_match(PASS_FSCK) : 0) |
  c->opts.recovery_passes |
  c->sb.recovery_passes_required;

 if (c->opts.recovery_pass_last)
  passes &= BIT_ULL(c->opts.recovery_pass_last + 1) - 1;

 /*
 * We can't allow set_may_go_rw to be excluded; that would cause us to
 * use the journal replay keys for updates where it's not expected.
 */
 c->opts.recovery_passes_exclude &= ~BCH_RECOVERY_PASS_set_may_go_rw;
 passes &= ~c->opts.recovery_passes_exclude;

 passes &= ~(BIT_ULL(from) - 1);

 down(&c->recovery.run_lock);
 int ret = __bch2_run_recovery_passes(c, passes, false);
 up(&c->recovery.run_lock);

 return ret;
}

static void prt_passes(struct printbuf *out, const char *msg, u64 passes)
{
 prt_printf(out, "%s:\t", msg);
 prt_bitflags(out, bch2_recovery_passes, passes);
 prt_newline(out);
}

void bch2_recovery_pass_status_to_text(struct printbuf *out, struct bch_fs *c)
{
 struct bch_fs_recovery *r = &c->recovery;

 printbuf_tabstop_push(out, 32);
 prt_passes(out, "Scheduled passes", c->sb.recovery_passes_required);
 prt_passes(out, "Scheduled online passes", c->sb.recovery_passes_required &
     bch2_recovery_passes_match(PASS_ONLINE));
 prt_passes(out, "Complete passes", r->passes_complete);
 prt_passes(out, "Failing passes", r->passes_failing);

 if (r->curr_pass) {
  prt_printf(out, "Current pass:\t%s\n", bch2_recovery_passes[r->curr_pass]);
  prt_passes(out, "Current passes", r->passes_to_run);
 }
}

void bch2_fs_recovery_passes_init(struct bch_fs *c)
{
 spin_lock_init(&c->recovery.lock);
 sema_init(&c->recovery.run_lock, 1);

 INIT_WORK(&c->recovery.work, bch2_async_recovery_passes_work);
}