btrfs: handle ENOMEM from btrfs_insert_dir_item() without aborting

Now that btrfs_insert_dir_item() returns -ENOMEM before modifying the
btree (thanks to delayed dir index pre-allocation), callers can handle
ENOMEM gracefully instead of aborting the transaction.

- btrfs_add_link(): add -ENOMEM to the recoverable errors alongside
  -EEXIST and -EOVERFLOW.
- btrfs_create_new_inode(): on -ENOMEM from btrfs_add_link(), orphan the
  newly-created inode instead of aborting. The inode item was already
  written with nlink 1, and discard_new_inode() marks it bad so eviction
  won't delete it. So clear_nlink() alone is not enough: persist nlink 0
  via btrfs_update_inode(), otherwise orphan cleanup would see nlink > 0,
  drop the orphan item, and leak the inode. Fall back to aborting only if
  that update also fails.

This turns a filesystem-killing abort into a graceful -ENOMEM return for
create(), mkdir(), mknod(), symlink(), and link() under memory pressure.

Assisted-by: LLM
Signed-off-by: Jeff Layton <jlayton@kernel.org>
Reviewed-by: David Sterba <dsterba@suse.com>
Signed-off-by: David Sterba <dsterba@suse.com>

diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c
index babc291751e96fd9eb34c28ece2d2073d40aaeec..67edf6618bda3476b3f743d755330be2bdac03bb 100644 (file)
--- a/fs/btrfs/inode.c
+++ b/fs/btrfs/inode.c
@@ -6828,7 +6828,27 @@ int btrfs_create_new_inode(struct btrfs_trans_handle *trans,
        } else {
                ret = btrfs_add_link(trans, BTRFS_I(dir), BTRFS_I(inode), name,
                                     false, BTRFS_I(inode)->dir_index);
-               if (unlikely(ret)) {
+               if (ret == -ENOMEM) {
+                       /*
+                        * Orphan the new inode instead of aborting. The inode
+                        * item was already written with nlink 1, and discard's
+                        * eviction won't delete a bad inode, so nlink 0 must be
+                        * persisted here or orphan cleanup would see nlink > 0,
+                        * drop the orphan item, and leak the inode.
+                        */
+                       clear_nlink(inode);
+                       /* btrfs_orphan_add() aborts the transaction on failure. */
+                       ret = btrfs_orphan_add(trans, BTRFS_I(inode));
+                       if (ret)
+                               goto discard;
+                       ret = btrfs_update_inode(trans, BTRFS_I(inode));
+                       if (ret) {
+                               btrfs_abort_transaction(trans, ret);
+                               goto discard;
+                       }
+                       ret = -ENOMEM;
+                       goto discard;
+               } else if (unlikely(ret)) {
                        btrfs_abort_transaction(trans, ret);
                        goto discard;
                }
@@ -6890,7 +6910,7 @@ int btrfs_add_link(struct btrfs_trans_handle *trans,
 
        ret = btrfs_insert_dir_item(trans, name, parent_inode, &key,
                                    btrfs_inode_type(inode), index, NULL);
-       if (ret == -EEXIST || ret == -EOVERFLOW)
+       if (ret == -EEXIST || ret == -EOVERFLOW || ret == -ENOMEM)
                goto fail_dir_item;
        else if (unlikely(ret)) {
                btrfs_abort_transaction(trans, ret);