Add some printk's to tell us why and/or where ReiserFS is giving us

apparently spurious I/O errors in readdir().

To reproduce, create a stack of the following:

	ReiserFS	(noatime)
	dm-crypt	(aes256-cbc-essiv:sha256)
	lvm2
	md RAID5	(5 disks)
	SATA disks

When the md-RAID5 array is in recovery mode, readdir() will occasionally
return -EIO, especially when the system is under heavy load from other
operations on the same filesystem.  No data ever seems to go permanently
missing, and retrying the readdir() operation usually succeeds.

dm-crypt seems to be critical to reproducing the problem, since other
filesystems on the same machine at the same time do not show these
symptoms, or show them much less often.

RAID5 recovery seems to be important as well, as I have observed no
spurious errors while recovery is not running.  That could mean that
either they occur much less often, or not at all.

diff --git a/fs/reiserfs/dir.c b/fs/reiserfs/dir.c
index e6b03d2020c1c51aedbe6e443ccc7e236c50d242..f5884e20ab77c49d7c3596b5402374e154887332 100644 (file)
--- a/fs/reiserfs/dir.c
+++ b/fs/reiserfs/dir.c
@@ -127,6 +127,7 @@ static int reiserfs_readdir(struct file *filp, void *dirent, filldir_t filldir)
                                        /* There is corrupted data in entry,
                                         * We'd better stop here */
                                        pathrelse(&path_to_entry);
+                                       printk("reiserfs: returning EIO at %s:%d\n", __FILE__, __LINE__);
                                        ret = -EIO;
                                        goto out;
                                }

diff --git a/fs/reiserfs/file.c b/fs/reiserfs/file.c
index 33408417038c2c93c576534c8d946deaa66ef4e9..7d200b1b1b43bdf0112ac9307f51e85010f0f72e 100644 (file)
--- a/fs/reiserfs/file.c
+++ b/fs/reiserfs/file.c
@@ -150,7 +150,12 @@ static int reiserfs_sync_file(struct file *p_s_filp,
                blkdev_issue_flush(p_s_inode->i_sb->s_bdev, NULL);
        if (barrier_done < 0)
                return barrier_done;
-       return (n_err < 0) ? -EIO : 0;
+       if (n_err < 0) {
+               printk("reiserfs: returning EIO at %s:%d\n", __FILE__, __LINE__);
+               return -EIO;
+       } else {
+               return 0;
+       }
 }
 
 /* taken fs/buffer.c:__block_commit_write */

diff --git a/fs/reiserfs/journal.c b/fs/reiserfs/journal.c
index 9643c3bbeb3b611d3817bc17698d76c5d54e89b5..6eb136fb1068b539ba0752b383646972bb7e59d9 100644 (file)
--- a/fs/reiserfs/journal.c
+++ b/fs/reiserfs/journal.c
@@ -874,6 +874,7 @@ static int write_ordered_buffers(spinlock_t * lock,
                 */
                if (!buffer_uptodate(bh) && buffer_dirty(bh)) {
                        clear_buffer_dirty(bh);
+                       printk("reiserfs: returning EIO at %s:%d\n", __FILE__, __LINE__);
                        ret = -EIO;
                }
                if (buffer_dirty(bh)) {
@@ -904,6 +905,7 @@ static int write_ordered_buffers(spinlock_t * lock,
                        spin_lock(lock);
                }
                if (!buffer_uptodate(bh)) {
+                       printk("reiserfs: returning EIO at %s:%d\n", __FILE__, __LINE__);
                        ret = -EIO;
                }
                /* ugly interaction with invalidatepage here.
@@ -1124,6 +1126,7 @@ static int flush_commit_list(struct super_block *s,
 #ifdef CONFIG_REISERFS_CHECK
                        reiserfs_warning(s, "journal-601, buffer write failed");
 #endif
+                       printk("reiserfs: returning EIO at %s:%d\n", __FILE__, __LINE__);
                        retval = -EIO;
                }
                put_bh(tbh);    /* once for journal_find_get_block */
@@ -1156,6 +1159,7 @@ static int flush_commit_list(struct super_block *s,
 #ifdef CONFIG_REISERFS_CHECK
                reiserfs_warning(s, "journal-615: buffer write failed");
 #endif
+               printk("reiserfs: returning EIO at %s:%d\n", __FILE__, __LINE__);
                retval = -EIO;
        }
        bforget(jl->j_commit_bh);
@@ -1280,8 +1284,10 @@ static int _update_journal_header_block(struct super_block *p_s_sb,
        struct reiserfs_journal_header *jh;
        struct reiserfs_journal *journal = SB_JOURNAL(p_s_sb);
 
-       if (reiserfs_is_journal_aborted(journal))
+       if (reiserfs_is_journal_aborted(journal)) {
+               printk("reiserfs: returning EIO at %s:%d\n", __FILE__, __LINE__);
                return -EIO;
+       }
 
        if (trans_id >= journal->j_last_flush_trans_id) {
                if (buffer_locked((journal->j_header_bh))) {
@@ -3145,10 +3151,12 @@ int reiserfs_end_persistent_transaction(struct reiserfs_transaction_handle *th)
 {
        struct super_block *s = th->t_super;
        int ret = 0;
-       if (th->t_trans_id)
+       if (th->t_trans_id) {
                ret = journal_end(th, th->t_super, th->t_blocks_allocated);
-       else
+       } else {
                ret = -EIO;
+               printk("reiserfs: returning EIO at %s:%d\n", __FILE__, __LINE__);
+       }
        if (th->t_refcount == 0) {
                SB_JOURNAL(s)->j_persistent_trans--;
                kfree(th);
@@ -3347,6 +3355,7 @@ int journal_end(struct reiserfs_transaction_handle *th,
 
        if (!th->t_trans_id) {
                WARN_ON(1);
+               printk("reiserfs: returning EIO at %s:%d\n", __FILE__, __LINE__);
                return -EIO;
        }
 

diff --git a/fs/reiserfs/namei.c b/fs/reiserfs/namei.c
index f89ebb943f3f1f6a2f219db1a9ec9fd2275e5f0e..75c1776e7e8a86f5fb52d7989681798b844bf600 100644 (file)
--- a/fs/reiserfs/namei.c
+++ b/fs/reiserfs/namei.c
@@ -367,6 +367,7 @@ static struct dentry *reiserfs_lookup(struct inode *dir, struct dentry *dentry,
        }
        reiserfs_write_unlock(dir->i_sb);
        if (retval == IO_ERROR) {
+               printk("reiserfs: returning EIO at %s:%d\n", __FILE__, __LINE__);
                return ERR_PTR(-EIO);
        }
 
@@ -480,6 +481,7 @@ static int reiserfs_add_entry(struct reiserfs_transaction_handle *th,
                pathrelse(&path);
 
                if (retval == IO_ERROR) {
+                       printk("reiserfs: returning EIO at %s:%d\n", __FILE__, __LINE__);
                        return -EIO;
                }
 
@@ -870,6 +872,7 @@ static int reiserfs_rmdir(struct inode *dir, struct dentry *dentry)
                retval = -ENOENT;
                goto end_rmdir;
        } else if (retval == IO_ERROR) {
+               printk("reiserfs: returning EIO at %s:%d\n", __FILE__, __LINE__);
                retval = -EIO;
                goto end_rmdir;
        }
@@ -882,6 +885,7 @@ static int reiserfs_rmdir(struct inode *dir, struct dentry *dentry)
        if (de.de_objectid != inode->i_ino) {
                // FIXME: compare key of an object and a key found in the
                // entry
+               printk("reiserfs: returning EIO at %s:%d\n", __FILE__, __LINE__);
                retval = -EIO;
                goto end_rmdir;
        }
@@ -959,6 +963,7 @@ static int reiserfs_unlink(struct inode *dir, struct dentry *dentry)
                retval = -ENOENT;
                goto end_unlink;
        } else if (retval == IO_ERROR) {
+               printk("reiserfs: returning EIO at %s:%d\n", __FILE__, __LINE__);
                retval = -EIO;
                goto end_unlink;
        }
@@ -969,6 +974,7 @@ static int reiserfs_unlink(struct inode *dir, struct dentry *dentry)
        if (de.de_objectid != inode->i_ino) {
                // FIXME: compare key of an object and a key found in the
                // entry
+               printk("reiserfs: returning EIO at %s:%d\n", __FILE__, __LINE__);
                retval = -EIO;
                goto end_unlink;
        }
@@ -1257,6 +1263,7 @@ static int reiserfs_rename(struct inode *old_dir, struct dentry *old_dentry,
        pathrelse(&old_entry_path);
        if (retval == IO_ERROR) {
                reiserfs_write_unlock(old_dir->i_sb);
+               printk("reiserfs: returning EIO at %s:%d\n", __FILE__, __LINE__);
                return -EIO;
        }
 
@@ -1288,12 +1295,14 @@ static int reiserfs_rename(struct inode *old_dir, struct dentry *old_dentry,
                pathrelse(&dot_dot_entry_path);
                if (retval != NAME_FOUND) {
                        reiserfs_write_unlock(old_dir->i_sb);
+                       printk("reiserfs: returning EIO at %s:%d\n", __FILE__, __LINE__);
                        return -EIO;
                }
 
                /* inode number of .. must equal old_dir->i_ino */
                if (dot_dot_de.de_objectid != old_dir->i_ino) {
                        reiserfs_write_unlock(old_dir->i_sb);
+                       printk("reiserfs: returning EIO at %s:%d\n", __FILE__, __LINE__);
                        return -EIO;
                }
        }
@@ -1339,6 +1348,7 @@ static int reiserfs_rename(struct inode *old_dir, struct dentry *old_dentry,
                        pathrelse(&old_entry_path);
                        journal_end(&th, old_dir->i_sb, jbegin_count);
                        reiserfs_write_unlock(old_dir->i_sb);
+                       printk("reiserfs: returning EIO at %s:%d\n", __FILE__, __LINE__);
                        return -EIO;
                }
 
@@ -1359,6 +1369,7 @@ static int reiserfs_rename(struct inode *old_dir, struct dentry *old_dentry,
                        pathrelse(&old_entry_path);
                        journal_end(&th, old_dir->i_sb, jbegin_count);
                        reiserfs_write_unlock(old_dir->i_sb);
+                       printk("reiserfs: returning EIO at %s:%d\n", __FILE__, __LINE__);
                        return -EIO;
                }
 
@@ -1377,6 +1388,7 @@ static int reiserfs_rename(struct inode *old_dir, struct dentry *old_dentry,
                                pathrelse(&old_entry_path);
                                journal_end(&th, old_dir->i_sb, jbegin_count);
                                reiserfs_write_unlock(old_dir->i_sb);
+                               printk("reiserfs: returning EIO at %s:%d\n", __FILE__, __LINE__);
                                return -EIO;
                        }
                        copy_item_head(&dot_dot_ih,

diff --git a/fs/reiserfs/resize.c b/fs/reiserfs/resize.c
index f71c3948edef8324ac68a71042fbc35209cb5ea3..21d838d9cefd45747de0f1dff42763961df73975 100644 (file)
--- a/fs/reiserfs/resize.c
+++ b/fs/reiserfs/resize.c
@@ -134,6 +134,7 @@ int reiserfs_resize(struct super_block *s, unsigned long block_count_new)
                        bh = sb_bread(s, i * s->s_blocksize * 8);
                        if (!bh) {
                                vfree(bitmap);
+                               printk("reiserfs: returning EIO at %s:%d\n", __FILE__, __LINE__);
                                return -EIO;
                        }
                        memset(bh->b_data, 0, sb_blocksize(sb));
@@ -166,6 +167,7 @@ int reiserfs_resize(struct super_block *s, unsigned long block_count_new)
                int jerr = journal_end(&th, s, 10);
                if (jerr)
                        return jerr;
+               printk("reiserfs: returning EIO at %s:%d\n", __FILE__, __LINE__);
                return -EIO;
        }
 
@@ -184,6 +186,7 @@ int reiserfs_resize(struct super_block *s, unsigned long block_count_new)
                int jerr = journal_end(&th, s, 10);
                if (jerr)
                        return jerr;
+               printk("reiserfs: returning EIO at %s:%d\n", __FILE__, __LINE__);
                return -EIO;
        }
 

diff --git a/fs/reiserfs/stree.c b/fs/reiserfs/stree.c
index abbc64dcc8d453cd8bd2281e382c85c37bd4fbf6..0af47671885ed4a5f4e4c5f9fcfb44e52908b3a7 100644 (file)
--- a/fs/reiserfs/stree.c
+++ b/fs/reiserfs/stree.c
@@ -1656,6 +1656,7 @@ int reiserfs_cut_from_item(struct reiserfs_transaction_handle *th,
                if (n_ret_value == NO_DISK_SPACE)
                        reiserfs_warning(p_s_sb, "NO_DISK_SPACE");
                unfix_nodes(&s_cut_balance);
+               printk("reiserfs: returning EIO at %s:%d\n", __FILE__, __LINE__);
                return -EIO;
        }
 
@@ -1996,6 +1997,7 @@ int reiserfs_paste_into_item(struct reiserfs_transaction_handle *th, struct tree
                    search_for_position_by_key(th->t_super, p_s_key,
                                               p_s_search_path);
                if (retval == IO_ERROR) {
+                       printk("reiserfs: returning EIO at %s:%d\n", __FILE__, __LINE__);
                        retval = -EIO;
                        goto error_out;
                }
@@ -2083,6 +2085,7 @@ int reiserfs_insert_item(struct reiserfs_transaction_handle *th, struct treepath
                PROC_INFO_INC(th->t_super, insert_item_restarted);
                retval = search_item(th->t_super, key, p_s_path);
                if (retval == IO_ERROR) {
+                       printk("reiserfs: returning EIO at %s:%d\n", __FILE__, __LINE__);
                        retval = -EIO;
                        goto error_out;
                }

diff --git a/fs/reiserfs/super.c b/fs/reiserfs/super.c
index 663a91f5dce8a33d4f12c85658122a62da216bc8..40d25216122b30df7fbbda6364cdcaf4d00cacc4 100644 (file)
--- a/fs/reiserfs/super.c
+++ b/fs/reiserfs/super.c
@@ -2153,8 +2153,10 @@ static ssize_t reiserfs_quota_read(struct super_block *sb, int type, char *data,
                        memset(data, 0, tocopy);
                else {
                        bh = sb_bread(sb, tmp_bh.b_blocknr);
-                       if (!bh)
+                       if (!bh) {
+                               printk("reiserfs: returning EIO at %s:%d\n", __FILE__, __LINE__);
                                return -EIO;
+                       }
                        memcpy(data, bh->b_data + offset, tocopy);
                        brelse(bh);
                }
@@ -2197,6 +2199,7 @@ static ssize_t reiserfs_quota_write(struct super_block *sb, int type,
                else
                        bh = sb_getblk(sb, tmp_bh.b_blocknr);
                if (!bh) {
+                       printk("reiserfs: returning EIO at %s:%d\n", __FILE__, __LINE__);
                        err = -EIO;
                        goto out;
                }

diff --git a/fs/reiserfs/tail_conversion.c b/fs/reiserfs/tail_conversion.c
index f8121a1147e863e80039ba676939b9c5abd7bd7f..181e438bd078675a99bd6d1814e55e8f56018c07 100644 (file)
--- a/fs/reiserfs/tail_conversion.c
+++ b/fs/reiserfs/tail_conversion.c
@@ -52,6 +52,7 @@ int direct2indirect(struct reiserfs_transaction_handle *th, struct inode *inode,
                                 "pasted or inserted byte exists in the tree %K. "
                                 "Use fsck to repair.", &end_key);
                pathrelse(path);
+               printk("reiserfs: returning EIO at %s:%d\n", __FILE__, __LINE__);
                return -EIO;
        }
 

diff --git a/fs/reiserfs/xattr.c b/fs/reiserfs/xattr.c
index ad92461cbfc3cadb5b2dcbb32725f108cc0f08a9..179bad5c901481e58b86f14535119cdb3d30031a 100644 (file)
--- a/fs/reiserfs/xattr.c
+++ b/fs/reiserfs/xattr.c
@@ -245,6 +245,7 @@ static int __xattr_readdir(struct inode *inode, void *dirent, filldir_t filldir)
                        // FIXME: we could just skip part of directory which could
                        // not be read
                        pathrelse(&path_to_entry);
+                       printk("reiserfs: returning EIO at %s:%d\n", __FILE__, __LINE__);
                        return -EIO;
                }
 
@@ -392,6 +393,7 @@ static struct page *reiserfs_get_page(struct inode *dir, unsigned long n)
 
       fail:
        reiserfs_put_page(page);
+       printk("reiserfs: returning EIO at %s:%d\n", __FILE__, __LINE__);
        return ERR_PTR(-EIO);
 }