[PATCH] AIO support for raw/O_DIRECT

Patch from Badari Pulavarty <pbadari@us.ibm.com> and myself

This patch adds the infrastructure for performing asynchronous (AIO) blockdev
direct-IO.

- Adds generic_file_aio_write_nolock() and make other
  generic_file_*_write() to use it.

- Modify generic_file_direct_IO() and ->direct_IO() functions to take
  "kiocb *" instead of "file *".

- Renames generic_direct_IO() to blockdev_direct_IO().

- Move generic_file_direct_IO() to mm/filemap.c (it is not
  blockdev-specific, whereas the rest of fs/direct-io.c is).

- Add AIO read/write support to the raw driver.

diff --git a/drivers/char/raw.c b/drivers/char/raw.c
index 8f764ad320bf0f921e69a35d42a1a5c151332089..8a43fb191dc0aa01810c9efdc3c4ce8a07f51a07 100644 (file)
--- a/drivers/char/raw.c
+++ b/drivers/char/raw.c
@@ -212,9 +212,20 @@ static ssize_t raw_file_write(struct file *file, const char *buf,
        return generic_file_write_nolock(file, &local_iov, 1, ppos);
 }
 
+static ssize_t raw_file_aio_write(struct kiocb *iocb, const char *buf,
+                                       size_t count, loff_t pos)
+{
+       struct iovec local_iov = { .iov_base = (void *)buf, .iov_len = count };
+
+       return generic_file_aio_write_nolock(iocb, &local_iov, 1, &iocb->ki_pos);
+}
+
+
 static struct file_operations raw_fops = {
        .read   =       generic_file_read,
+       .aio_read =     generic_file_aio_read,
        .write  =       raw_file_write,
+       .aio_write =    raw_file_aio_write,
        .open   =       raw_open,
        .release=       raw_release,
        .ioctl  =       raw_ioctl,

diff --git a/fs/block_dev.c b/fs/block_dev.c
index 2e0b69ff70a19cf0b2e795b017b9230b16ebb36a..e1692fed1824ed7d4d8fed4ef999b8501cee447c 100644 (file)
--- a/fs/block_dev.c
+++ b/fs/block_dev.c
@@ -118,12 +118,13 @@ blkdev_get_blocks(struct inode *inode, sector_t iblock,
 }
 
 static int
-blkdev_direct_IO(int rw, struct file *file, const struct iovec *iov,
+blkdev_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov,
                        loff_t offset, unsigned long nr_segs)
 {
+       struct file *file = iocb->ki_filp;
        struct inode *inode = file->f_dentry->d_inode->i_mapping->host;
 
-       return generic_direct_IO(rw, inode, inode->i_bdev, iov, offset,
+       return blockdev_direct_IO(rw, iocb, inode, inode->i_bdev, iov, offset,
                                nr_segs, blkdev_get_blocks);
 }
 

diff --git a/fs/direct-io.c b/fs/direct-io.c
index 643f03c26d5535dfca4692bab43b1bd916952844..9aba0dd89d1a44e26a30c99abcd645329969986b 100644 (file)
--- a/fs/direct-io.c
+++ b/fs/direct-io.c
@@ -845,9 +845,9 @@ direct_io_worker(int rw, struct inode *inode, const struct iovec *iov,
  * This is a library function for use by filesystem drivers.
  */
 int
-generic_direct_IO(int rw, struct inode *inode, struct block_device *bdev, 
-       const struct iovec *iov, loff_t offset, unsigned long nr_segs, 
-       get_blocks_t get_blocks)
+blockdev_direct_IO(int rw, struct kiocb *iocb, struct inode *inode, 
+       struct block_device *bdev, const struct iovec *iov, loff_t offset, 
+       unsigned long nr_segs, get_blocks_t get_blocks)
 {
        int seg;
        size_t size;
@@ -886,25 +886,3 @@ generic_direct_IO(int rw, struct inode *inode, struct block_device *bdev,
 out:
        return retval;
 }
-
-ssize_t
-generic_file_direct_IO(int rw, struct file *file, const struct iovec *iov,
-       loff_t offset, unsigned long nr_segs)
-{
-       struct address_space *mapping = file->f_dentry->d_inode->i_mapping;
-       ssize_t retval;
-
-       if (mapping->nrpages) {
-               retval = filemap_fdatawrite(mapping);
-               if (retval == 0)
-                       retval = filemap_fdatawait(mapping);
-               if (retval)
-                       goto out;
-       }
-
-       retval = mapping->a_ops->direct_IO(rw, file, iov, offset, nr_segs);
-       if (rw == WRITE && mapping->nrpages)
-               invalidate_inode_pages2(mapping);
-out:
-       return retval;
-}

diff --git a/fs/ext2/inode.c b/fs/ext2/inode.c
index 879944b4f1e3d786a2ded4c09060c46624ac340e..c2132b72ea246ab7d058adb431ade0347860a3a8 100644 (file)
--- a/fs/ext2/inode.c
+++ b/fs/ext2/inode.c
@@ -652,12 +652,13 @@ ext2_get_blocks(struct inode *inode, sector_t iblock, unsigned long max_blocks,
 }
 
 static int
-ext2_direct_IO(int rw, struct file *file, const struct iovec *iov,
+ext2_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov,
                        loff_t offset, unsigned long nr_segs)
 {
+       struct file *file = iocb->ki_filp;
        struct inode *inode = file->f_dentry->d_inode->i_mapping->host;
 
-       return generic_direct_IO(rw, inode, inode->i_sb->s_bdev, iov,
+       return blockdev_direct_IO(rw, iocb, inode, inode->i_sb->s_bdev, iov,
                                offset, nr_segs, ext2_get_blocks);
 }
 

diff --git a/fs/ext3/inode.c b/fs/ext3/inode.c
index 9cd551930e3c5a6960548fe16925e16dc5a13507..a5e85ef29d2c796780510251abc7c960fe7c48f1 100644 (file)
--- a/fs/ext3/inode.c
+++ b/fs/ext3/inode.c
@@ -1405,10 +1405,11 @@ static int ext3_releasepage(struct page *page, int wait)
  * If the O_DIRECT write is intantiating holes inside i_size and the machine
  * crashes then stale disk data _may_ be exposed inside the file.
  */
-static int ext3_direct_IO(int rw, struct file *file,
+static int ext3_direct_IO(int rw, struct kiocb *iocb,
                        const struct iovec *iov, loff_t offset,
                        unsigned long nr_segs)
 {
+       struct file *file = iocb->ki_filp;
        struct inode *inode = file->f_dentry->d_inode->i_mapping->host;
        struct ext3_inode_info *ei = EXT3_I(inode);
        handle_t *handle = NULL;
@@ -1437,8 +1438,8 @@ static int ext3_direct_IO(int rw, struct file *file,
                }
        }
 
-       ret = generic_direct_IO(rw, inode, inode->i_sb->s_bdev, iov, offset,
-                               nr_segs, ext3_direct_io_get_blocks);
+       ret = blockdev_direct_IO(rw, iocb, inode, inode->i_sb->s_bdev, iov, 
+                               offset, nr_segs, ext3_direct_io_get_blocks);
 
 out_stop:
        if (handle) {

diff --git a/fs/jfs/inode.c b/fs/jfs/inode.c
index 454b27a20f58ff3e40fe2864950806e1632975ec..ee234724e7a5c0fe5e902069572cc706d2f7da27 100644 (file)
--- a/fs/jfs/inode.c
+++ b/fs/jfs/inode.c
@@ -310,12 +310,13 @@ static sector_t jfs_bmap(struct address_space *mapping, sector_t block)
        return generic_block_bmap(mapping, block, jfs_get_block);
 }
 
-static int jfs_direct_IO(int rw, struct file *file, const struct iovec *iov,
+static int jfs_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov,
                        loff_t offset, unsigned long nr_segs)
 {
+       struct file *file = iocb->ki_filp;
        struct inode *inode = file->f_dentry->d_inode->i_mapping->host;
 
-       return generic_direct_IO(rw, inode, inode->i_sb->s_bdev, iov,
+       return blockdev_direct_IO(rw, iocb, inode, inode->i_sb->s_bdev, iov,
                                offset, nr_segs, jfs_get_blocks);
 }
 

diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c
index 7512320e40858d4b60f78f3a877b56ba084a4e70..70c1299621a0b976bfca1a1f2b3db2b17b081990 100644 (file)
--- a/fs/nfs/direct.c
+++ b/fs/nfs/direct.c
@@ -235,7 +235,7 @@ do_nfs_direct_IO(int rw, const struct inode *inode,
  * this function to do a write.
  */
 int
-nfs_direct_IO(int rw, struct file *file, const struct iovec *iov,
+nfs_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov,
                loff_t offset, unsigned long nr_segs)
 {
        /* None of this works yet, so prevent it from compiling. */

diff --git a/fs/xfs/linux/xfs_aops.c b/fs/xfs/linux/xfs_aops.c
index 5638a2ca26f43ec3c413e0ac1f59609ce242d844..9398993ec4d4ff9a0d0f57c3fb4214dc0cd7e8f7 100644 (file)
--- a/fs/xfs/linux/xfs_aops.c
+++ b/fs/xfs/linux/xfs_aops.c
@@ -625,14 +625,15 @@ linvfs_get_blocks_direct(
 STATIC int
 linvfs_direct_IO(
        int                     rw,
-       struct file             *file,
+       struct kiocb            *iocb,
        const struct iovec      *iov,
        loff_t                  offset,
        unsigned long           nr_segs)
 {
+       struct file *file = iocb->ki_filp;
        struct inode *inode = file->f_dentry->d_inode->i_mapping->host;
 
-        return generic_direct_IO(rw, inode, NULL,
+        return blockdev_direct_IO(rw, iocb, inode, NULL,
                        iov, offset, nr_segs, linvfs_get_blocks_direct);
 }
 

diff --git a/include/linux/fs.h b/include/linux/fs.h
index 878f2d2e4a9f9a396118f92d0d262d0b783706be..821034d8745bf595546750c1de1fdea29009f99b 100644 (file)
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -276,6 +276,7 @@ struct iattr {
 struct page;
 struct address_space;
 struct writeback_control;
+struct kiocb;
 
 struct address_space_operations {
        int (*writepage)(struct page *page, struct writeback_control *wbc);
@@ -301,7 +302,7 @@ struct address_space_operations {
        sector_t (*bmap)(struct address_space *, sector_t);
        int (*invalidatepage) (struct page *, unsigned long);
        int (*releasepage) (struct page *, int);
-       int (*direct_IO)(int, struct file *, const struct iovec *iov,
+       int (*direct_IO)(int, struct kiocb *, const struct iovec *iov,
                        loff_t offset, unsigned long nr_segs);
 };
 
@@ -736,7 +737,6 @@ typedef int (*read_actor_t)(read_descriptor_t *, struct page *, unsigned long, u
  * read, write, poll, fsync, readv, writev can be called
  *   without the big kernel lock held in all filesystems.
  */
-struct kiocb;
 struct file_operations {
        struct module *owner;
        loff_t (*llseek) (struct file *, loff_t, int);
@@ -1234,6 +1234,8 @@ extern ssize_t generic_file_read(struct file *, char *, size_t, loff_t *);
 extern ssize_t generic_file_write(struct file *, const char *, size_t, loff_t *);
 extern ssize_t generic_file_aio_read(struct kiocb *, char *, size_t, loff_t);
 extern ssize_t generic_file_aio_write(struct kiocb *, const char *, size_t, loff_t);
+extern ssize_t generic_file_aio_write_nolock(struct kiocb *, const struct iovec *,
+                               unsigned long, loff_t *);
 extern ssize_t do_sync_read(struct file *filp, char *buf, size_t len, loff_t *ppos);
 extern ssize_t do_sync_write(struct file *filp, const char *buf, size_t len, loff_t *ppos);
 ssize_t generic_file_write_nolock(struct file *file, const struct iovec *iov,
@@ -1243,10 +1245,11 @@ extern void do_generic_mapping_read(struct address_space *, struct file_ra_state
                                    loff_t *, read_descriptor_t *, read_actor_t);
 extern void
 file_ra_state_init(struct file_ra_state *ra, struct address_space *mapping);
-extern ssize_t generic_file_direct_IO(int rw, struct file *file,
+extern ssize_t generic_file_direct_IO(int rw, struct kiocb *iocb,
        const struct iovec *iov, loff_t offset, unsigned long nr_segs);
-extern int generic_direct_IO(int rw, struct inode *inode, struct block_device *bdev,
-       const struct iovec *iov, loff_t offset, unsigned long nr_segs, get_blocks_t *get_blocks);
+extern int blockdev_direct_IO(int rw, struct kiocb *iocb, struct inode *inode, 
+       struct block_device *bdev, const struct iovec *iov, loff_t offset, 
+       unsigned long nr_segs, get_blocks_t *get_blocks);
 extern ssize_t generic_file_readv(struct file *filp, const struct iovec *iov, 
        unsigned long nr_segs, loff_t *ppos);
 ssize_t generic_file_writev(struct file *filp, const struct iovec *iov, 

diff --git a/kernel/ksyms.c b/kernel/ksyms.c
index 414ab402325e8af5d228b49984e351874a77a912..d5ad60d89be0018ed0eeede921d84d96a6ded26f 100644 (file)
--- a/kernel/ksyms.c
+++ b/kernel/ksyms.c
@@ -215,7 +215,7 @@ EXPORT_SYMBOL(ll_rw_block);
 EXPORT_SYMBOL(submit_bh);
 EXPORT_SYMBOL(unlock_buffer);
 EXPORT_SYMBOL(__wait_on_buffer);
-EXPORT_SYMBOL(generic_direct_IO);
+EXPORT_SYMBOL(blockdev_direct_IO);
 EXPORT_SYMBOL(block_write_full_page);
 EXPORT_SYMBOL(block_read_full_page);
 EXPORT_SYMBOL(block_prepare_write);

diff --git a/mm/filemap.c b/mm/filemap.c
index 18fddd3026f1b1c285b5888552da89c82c9fae52..44c18491b6691846cabc2dd53c0698536f12518a 100644 (file)
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -805,7 +805,7 @@ __generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov,
                                nr_segs = iov_shorten((struct iovec *)iov,
                                                        nr_segs, count);
                        }
-                       retval = generic_file_direct_IO(READ, filp,
+                       retval = generic_file_direct_IO(READ, iocb,
                                        iov, pos, nr_segs);
                        if (retval > 0)
                                *ppos = pos + retval;
@@ -1532,9 +1532,10 @@ filemap_set_next_iovec(const struct iovec **iovp, size_t *basep, size_t bytes)
  *                                                     okir@monad.swb.de
  */
 ssize_t
-generic_file_write_nolock(struct file *file, const struct iovec *iov,
+generic_file_aio_write_nolock(struct kiocb *iocb, const struct iovec *iov,
                                unsigned long nr_segs, loff_t *ppos)
 {
+       struct file *file = iocb->ki_filp;
        struct address_space * mapping = file->f_dentry->d_inode->i_mapping;
        struct address_space_operations *a_ops = mapping->a_ops;
        size_t ocount;          /* original count */
@@ -1674,7 +1675,7 @@ generic_file_write_nolock(struct file *file, const struct iovec *iov,
                if (count != ocount)
                        nr_segs = iov_shorten((struct iovec *)iov,
                                                nr_segs, count);
-               written = generic_file_direct_IO(WRITE, file,
+               written = generic_file_direct_IO(WRITE, iocb,
                                        iov, pos, nr_segs);
                if (written > 0) {
                        loff_t end = pos + written;
@@ -1786,12 +1787,39 @@ out:
        return err;
 }
 
+ssize_t
+generic_file_write_nolock(struct file *file, const struct iovec *iov,
+                               unsigned long nr_segs, loff_t *ppos)
+{
+       struct kiocb kiocb;
+       ssize_t ret;
+
+       init_sync_kiocb(&kiocb, file);
+       ret = generic_file_aio_write_nolock(&kiocb, iov, nr_segs, ppos);
+       if (-EIOCBQUEUED == ret)
+               ret = wait_on_sync_kiocb(&kiocb);
+       return ret;
+}
+
 ssize_t generic_file_aio_write(struct kiocb *iocb, const char *buf,
                               size_t count, loff_t pos)
 {
-       return generic_file_write(iocb->ki_filp, buf, count, &iocb->ki_pos);
+       struct file *file = iocb->ki_filp;
+       struct inode *inode = file->f_dentry->d_inode->i_mapping->host;
+       int err;
+       struct iovec local_iov = { .iov_base = (void *)buf, .iov_len = count };
+
+       BUG_ON(iocb->ki_pos != pos);
+
+       down(&inode->i_sem);
+       err = generic_file_aio_write_nolock(iocb, &local_iov, 1, 
+                                               &iocb->ki_pos);
+       up(&inode->i_sem);
+
+       return err;
 }
 EXPORT_SYMBOL(generic_file_aio_write);
+EXPORT_SYMBOL(generic_file_aio_write_nolock);
 
 ssize_t generic_file_write(struct file *file, const char *buf,
                           size_t count, loff_t *ppos)
@@ -1831,3 +1859,26 @@ ssize_t generic_file_writev(struct file *file, const struct iovec *iov,
        up(&inode->i_sem);
        return ret;
 }
+
+ssize_t
+generic_file_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov,
+       loff_t offset, unsigned long nr_segs)
+{
+       struct file *file = iocb->ki_filp;
+       struct address_space *mapping = file->f_dentry->d_inode->i_mapping;
+       ssize_t retval;
+
+       if (mapping->nrpages) {
+               retval = filemap_fdatawrite(mapping);
+               if (retval == 0)
+                       retval = filemap_fdatawait(mapping);
+               if (retval)
+                       goto out;
+       }
+
+       retval = mapping->a_ops->direct_IO(rw, iocb, iov, offset, nr_segs);
+       if (rw == WRITE && mapping->nrpages)
+               invalidate_inode_pages2(mapping);
+out:
+       return retval;
+}