[PATCH] Make diskstats per-cpu using kmalloc_percpu

Patch from Ravikiran G Thirumalai <kiran@in.ibm.com>

Makes the disk stats on struct gendisk per-cpu.

diff --git a/drivers/block/genhd.c b/drivers/block/genhd.c
index 7d7af54b0ba5dc6b1d55d8820b6d92c0c5845701..eee503409ebcdb9690e4c33a0c14d21012e1d943 100644 (file)
--- a/drivers/block/genhd.c
+++ b/drivers/block/genhd.c
@@ -450,7 +450,7 @@ static inline unsigned jiffies_to_msec(unsigned jif)
        return (jif / HZ) * 1000 + (jif % HZ) * 1000 / HZ;
 #endif
 }
-static ssize_t disk_stat_read(struct gendisk * disk, char *page)
+static ssize_t disk_stats_read(struct gendisk * disk, char *page)
 {
        disk_round_stats(disk);
        return sprintf(page,
@@ -458,14 +458,16 @@ static ssize_t disk_stat_read(struct gendisk * disk, char *page)
                "%8u %8u %8llu %8u "
                "%8u %8u %8u"
                "\n",
-               disk->reads, disk->read_merges,
-               (unsigned long long)disk->read_sectors,
-               jiffies_to_msec(disk->read_ticks),
-               disk->writes, disk->write_merges,
-               (unsigned long long)disk->write_sectors,
-               jiffies_to_msec(disk->write_ticks),
-               disk->in_flight, jiffies_to_msec(disk->io_ticks),
-               jiffies_to_msec(disk->time_in_queue));
+               disk_stat_read(disk, reads), disk_stat_read(disk, read_merges),
+               (unsigned long long)disk_stat_read(disk, read_sectors),
+               jiffies_to_msec(disk_stat_read(disk, read_ticks)),
+               disk_stat_read(disk, writes), 
+               disk_stat_read(disk, write_merges),
+               (unsigned long long)disk_stat_read(disk, write_sectors),
+               jiffies_to_msec(disk_stat_read(disk, write_ticks)),
+               disk_stat_read(disk, in_flight), 
+               jiffies_to_msec(disk_stat_read(disk, io_ticks)),
+               jiffies_to_msec(disk_stat_read(disk, time_in_queue)));
 }
 static struct disk_attribute disk_attr_dev = {
        .attr = {.name = "dev", .mode = S_IRUGO },
@@ -481,7 +483,7 @@ static struct disk_attribute disk_attr_size = {
 };
 static struct disk_attribute disk_attr_stat = {
        .attr = {.name = "stat", .mode = S_IRUGO },
-       .show   = disk_stat_read
+       .show   = disk_stats_read
 };
 
 static struct attribute * default_attrs[] = {
@@ -497,6 +499,7 @@ static void disk_release(struct kobject * kobj)
        struct gendisk *disk = to_disk(kobj);
        kfree(disk->random);
        kfree(disk->part);
+       free_disk_stats(disk);
        kfree(disk);
 }
 
@@ -516,6 +519,10 @@ struct gendisk *alloc_disk(int minors)
        struct gendisk *disk = kmalloc(sizeof(struct gendisk), GFP_KERNEL);
        if (disk) {
                memset(disk, 0, sizeof(struct gendisk));
+               if (!init_disk_stats(disk)) {
+                       kfree(disk);
+                       return NULL;
+               }
                if (minors > 1) {
                        int size = (minors - 1) * sizeof(struct hd_struct);
                        disk->part = kmalloc(size, GFP_KERNEL);

diff --git a/drivers/block/ll_rw_blk.c b/drivers/block/ll_rw_blk.c
index b7f765e80f16c80cd5985dc8f7c0a936836c5451..691ccc1f0d60a27c341b40b18421521955c343df 100644 (file)
--- a/drivers/block/ll_rw_blk.c
+++ b/drivers/block/ll_rw_blk.c
@@ -1475,17 +1475,17 @@ void drive_stat_acct(struct request *rq, int nr_sectors, int new_io)
                return;
 
        if (rw == READ) {
-               rq->rq_disk->read_sectors += nr_sectors;
+               disk_stat_add(rq->rq_disk, read_sectors, nr_sectors);
                if (!new_io)
-                       rq->rq_disk->read_merges++;
+                       disk_stat_inc(rq->rq_disk, read_merges);
        } else if (rw == WRITE) {
-               rq->rq_disk->write_sectors += nr_sectors;
+               disk_stat_add(rq->rq_disk, write_sectors, nr_sectors);
                if (!new_io)
-                       rq->rq_disk->write_merges++;
+                       disk_stat_inc(rq->rq_disk, write_merges);
        }
        if (new_io) {
                disk_round_stats(rq->rq_disk);
-               rq->rq_disk->in_flight++;
+               disk_stat_inc(rq->rq_disk, in_flight);
        }
 }
 
@@ -1525,11 +1525,12 @@ void disk_round_stats(struct gendisk *disk)
 {
        unsigned long now = jiffies;
 
-       disk->time_in_queue += disk->in_flight * (now - disk->stamp);
+       disk_stat_add(disk, time_in_queue, 
+                       disk_stat_read(disk, in_flight) * (now - disk->stamp));
        disk->stamp = now;
 
-       if (disk->in_flight)
-               disk->io_ticks += (now - disk->stamp_idle);
+       if (disk_stat_read(disk, in_flight))
+               disk_stat_add(disk, io_ticks, (now - disk->stamp_idle));
        disk->stamp_idle = now;
 }
 
@@ -1647,7 +1648,7 @@ static int attempt_merge(request_queue_t *q, struct request *req,
 
                if (req->rq_disk) {
                        disk_round_stats(req->rq_disk);
-                       req->rq_disk->in_flight--;
+                       disk_stat_dec(req->rq_disk, in_flight);
                }
 
                __blk_put_request(q, next);
@@ -2199,16 +2200,16 @@ void end_that_request_last(struct request *req)
                unsigned long duration = jiffies - req->start_time;
                switch (rq_data_dir(req)) {
                    case WRITE:
-                       disk->writes++;
-                       disk->write_ticks += duration;
+                       disk_stat_inc(disk, writes);
+                       disk_stat_add(disk, write_ticks, duration);
                        break;
                    case READ:
-                       disk->reads++;
-                       disk->read_ticks += duration;
+                       disk_stat_inc(disk, reads);
+                       disk_stat_add(disk, read_ticks, duration);
                        break;
                }
                disk_round_stats(disk);
-               disk->in_flight--;
+               disk_stat_dec(disk, in_flight);
        }
        __blk_put_request(req->q, req);
 }

diff --git a/drivers/md/md.c b/drivers/md/md.c
index a3d5a2a29bd614ff32fd2958ee68a480436d6715..cd65d8960c399c691fbf4ce21358280ad37475f7 100644 (file)
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -2803,7 +2803,9 @@ static int is_mddev_idle(mddev_t *mddev)
        idle = 1;
        ITERATE_RDEV(mddev,rdev,tmp) {
                struct gendisk *disk = rdev->bdev->bd_contains->bd_disk;
-               curr_events = disk->read_sectors + disk->write_sectors - disk->sync_io;
+               curr_events = disk_stat_read(disk, read_sectors) + 
+                               disk_stat_read(disk, write_sectors) - 
+                               disk->sync_io;
                if ((curr_events - rdev->last_events) > 32) {
                        rdev->last_events = curr_events;
                        idle = 0;

diff --git a/fs/partitions/check.c b/fs/partitions/check.c
index 32ffc95f286a1a04a7e9ca89da54cb935b6fb879..2051264d725ed0575c9fc330330841bcc8711360 100644 (file)
--- a/fs/partitions/check.c
+++ b/fs/partitions/check.c
@@ -503,13 +503,7 @@ void del_gendisk(struct gendisk *disk)
        disk->capacity = 0;
        disk->flags &= ~GENHD_FL_UP;
        unlink_gendisk(disk);
-       disk->reads = disk->writes = 0;
-       disk->read_sectors = disk->write_sectors = 0;
-       disk->read_merges = disk->write_merges = 0;
-       disk->read_ticks = disk->write_ticks = 0;
-       disk->in_flight = 0;
-       disk->io_ticks = 0;
-       disk->time_in_queue = 0;
+       disk_stat_set_all(disk, 0);
        disk->stamp = disk->stamp_idle = 0;
        devfs_remove_partitions(disk);
        if (disk->driverfs_dev) {

diff --git a/include/linux/genhd.h b/include/linux/genhd.h
index 5739137a54ba8ffa81c1339bdbe3de94ac09d059..e015f567e226441574eb509334f87bbac2f4d759 100644 (file)
--- a/include/linux/genhd.h
+++ b/include/linux/genhd.h
@@ -13,6 +13,8 @@
 #include <linux/types.h>
 #include <linux/major.h>
 #include <linux/device.h>
+#include <linux/smp.h>
+#include <linux/string.h>
 
 enum {
 /* These three have identical behaviour; use the second one if DOS FDISK gets
@@ -70,6 +72,16 @@ struct hd_struct {
 #define GENHD_FL_CD    8
 #define GENHD_FL_UP    16
 
+struct disk_stats {
+       unsigned read_sectors, write_sectors;
+       unsigned reads, writes;
+       unsigned read_merges, write_merges;
+       unsigned read_ticks, write_ticks;
+       unsigned io_ticks;
+       int in_flight;
+       unsigned time_in_queue;
+};
+       
 struct gendisk {
        int major;                      /* major number of driver */
        int first_minor;
@@ -94,16 +106,86 @@ struct gendisk {
        int policy;
 
        unsigned sync_io;               /* RAID */
-       unsigned read_sectors, write_sectors;
-       unsigned reads, writes;
-       unsigned read_merges, write_merges;
-       unsigned read_ticks, write_ticks;
-       unsigned io_ticks;
-       int in_flight;
        unsigned long stamp, stamp_idle;
-       unsigned time_in_queue;
+#ifdef CONFIG_SMP
+       struct disk_stats *dkstats;
+#else
+       struct disk_stats dkstats;
+#endif
 };
 
+/* 
+ * Macros to operate on percpu disk statistics:
+ * Since writes to disk_stats are serialised through the queue_lock,
+ * smp_processor_id() should be enough to get to the per_cpu versions
+ * of statistics counters
+ */
+#ifdef CONFIG_SMP
+#define disk_stat_add(gendiskp, field, addnd)  \
+       (per_cpu_ptr(gendiskp->dkstats, smp_processor_id())->field += addnd)
+#define disk_stat_read(gendiskp, field)                                        \
+({                                                                     \
+       typeof(gendiskp->dkstats->field) res = 0;                       \
+       int i;                                                          \
+       for (i=0; i < NR_CPUS; i++) {                                   \
+               if (!cpu_possible(i))                                   \
+                       continue;                                       \
+               res += per_cpu_ptr(gendiskp->dkstats, i)->field;        \
+       }                                                               \
+       res;                                                            \
+})
+
+static inline void disk_stat_set_all(struct gendisk *gendiskp, int value)      {
+       int i;
+       for (i=0; i < NR_CPUS; i++) {
+               if (cpu_possible(i)) {
+                       memset(per_cpu_ptr(gendiskp->dkstats, i), value,        
+                                       sizeof (struct disk_stats));
+               }
+       }
+}              
+                               
+#else
+#define disk_stat_add(gendiskp, field, addnd) (gendiskp->dkstats.field += addnd)
+#define disk_stat_read(gendiskp, field)        (gendiskp->dkstats.field)
+
+static inline void disk_stat_set_all(struct gendisk *gendiskp, int value)      {
+       memset(&gendiskp->dkstats, value, sizeof (struct disk_stats));
+}
+#endif
+
+#define disk_stat_inc(gendiskp, field) disk_stat_add(gendiskp, field, 1)
+#define disk_stat_dec(gendiskp, field) disk_stat_add(gendiskp, field, -1)
+#define disk_stat_sub(gendiskp, field, subnd) \
+               disk_stat_add(gendiskp, field, -subnd)
+
+
+/* Inlines to alloc and free disk stats in struct gendisk */
+#ifdef  CONFIG_SMP
+static inline int init_disk_stats(struct gendisk *disk)
+{
+       disk->dkstats = kmalloc_percpu(sizeof (struct disk_stats), GFP_KERNEL);
+       if (!disk->dkstats)
+               return 0;
+       disk_stat_set_all(disk, 0);
+       return 1;
+}
+
+static inline void free_disk_stats(struct gendisk *disk)
+{
+       kfree_percpu(disk->dkstats);
+}
+#else  /* CONFIG_SMP */
+static inline int init_disk_stats(struct gendisk *disk)
+{
+       return 1;
+}
+
+static inline void free_disk_stats(struct gendisk *disk)
+{
+}
+#endif /* CONFIG_SMP */
+
 /* drivers/block/ll_rw_blk.c */
 extern void disk_round_stats(struct gendisk *disk);