[PATCH] convert node/zone_start_paddr to pfns

I've had ia32-discontigmem under test for a month, uneventfully.  Possibly
because I don't have a machine to test it on....

A major part of this work is a general move to convert the low-level
memory management to consistently use pageframe numbers.  It's a bit
schizo at present..

This patch was written by Martin Bligh.  A version of this patch is in
the 2.4 aa tree.

It changes the unsigned longs node_start_paddr and zone_start_paddr to
page frame numbers.  This is necessary because a PAE address is 36 bits
and cannot be represented in an unsigned long.

 - The per-node physical memory start address node_start_paddr becomes
   a pfn, node_start_pfn.

 - The per-zone physical memory start address zone_start_paddr becomes
   a pfn, zone_start_pfn.

 - free_area_init_node() takes a pfn rather than a physical address.

Patricia has tested this patch on the following configurations: UP,
SMP, SMP PAE, multiquad, multiquad PAE, multiquad DISCONTIGMEM,
multiquad DISCONTIGMEM PAE.

diff --git a/arch/alpha/mm/numa.c b/arch/alpha/mm/numa.c
index 8a86daca887fe7c10f5bcc7e3913222938512fe0..2458576ec8ae4bbc3eda60ef03155ce868a1ab70 100644 (file)
--- a/arch/alpha/mm/numa.c
+++ b/arch/alpha/mm/numa.c
@@ -294,7 +294,7 @@ void __init paging_init(void)
                        zones_size[ZONE_DMA] = dma_local_pfn;
                        zones_size[ZONE_NORMAL] = (end_pfn - start_pfn) - dma_local_pfn;
                }
-               free_area_init_node(nid, NODE_DATA(nid), NULL, zones_size, start_pfn<<PAGE_SHIFT, NULL);
+               free_area_init_node(nid, NODE_DATA(nid), NULL, zones_size, start_pfn, NULL);
                lmax_mapnr = PLAT_NODE_DATA_STARTNR(nid) + PLAT_NODE_DATA_SIZE(nid);
                if (lmax_mapnr > max_mapnr) {
                        max_mapnr = lmax_mapnr;
@@ -371,7 +371,7 @@ void __init mem_init(void)
                totalram_pages += free_all_bootmem_node(NODE_DATA(nid));
 
                lmem_map = NODE_MEM_MAP(nid);
-               pfn = NODE_DATA(nid)->node_start_paddr >> PAGE_SHIFT;
+               pfn = NODE_DATA(nid)->node_start_pfn;
                for (i = 0; i < PLAT_NODE_DATA_SIZE(nid); i++, pfn++)
                        if (page_is_ram(pfn) && PageReserved(lmem_map+i))
                                reservedpages++;

diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c
index 019a0d4185a2d981de49951ab43d90438ffea30b..d8fef49ded9c61701750fcade848ad6bbb813327 100644 (file)
--- a/arch/arm/mm/init.c
+++ b/arch/arm/mm/init.c
@@ -512,7 +512,7 @@ void __init paging_init(struct meminfo *mi, struct machine_desc *mdesc)
                arch_adjust_zones(node, zone_size, zhole_size);
 
                free_area_init_node(node, pgdat, 0, zone_size,
-                               bdata->node_boot_start, zhole_size);
+                               bdata->node_boot_start >> PAGE_SHIFT, zhole_size);
        }
 
        /*

diff --git a/arch/cris/mm/init.c b/arch/cris/mm/init.c
index 35a7db18886358213472dcbab0cfe18019005fdd..37c8d7a3c8bd9cc199265e43c68a67d8f90dee74 100644 (file)
--- a/arch/cris/mm/init.c
+++ b/arch/cris/mm/init.c
@@ -345,7 +345,7 @@ paging_init(void)
         * mem_map page array.
         */
 
-       free_area_init_node(0, 0, 0, zones_size, PAGE_OFFSET, 0);
+       free_area_init_node(0, 0, 0, zones_size, PAGE_OFFSET >> PAGE_SHIFT, 0);
 
 }
 

diff --git a/arch/mips64/sgi-ip27/ip27-memory.c b/arch/mips64/sgi-ip27/ip27-memory.c
index 41f13c3b63dccc5b32f0e2bacb85815ac6a619be..e5f79e031816ade1efb35d11bb8f9f0c1f52af91 100644 (file)
--- a/arch/mips64/sgi-ip27/ip27-memory.c
+++ b/arch/mips64/sgi-ip27/ip27-memory.c
@@ -253,7 +253,7 @@ void __init paging_init(void)
 
                zones_size[ZONE_DMA] = end_pfn + 1 - start_pfn;
                free_area_init_node(node, NODE_DATA(node), 0, zones_size, 
-                                               start_pfn << PAGE_SHIFT, 0);
+                                               start_pfn, 0);
                if ((PLAT_NODE_DATA_STARTNR(node) + 
                                        PLAT_NODE_DATA_SIZE(node)) > pagenr)
                        pagenr = PLAT_NODE_DATA_STARTNR(node) +

diff --git a/arch/sh/mm/init.c b/arch/sh/mm/init.c
index c02c1ebefe4f8e4d553c120a666509852da87466..1bc8f5ce978b84d7224ffbfa576554dc29b457f8 100644 (file)
--- a/arch/sh/mm/init.c
+++ b/arch/sh/mm/init.c
@@ -123,11 +123,11 @@ void __init paging_init(void)
                        zones_size[ZONE_DMA] = max_dma - start_pfn;
                        zones_size[ZONE_NORMAL] = low - max_dma;
                }
-               free_area_init_node(0, NODE_DATA(0), 0, zones_size, __MEMORY_START, 0);
+               free_area_init_node(0, NODE_DATA(0), 0, zones_size, __MEMORY_START >> PAGE_SHIFT, 0);
 #ifdef CONFIG_DISCONTIGMEM
                zones_size[ZONE_DMA] = __MEMORY_SIZE_2ND >> PAGE_SHIFT;
                zones_size[ZONE_NORMAL] = 0;
-               free_area_init_node(1, NODE_DATA(1), 0, zones_size, __MEMORY_START_2ND, 0);
+               free_area_init_node(1, NODE_DATA(1), 0, zones_size, __MEMORY_START_2ND >> PAGE_SHIFT, 0);
 #endif
        }
 }

diff --git a/arch/sparc/mm/srmmu.c b/arch/sparc/mm/srmmu.c
index 3a6dda3db3e47e936375144a3cbd6d6a95f1e7e2..f7dcd0932bec378de077e1a0cdfd1d31afa1454f 100644 (file)
--- a/arch/sparc/mm/srmmu.c
+++ b/arch/sparc/mm/srmmu.c
@@ -1311,7 +1311,7 @@ void __init srmmu_paging_init(void)
                zholes_size[ZONE_HIGHMEM] = npages - calc_highpages();
 
                free_area_init_node(0, NULL, NULL, zones_size,
-                                   phys_base, zholes_size);
+                                   phys_base >> PAGE_SHIFT, zholes_size);
        }
 
 /* P3: easy to fix, todo. Current code is utterly broken, though. */

diff --git a/arch/sparc/mm/sun4c.c b/arch/sparc/mm/sun4c.c
index 782834be8b8cbd65906ed2dee72589976c7f5f14..2fb85bac38b496706da91b60f7b936839d9cd027 100644 (file)
--- a/arch/sparc/mm/sun4c.c
+++ b/arch/sparc/mm/sun4c.c
@@ -2074,7 +2074,7 @@ void __init sun4c_paging_init(void)
                zholes_size[ZONE_HIGHMEM] = npages - calc_highpages();
 
                free_area_init_node(0, NULL, NULL, zones_size,
-                                   phys_base, zholes_size);
+                                   phys_base >> PAGE_SHIFT, zholes_size);
        }
 
        cnt = 0;

diff --git a/arch/sparc64/mm/init.c b/arch/sparc64/mm/init.c
index 5c1eda8afd5ad6035bdb2c6cc559c08635e4b150..68d33e579009b58a333380cce34a101cdb25cb62 100644 (file)
--- a/arch/sparc64/mm/init.c
+++ b/arch/sparc64/mm/init.c
@@ -1559,7 +1559,7 @@ void __init paging_init(void)
                zholes_size[ZONE_DMA] = npages - pages_avail;
 
                free_area_init_node(0, NULL, NULL, zones_size,
-                                   phys_base, zholes_size);
+                                   phys_base >> PAGE_SHIFT, zholes_size);
        }
 
        device_scan();

diff --git a/include/asm-alpha/mmzone.h b/include/asm-alpha/mmzone.h
index 415678ec774615801e6fcf5023570be7bb75992a..572569df5dd40b9a9248e7173ac9d996e0304970 100644 (file)
--- a/include/asm-alpha/mmzone.h
+++ b/include/asm-alpha/mmzone.h
@@ -52,14 +52,14 @@ extern plat_pg_data_t *plat_node_data[];
 
 #if 1
 #define PLAT_NODE_DATA_LOCALNR(p, n)   \
-       (((p) - PLAT_NODE_DATA(n)->gendata.node_start_paddr) >> PAGE_SHIFT)
+       (((p) >> PAGE_SHIFT) - PLAT_NODE_DATA(n)->gendata.node_start_pfn)
 #else
 static inline unsigned long
 PLAT_NODE_DATA_LOCALNR(unsigned long p, int n)
 {
        unsigned long temp;
-       temp = p - PLAT_NODE_DATA(n)->gendata.node_start_paddr;
-       return (temp >> PAGE_SHIFT);
+       temp = p >> PAGE_SHIFT;
+       return temp - PLAT_NODE_DATA(n)->gendata.node_start_pfn;
 }
 #endif
 
@@ -96,7 +96,7 @@ PLAT_NODE_DATA_LOCALNR(unsigned long p, int n)
  * and returns the kaddr corresponding to first physical page in the
  * node's mem_map.
  */
-#define LOCAL_BASE_ADDR(kaddr) ((unsigned long)__va(NODE_DATA(KVADDR_TO_NID(kaddr))->node_start_paddr))
+#define LOCAL_BASE_ADDR(kaddr) ((unsigned long)__va(NODE_DATA(KVADDR_TO_NID(kaddr))->node_start_pfn << PAGE_SHIFT))
 
 #define LOCAL_MAP_NR(kvaddr) \
        (((unsigned long)(kvaddr)-LOCAL_BASE_ADDR(kvaddr)) >> PAGE_SHIFT)

diff --git a/include/asm-alpha/pgtable.h b/include/asm-alpha/pgtable.h
index a52575213a9643a1c0941ae1e24b376563cf3f83..bd7b7c0724e2cfb12c4e7badc6fa72f2be128b71 100644 (file)
--- a/include/asm-alpha/pgtable.h
+++ b/include/asm-alpha/pgtable.h
@@ -195,8 +195,8 @@ extern unsigned long __zero_page(void);
 #define PAGE_TO_PA(page)       ((page - mem_map) << PAGE_SHIFT)
 #else
 #define PAGE_TO_PA(page) \
-               ((((page)-(page)->zone->zone_mem_map) << PAGE_SHIFT) \
-               + (page)->zone->zone_start_paddr)
+               ((( (page) - (page)->zone->zone_mem_map ) \
+               + (page)->zone->zone_start_pfn) << PAGE_SHIFT)
 #endif
 
 #ifndef CONFIG_DISCONTIGMEM
@@ -216,7 +216,7 @@ extern unsigned long __zero_page(void);
        unsigned long pfn;                                                      \
                                                                                \
        pfn = ((unsigned long)((page)-(page)->zone->zone_mem_map)) << 32;       \
-       pfn += (page)->zone->zone_start_paddr << (32-PAGE_SHIFT);               \
+       pfn += (page)->zone->zone_start_pfn << 32);                             \
        pte_val(pte) = pfn | pgprot_val(pgprot);                                \
                                                                                \
        pte;                                                                    \

diff --git a/include/asm-arm/memory.h b/include/asm-arm/memory.h
index 57f6b758164ad71e98a8cfe4d971ceda61c76719..b46474f46b74a5366c55ef9478d6c47d35a512bc 100644 (file)
--- a/include/asm-arm/memory.h
+++ b/include/asm-arm/memory.h
@@ -80,8 +80,8 @@ static inline void *phys_to_virt(unsigned long x)
  * around in memory.
  */
 #define page_to_pfn(page)                                      \
-       (((page) - page_zone(page)->zone_mem_map)               \
-         + (page_zone(page)->zone_start_paddr >> PAGE_SHIFT))
+       (( (page) - page_zone(page)->zone_mem_map)              \
+         + page_zone(page)->zone_start_pfn)
 
 #define pfn_to_page(pfn)                                       \
        (PFN_TO_MAPBASE(pfn) + LOCAL_MAP_NR((pfn) << PAGE_SHIFT))

diff --git a/include/asm-mips64/mmzone.h b/include/asm-mips64/mmzone.h
index 351670f4655df54cc7ea71140dca0df990070cf2..5e643b114269c26b35cf328e2cc19879d022afe6 100644 (file)
--- a/include/asm-mips64/mmzone.h
+++ b/include/asm-mips64/mmzone.h
@@ -27,7 +27,7 @@ extern plat_pg_data_t *plat_node_data[];
 #define PLAT_NODE_DATA_STARTNR(n)    (PLAT_NODE_DATA(n)->gendata.node_start_mapnr)
 #define PLAT_NODE_DATA_SIZE(n)      (PLAT_NODE_DATA(n)->gendata.node_size)
 #define PLAT_NODE_DATA_LOCALNR(p, n) \
-               (((p) - PLAT_NODE_DATA(n)->gendata.node_start_paddr) >> PAGE_SHIFT)
+               (((p) >> PAGE_SHIFT) - PLAT_NODE_DATA(n)->gendata.node_start_pfn)
 
 #define numa_node_id() cputocnode(current->processor)
 

diff --git a/include/asm-mips64/pgtable.h b/include/asm-mips64/pgtable.h
index d3118912a2a5432a23316c97a0c46e44633bcab1..e8ec5740d3b0b17c2e8e1c63aacd4aa02b2b7be3 100644 (file)
--- a/include/asm-mips64/pgtable.h
+++ b/include/asm-mips64/pgtable.h
@@ -484,8 +484,8 @@ static inline pgprot_t pgprot_noncached(pgprot_t _prot)
 #define PAGE_TO_PA(page)       ((page - mem_map) << PAGE_SHIFT)
 #else
 #define PAGE_TO_PA(page) \
-               ((((page)-(page)->zone->zone_mem_map) << PAGE_SHIFT) \
-               + ((page)->zone->zone_start_paddr))
+               (( ((page)-(page)->zone->zone_mem_map) + \
+                  (page)->zone->zone_start_pfn) << PAGE_SHIFT)
 #endif
 #define mk_pte(page, pgprot)                                           \
 ({                                                                     \

diff --git a/include/asm-ppc64/mmzone.h b/include/asm-ppc64/mmzone.h
index f8af72a4e0ca2172ccda56b6adc8c99422289259..8129072d3b507dba2fde57a01849ed0962e10552 100644 (file)
--- a/include/asm-ppc64/mmzone.h
+++ b/include/asm-ppc64/mmzone.h
@@ -31,7 +31,7 @@ extern plat_pg_data_t plat_node_data[];
        (PLAT_NODE_DATA(n)->gendata.node_start_mapnr)
 #define PLAT_NODE_DATA_SIZE(n)         (PLAT_NODE_DATA(n)->gendata.node_size)
 #define PLAT_NODE_DATA_LOCALNR(p, n)   \
-       (((p) - PLAT_NODE_DATA(n)->gendata.node_start_paddr) >> PAGE_SHIFT)
+       (((p) >> PAGE_SHIFT) - PLAT_NODE_DATA(n)->gendata.node_start_pfn)
 
 #ifdef CONFIG_DISCONTIGMEM
 
@@ -67,7 +67,7 @@ extern plat_pg_data_t plat_node_data[];
  * node's mem_map.
  */
 #define LOCAL_BASE_ADDR(kaddr) \
-       ((unsigned long)__va(NODE_DATA(KVADDR_TO_NID(kaddr))->node_start_paddr))
+       ((unsigned long)__va(NODE_DATA(KVADDR_TO_NID(kaddr))->node_start_pfn << PAGE_SHIFT))
 
 #define LOCAL_MAP_NR(kvaddr) \
        (((unsigned long)(kvaddr)-LOCAL_BASE_ADDR(kvaddr)) >> PAGE_SHIFT)

diff --git a/include/asm-ppc64/page.h b/include/asm-ppc64/page.h
index bf0c552f61a59c30fb6546d9d97e3a5e35c0db14..ebcd92b05f851a06812a0d0a313a537186f20f97 100644 (file)
--- a/include/asm-ppc64/page.h
+++ b/include/asm-ppc64/page.h
@@ -207,7 +207,7 @@ static inline int get_order(unsigned long size)
 #ifdef CONFIG_DISCONTIGMEM
 #define page_to_pfn(page) \
                ((page) - page_zone(page)->zone_mem_map + \
-               (page_zone(page)->zone_start_paddr >> PAGE_SHIFT))
+               (page_zone(page)->zone_start_pfn))
 #define pfn_to_page(pfn)       discontigmem_pfn_to_page(pfn)
 #else
 #define pfn_to_page(pfn)       (mem_map + (pfn))

diff --git a/include/linux/mm.h b/include/linux/mm.h
index cdfaea4e330793f4dfb821cd24d4e9b9539848db..84dc4bba425e72fc5dfe8088254d58a58f0e11e6 100644 (file)
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -316,8 +316,8 @@ static inline void set_page_zone(struct page *page, unsigned long zone_num)
 #else /* CONFIG_HIGHMEM || WANT_PAGE_VIRTUAL */
 
 #define page_address(page)                                             \
-       __va( (((page) - page_zone(page)->zone_mem_map) << PAGE_SHIFT)  \
-                       + page_zone(page)->zone_start_paddr)
+       __va( ( ((page) - page_zone(page)->zone_mem_map)                \
+                       + page_zone(page)->zone_start_pfn) << PAGE_SHIFT)
 
 #endif /* CONFIG_HIGHMEM || WANT_PAGE_VIRTUAL */
 
@@ -398,7 +398,7 @@ static inline pmd_t *pmd_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long a
 
 extern void free_area_init(unsigned long * zones_size);
 extern void free_area_init_node(int nid, pg_data_t *pgdat, struct page *pmap,
-       unsigned long * zones_size, unsigned long zone_start_paddr, 
+       unsigned long * zones_size, unsigned long zone_start_pfn, 
        unsigned long *zholes_size);
 extern void mem_init(void);
 extern void show_mem(void);

diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index f30bd4f44863a302b69a37604c430b3c9e56ad37..e5b6bc1111b89bb8d22e47fcc2a80ff8e31f2771 100644 (file)
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -110,7 +110,8 @@ struct zone {
         */
        struct pglist_data      *zone_pgdat;
        struct page             *zone_mem_map;
-       unsigned long           zone_start_paddr;
+       /* zone_start_pfn == zone_start_paddr >> PAGE_SHIFT */
+       unsigned long           zone_start_pfn;
        unsigned long           zone_start_mapnr;
 
        /*
@@ -161,7 +162,7 @@ typedef struct pglist_data {
        struct page *node_mem_map;
        unsigned long *valid_addr_bitmap;
        struct bootmem_data *bdata;
-       unsigned long node_start_paddr;
+       unsigned long node_start_pfn;
        unsigned long node_start_mapnr;
        unsigned long node_size;
        int node_id;

diff --git a/mm/highmem.c b/mm/highmem.c
index ae9c5a26376b26bb4878b1ae0dd76b6f571d81b6..70548f87d9c5dd86c0863a947cdce82e1fef9fd1 100644 (file)
--- a/mm/highmem.c
+++ b/mm/highmem.c
@@ -383,7 +383,7 @@ void blk_queue_bounce(request_queue_t *q, struct bio **bio_orig)
                /*
                 * is destination page below bounce pfn?
                 */
-               if ((page - page_zone(page)->zone_mem_map) + (page_zone(page)->zone_start_paddr >> PAGE_SHIFT) < pfn)
+               if ((page - page_zone(page)->zone_mem_map) + (page_zone(page)->zone_start_pfn) < pfn)
                        continue;
 
                /*

diff --git a/mm/numa.c b/mm/numa.c
index a7605cdae51bc922cea3298bea27dd4469a4b022..c293d9ae2df02f3890c2b2bd4abd60f6a7c1cd9c 100644 (file)
--- a/mm/numa.c
+++ b/mm/numa.c
@@ -22,11 +22,11 @@ pg_data_t contig_page_data = { .bdata = &contig_bootmem_data };
  * Should be invoked with paramters (0, 0, unsigned long *[], start_paddr).
  */
 void __init free_area_init_node(int nid, pg_data_t *pgdat, struct page *pmap,
-       unsigned long *zones_size, unsigned long zone_start_paddr, 
+       unsigned long *zones_size, unsigned long zone_start_pfn, 
        unsigned long *zholes_size)
 {
        free_area_init_core(0, &contig_page_data, &mem_map, zones_size, 
-                               zone_start_paddr, zholes_size, pmap);
+                               zone_start_pfn, zholes_size, pmap);
 }
 
 #endif /* !CONFIG_DISCONTIGMEM */
@@ -48,7 +48,7 @@ struct page * alloc_pages_node(int nid, unsigned int gfp_mask, unsigned int orde
  * Nodes can be initialized parallely, in no particular order.
  */
 void __init free_area_init_node(int nid, pg_data_t *pgdat, struct page *pmap,
-       unsigned long *zones_size, unsigned long zone_start_paddr, 
+       unsigned long *zones_size, unsigned long zone_start_pfn, 
        unsigned long *zholes_size)
 {
        int i, size = 0;
@@ -57,7 +57,7 @@ void __init free_area_init_node(int nid, pg_data_t *pgdat, struct page *pmap,
        if (mem_map == NULL)
                mem_map = (struct page *)PAGE_OFFSET;
 
-       free_area_init_core(nid, pgdat, &discard, zones_size, zone_start_paddr,
+       free_area_init_core(nid, pgdat, &discard, zones_size, zone_start_pfn,
                                        zholes_size, pmap);
        pgdat->node_id = nid;
 

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index bc37f860b0cf9c20978f6b7717995904df7377ad..a017799cba1def1404c8f82fb13423885ceccbef 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -773,7 +773,7 @@ static inline unsigned long wait_table_bits(unsigned long size)
  *   - clear the memory bitmaps
  */
 void __init free_area_init_core(int nid, pg_data_t *pgdat, struct page **gmap,
-       unsigned long *zones_size, unsigned long zone_start_paddr, 
+       unsigned long *zones_size, unsigned long zone_start_pfn, 
        unsigned long *zholes_size, struct page *lmem_map)
 {
        unsigned long i, j;
@@ -781,8 +781,6 @@ void __init free_area_init_core(int nid, pg_data_t *pgdat, struct page **gmap,
        unsigned long totalpages, offset, realtotalpages;
        const unsigned long zone_required_alignment = 1UL << (MAX_ORDER-1);
 
-       BUG_ON(zone_start_paddr & ~PAGE_MASK);
-
        totalpages = 0;
        for (i = 0; i < MAX_NR_ZONES; i++) {
                unsigned long size = zones_size[i];
@@ -810,7 +808,7 @@ void __init free_area_init_core(int nid, pg_data_t *pgdat, struct page **gmap,
        }
        *gmap = pgdat->node_mem_map = lmem_map;
        pgdat->node_size = totalpages;
-       pgdat->node_start_paddr = zone_start_paddr;
+       pgdat->node_start_pfn = zone_start_pfn;
        pgdat->node_start_mapnr = (lmem_map - mem_map);
        pgdat->nr_zones = 0;
 
@@ -868,9 +866,9 @@ void __init free_area_init_core(int nid, pg_data_t *pgdat, struct page **gmap,
 
                zone->zone_mem_map = mem_map + offset;
                zone->zone_start_mapnr = offset;
-               zone->zone_start_paddr = zone_start_paddr;
+               zone->zone_start_pfn = zone_start_pfn;
 
-               if ((zone_start_paddr >> PAGE_SHIFT) & (zone_required_alignment-1))
+               if ((zone_start_pfn) & (zone_required_alignment-1))
                        printk("BUG: wrong zone alignment, it will crash\n");
 
                /*
@@ -885,8 +883,12 @@ void __init free_area_init_core(int nid, pg_data_t *pgdat, struct page **gmap,
                        SetPageReserved(page);
                        INIT_LIST_HEAD(&page->list);
                        if (j != ZONE_HIGHMEM)
-                               set_page_address(page, __va(zone_start_paddr));
-                       zone_start_paddr += PAGE_SIZE;
+                               /*
+                                * The shift left won't overflow because the
+                                * ZONE_NORMAL is below 4G.
+                                */
+                               set_page_address(page, __va(zone_start_pfn << PAGE_SHIFT));
+                       zone_start_pfn++;
                }
 
                offset += size;