tristate '/dev/cpu/*/cpuid - CPU information support' CONFIG_X86_CPUID
choice 'High Memory Support' \
- "off CONFIG_NOHIGHMEM \
- 4GB CONFIG_HIGHMEM4G \
- 64GB CONFIG_HIGHMEM64G" off
+ "off CONFIG_NOHIGHMEM \
+ 4GB CONFIG_HIGHMEM4G \
+ 4GB-highpte CONFIG_HIGHMEM4G_HIGHPTE \
+ 64GB CONFIG_HIGHMEM64G \
+ 64GB-highpte CONFIG_HIGHMEM64G_HIGHPTE" off
if [ "$CONFIG_HIGHMEM4G" = "y" ]; then
define_bool CONFIG_HIGHMEM y
fi
+if [ "$CONFIG_HIGHMEM4G_HIGHPTE" = "y" ]; then
+ define_bool CONFIG_HIGHMEM y
+ define_bool CONFIG_HIGHPTE y
+fi
if [ "$CONFIG_HIGHMEM64G" = "y" ]; then
define_bool CONFIG_HIGHMEM y
define_bool CONFIG_X86_PAE y
fi
+if [ "$CONFIG_HIGHMEM64G_HIGHPTE" = "y" ]; then
+ define_bool CONFIG_HIGHMEM y
+ define_bool CONFIG_HIGHPTE y
+ define_bool CONFIG_X86_PAE y
+fi
bool 'Math emulation' CONFIG_MATH_EMULATION
bool 'MTRR (Memory Type Range Register) support' CONFIG_MTRR
bool 'Kernel debugging' CONFIG_DEBUG_KERNEL
if [ "$CONFIG_DEBUG_KERNEL" != "n" ]; then
- bool ' Debug high memory support' CONFIG_DEBUG_HIGHMEM
bool ' Debug memory allocations' CONFIG_DEBUG_SLAB
bool ' Memory mapped I/O debugging' CONFIG_DEBUG_IOVIRT
bool ' Magic SysRq key' CONFIG_MAGIC_SYSRQ
bool ' Spinlock debugging' CONFIG_DEBUG_SPINLOCK
+ if [ "$CONFIG_HIGHMEM" = "y" ]; then
+ bool ' Highmem debugging' CONFIG_DEBUG_HIGHMEM
+ fi
fi
endmenu
while (!need_resched())
idle();
schedule();
- check_pgt_cache();
}
}
struct cpuinfo_x86 *c = cpu_data + id;
*c = boot_cpu_data;
- c->pte_quick = 0;
- c->pmd_quick = 0;
- c->pgd_quick = 0;
- c->pgtable_cache_sz = 0;
identify_cpu(c);
/*
* Mask B, Pentium, but not Pentium MMX
page = (unsigned long) vmalloc(PAGE_SIZE);
pgd = pgd_offset(&init_mm, page);
pmd = pmd_offset(pgd, page);
- pte = pte_offset(pmd, page);
+ pte = pte_offset_kernel(pmd, page);
__free_page(pte_page(*pte));
*pte = mk_pte_phys(__pa(&idt_table), PAGE_KERNEL_RO);
/*
{
pgd_t *pgd;
pmd_t *pmd;
- pte_t *pte;
+ pte_t *pte, *mapped;
int i;
pgd = pgd_offset(tsk->mm, 0xA0000);
pmd_clear(pmd);
return;
}
- pte = pte_offset(pmd, 0xA0000);
+ preempt_disable();
+ pte = mapped = pte_offset_map(pmd, 0xA0000);
for (i = 0; i < 32; i++) {
if (pte_present(*pte))
set_pte(pte, pte_wrprotect(*pte));
pte++;
}
+ pte_unmap(mapped);
+ preempt_enable();
flush_tlb();
}
asm("movl %%cr3,%0":"=r" (page));
page = ((unsigned long *) __va(page))[address >> 22];
printk(KERN_ALERT "*pde = %08lx\n", page);
+ /*
+ * We must not directly access the pte in the highpte
+ * case, the page table might be allocated in highmem.
+ * And lets rather not kmap-atomic the pte, just in case
+ * it's allocated already.
+ */
+#ifndef CONFIG_HIGHPTE
if (page & 1) {
page &= PAGE_MASK;
address &= 0x003ff000;
page = ((unsigned long *) __va(page))[address >> PAGE_SHIFT];
printk(KERN_ALERT "*pte = %08lx\n", page);
}
+#endif
die("Oops", regs, error_code);
bust_spinlocks(0);
do_exit(SIGKILL);
goto no_context;
set_pmd(pmd, *pmd_k);
- pte_k = pte_offset(pmd_k, address);
+ pte_k = pte_offset_kernel(pmd_k, address);
if (!pte_present(*pte_k))
goto no_context;
return;
static unsigned long totalram_pages;
static unsigned long totalhigh_pages;
-int do_check_pgt_cache(int low, int high)
-{
- int freed = 0;
- if(pgtable_cache_size > high) {
- do {
- if (pgd_quicklist) {
- free_pgd_slow(get_pgd_fast());
- freed++;
- }
- if (pmd_quicklist) {
- pmd_free_slow(pmd_alloc_one_fast(NULL, 0));
- freed++;
- }
- if (pte_quicklist) {
- pte_free_slow(pte_alloc_one_fast(NULL, 0));
- freed++;
- }
- } while(pgtable_cache_size > low);
- }
- return freed;
-}
-
/*
* NOTE: pagetable_init alloc all the fixmap pagetables contiguous on the
* physical space so we can cache the place of the first one and move
pgprot_t kmap_prot;
#define kmap_get_fixmap_pte(vaddr) \
- pte_offset(pmd_offset(pgd_offset_k(vaddr), (vaddr)), (vaddr))
+ pte_offset_kernel(pmd_offset(pgd_offset_k(vaddr), (vaddr)), (vaddr))
void __init kmap_init(void)
{
printk("%d reserved pages\n",reserved);
printk("%d pages shared\n",shared);
printk("%d pages swap cached\n",cached);
- printk("%ld pages in page table cache\n",pgtable_cache_size);
show_buffers();
}
printk("PAE BUG #01!\n");
return;
}
- pte = pte_offset(pmd, vaddr);
+ pte = pte_offset_kernel(pmd, vaddr);
if (pte_val(*pte))
pte_ERROR(*pte);
pgprot_val(prot) = pgprot_val(PAGE_KERNEL) | pgprot_val(flags);
if (pmd_none(*pmd)) {
pte = (pte_t *) alloc_bootmem_low_pages(PAGE_SIZE);
set_pmd(pmd, __pmd(_KERNPG_TABLE + __pa(pte)));
- if (pte != pte_offset(pmd, 0))
+ if (pte != pte_offset_kernel(pmd, 0))
BUG();
}
vaddr += PMD_SIZE;
*pte = mk_pte_phys(__pa(vaddr), PAGE_KERNEL);
}
set_pmd(pmd, __pmd(_KERNPG_TABLE + __pa(pte_base)));
- if (pte_base != pte_offset(pmd, 0))
+ if (pte_base != pte_offset_kernel(pmd, 0))
BUG();
}
pgd = swapper_pg_dir + __pgd_offset(vaddr);
pmd = pmd_offset(pgd, vaddr);
- pte = pte_offset(pmd, vaddr);
+ pte = pte_offset_kernel(pmd, vaddr);
pkmap_page_table = pte;
#endif
pgd = swapper_pg_dir + __pgd_offset(vaddr);
pmd = pmd_offset(pgd, vaddr);
- pte = pte_offset(pmd, vaddr);
+ pte = pte_offset_kernel(pmd, vaddr);
old_pte = *pte;
*pte = mk_pte_phys(0, PAGE_READONLY);
local_flush_tlb();
if (address >= end)
BUG();
do {
- pte_t * pte = pte_alloc(&init_mm, pmd, address);
+ pte_t * pte = pte_alloc_kernel(&init_mm, pmd, address);
if (!pte)
return -ENOMEM;
remap_area_pte(pte, address, end - address, address + phys_addr, flags);
for (i = vma->vm_start; i < vma->vm_end; i += PAGE_SIZE) {
pgd = pgd_offset(vma->vm_mm, i);
pmd = pmd_offset(pgd, i);
- pte = pte_offset(pmd, i);
+ preempt_disable();
+ pte = pte_offset_map(pmd, i);
if (pte_present(*pte)) {
address = __pa(pte_page(*pte))
+ (i & (PAGE_SIZE-1));
} else {
DRM_PROC_PRINT(" 0x%08lx\n", i);
}
+ pte_unmap(pte);
+ preempt_enable();
}
#endif
}
if ( !pmd_present( *pmd ) )
goto failed;
- pte = pte_offset( pmd, i );
- if ( !pte_present( *pte ) )
+ preempt_disable();
+ pte = pte_offset_map( pmd, i );
+ if ( !pte_present( *pte ) ) {
+ pte_unmap(pte);
+ preempt_enable();
goto failed;
+ }
+ pte_unmap(pte);
+ preempt_enable();
entry->pagelist[j] = pte_page( *pte );
if( !pgd_present( *pgd ) ) return NOPAGE_OOM;
pmd = pmd_offset( pgd, i );
if( !pmd_present( *pmd ) ) return NOPAGE_OOM;
- pte = pte_offset( pmd, i );
- if( !pte_present( *pte ) ) return NOPAGE_OOM;
+ preempt_disable();
+ pte = pte_offset_map( pmd, i );
+ if( !pte_present( *pte ) ) {
+ pte_unmap(pte);
+ preempt_enable();
+ return NOPAGE_OOM;
+ }
+ pte_unmap(pte);
+ preempt_enable();
page = pte_page(*pte);
get_page(page);
if (!pgd_none(*pgd)) {
pmd = pmd_offset(pgd, adr);
if (!pmd_none(*pmd)) {
- ptep = pte_offset(pmd, adr);
+ preempt_disable();
+ ptep = pte_offset_map(pmd, adr);
pte = *ptep;
+ pte_unmap(pte);
+ preempt_enable();
if(pte_present(pte))
ret = pte_page(pte);
}
int board = GRAPHICS_CARD (vma->vm_dentry->d_inode->i_rdev);
unsigned long virt_add, phys_add;
+ struct page * page;
#ifdef DEBUG
printk ("Got a page fault for board %d address=%lx guser=%lx\n", board,
pgd = pgd_offset(current->mm, address);
pmd = pmd_offset(pgd, address);
- pte = pte_offset(pmd, address);
- return pte_page(*pte);
+ pte = pte_kmap_offset(pmd, address);
+ page = pte_page(*pte);
+ pte_kunmap(pte);
+ return page;
}
/*
if (!pgd_none (*pgd)) {
pmd = pmd_offset (pgd, adr);
if (!pmd_none (*pmd)) {
- ptep = pte_offset (pmd, adr);
+ preempt_disable();
+ ptep = pte_offset_map (pmd, adr);
pte = *ptep;
+ pte_unmap(pte);
+ preempt_enable();
if (pte_present (pte)) {
ret = (unsigned long) page_address (pte_page (pte));
ret |= (adr & (PAGE_SIZE - 1));
if (!pgd_none(*pgd)) {
pmd = pmd_offset(pgd, adr);
if (!pmd_none(*pmd)) {
- ptep = pte_offset(pmd, adr);
+ preempt_disable();
+ ptep = pte_offset_map(pmd, adr);
pte = *ptep;
+ pte_unmap(pte);
+ preempt_enable();
if(pte_present(pte)) {
ret = (unsigned long) page_address(pte_page(pte));
ret |= (adr & (PAGE_SIZE - 1));
pmd = pmd_alloc(tsk->mm, pgd, address);
if (!pmd)
goto out;
- pte = pte_alloc(tsk->mm, pmd, address);
+ pte = pte_alloc_map(tsk->mm, pmd, address);
if (!pte)
goto out;
- if (!pte_none(*pte))
+ if (!pte_none(*pte)) {
+ pte_unmap(pte);
goto out;
+ }
lru_cache_add(page);
flush_dcache_page(page);
flush_page_to_ram(page);
set_pte(pte, pte_mkdirty(pte_mkwrite(mk_pte(page, PAGE_COPY))));
+ pte_unmap(pte);
tsk->mm->rss++;
spin_unlock(&tsk->mm->page_table_lock);
return res;
}
-static inline void statm_pte_range(pmd_t * pmd, unsigned long address, unsigned long size,
- int * pages, int * shared, int * dirty, int * total)
+static inline void statm_pte_range(pmd_t * pmd, unsigned long address, unsigned long size, int * pages, int * shared, int * dirty, int * total)
{
- pte_t * pte;
- unsigned long end;
+ unsigned long end, pmd_end;
+ pte_t *pte;
if (pmd_none(*pmd))
return;
pmd_clear(pmd);
return;
}
- pte = pte_offset(pmd, address);
- address &= ~PMD_MASK;
+ preempt_disable();
+ pte = pte_offset_map(pmd, address);
end = address + size;
- if (end > PMD_SIZE)
- end = PMD_SIZE;
+ pmd_end = (address + PMD_SIZE) & PMD_MASK;
+ if (end > pmd_end)
+ end = pmd_end;
do {
pte_t page = *pte;
struct page *ptpage;
if (page_count(pte_page(page)) > 1)
++*shared;
} while (address < end);
+ pte_unmap(pte - 1);
+ preempt_enable();
}
static inline void statm_pmd_range(pgd_t * pgd, unsigned long address, unsigned long size,
#include <asm/kmap_types.h>
#include <asm/pgtable.h>
-#ifdef CONFIG_DEBUG_HIGHMEM
-#define HIGHMEM_DEBUG 1
-#else
-#define HIGHMEM_DEBUG 0
-#endif
-
/* declarations for highmem.c */
extern unsigned long highstart_pfn, highend_pfn;
idx = type + KM_TYPE_NR*smp_processor_id();
vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
-#if HIGHMEM_DEBUG
+#if CONFIG_DEBUG_HIGHMEM
if (!pte_none(*(kmap_pte-idx)))
BUG();
#endif
static inline void kunmap_atomic(void *kvaddr, enum km_type type)
{
-#if HIGHMEM_DEBUG
- unsigned long vaddr = (unsigned long) kvaddr;
+#if CONFIG_DEBUG_HIGHMEM
+ unsigned long vaddr = (unsigned long) kvaddr & PAGE_MASK;
enum fixed_addresses idx = type + KM_TYPE_NR*smp_processor_id();
if (vaddr < FIXADDR_START) { // FIXME
#ifndef _ASM_KMAP_TYPES_H
#define _ASM_KMAP_TYPES_H
+#include <linux/config.h>
+
+#if CONFIG_DEBUG_HIGHMEM
+# define D(n) __KM_FENCE_##n ,
+#else
+# define D(n)
+#endif
+
enum km_type {
- KM_BOUNCE_READ,
- KM_SKB_DATA,
- KM_SKB_DATA_SOFTIRQ,
- KM_USER0,
- KM_USER1,
- KM_BIO_IRQ,
- KM_TYPE_NR
+D(0) KM_BOUNCE_READ,
+D(1) KM_SKB_DATA,
+D(2) KM_SKB_DATA_SOFTIRQ,
+D(3) KM_USER0,
+D(4) KM_USER1,
+D(5) KM_BIO_IRQ,
+D(6) KM_PTE0,
+D(7) KM_PTE1,
+D(8) KM_TYPE_NR
};
+#undef D
+
#endif
#include <asm/processor.h>
#include <asm/fixmap.h>
#include <linux/threads.h>
+#include <linux/highmem.h>
-#define pgd_quicklist (current_cpu_data.pgd_quick)
-#define pmd_quicklist (current_cpu_data.pmd_quick)
-#define pte_quicklist (current_cpu_data.pte_quick)
-#define pgtable_cache_size (current_cpu_data.pgtable_cache_sz)
-
-#define pmd_populate(mm, pmd, pte) \
+#define pmd_populate_kernel(mm, pmd, pte) \
set_pmd(pmd, __pmd(_PAGE_TABLE + __pa(pte)))
+static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, struct page *pte)
+{
+ set_pmd(pmd, __pmd(_PAGE_TABLE +
+ ((unsigned long long)(pte - mem_map) <<
+ (unsigned long long) PAGE_SHIFT)));
+}
/*
* Allocate and free page tables.
*/
extern void kmem_cache_free(struct kmem_cache_s *, void *);
-static inline pgd_t *get_pgd_slow(void)
+static inline pgd_t *pgd_alloc(struct mm_struct *mm)
{
int i;
pgd_t *pgd = kmem_cache_alloc(pae_pgd_cachep, GFP_KERNEL);
#else
-static inline pgd_t *get_pgd_slow(void)
+static inline pgd_t *pgd_alloc(struct mm_struct *mm)
{
pgd_t *pgd = (pgd_t *)__get_free_page(GFP_KERNEL);
#endif /* CONFIG_X86_PAE */
-static inline pgd_t *get_pgd_fast(void)
-{
- unsigned long *ret;
-
- preempt_disable();
- if ((ret = pgd_quicklist) != NULL) {
- pgd_quicklist = (unsigned long *)(*ret);
- ret[0] = 0;
- pgtable_cache_size--;
- preempt_enable();
- } else {
- preempt_enable();
- ret = (unsigned long *)get_pgd_slow();
- }
- return (pgd_t *)ret;
-}
-
-static inline void free_pgd_fast(pgd_t *pgd)
-{
- preempt_disable();
- *(unsigned long *)pgd = (unsigned long) pgd_quicklist;
- pgd_quicklist = (unsigned long *) pgd;
- pgtable_cache_size++;
- preempt_enable();
-}
-
-static inline void free_pgd_slow(pgd_t *pgd)
+static inline void pgd_free(pgd_t *pgd)
{
#if defined(CONFIG_X86_PAE)
int i;
#endif
}
-static inline pte_t *pte_alloc_one(struct mm_struct *mm, unsigned long address)
+static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address)
{
+ int count = 0;
pte_t *pte;
-
- pte = (pte_t *) __get_free_page(GFP_KERNEL);
- if (pte)
- clear_page(pte);
+
+ do {
+ pte = (pte_t *) __get_free_page(GFP_KERNEL);
+ if (pte)
+ clear_page(pte);
+ else {
+ current->state = TASK_UNINTERRUPTIBLE;
+ schedule_timeout(HZ);
+ }
+ } while (!pte && (count++ < 10));
return pte;
}
-static inline pte_t *pte_alloc_one_fast(struct mm_struct *mm,
- unsigned long address)
+static inline struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address)
{
- unsigned long *ret;
-
- preempt_disable();
- if ((ret = (unsigned long *)pte_quicklist) != NULL) {
- pte_quicklist = (unsigned long *)(*ret);
- ret[0] = ret[1];
- pgtable_cache_size--;
- }
- preempt_enable();
- return (pte_t *)ret;
+ int count = 0;
+ struct page *pte;
+
+ do {
+#if CONFIG_HIGHPTE
+ pte = alloc_pages(GFP_KERNEL | __GFP_HIGHMEM, 0);
+#else
+ pte = alloc_pages(GFP_KERNEL, 0);
+#endif
+ if (pte)
+ clear_highpage(pte);
+ else {
+ current->state = TASK_UNINTERRUPTIBLE;
+ schedule_timeout(HZ);
+ }
+ } while (!pte && (count++ < 10));
+ return pte;
}
-static inline void pte_free_fast(pte_t *pte)
+static inline void pte_free_kernel(pte_t *pte)
{
- preempt_disable();
- *(unsigned long *)pte = (unsigned long) pte_quicklist;
- pte_quicklist = (unsigned long *) pte;
- pgtable_cache_size++;
- preempt_enable();
+ free_page((unsigned long)pte);
}
-static __inline__ void pte_free_slow(pte_t *pte)
+static inline void pte_free(struct page *pte)
{
- free_page((unsigned long)pte);
+ __free_page(pte);
}
-#define pte_free(pte) pte_free_slow(pte)
-#define pgd_free(pgd) free_pgd_slow(pgd)
-#define pgd_alloc(mm) get_pgd_fast()
-
/*
* allocating and freeing a pmd is trivial: the 1-entry pmd is
* inside the pgd, so has no extra memory associated with it.
* (In the PAE case we free the pmds as part of the pgd.)
*/
-#define pmd_alloc_one_fast(mm, addr) ({ BUG(); ((pmd_t *)1); })
#define pmd_alloc_one(mm, addr) ({ BUG(); ((pmd_t *)2); })
-#define pmd_free_slow(x) do { } while (0)
-#define pmd_free_fast(x) do { } while (0)
#define pmd_free(x) do { } while (0)
#define pgd_populate(mm, pmd, pte) BUG()
-extern int do_check_pgt_cache(int, int);
-
/*
* TLB flushing:
*
#define page_pte(page) page_pte_prot(page, __pgprot(0))
-#define pmd_page(pmd) \
+#define pmd_page_kernel(pmd) \
((unsigned long) __va(pmd_val(pmd) & PAGE_MASK))
+#define pmd_page(pmd) \
+ (mem_map + (pmd_val(pmd) >> PAGE_SHIFT))
+
/* to find an entry in a page-table-directory. */
#define pgd_index(address) ((address >> PGDIR_SHIFT) & (PTRS_PER_PGD-1))
/* Find an entry in the third-level page table.. */
#define __pte_offset(address) \
((address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1))
-#define pte_offset(dir, address) ((pte_t *) pmd_page(*(dir)) + \
- __pte_offset(address))
+#define pte_offset_kernel(dir, address) \
+ ((pte_t *) pmd_page_kernel(*(dir)) + __pte_offset(address))
+#define pte_offset_map(dir, address) \
+ ((pte_t *)kmap_atomic(pmd_page(*(dir)),KM_PTE0) + __pte_offset(address))
+#define pte_offset_map2(dir, address) \
+ ((pte_t *)kmap_atomic(pmd_page(*(dir)),KM_PTE1) + __pte_offset(address))
+#define pte_unmap(pte) kunmap_atomic(pte, KM_PTE0)
+#define pte_unmap2(pte) kunmap_atomic(pte, KM_PTE1)
/*
* The i386 doesn't have any external MMU info: the kernel page
int f00f_bug;
int coma_bug;
unsigned long loops_per_jiffy;
- unsigned long *pgd_quick;
- unsigned long *pmd_quick;
- unsigned long *pte_quick;
- unsigned long pgtable_cache_sz;
} __attribute__((__aligned__(SMP_CACHE_BYTES)));
#define X86_VENDOR_INTEL 0
#include <linux/config.h>
#include <linux/bio.h>
#include <linux/fs.h>
-#include <asm/pgalloc.h>
#ifdef CONFIG_HIGHMEM
unsigned int nr_free_highpages(void);
extern void create_bounce(unsigned long pfn, int gfp, struct bio **bio_orig);
+extern void check_highmem_ptes(void);
static inline char *bh_kmap(struct buffer_head *bh)
{
static inline void clear_highpage(struct page *page)
{
- clear_page(kmap(page));
- kunmap(page);
+ void *kaddr = kmap_atomic(page, KM_USER0);
+ clear_page(kaddr);
+ kunmap_atomic(kaddr, KM_USER0);
}
/*
*/
static inline void memclear_highpage_flush(struct page *page, unsigned int offset, unsigned int size)
{
- char *kaddr;
+ void *kaddr;
if (offset + size > PAGE_SIZE)
BUG();
- kaddr = kmap(page);
- memset(kaddr + offset, 0, size);
+
+ kaddr = kmap_atomic(page, KM_USER0);
+ memset((char *)kaddr + offset, 0, size);
flush_dcache_page(page);
flush_page_to_ram(page);
- kunmap(page);
+ kunmap_atomic(kaddr, KM_USER0);
}
static inline void copy_user_highpage(struct page *to, struct page *from, unsigned long vaddr)
extern int vmtruncate(struct inode * inode, loff_t offset);
extern pmd_t *FASTCALL(__pmd_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address));
-extern pte_t *FASTCALL(pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address));
+extern pte_t *FASTCALL(pte_alloc_kernel(struct mm_struct *mm, pmd_t *pmd, unsigned long address));
+extern pte_t *FASTCALL(pte_alloc_map(struct mm_struct *mm, pmd_t *pmd, unsigned long address));
extern int handle_mm_fault(struct mm_struct *mm,struct vm_area_struct *vma, unsigned long address, int write_access);
extern int make_pages_present(unsigned long addr, unsigned long end);
extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write);
/*
* On a two-level page table, this ends up being trivial. Thus the
- * inlining and the symmetry break with pte_alloc() that does all
+ * inlining and the symmetry break with pte_alloc_map() that does all
* of this out-of-line.
*/
static inline pmd_t *pmd_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
return pmd_offset(pgd, address);
}
-extern int pgt_cache_water[2];
-extern int check_pgt_cache(void);
-
extern void free_area_init(unsigned long * zones_size);
extern void free_area_init_node(int nid, pg_data_t *pgdat, struct page *pmap,
unsigned long * zones_size, unsigned long zone_start_paddr,
#include <linux/completion.h>
#include <asm/mmu_context.h>
#include <linux/kernel_stat.h>
+#include <linux/highmem.h>
/*
* Priority of a process goes from 0 to 139. The 0-99
if (unlikely(in_interrupt()))
BUG();
+#if CONFIG_DEBUG_HIGHMEM
+ check_highmem_ptes();
+#endif
need_resched:
preempt_disable();
prev = current;
extern int acct_parm[];
#endif
-extern int pgt_cache_water[];
-
static int parse_table(int *, int, void *, size_t *, void *, size_t,
ctl_table *, void **);
static int proc_doutsstring(ctl_table *table, int write, struct file *filp,
sizeof(sysctl_overcommit_memory), 0644, NULL, &proc_dointvec},
{VM_PAGERDAEMON, "kswapd",
&pager_daemon, sizeof(pager_daemon_t), 0644, NULL, &proc_dointvec},
- {VM_PGT_CACHE, "pagetable_cache",
- &pgt_cache_water, 2*sizeof(int), 0644, NULL, &proc_dointvec},
{VM_PAGE_CLUSTER, "page-cluster",
&page_cluster, sizeof(int), 0644, NULL, &proc_dointvec},
{0}
/* Called with mm->page_table_lock held to protect against other
* threads/the swapper from ripping pte's out from under us.
*/
-static inline int filemap_sync_pte(pte_t * ptep, struct vm_area_struct *vma,
+static inline int filemap_sync_pte(pte_t *ptep, pmd_t *pmdp, struct vm_area_struct *vma,
unsigned long address, unsigned int flags)
{
pte_t pte = *ptep;
}
static inline int filemap_sync_pte_range(pmd_t * pmd,
- unsigned long address, unsigned long size,
- struct vm_area_struct *vma, unsigned long offset, unsigned int flags)
+ unsigned long address, unsigned long end,
+ struct vm_area_struct *vma, unsigned int flags)
{
- pte_t * pte;
- unsigned long end;
+ pte_t *pte;
int error;
if (pmd_none(*pmd))
pmd_clear(pmd);
return 0;
}
- pte = pte_offset(pmd, address);
- offset += address & PMD_MASK;
- address &= ~PMD_MASK;
- end = address + size;
- if (end > PMD_SIZE)
- end = PMD_SIZE;
+ pte = pte_offset_map(pmd, address);
+ if ((address & PMD_MASK) != (end & PMD_MASK))
+ end = (address & PMD_MASK) + PMD_SIZE;
error = 0;
do {
- error |= filemap_sync_pte(pte, vma, address + offset, flags);
+ error |= filemap_sync_pte(pte, pmd, vma, address, flags);
address += PAGE_SIZE;
pte++;
} while (address && (address < end));
+
+ pte_unmap(pte - 1);
+
return error;
}
static inline int filemap_sync_pmd_range(pgd_t * pgd,
- unsigned long address, unsigned long size,
+ unsigned long address, unsigned long end,
struct vm_area_struct *vma, unsigned int flags)
{
pmd_t * pmd;
- unsigned long offset, end;
int error;
if (pgd_none(*pgd))
return 0;
}
pmd = pmd_offset(pgd, address);
- offset = address & PGDIR_MASK;
- address &= ~PGDIR_MASK;
- end = address + size;
- if (end > PGDIR_SIZE)
- end = PGDIR_SIZE;
+ if ((address & PGDIR_MASK) != (end & PGDIR_MASK))
+ end = (address & PGDIR_MASK) + PGDIR_SIZE;
error = 0;
do {
- error |= filemap_sync_pte_range(pmd, address, end - address, vma, offset, flags);
+ error |= filemap_sync_pte_range(pmd, address, end, vma, flags);
address = (address + PMD_SIZE) & PMD_MASK;
pmd++;
} while (address && (address < end));
spin_lock(&vma->vm_mm->page_table_lock);
dir = pgd_offset(vma->vm_mm, address);
- flush_cache_range(vma, end - size, end);
+ flush_cache_range(vma, address, end);
if (address >= end)
BUG();
do {
- error |= filemap_sync_pmd_range(dir, address, end - address, vma, flags);
+ error |= filemap_sync_pmd_range(dir, address, end, vma, flags);
address = (address + PGDIR_SIZE) & PGDIR_MASK;
dir++;
} while (address && (address < end));
#include <linux/pagemap.h>
#include <linux/mempool.h>
#include <linux/blkdev.h>
+#include <asm/pgalloc.h>
static mempool_t *page_pool, *isa_page_pool;
bio->bi_private = *bio_orig;
*bio_orig = bio;
}
+
+#if CONFIG_DEBUG_HIGHMEM
+void check_highmem_ptes(void)
+{
+ int idx, type;
+
+ for (type = 0; type < KM_TYPE_NR; type++) {
+ idx = type + KM_TYPE_NR*smp_processor_id();
+ if (!pte_none(*(kmap_pte-idx))) {
+ printk("scheduling with KM_TYPE %d held!\n", type);
+ BUG();
+ }
+ }
+}
+#endif
+
*/
static inline void free_one_pmd(pmd_t * dir)
{
- pte_t * pte;
+ struct page *pte;
if (pmd_none(*dir))
return;
pmd_clear(dir);
return;
}
- pte = pte_offset(dir, 0);
+ pte = pmd_page(*dir);
pmd_clear(dir);
pte_free(pte);
}
pmd_free(pmd);
}
-/* Low and high watermarks for page table cache.
- The system should try to have pgt_water[0] <= cache elements <= pgt_water[1]
- */
-int pgt_cache_water[2] = { 25, 50 };
-
-/* Returns the number of pages freed */
-int check_pgt_cache(void)
-{
- return do_check_pgt_cache(pgt_cache_water[0], pgt_cache_water[1]);
-}
-
-
/*
* This function clears all user-level page tables of a process - this
* is needed by execve(), so that old pages aren't in the way.
page_dir++;
} while (--nr);
spin_unlock(&mm->page_table_lock);
+}
+
+pte_t * pte_alloc_map(struct mm_struct *mm, pmd_t *pmd, unsigned long address)
+{
+ if (!pmd_present(*pmd)) {
+ struct page *new;
- /* keep the page table cache within bounds */
- check_pgt_cache();
+ spin_unlock(&mm->page_table_lock);
+ new = pte_alloc_one(mm, address);
+ spin_lock(&mm->page_table_lock);
+ if (!new)
+ return NULL;
+
+ /*
+ * Because we dropped the lock, we should re-check the
+ * entry, as somebody else could have populated it..
+ */
+ if (pmd_present(*pmd)) {
+ pte_free(new);
+ goto out;
+ }
+ pmd_populate(mm, pmd, new);
+ }
+out:
+ if (pmd_present(*pmd))
+ return pte_offset_map(pmd, address);
+ return NULL;
}
+pte_t * pte_alloc_kernel(struct mm_struct *mm, pmd_t *pmd, unsigned long address)
+{
+ if (!pmd_present(*pmd)) {
+ pte_t *new;
+
+ spin_unlock(&mm->page_table_lock);
+ new = pte_alloc_one_kernel(mm, address);
+ spin_lock(&mm->page_table_lock);
+ if (!new)
+ return NULL;
+
+ /*
+ * Because we dropped the lock, we should re-check the
+ * entry, as somebody else could have populated it..
+ */
+ if (pmd_present(*pmd)) {
+ pte_free_kernel(new);
+ goto out;
+ }
+ pmd_populate_kernel(mm, pmd, new);
+ }
+out:
+ return pte_offset_kernel(pmd, address);
+}
#define PTE_TABLE_MASK ((PTRS_PER_PTE-1) * sizeof(pte_t))
#define PMD_TABLE_MASK ((PTRS_PER_PMD-1) * sizeof(pmd_t))
* variable count and make things faster. -jj
*
* dst->page_table_lock is held on entry and exit,
- * but may be dropped within pmd_alloc() and pte_alloc().
+ * but may be dropped within pmd_alloc() and pte_alloc_map().
*/
int copy_page_range(struct mm_struct *dst, struct mm_struct *src,
struct vm_area_struct *vma)
goto cont_copy_pmd_range;
}
- src_pte = pte_offset(src_pmd, address);
- dst_pte = pte_alloc(dst, dst_pmd, address);
+ dst_pte = pte_alloc_map(dst, dst_pmd, address);
if (!dst_pte)
goto nomem;
-
spin_lock(&src->page_table_lock);
+ src_pte = pte_offset_map2(src_pmd, address);
do {
pte_t pte = *src_pte;
struct page *ptepage;
cont_copy_pte_range: set_pte(dst_pte, pte);
cont_copy_pte_range_noset: address += PAGE_SIZE;
- if (address >= end)
+ if (address >= end) {
+ pte_unmap2(src_pte);
+ pte_unmap(dst_pte);
goto out_unlock;
+ }
src_pte++;
dst_pte++;
} while ((unsigned long)src_pte & PTE_TABLE_MASK);
+ pte_unmap2(src_pte-1);
+ pte_unmap(dst_pte-1);
spin_unlock(&src->page_table_lock);
cont_copy_pmd_range: src_pmd++;
static inline int zap_pte_range(mmu_gather_t *tlb, pmd_t * pmd, unsigned long address, unsigned long size)
{
unsigned long offset;
- pte_t * ptep;
+ pte_t *ptep;
int freed = 0;
if (pmd_none(*pmd))
pmd_clear(pmd);
return 0;
}
- ptep = pte_offset(pmd, address);
+ ptep = pte_offset_map(pmd, address);
offset = address & ~PMD_MASK;
if (offset + size > PMD_SIZE)
size = PMD_SIZE - offset;
pte_clear(ptep);
}
}
+ pte_unmap(ptep-1);
return freed;
}
if (pmd_none(*pmd) || pmd_bad(*pmd))
goto out;
- ptep = pte_offset(pmd, address);
- if (!ptep)
+ preempt_disable();
+ ptep = pte_offset_map(pmd, address);
+ if (!ptep) {
+ preempt_enable();
goto out;
+ }
pte = *ptep;
+ pte_unmap(ptep);
+ preempt_enable();
if (pte_present(pte)) {
if (!write ||
(pte_write(pte) && pte_dirty(pte)))
if (end > PGDIR_SIZE)
end = PGDIR_SIZE;
do {
- pte_t * pte = pte_alloc(mm, pmd, address);
+ pte_t * pte = pte_alloc_map(mm, pmd, address);
if (!pte)
return -ENOMEM;
zeromap_pte_range(pte, address, end - address, prot);
+ pte_unmap(pte);
address = (address + PMD_SIZE) & PMD_MASK;
pmd++;
} while (address && (address < end));
end = PGDIR_SIZE;
phys_addr -= address;
do {
- pte_t * pte = pte_alloc(mm, pmd, address);
+ pte_t * pte = pte_alloc_map(mm, pmd, address);
if (!pte)
return -ENOMEM;
remap_pte_range(pte, address, end - address, address + phys_addr, prot);
+ pte_unmap(pte);
address = (address + PMD_SIZE) & PMD_MASK;
pmd++;
} while (address && (address < end));
* with the page_table_lock released.
*/
static int do_wp_page(struct mm_struct *mm, struct vm_area_struct * vma,
- unsigned long address, pte_t *page_table, pte_t pte)
+ unsigned long address, pte_t *page_table, pmd_t *pmd, pte_t pte)
{
struct page *old_page, *new_page;
if (reuse) {
flush_cache_page(vma, address);
establish_pte(vma, address, page_table, pte_mkyoung(pte_mkdirty(pte_mkwrite(pte))));
+ pte_unmap(page_table);
spin_unlock(&mm->page_table_lock);
return 1; /* Minor fault */
}
}
+ pte_unmap(page_table);
/*
* Ok, we need to copy. Oh, well..
* Re-check the pte - we dropped the lock
*/
spin_lock(&mm->page_table_lock);
+ page_table = pte_offset_map(pmd, address);
if (pte_same(*page_table, pte)) {
if (PageReserved(old_page))
++mm->rss;
/* Free the old page.. */
new_page = old_page;
}
+ pte_unmap(page_table);
spin_unlock(&mm->page_table_lock);
page_cache_release(new_page);
page_cache_release(old_page);
return 1; /* Minor fault */
bad_wp_page:
+ pte_unmap(page_table);
spin_unlock(&mm->page_table_lock);
printk("do_wp_page: bogus page at address %08lx (page 0x%lx)\n",address,(unsigned long)old_page);
return -1;
*/
static int do_swap_page(struct mm_struct * mm,
struct vm_area_struct * vma, unsigned long address,
- pte_t * page_table, pte_t orig_pte, int write_access)
+ pte_t *page_table, pmd_t *pmd, pte_t orig_pte, int write_access)
{
struct page *page;
swp_entry_t entry = pte_to_swp_entry(orig_pte);
pte_t pte;
int ret = 1;
+ pte_unmap(page_table);
spin_unlock(&mm->page_table_lock);
page = lookup_swap_cache(entry);
if (!page) {
*/
int retval;
spin_lock(&mm->page_table_lock);
+ page_table = pte_offset_map(pmd, address);
retval = pte_same(*page_table, orig_pte) ? -1 : 1;
+ pte_unmap(page_table);
spin_unlock(&mm->page_table_lock);
return retval;
}
* released the page table lock.
*/
spin_lock(&mm->page_table_lock);
+ page_table = pte_offset_map(pmd, address);
if (!pte_same(*page_table, orig_pte)) {
+ pte_unmap(page_table);
spin_unlock(&mm->page_table_lock);
unlock_page(page);
page_cache_release(page);
/* No need to invalidate - it was non-present before */
update_mmu_cache(vma, address, pte);
+ pte_unmap(page_table);
spin_unlock(&mm->page_table_lock);
return ret;
}
* spinlock held to protect against concurrent faults in
* multithreaded programs.
*/
-static int do_anonymous_page(struct mm_struct * mm, struct vm_area_struct * vma, pte_t *page_table, int write_access, unsigned long addr)
+static int do_anonymous_page(struct mm_struct * mm, struct vm_area_struct * vma, pte_t *page_table, pmd_t *pmd, int write_access, unsigned long addr)
{
pte_t entry;
struct page *page;
/* Allocate our own private page. */
+ pte_unmap(page_table);
spin_unlock(&mm->page_table_lock);
page = alloc_page(GFP_HIGHUSER);
clear_user_highpage(page, addr);
spin_lock(&mm->page_table_lock);
+ page_table = pte_offset_map(pmd, addr);
+
if (!pte_none(*page_table)) {
+ pte_unmap(page_table);
page_cache_release(page);
spin_unlock(&mm->page_table_lock);
return 1;
}
set_pte(page_table, entry);
+ pte_unmap(page_table);
/* No need to invalidate - it was non-present before */
update_mmu_cache(vma, addr, entry);
* spinlock held. Exit with the spinlock released.
*/
static int do_no_page(struct mm_struct * mm, struct vm_area_struct * vma,
- unsigned long address, int write_access, pte_t *page_table)
+ unsigned long address, int write_access, pte_t *page_table, pmd_t *pmd)
{
struct page * new_page;
pte_t entry;
if (!vma->vm_ops || !vma->vm_ops->nopage)
- return do_anonymous_page(mm, vma, page_table, write_access, address);
+ return do_anonymous_page(mm, vma, page_table, pmd, write_access, address);
+ pte_unmap(page_table);
spin_unlock(&mm->page_table_lock);
new_page = vma->vm_ops->nopage(vma, address & PAGE_MASK, 0);
}
spin_lock(&mm->page_table_lock);
+ page_table = pte_offset_map(pmd, address);
+
/*
* This silly early PAGE_DIRTY setting removes a race
* due to the bad i386 page protection. But it's valid
if (write_access)
entry = pte_mkwrite(pte_mkdirty(entry));
set_pte(page_table, entry);
+ pte_unmap(page_table);
} else {
/* One of our sibling threads was faster, back out. */
+ pte_unmap(page_table);
page_cache_release(new_page);
spin_unlock(&mm->page_table_lock);
return 1;
*/
static inline int handle_pte_fault(struct mm_struct *mm,
struct vm_area_struct * vma, unsigned long address,
- int write_access, pte_t * pte)
+ int write_access, pte_t *pte, pmd_t *pmd)
{
pte_t entry;
* drop the lock.
*/
if (pte_none(entry))
- return do_no_page(mm, vma, address, write_access, pte);
- return do_swap_page(mm, vma, address, pte, entry, write_access);
+ return do_no_page(mm, vma, address, write_access, pte, pmd);
+ return do_swap_page(mm, vma, address, pte, pmd, entry, write_access);
}
if (write_access) {
if (!pte_write(entry))
- return do_wp_page(mm, vma, address, pte, entry);
+ return do_wp_page(mm, vma, address, pte, pmd, entry);
entry = pte_mkdirty(entry);
}
entry = pte_mkyoung(entry);
establish_pte(vma, address, pte, entry);
+ pte_unmap(pte);
spin_unlock(&mm->page_table_lock);
return 1;
}
pmd = pmd_alloc(mm, pgd, address);
if (pmd) {
- pte_t * pte = pte_alloc(mm, pmd, address);
+ pte_t * pte = pte_alloc_map(mm, pmd, address);
if (pte)
- return handle_pte_fault(mm, vma, address, write_access, pte);
+ return handle_pte_fault(mm, vma, address, write_access, pte, pmd);
}
spin_unlock(&mm->page_table_lock);
return -1;
{
pmd_t *new;
- /* "fast" allocation can happen without dropping the lock.. */
- new = pmd_alloc_one_fast(mm, address);
- if (!new) {
- spin_unlock(&mm->page_table_lock);
- new = pmd_alloc_one(mm, address);
- spin_lock(&mm->page_table_lock);
- if (!new)
- return NULL;
+ spin_unlock(&mm->page_table_lock);
+ new = pmd_alloc_one(mm, address);
+ spin_lock(&mm->page_table_lock);
+ if (!new)
+ return NULL;
- /*
- * Because we dropped the lock, we should re-check the
- * entry, as somebody else could have populated it..
- */
- if (!pgd_none(*pgd)) {
- pmd_free(new);
- goto out;
- }
+ /*
+ * Because we dropped the lock, we should re-check the
+ * entry, as somebody else could have populated it..
+ */
+ if (pgd_present(*pgd)) {
+ pmd_free(new);
+ goto out;
}
pgd_populate(mm, pgd, new);
out:
return pmd_offset(pgd, address);
}
-/*
- * Allocate the page table directory.
- *
- * We've already handled the fast-path in-line, and we own the
- * page table lock.
- */
-pte_t *pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address)
-{
- if (pmd_none(*pmd)) {
- pte_t *new;
-
- /* "fast" allocation can happen without dropping the lock.. */
- new = pte_alloc_one_fast(mm, address);
- if (!new) {
- spin_unlock(&mm->page_table_lock);
- new = pte_alloc_one(mm, address);
- spin_lock(&mm->page_table_lock);
- if (!new)
- return NULL;
-
- /*
- * Because we dropped the lock, we should re-check the
- * entry, as somebody else could have populated it..
- */
- if (!pmd_none(*pmd)) {
- pte_free(new);
- goto out;
- }
- }
- pmd_populate(mm, pmd, new);
- }
-out:
- return pte_offset(pmd, address);
-}
-
int make_pages_present(unsigned long addr, unsigned long end)
{
int ret, len, write;
#include <asm/uaccess.h>
#include <asm/pgalloc.h>
#include <asm/pgtable.h>
+#include <linux/highmem.h>
static inline void change_pte_range(pmd_t * pmd, unsigned long address,
unsigned long size, pgprot_t newprot)
pmd_clear(pmd);
return;
}
- pte = pte_offset(pmd, address);
+ pte = pte_offset_map(pmd, address);
address &= ~PMD_MASK;
end = address + size;
if (end > PMD_SIZE)
address += PAGE_SIZE;
pte++;
} while (address && (address < end));
+ pte_unmap(pte - 1);
}
static inline void change_pmd_range(pgd_t * pgd, unsigned long address,
extern int vm_enough_memory(long pages);
-static inline pte_t *get_one_pte(struct mm_struct *mm, unsigned long addr)
+static inline pte_t *get_one_pte_map2(struct mm_struct *mm, unsigned long addr)
{
pgd_t * pgd;
pmd_t * pmd;
goto end;
}
- pte = pte_offset(pmd, addr);
- if (pte_none(*pte))
+ pte = pte_offset_map2(pmd, addr);
+ if (pte_none(*pte)) {
+ pte_unmap2(pte);
pte = NULL;
+ }
end:
return pte;
}
-static inline pte_t *alloc_one_pte(struct mm_struct *mm, unsigned long addr)
+static inline pte_t *alloc_one_pte_map(struct mm_struct *mm, unsigned long addr)
{
pmd_t * pmd;
pte_t * pte = NULL;
pmd = pmd_alloc(mm, pgd_offset(mm, addr), addr);
if (pmd)
- pte = pte_alloc(mm, pmd, addr);
+ pte = pte_alloc_map(mm, pmd, addr);
return pte;
}
static int move_one_page(struct mm_struct *mm, unsigned long old_addr, unsigned long new_addr)
{
int error = 0;
- pte_t * src;
+ pte_t *src, *dst;
spin_lock(&mm->page_table_lock);
- src = get_one_pte(mm, old_addr);
- if (src)
- error = copy_one_pte(mm, src, alloc_one_pte(mm, new_addr));
+ src = get_one_pte_map2(mm, old_addr);
+ if (src) {
+ dst = alloc_one_pte_map(mm, new_addr);
+ error = copy_one_pte(mm, src, dst);
+ pte_unmap2(src);
+ pte_unmap(dst);
+ }
spin_unlock(&mm->page_table_lock);
return error;
}
pmd_clear(dir);
return;
}
- pte = pte_offset(dir, address);
+ pte = pte_offset_map(dir, address);
offset += address & PMD_MASK;
address &= ~PMD_MASK;
end = address + size;
address += PAGE_SIZE;
pte++;
} while (address && (address < end));
+ pte_unmap(pte - 1);
}
/* mmlist_lock and vma->vm_mm->page_table_lock are held */
pmd_clear(pmd);
return;
}
- pte = pte_offset(pmd, address);
+ pte = pte_offset_kernel(pmd, address);
address &= ~PMD_MASK;
end = address + size;
if (end > PMD_SIZE)
if (end > PGDIR_SIZE)
end = PGDIR_SIZE;
do {
- pte_t * pte = pte_alloc(&init_mm, pmd, address);
+ pte_t * pte = pte_alloc_kernel(&init_mm, pmd, address);
if (!pte)
return -ENOMEM;
if (alloc_area_pte(pte, address, end - address, gfp_mask, prot))
return count;
}
- pte = pte_offset(dir, address);
+ pte = pte_offset_map(dir, address);
pmd_end = (address + PMD_SIZE) & PMD_MASK;
if (end > pmd_end)
count -= try_to_swap_out(mm, vma, address, pte, page, classzone);
if (!count) {
address += PAGE_SIZE;
+ pte++;
break;
}
}
address += PAGE_SIZE;
pte++;
} while (address && (address < end));
+ pte_unmap(pte - 1);
mm->swap_address = address;
return count;
}
projects / linux / commitdiff