#include <asm/tlb.h>
#include <asm/hvcall.h>
-
-long plpar_pte_enter(unsigned long flags,
- unsigned long ptex,
- unsigned long new_pteh, unsigned long new_ptel,
- unsigned long *old_pteh_ret, unsigned long *old_ptel_ret)
-{
- unsigned long dummy, ret;
- ret = plpar_hcall(H_ENTER, flags, ptex, new_pteh, new_ptel,
- old_pteh_ret, old_ptel_ret, &dummy);
- return(ret);
-}
-
long plpar_pte_remove(unsigned long flags,
unsigned long ptex,
unsigned long avpn,
tce_ret, &dummy, &dummy);
}
-
long plpar_tce_put(unsigned long liobn,
unsigned long ioba,
unsigned long tceval)
unsigned long len,
const char *buffer)
{
- unsigned long dummy;
unsigned long *lbuf = (unsigned long *)buffer; /* ToDo: alignment? */
- return plpar_hcall(H_PUT_TERM_CHAR, termno, len,
- lbuf[0], lbuf[1], &dummy, &dummy, &dummy);
+ return plpar_hcall_norets(H_PUT_TERM_CHAR, termno, len, lbuf[0],
+ lbuf[1]);
}
static void tce_build_pSeriesLP(struct TceTable *tbl, long tcenum,
int hvc_put_chars(int index, const char *buf, int count)
{
- unsigned long dummy;
unsigned long *lbuf = (unsigned long *) buf;
long ret;
- ret = plpar_hcall(H_PUT_TERM_CHAR, index, count, lbuf[0], lbuf[1],
- &dummy, &dummy, &dummy);
+ ret = plpar_hcall_norets(H_PUT_TERM_CHAR, index, count, lbuf[0],
+ lbuf[1]);
if (ret == H_Success)
return count;
if (ret == H_Busy)
-
long pSeries_lpar_hpte_insert(unsigned long hpte_group,
unsigned long va, unsigned long prpn,
int secondary, unsigned long hpteflags,
unsigned long flags;
unsigned long slot;
HPTE lhpte;
+ unsigned long dummy0, dummy1;
/* Fill in the local HPTE with absolute rpn, avpn and flags */
lhpte.dw1.dword1 = 0;
/* Now fill in the actual HPTE */
/* Set CEC cookie to 0 */
- /* Large page = 0 */
/* Zero page = 0 */
/* I-cache Invalidate = 0 */
/* I-cache synchronize = 0 */
if (hpteflags & (_PAGE_GUARDED|_PAGE_NO_CACHE))
lhpte.dw1.flags.flags &= ~_PAGE_COHERENT;
- __asm__ __volatile__ (
- H_ENTER_r3
- "mr 4, %2\n"
- "mr 5, %3\n"
- "mr 6, %4\n"
- "mr 7, %5\n"
- HSC
- "mr %0, 3\n"
- "mr %1, 4\n"
- : "=r" (lpar_rc), "=r" (slot)
- : "r" (flags), "r" (hpte_group), "r" (lhpte.dw0.dword0),
- "r" (lhpte.dw1.dword1)
- : "r3", "r4", "r5", "r6", "r7", "cc");
+ lpar_rc = plpar_hcall(H_ENTER, flags, hpte_group, lhpte.dw0.dword0,
+ lhpte.dw1.dword1, &slot, &dummy0, &dummy1);
if (lpar_rc == H_PTEG_Full)
return -1;
static int htlbpage_free; /* = 0 */
static int htlbpage_total; /* = 0 */
-static LIST_HEAD(htlbpage_freelist);
+static struct list_head hugepage_freelists[MAX_NUMNODES];
+
+static void enqueue_huge_page(struct page *page)
+{
+ list_add(&page->list,
+ &hugepage_freelists[page_zone(page)->zone_pgdat->node_id]);
+}
+
+/* XXX make this a sysctl */
+unsigned long largepage_roundrobin = 1;
+
+static struct page *dequeue_huge_page(void)
+{
+ static int nid = 0;
+ struct page *page = NULL;
+ int i;
+
+ if (!largepage_roundrobin)
+ nid = numa_node_id();
+
+ for (i = 0; i < numnodes; i++) {
+ if (!list_empty(&hugepage_freelists[nid]))
+ break;
+ nid = (nid + 1) % numnodes;
+ }
+
+ if (!list_empty(&hugepage_freelists[nid])) {
+ page = list_entry(hugepage_freelists[nid].next, struct page, list);
+ list_del(&page->list);
+ }
+
+ if (largepage_roundrobin)
+ nid = (nid + 1) % numnodes;
+
+ return page;
+}
+
+static struct page *alloc_fresh_huge_page(void)
+{
+ static int nid = 0;
+ struct page *page;
+
+ page = alloc_pages_node(nid, GFP_HIGHUSER, HUGETLB_PAGE_ORDER);
+ if (!page)
+ return NULL;
+
+ nid = page_zone(page)->zone_pgdat->node_id;
+ nid = (nid + 1) % numnodes;
+ return page;
+}
/* HugePTE layout:
*
struct page *page;
spin_lock(&htlbpage_lock);
- if (list_empty(&htlbpage_freelist)) {
+ page = dequeue_huge_page();
+ if (!page) {
spin_unlock(&htlbpage_lock);
return NULL;
}
- page = list_entry(htlbpage_freelist.next, struct page, list);
- list_del(&page->list);
htlbpage_free--;
spin_unlock(&htlbpage_lock);
set_page_count(page, 1);
INIT_LIST_HEAD(&page->list);
spin_lock(&htlbpage_lock);
- list_add(&page->list, &htlbpage_freelist);
+ enqueue_huge_page(page);
htlbpage_free++;
spin_unlock(&htlbpage_lock);
}
if (!in_hugepage_area(mm->context, ea))
return -1;
+ ea &= ~(HPAGE_SIZE-1);
+
/* We have to find the first hugepte in the batch, since
* that's the one that will store the HPTE flags */
- ptep = hugepte_offset(mm, ea & ~(HPAGE_SIZE-1));
+ ptep = hugepte_offset(mm, ea);
/* Search the Linux page table for a match with va */
va = (vsid << 28) | (ea & 0x0fffffff);
hugepte_val(new_pte) &= ~_HUGEPAGE_HPTEFLAGS;
hugepte_val(new_pte) |= _HUGEPAGE_HASHPTE;
+ /* Add in WIMG bits */
+ /* XXX We should store these in the pte */
+ hpteflags |= _PAGE_COHERENT;
+
slot = ppc_md.hpte_insert(hpte_group, va, prpn, 0,
hpteflags, 0, 1);
}
if (unlikely(slot == -2))
- panic("hash_page: pte_insert failed\n");
+ panic("hash_huge_page: pte_insert failed\n");
hugepte_val(new_pte) |= (slot<<5) & _HUGEPAGE_GROUP_IX;
return htlbpage_total;
if (lcount > 0) { /* Increase the mem size. */
while (lcount--) {
- page = alloc_pages(__GFP_HIGHMEM, HUGETLB_PAGE_ORDER);
+ page = alloc_fresh_huge_page();
if (page == NULL)
break;
spin_lock(&htlbpage_lock);
- list_add(&page->list, &htlbpage_freelist);
+ enqueue_huge_page(page);
htlbpage_free++;
htlbpage_total++;
spin_unlock(&htlbpage_lock);
struct page *page;
if (cur_cpu_spec->cpu_features & CPU_FTR_16M_PAGE) {
+ for (i = 0; i < MAX_NUMNODES; ++i)
+ INIT_LIST_HEAD(&hugepage_freelists[i]);
+
for (i = 0; i < htlbpage_max; ++i) {
- page = alloc_pages(__GFP_HIGHMEM, HUGETLB_PAGE_ORDER);
+ page = alloc_fresh_huge_page();
if (!page)
break;
spin_lock(&htlbpage_lock);
- list_add(&page->list, &htlbpage_freelist);
+ enqueue_huge_page(page);
spin_unlock(&htlbpage_lock);
}
htlbpage_max = htlbpage_free = htlbpage_total = i;
htlbpage_max = 0;
printk("CPU does not support HugeTLB\n");
}
-
+
return 0;
}
module_init(hugetlb_init);
projects / linux / commitdiff