[SPARC64]: Fix PCI IOMMU invalid iopte handling.

Instead of marking them as invalid, point them
at a dummy page.  This handles buggy third-party
bridges that erroneously prefetch sometimes.

Signed-off-by: David S. Miller <davem@redhat.com>

diff --git a/arch/sparc64/kernel/pci_iommu.c b/arch/sparc64/kernel/pci_iommu.c
index cca82a12ed22340bf1c5938621c074149d5cf970..94a52983c45269d305e7531b0229ef8f040e28cd 100644 (file)
--- a/arch/sparc64/kernel/pci_iommu.c
+++ b/arch/sparc64/kernel/pci_iommu.c
@@ -56,6 +56,39 @@ static void __iommu_flushall(struct pci_iommu *iommu)
        }
 }
 
+#define IOPTE_CONSISTENT(CTX) \
+       (IOPTE_VALID | IOPTE_CACHE | \
+        (((CTX) << 47) & IOPTE_CONTEXT))
+
+#define IOPTE_STREAMING(CTX) \
+       (IOPTE_CONSISTENT(CTX) | IOPTE_STBUF)
+
+/* Existing mappings are never marked invalid, instead they
+ * are pointed to a dummy page.
+ */
+#define IOPTE_IS_DUMMY(iommu, iopte)   \
+       ((iopte_val(*iopte) & IOPTE_PAGE) == (iommu)->dummy_page_pa)
+
+static void inline iopte_make_dummy(struct pci_iommu *iommu, iopte_t *iopte)
+{
+       unsigned long val = iopte_val(*iopte);
+
+       val &= ~IOPTE_PAGE;
+       val |= iommu->dummy_page_pa;
+
+       iopte_val(*iopte) = val;
+}
+
+void pci_iommu_table_init(struct pci_iommu *iommu, int tsbsize)
+{
+       int i;
+
+       tsbsize /= sizeof(iopte_t);
+
+       for (i = 0; i < tsbsize; i++)
+               iopte_make_dummy(iommu, &iommu->page_table[i]);
+}
+
 static iopte_t *alloc_streaming_cluster(struct pci_iommu *iommu, unsigned long npages)
 {
        iopte_t *iopte, *limit, *first;
@@ -79,7 +112,7 @@ static iopte_t *alloc_streaming_cluster(struct pci_iommu *iommu, unsigned long n
        
        first = iopte;
        for (;;) {
-               if (iopte_val(*iopte) == 0UL) {
+               if (IOPTE_IS_DUMMY(iommu, iopte)) {
                        if ((iopte + (1 << cnum)) >= limit)
                                ent = 0;
                        else
@@ -142,12 +175,12 @@ static iopte_t *alloc_consistent_cluster(struct pci_iommu *iommu, unsigned long
        iopte = iommu->page_table + (1 << (iommu->page_table_sz_bits - PBM_LOGCLUSTERS));
        while (iopte > iommu->page_table) {
                iopte--;
-               if (!(iopte_val(*iopte) & IOPTE_VALID)) {
+               if (IOPTE_IS_DUMMY(iommu, iopte)) {
                        unsigned long tmp = npages;
 
                        while (--tmp) {
                                iopte--;
-                               if (iopte_val(*iopte) & IOPTE_VALID)
+                               if (!IOPTE_IS_DUMMY(iommu, iopte))
                                        break;
                        }
                        if (tmp == 0) {
@@ -162,15 +195,6 @@ static iopte_t *alloc_consistent_cluster(struct pci_iommu *iommu, unsigned long
        return NULL;
 }
 
-#define IOPTE_CONSISTENT(CTX) \
-       (IOPTE_VALID | IOPTE_CACHE | \
-        (((CTX) << 47) & IOPTE_CONTEXT))
-
-#define IOPTE_STREAMING(CTX) \
-       (IOPTE_CONSISTENT(CTX) | IOPTE_STBUF)
-
-#define IOPTE_INVALID  0UL
-
 /* Allocate and map kernel buffer of size SIZE using consistent mode
  * DMA for PCI device PDEV.  Return non-NULL cpu-side address if
  * successful and set *DMA_ADDRP to the PCI side dma address.
@@ -261,7 +285,7 @@ void pci_free_consistent(struct pci_dev *pdev, size_t size, void *cpu, dma_addr_
                limit = (iommu->page_table +
                         (1 << (iommu->page_table_sz_bits - PBM_LOGCLUSTERS)));
                while (walk < limit) {
-                       if (iopte_val(*walk) != IOPTE_INVALID)
+                       if (!IOPTE_IS_DUMMY(iommu, walk))
                                break;
                        walk++;
                }
@@ -280,7 +304,7 @@ void pci_free_consistent(struct pci_dev *pdev, size_t size, void *cpu, dma_addr_
                ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL;
 
        for (i = 0; i < npages; i++, iopte++)
-               iopte_val(*iopte) = IOPTE_INVALID;
+               iopte_make_dummy(iommu, iopte);
 
        if (iommu->iommu_ctxflush) {
                pci_iommu_write(iommu->iommu_ctxflush, ctx);
@@ -376,7 +400,7 @@ void pci_unmap_single(struct pci_dev *pdev, dma_addr_t bus_addr, size_t sz, int
        base = iommu->page_table +
                ((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT);
 #ifdef DEBUG_PCI_IOMMU
-       if (iopte_val(*base) == IOPTE_INVALID)
+       if (IOPTE_IS_DUMMY(iommu, base))
                printk("pci_unmap_single called on non-mapped region %08x,%08x from %016lx\n",
                       bus_addr, sz, __builtin_return_address(0));
 #endif
@@ -415,7 +439,7 @@ void pci_unmap_single(struct pci_dev *pdev, dma_addr_t bus_addr, size_t sz, int
        }
 
        /* Step 2: Clear out first TSB entry. */
-       iopte_val(*base) = IOPTE_INVALID;
+       iopte_make_dummy(iommu, base);
 
        free_streaming_cluster(iommu, bus_addr - iommu->page_table_map_base,
                               npages, ctx);
@@ -611,7 +635,7 @@ void pci_unmap_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems,
                ((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT);
 
 #ifdef DEBUG_PCI_IOMMU
-       if (iopte_val(*base) == IOPTE_INVALID)
+       if (IOPTE_IS_DUMMY(iommu, base))
                printk("pci_unmap_sg called on non-mapped region %016lx,%d from %016lx\n", sglist->dma_address, nelems, __builtin_return_address(0));
 #endif
 
@@ -648,7 +672,7 @@ void pci_unmap_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems,
        }
 
        /* Step 2: Clear out first TSB entry. */
-       iopte_val(*base) = IOPTE_INVALID;
+       iopte_make_dummy(iommu, base);
 
        free_streaming_cluster(iommu, bus_addr - iommu->page_table_map_base,
                               npages, ctx);

diff --git a/arch/sparc64/kernel/pci_psycho.c b/arch/sparc64/kernel/pci_psycho.c
index dc4d63084daee99a49fa295144bb4c0f3789af2d..3e1e1dae35162117b13826a56b2f21cca5a9ad2a 100644 (file)
--- a/arch/sparc64/kernel/pci_psycho.c
+++ b/arch/sparc64/kernel/pci_psycho.c
@@ -1241,6 +1241,14 @@ static void __init psycho_iommu_init(struct pci_controller_info *p)
         * in pci_iommu.c
         */
 
+       iommu->dummy_page = __get_free_pages(GFP_KERNEL, 0);
+       if (!iommu->dummy_page) {
+               prom_printf("PSYCHO_IOMMU: Error, gfp(dummy_page) failed.\n");
+               prom_halt();
+       }
+       memset((void *)iommu->dummy_page, 0, PAGE_SIZE);
+       iommu->dummy_page_pa = (unsigned long) __pa(iommu->dummy_page);
+
        /* Using assumed page size 8K with 128K entries we need 1MB iommu page
         * table (128K ioptes * 8 bytes per iopte).  This is
         * page order 7 on UltraSparc.
@@ -1254,7 +1262,7 @@ static void __init psycho_iommu_init(struct pci_controller_info *p)
        iommu->page_table_sz_bits = 17;
        iommu->page_table_map_base = 0xc0000000;
        iommu->dma_addr_mask = 0xffffffff;
-       memset((char *)tsbbase, 0, IO_TSB_SIZE);
+       pci_iommu_table_init(iommu, IO_TSB_SIZE);
 
        /* We start with no consistent mappings. */
        iommu->lowest_consistent_map =

diff --git a/arch/sparc64/kernel/pci_sabre.c b/arch/sparc64/kernel/pci_sabre.c
index 6a266a7bf0a16a2aa33ed76934e9963a4c61bc3f..7d6e5149a4e9a94e9b70bef175082ddc005f4b57 100644 (file)
--- a/arch/sparc64/kernel/pci_sabre.c
+++ b/arch/sparc64/kernel/pci_sabre.c
@@ -1293,6 +1293,14 @@ static void __init sabre_iommu_init(struct pci_controller_info *p,
         * in pci_iommu.c
         */
 
+       iommu->dummy_page = __get_free_pages(GFP_KERNEL, 0);
+       if (!iommu->dummy_page) {
+               prom_printf("PSYCHO_IOMMU: Error, gfp(dummy_page) failed.\n");
+               prom_halt();
+       }
+       memset((void *)iommu->dummy_page, 0, PAGE_SIZE);
+       iommu->dummy_page_pa = (unsigned long) __pa(iommu->dummy_page);
+
        tsbbase = __get_free_pages(GFP_KERNEL, order = get_order(tsbsize * 1024 * 8));
        if (!tsbbase) {
                prom_printf("SABRE_IOMMU: Error, gfp(tsb) failed.\n");
@@ -1301,7 +1309,7 @@ static void __init sabre_iommu_init(struct pci_controller_info *p,
        iommu->page_table = (iopte_t *)tsbbase;
        iommu->page_table_map_base = dvma_offset;
        iommu->dma_addr_mask = dma_mask;
-       memset((char *)tsbbase, 0, PAGE_SIZE << order);
+       pci_iommu_table_init(iommu, PAGE_SIZE << order);
 
        sabre_write(p->pbm_A.controller_regs + SABRE_IOMMU_TSBBASE, __pa(tsbbase));
 

diff --git a/arch/sparc64/kernel/pci_schizo.c b/arch/sparc64/kernel/pci_schizo.c
index be6fa906a54d30b941a06da7c1c023c0430ba343..3f90de646f7f6a46d12eeca704c4989e03adeb15 100644 (file)
--- a/arch/sparc64/kernel/pci_schizo.c
+++ b/arch/sparc64/kernel/pci_schizo.c
@@ -1766,6 +1766,14 @@ static void schizo_pbm_iommu_init(struct pci_pbm_info *pbm)
         * in pci_iommu.c
         */
 
+       iommu->dummy_page = __get_free_pages(GFP_KERNEL, 0);
+       if (!iommu->dummy_page) {
+               prom_printf("PSYCHO_IOMMU: Error, gfp(dummy_page) failed.\n");
+               prom_halt();
+       }
+       memset((void *)iommu->dummy_page, 0, PAGE_SIZE);
+       iommu->dummy_page_pa = (unsigned long) __pa(iommu->dummy_page);
+
        /* Using assumed page size 8K with 128K entries we need 1MB iommu page
         * table (128K ioptes * 8 bytes per iopte).  This is
         * page order 7 on UltraSparc.
@@ -1780,7 +1788,7 @@ static void schizo_pbm_iommu_init(struct pci_pbm_info *pbm)
        iommu->page_table = (iopte_t *)tsbbase;
        iommu->page_table_map_base = vdma[0];
        iommu->dma_addr_mask = dma_mask;
-       memset((char *)tsbbase, 0, PAGE_SIZE << order);
+       pci_iommu_table_init(iommu, PAGE_SIZE << order);
 
        switch (tsbsize) {
        case 64:

diff --git a/include/asm-sparc64/pbm.h b/include/asm-sparc64/pbm.h
index 0cddbd2bf2c6e1caf62bd9fcce1e4f9ef8e70821..92999631c81966a7a310d6b718955b83fd378fbe 100644 (file)
--- a/include/asm-sparc64/pbm.h
+++ b/include/asm-sparc64/pbm.h
@@ -70,6 +70,13 @@ struct pci_iommu {
         */
        u32             lowest_consistent_map;
 
+       /* In order to deal with some buggy third-party PCI bridges that
+        * do wrong prefetching, we never mark valid mappings as invalid.
+        * Instead we point them at this dummy page.
+        */
+       unsigned long   dummy_page;
+       unsigned long   dummy_page_pa;
+
        /* If PBM_NCLUSTERS is ever decreased to 4 or lower,
         * or if largest supported page_table_sz * 8K goes above
         * 2GB, you must increase the size of the type of
@@ -93,6 +100,8 @@ struct pci_iommu {
        u32 dma_addr_mask;
 };
 
+extern void pci_iommu_table_init(struct pci_iommu *, int);
+
 /* This describes a PCI bus module's streaming buffer. */
 struct pci_strbuf {
        int             strbuf_enabled;         /* Present and using it? */