From d360f1ee0f1b4a45cf9ac061eddf0c10596473d4 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Mon, 23 Aug 2004 00:28:39 -0700 Subject: [PATCH] [SPARC64]: Fix PCI IOMMU invalid iopte handling. Instead of marking them as invalid, point them at a dummy page. This handles buggy third-party bridges that erroneously prefetch sometimes. Signed-off-by: David S. Miller --- arch/sparc64/kernel/pci_iommu.c | 60 ++++++++++++++++++++++---------- arch/sparc64/kernel/pci_psycho.c | 10 +++++- arch/sparc64/kernel/pci_sabre.c | 10 +++++- arch/sparc64/kernel/pci_schizo.c | 10 +++++- include/asm-sparc64/pbm.h | 9 +++++ 5 files changed, 78 insertions(+), 21 deletions(-) diff --git a/arch/sparc64/kernel/pci_iommu.c b/arch/sparc64/kernel/pci_iommu.c index cca82a12ed223..94a52983c4526 100644 --- a/arch/sparc64/kernel/pci_iommu.c +++ b/arch/sparc64/kernel/pci_iommu.c @@ -56,6 +56,39 @@ static void __iommu_flushall(struct pci_iommu *iommu) } } +#define IOPTE_CONSISTENT(CTX) \ + (IOPTE_VALID | IOPTE_CACHE | \ + (((CTX) << 47) & IOPTE_CONTEXT)) + +#define IOPTE_STREAMING(CTX) \ + (IOPTE_CONSISTENT(CTX) | IOPTE_STBUF) + +/* Existing mappings are never marked invalid, instead they + * are pointed to a dummy page. + */ +#define IOPTE_IS_DUMMY(iommu, iopte) \ + ((iopte_val(*iopte) & IOPTE_PAGE) == (iommu)->dummy_page_pa) + +static void inline iopte_make_dummy(struct pci_iommu *iommu, iopte_t *iopte) +{ + unsigned long val = iopte_val(*iopte); + + val &= ~IOPTE_PAGE; + val |= iommu->dummy_page_pa; + + iopte_val(*iopte) = val; +} + +void pci_iommu_table_init(struct pci_iommu *iommu, int tsbsize) +{ + int i; + + tsbsize /= sizeof(iopte_t); + + for (i = 0; i < tsbsize; i++) + iopte_make_dummy(iommu, &iommu->page_table[i]); +} + static iopte_t *alloc_streaming_cluster(struct pci_iommu *iommu, unsigned long npages) { iopte_t *iopte, *limit, *first; @@ -79,7 +112,7 @@ static iopte_t *alloc_streaming_cluster(struct pci_iommu *iommu, unsigned long n first = iopte; for (;;) { - if (iopte_val(*iopte) == 0UL) { + if (IOPTE_IS_DUMMY(iommu, iopte)) { if ((iopte + (1 << cnum)) >= limit) ent = 0; else @@ -142,12 +175,12 @@ static iopte_t *alloc_consistent_cluster(struct pci_iommu *iommu, unsigned long iopte = iommu->page_table + (1 << (iommu->page_table_sz_bits - PBM_LOGCLUSTERS)); while (iopte > iommu->page_table) { iopte--; - if (!(iopte_val(*iopte) & IOPTE_VALID)) { + if (IOPTE_IS_DUMMY(iommu, iopte)) { unsigned long tmp = npages; while (--tmp) { iopte--; - if (iopte_val(*iopte) & IOPTE_VALID) + if (!IOPTE_IS_DUMMY(iommu, iopte)) break; } if (tmp == 0) { @@ -162,15 +195,6 @@ static iopte_t *alloc_consistent_cluster(struct pci_iommu *iommu, unsigned long return NULL; } -#define IOPTE_CONSISTENT(CTX) \ - (IOPTE_VALID | IOPTE_CACHE | \ - (((CTX) << 47) & IOPTE_CONTEXT)) - -#define IOPTE_STREAMING(CTX) \ - (IOPTE_CONSISTENT(CTX) | IOPTE_STBUF) - -#define IOPTE_INVALID 0UL - /* Allocate and map kernel buffer of size SIZE using consistent mode * DMA for PCI device PDEV. Return non-NULL cpu-side address if * successful and set *DMA_ADDRP to the PCI side dma address. @@ -261,7 +285,7 @@ void pci_free_consistent(struct pci_dev *pdev, size_t size, void *cpu, dma_addr_ limit = (iommu->page_table + (1 << (iommu->page_table_sz_bits - PBM_LOGCLUSTERS))); while (walk < limit) { - if (iopte_val(*walk) != IOPTE_INVALID) + if (!IOPTE_IS_DUMMY(iommu, walk)) break; walk++; } @@ -280,7 +304,7 @@ void pci_free_consistent(struct pci_dev *pdev, size_t size, void *cpu, dma_addr_ ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL; for (i = 0; i < npages; i++, iopte++) - iopte_val(*iopte) = IOPTE_INVALID; + iopte_make_dummy(iommu, iopte); if (iommu->iommu_ctxflush) { pci_iommu_write(iommu->iommu_ctxflush, ctx); @@ -376,7 +400,7 @@ void pci_unmap_single(struct pci_dev *pdev, dma_addr_t bus_addr, size_t sz, int base = iommu->page_table + ((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT); #ifdef DEBUG_PCI_IOMMU - if (iopte_val(*base) == IOPTE_INVALID) + if (IOPTE_IS_DUMMY(iommu, base)) printk("pci_unmap_single called on non-mapped region %08x,%08x from %016lx\n", bus_addr, sz, __builtin_return_address(0)); #endif @@ -415,7 +439,7 @@ void pci_unmap_single(struct pci_dev *pdev, dma_addr_t bus_addr, size_t sz, int } /* Step 2: Clear out first TSB entry. */ - iopte_val(*base) = IOPTE_INVALID; + iopte_make_dummy(iommu, base); free_streaming_cluster(iommu, bus_addr - iommu->page_table_map_base, npages, ctx); @@ -611,7 +635,7 @@ void pci_unmap_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems, ((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT); #ifdef DEBUG_PCI_IOMMU - if (iopte_val(*base) == IOPTE_INVALID) + if (IOPTE_IS_DUMMY(iommu, base)) printk("pci_unmap_sg called on non-mapped region %016lx,%d from %016lx\n", sglist->dma_address, nelems, __builtin_return_address(0)); #endif @@ -648,7 +672,7 @@ void pci_unmap_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems, } /* Step 2: Clear out first TSB entry. */ - iopte_val(*base) = IOPTE_INVALID; + iopte_make_dummy(iommu, base); free_streaming_cluster(iommu, bus_addr - iommu->page_table_map_base, npages, ctx); diff --git a/arch/sparc64/kernel/pci_psycho.c b/arch/sparc64/kernel/pci_psycho.c index dc4d63084daee..3e1e1dae35162 100644 --- a/arch/sparc64/kernel/pci_psycho.c +++ b/arch/sparc64/kernel/pci_psycho.c @@ -1241,6 +1241,14 @@ static void __init psycho_iommu_init(struct pci_controller_info *p) * in pci_iommu.c */ + iommu->dummy_page = __get_free_pages(GFP_KERNEL, 0); + if (!iommu->dummy_page) { + prom_printf("PSYCHO_IOMMU: Error, gfp(dummy_page) failed.\n"); + prom_halt(); + } + memset((void *)iommu->dummy_page, 0, PAGE_SIZE); + iommu->dummy_page_pa = (unsigned long) __pa(iommu->dummy_page); + /* Using assumed page size 8K with 128K entries we need 1MB iommu page * table (128K ioptes * 8 bytes per iopte). This is * page order 7 on UltraSparc. @@ -1254,7 +1262,7 @@ static void __init psycho_iommu_init(struct pci_controller_info *p) iommu->page_table_sz_bits = 17; iommu->page_table_map_base = 0xc0000000; iommu->dma_addr_mask = 0xffffffff; - memset((char *)tsbbase, 0, IO_TSB_SIZE); + pci_iommu_table_init(iommu, IO_TSB_SIZE); /* We start with no consistent mappings. */ iommu->lowest_consistent_map = diff --git a/arch/sparc64/kernel/pci_sabre.c b/arch/sparc64/kernel/pci_sabre.c index 6a266a7bf0a16..7d6e5149a4e9a 100644 --- a/arch/sparc64/kernel/pci_sabre.c +++ b/arch/sparc64/kernel/pci_sabre.c @@ -1293,6 +1293,14 @@ static void __init sabre_iommu_init(struct pci_controller_info *p, * in pci_iommu.c */ + iommu->dummy_page = __get_free_pages(GFP_KERNEL, 0); + if (!iommu->dummy_page) { + prom_printf("PSYCHO_IOMMU: Error, gfp(dummy_page) failed.\n"); + prom_halt(); + } + memset((void *)iommu->dummy_page, 0, PAGE_SIZE); + iommu->dummy_page_pa = (unsigned long) __pa(iommu->dummy_page); + tsbbase = __get_free_pages(GFP_KERNEL, order = get_order(tsbsize * 1024 * 8)); if (!tsbbase) { prom_printf("SABRE_IOMMU: Error, gfp(tsb) failed.\n"); @@ -1301,7 +1309,7 @@ static void __init sabre_iommu_init(struct pci_controller_info *p, iommu->page_table = (iopte_t *)tsbbase; iommu->page_table_map_base = dvma_offset; iommu->dma_addr_mask = dma_mask; - memset((char *)tsbbase, 0, PAGE_SIZE << order); + pci_iommu_table_init(iommu, PAGE_SIZE << order); sabre_write(p->pbm_A.controller_regs + SABRE_IOMMU_TSBBASE, __pa(tsbbase)); diff --git a/arch/sparc64/kernel/pci_schizo.c b/arch/sparc64/kernel/pci_schizo.c index be6fa906a54d3..3f90de646f7f6 100644 --- a/arch/sparc64/kernel/pci_schizo.c +++ b/arch/sparc64/kernel/pci_schizo.c @@ -1766,6 +1766,14 @@ static void schizo_pbm_iommu_init(struct pci_pbm_info *pbm) * in pci_iommu.c */ + iommu->dummy_page = __get_free_pages(GFP_KERNEL, 0); + if (!iommu->dummy_page) { + prom_printf("PSYCHO_IOMMU: Error, gfp(dummy_page) failed.\n"); + prom_halt(); + } + memset((void *)iommu->dummy_page, 0, PAGE_SIZE); + iommu->dummy_page_pa = (unsigned long) __pa(iommu->dummy_page); + /* Using assumed page size 8K with 128K entries we need 1MB iommu page * table (128K ioptes * 8 bytes per iopte). This is * page order 7 on UltraSparc. @@ -1780,7 +1788,7 @@ static void schizo_pbm_iommu_init(struct pci_pbm_info *pbm) iommu->page_table = (iopte_t *)tsbbase; iommu->page_table_map_base = vdma[0]; iommu->dma_addr_mask = dma_mask; - memset((char *)tsbbase, 0, PAGE_SIZE << order); + pci_iommu_table_init(iommu, PAGE_SIZE << order); switch (tsbsize) { case 64: diff --git a/include/asm-sparc64/pbm.h b/include/asm-sparc64/pbm.h index 0cddbd2bf2c6e..92999631c8196 100644 --- a/include/asm-sparc64/pbm.h +++ b/include/asm-sparc64/pbm.h @@ -70,6 +70,13 @@ struct pci_iommu { */ u32 lowest_consistent_map; + /* In order to deal with some buggy third-party PCI bridges that + * do wrong prefetching, we never mark valid mappings as invalid. + * Instead we point them at this dummy page. + */ + unsigned long dummy_page; + unsigned long dummy_page_pa; + /* If PBM_NCLUSTERS is ever decreased to 4 or lower, * or if largest supported page_table_sz * 8K goes above * 2GB, you must increase the size of the type of @@ -93,6 +100,8 @@ struct pci_iommu { u32 dma_addr_mask; }; +extern void pci_iommu_table_init(struct pci_iommu *, int); + /* This describes a PCI bus module's streaming buffer. */ struct pci_strbuf { int strbuf_enabled; /* Present and using it? */ -- 2.53.0