arm64: kvm: use inner-shareable barriers for inner-shareable maintenance

commit ee9e101c11478680d579bd20bb38a4d3e2514fe3 upstream.

In order to ensure completion of inner-shareable maintenance instructions
(cache and TLB) on AArch64, we can use the -ish suffix to the dsb
instruction.

This patch relaxes our dsb sy instructions to dsb ish where possible.

Acked-by: Catalin Marinas <catalin.marinas@arm.com>
Acked-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Shannon Zhao <shannon.zhao@linaro.org>
Signed-off-by: Jiri Slaby <jslaby@suse.cz>

diff --git a/arch/arm64/kvm/hyp.S b/arch/arm64/kvm/hyp.S
index d5581ffa70063049924d1a02d427b3b9faf09714..1144e4412838610bf57161fc3f93ae53c604121b 100644 (file)
--- a/arch/arm64/kvm/hyp.S
+++ b/arch/arm64/kvm/hyp.S
@@ -617,9 +617,15 @@ ENTRY(__kvm_tlb_flush_vmid_ipa)
         * whole of Stage-1. Weep...
         */
        tlbi    ipas2e1is, x1
-       dsb     sy
+       /*
+        * We have to ensure completion of the invalidation at Stage-2,
+        * since a table walk on another CPU could refill a TLB with a
+        * complete (S1 + S2) walk based on the old Stage-2 mapping if
+        * the Stage-1 invalidation happened first.
+        */
+       dsb     ish
        tlbi    vmalle1is
-       dsb     sy
+       dsb     ish
        isb
 
        msr     vttbr_el2, xzr
@@ -630,7 +636,7 @@ ENTRY(__kvm_flush_vm_context)
        dsb     ishst
        tlbi    alle1is
        ic      ialluis
-       dsb     sy
+       dsb     ish
        ret
 ENDPROC(__kvm_flush_vm_context)