x86/sev: Make enc_dec_hypercall() accept a size instead of npages

commit ac3f9c9f1b37edaa7d1a9b908bc79d843955a1a2 upstream.

enc_dec_hypercall() accepted a page count instead of a size, which
forced its callers to round up. As a result, non-page aligned
vaddrs caused pages to be spuriously marked as decrypted via the
encryption status hypercall, which in turn caused consistent
corruption of pages during live migration. Live migration requires
accurate encryption status information to avoid migrating pages
from the wrong perspective.

Fixes: 064ce6c550a0 ("mm: x86: Invoke hypercall when page encryption status is changed")
Signed-off-by: Steve Rutherford <srutherford@google.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Tom Lendacky <thomas.lendacky@amd.com>
Reviewed-by: Pankaj Gupta <pankaj.gupta@amd.com>
Tested-by: Ben Hillier <bhillier@google.com>
Cc: stable@vger.kernel.org
Link: https://lore.kernel.org/r/20230824223731.2055016-1-srutherford@google.com
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/arch/x86/include/asm/mem_encrypt.h b/arch/x86/include/asm/mem_encrypt.h
index 7f97a8a97e24a81ad56fc4421b30c38a5ba2c821..473b16d73b4710849349902b54d1ec1e725ff054 100644 (file)
--- a/arch/x86/include/asm/mem_encrypt.h
+++ b/arch/x86/include/asm/mem_encrypt.h
@@ -50,8 +50,8 @@ void __init sme_enable(struct boot_params *bp);
 
 int __init early_set_memory_decrypted(unsigned long vaddr, unsigned long size);
 int __init early_set_memory_encrypted(unsigned long vaddr, unsigned long size);
-void __init early_set_mem_enc_dec_hypercall(unsigned long vaddr, int npages,
-                                           bool enc);
+void __init early_set_mem_enc_dec_hypercall(unsigned long vaddr,
+                                           unsigned long size, bool enc);
 
 void __init mem_encrypt_free_decrypted_mem(void);
 
@@ -85,7 +85,7 @@ early_set_memory_decrypted(unsigned long vaddr, unsigned long size) { return 0;
 static inline int __init
 early_set_memory_encrypted(unsigned long vaddr, unsigned long size) { return 0; }
 static inline void __init
-early_set_mem_enc_dec_hypercall(unsigned long vaddr, int npages, bool enc) {}
+early_set_mem_enc_dec_hypercall(unsigned long vaddr, unsigned long size, bool enc) {}
 
 static inline void mem_encrypt_free_decrypted_mem(void) { }
 

diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c
index 1cceac5984daa902f311a4001ba1a028ab6f91b3..526d4da3dcd46b3e412aa8868753343fb76fbc16 100644 (file)
--- a/arch/x86/kernel/kvm.c
+++ b/arch/x86/kernel/kvm.c
@@ -966,10 +966,8 @@ static void __init kvm_init_platform(void)
                 * Ensure that _bss_decrypted section is marked as decrypted in the
                 * shared pages list.
                 */
-               nr_pages = DIV_ROUND_UP(__end_bss_decrypted - __start_bss_decrypted,
-                                       PAGE_SIZE);
                early_set_mem_enc_dec_hypercall((unsigned long)__start_bss_decrypted,
-                                               nr_pages, 0);
+                                               __end_bss_decrypted - __start_bss_decrypted, 0);
 
                /*
                 * If not booted using EFI, enable Live migration support.

diff --git a/arch/x86/mm/mem_encrypt_amd.c b/arch/x86/mm/mem_encrypt_amd.c
index 4f95c449a406e1196eb3bfe30468a46ae1811ade..bff4201c9db7c1ea32fdb829f7c26a5ac3195b11 100644 (file)
--- a/arch/x86/mm/mem_encrypt_amd.c
+++ b/arch/x86/mm/mem_encrypt_amd.c
@@ -288,11 +288,10 @@ static bool amd_enc_cache_flush_required(void)
        return !cpu_feature_enabled(X86_FEATURE_SME_COHERENT);
 }
 
-static void enc_dec_hypercall(unsigned long vaddr, int npages, bool enc)
+static void enc_dec_hypercall(unsigned long vaddr, unsigned long size, bool enc)
 {
 #ifdef CONFIG_PARAVIRT
-       unsigned long sz = npages << PAGE_SHIFT;
-       unsigned long vaddr_end = vaddr + sz;
+       unsigned long vaddr_end = vaddr + size;
 
        while (vaddr < vaddr_end) {
                int psize, pmask, level;
@@ -342,7 +341,7 @@ static bool amd_enc_status_change_finish(unsigned long vaddr, int npages, bool e
                snp_set_memory_private(vaddr, npages);
 
        if (!cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT))
-               enc_dec_hypercall(vaddr, npages, enc);
+               enc_dec_hypercall(vaddr, npages << PAGE_SHIFT, enc);
 
        return true;
 }
@@ -466,7 +465,7 @@ static int __init early_set_memory_enc_dec(unsigned long vaddr,
 
        ret = 0;
 
-       early_set_mem_enc_dec_hypercall(start, PAGE_ALIGN(size) >> PAGE_SHIFT, enc);
+       early_set_mem_enc_dec_hypercall(start, size, enc);
 out:
        __flush_tlb_all();
        return ret;
@@ -482,9 +481,9 @@ int __init early_set_memory_encrypted(unsigned long vaddr, unsigned long size)
        return early_set_memory_enc_dec(vaddr, size, true);
 }
 
-void __init early_set_mem_enc_dec_hypercall(unsigned long vaddr, int npages, bool enc)
+void __init early_set_mem_enc_dec_hypercall(unsigned long vaddr, unsigned long size, bool enc)
 {
-       enc_dec_hypercall(vaddr, npages, enc);
+       enc_dec_hypercall(vaddr, size, enc);
 }
 
 void __init sme_early_init(void)