Commit 1e8c5971 authored by Brijesh Singh's avatar Brijesh Singh Committed by Borislav Petkov
Browse files

x86/mm/cpa: Generalize __set_memory_enc_pgtable() 



The kernel provides infrastructure to set or clear the encryption mask
from the pages for AMD SEV, but TDX requires few tweaks.

- TDX and SEV have different requirements to the cache and TLB
  flushing.

- TDX has own routine to notify VMM about page encryption status change.

Modify __set_memory_enc_pgtable() and make it flexible enough to cover
both AMD SEV and Intel TDX. The AMD-specific behavior is isolated in the
callbacks under x86_platform.guest. TDX will provide own version of said
callbacks.

  [ bp: Beat into submission. ]

Signed-off-by: default avatarBrijesh Singh <brijesh.singh@amd.com>
Signed-off-by: default avatarBorislav Petkov <bp@suse.de>
Acked-by: default avatarKirill A. Shutemov <kirill.shutemov@linux.intel.com>
Link: https://lore.kernel.org/r/20220223043528.2093214-1-brijesh.singh@amd.com
parent b577f542

arch/x86/include/asm/set_memory.h

+0 −1
Original line number Diff line number Diff line
@@ -84,7 +84,6 @@ int set_pages_rw(struct page *page, int numpages); 
int set_direct_map_invalid_noflush(struct page *page);

int set_direct_map_default_noflush(struct page *page);

bool kernel_page_present(struct page *page);

void notify_range_enc_status_changed(unsigned long vaddr, int npages, bool enc);



extern int kernel_set_to_readonly;

arch/x86/include/asm/x86_init.h

+16 −0
Original line number Diff line number Diff line
@@ -141,6 +141,21 @@ struct x86_init_acpi { 
	void (*reduced_hw_early_init)(void);

};



/**

 * struct x86_guest - Functions used by misc guest incarnations like SEV, TDX, etc.

 *

 * @enc_status_change_prepare	Notify HV before the encryption status of a range is changed

 * @enc_status_change_finish	Notify HV after the encryption status of a range is changed

 * @enc_tlb_flush_required	Returns true if a TLB flush is needed before changing page encryption status

 * @enc_cache_flush_required	Returns true if a cache flush is needed before changing page encryption status

 */

struct x86_guest {

	void (*enc_status_change_prepare)(unsigned long vaddr, int npages, bool enc);

	bool (*enc_status_change_finish)(unsigned long vaddr, int npages, bool enc);

	bool (*enc_tlb_flush_required)(bool enc);

	bool (*enc_cache_flush_required)(void);

};



/**

 * struct x86_init_ops - functions for platform specific setup

 *
@@ -287,6 +302,7 @@ struct x86_platform_ops { 
	struct x86_legacy_features legacy;

	void (*set_legacy_features)(void);

	struct x86_hyper_runtime hyper;

	struct x86_guest guest;

};



struct x86_apic_ops {

arch/x86/kernel/x86_init.c

+14 −2
Original line number Diff line number Diff line
@@ -129,6 +129,11 @@ struct x86_cpuinit_ops x86_cpuinit = { 


static void default_nmi_init(void) { };



static void enc_status_change_prepare_noop(unsigned long vaddr, int npages, bool enc) { }

static bool enc_status_change_finish_noop(unsigned long vaddr, int npages, bool enc) { return false; }

static bool enc_tlb_flush_required_noop(bool enc) { return false; }

static bool enc_cache_flush_required_noop(void) { return false; }



struct x86_platform_ops x86_platform __ro_after_init = {

	.calibrate_cpu			= native_calibrate_cpu_early,

	.calibrate_tsc			= native_calibrate_tsc,
@@ -141,6 +146,13 @@ struct x86_platform_ops x86_platform __ro_after_init = { 
	.save_sched_clock_state		= tsc_save_sched_clock_state,

	.restore_sched_clock_state	= tsc_restore_sched_clock_state,

	.hyper.pin_vcpu			= x86_op_int_noop,



	.guest = {

		.enc_status_change_prepare = enc_status_change_prepare_noop,

		.enc_status_change_finish  = enc_status_change_finish_noop,

		.enc_tlb_flush_required	   = enc_tlb_flush_required_noop,

		.enc_cache_flush_required  = enc_cache_flush_required_noop,

	},

};



EXPORT_SYMBOL_GPL(x86_platform);

arch/x86/mm/mem_encrypt_amd.c

+50 −22
Original line number Diff line number Diff line
@@ -177,25 +177,6 @@ void __init sme_map_bootdata(char *real_mode_data) 
	__sme_early_map_unmap_mem(__va(cmdline_paddr), COMMAND_LINE_SIZE, true);

}



void __init sme_early_init(void)

{

	unsigned int i;



	if (!sme_me_mask)

		return;



	early_pmd_flags = __sme_set(early_pmd_flags);



	__supported_pte_mask = __sme_set(__supported_pte_mask);



	/* Update the protection map with memory encryption mask */

	for (i = 0; i < ARRAY_SIZE(protection_map); i++)

		protection_map[i] = pgprot_encrypted(protection_map[i]);



	if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT))

		swiotlb_force = SWIOTLB_FORCE;

}



void __init sev_setup_arch(void)

{

	phys_addr_t total_mem = memblock_phys_mem_size();
@@ -256,7 +237,17 @@ static unsigned long pg_level_to_pfn(int level, pte_t *kpte, pgprot_t *ret_prot) 
	return pfn;

}



void notify_range_enc_status_changed(unsigned long vaddr, int npages, bool enc)

static bool amd_enc_tlb_flush_required(bool enc)

{

	return true;

}



static bool amd_enc_cache_flush_required(void)

{

	return !cpu_feature_enabled(X86_FEATURE_SME_COHERENT);

}



static void enc_dec_hypercall(unsigned long vaddr, int npages, bool enc)

{

#ifdef CONFIG_PARAVIRT

	unsigned long sz = npages << PAGE_SHIFT;
@@ -287,6 +278,19 @@ void notify_range_enc_status_changed(unsigned long vaddr, int npages, bool enc) 
#endif

}



static void amd_enc_status_change_prepare(unsigned long vaddr, int npages, bool enc)

{

}



/* Return true unconditionally: return value doesn't matter for the SEV side */

static bool amd_enc_status_change_finish(unsigned long vaddr, int npages, bool enc)

{

	if (!cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT))

		enc_dec_hypercall(vaddr, npages, enc);



	return true;

}



static void __init __set_clr_pte_enc(pte_t *kpte, int level, bool enc)

{

	pgprot_t old_prot, new_prot;
@@ -392,7 +396,7 @@ static int __init early_set_memory_enc_dec(unsigned long vaddr, 


	ret = 0;



	notify_range_enc_status_changed(start, PAGE_ALIGN(size) >> PAGE_SHIFT, enc);

	early_set_mem_enc_dec_hypercall(start, PAGE_ALIGN(size) >> PAGE_SHIFT, enc);

out:

	__flush_tlb_all();

	return ret;
@@ -410,7 +414,31 @@ int __init early_set_memory_encrypted(unsigned long vaddr, unsigned long size) 


void __init early_set_mem_enc_dec_hypercall(unsigned long vaddr, int npages, bool enc)

{

	notify_range_enc_status_changed(vaddr, npages, enc);

	enc_dec_hypercall(vaddr, npages, enc);

}



void __init sme_early_init(void)

{

	unsigned int i;



	if (!sme_me_mask)

		return;



	early_pmd_flags = __sme_set(early_pmd_flags);



	__supported_pte_mask = __sme_set(__supported_pte_mask);



	/* Update the protection map with memory encryption mask */

	for (i = 0; i < ARRAY_SIZE(protection_map); i++)

		protection_map[i] = pgprot_encrypted(protection_map[i]);



	if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT))

		swiotlb_force = SWIOTLB_FORCE;



	x86_platform.guest.enc_status_change_prepare = amd_enc_status_change_prepare;

	x86_platform.guest.enc_status_change_finish  = amd_enc_status_change_finish;

	x86_platform.guest.enc_tlb_flush_required    = amd_enc_tlb_flush_required;

	x86_platform.guest.enc_cache_flush_required  = amd_enc_cache_flush_required;

}



void __init mem_encrypt_free_decrypted_mem(void)

arch/x86/mm/pat/set_memory.c

+11 −9
Original line number Diff line number Diff line
@@ -2008,10 +2008,12 @@ static int __set_memory_enc_pgtable(unsigned long addr, int numpages, bool enc) 
	kmap_flush_unused();

	vm_unmap_aliases();



	/*

	 * Before changing the encryption attribute, we need to flush caches.

	 */

	cpa_flush(&cpa, !this_cpu_has(X86_FEATURE_SME_COHERENT));

	/* Flush the caches as needed before changing the encryption attribute. */

	if (x86_platform.guest.enc_tlb_flush_required(enc))

		cpa_flush(&cpa, x86_platform.guest.enc_cache_flush_required());



	/* Notify hypervisor that we are about to set/clr encryption attribute. */

	x86_platform.guest.enc_status_change_prepare(addr, numpages, enc);



	ret = __change_page_attr_set_clr(&cpa, 1);
@@ -2024,11 +2026,11 @@ static int __set_memory_enc_pgtable(unsigned long addr, int numpages, bool enc) 
	 */

	cpa_flush(&cpa, 0);



	/*

	 * Notify hypervisor that a given memory range is mapped encrypted

	 * or decrypted.

	 */

	notify_range_enc_status_changed(addr, numpages, enc);

	/* Notify hypervisor that we have successfully set/clr encryption attribute. */

	if (!ret) {

		if (!x86_platform.guest.enc_status_change_finish(addr, numpages, enc))

			ret = -EIO;

	}



	return ret;

}