x86/bugs: Add a Transient Scheduler Attacks mitigation

Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
Co-developed-by: Kim Phillips <kim.phillips@amd.com>
Signed-off-by: Kim Phillips <kim.phillips@amd.com>
Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>

Orabug: 38172250
CVE: CVE-2024-36350, CVE-2024-36357

Signed-off-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
Reviewed-by: Ankur Arora <ankur.a.arora@oracle.com>

Conflicts:
	arch/x86/include/asm/cpufeatures.h
	- Use X86_BUG_AMD_APIC_C1E bit
	arch/x86/kernel/cpu/bugs.c
	- Don't set X86_FEATURE_CLEAR_CPU_BUF
	- Context
	arch/x86/include/asm/processor.h
	arch/x86/kernel/cpu/amd.c
	arch/x86/kernel/cpu/scattered.c
	include/linux/cpu.h
	- Context
	arch/x86/kernel/cpu/common.c
	- Use bit 10 for TSA
	- Context
	drivers/base/cpu.c
	- Opencode CPU_SHOW_VULN_FALLBACK
	arch/x86/kvm/svm/svm.c
	- Add the flush to svm_prepare_guest_switch()
Signed-off-by: Sherry Yang <sherry.yang@oracle.com>

Author: bp3tk0v

arch/x86/Kconfig

@@ -2584,6 +2584,15 @@ config SLS
 	  against straight line speculation. The kernel image might be slightly
 	  larger.
 
+config MITIGATION_TSA
+	bool "Mitigate Transient Scheduler Attacks"
+	depends on CPU_SUP_AMD
+	default y
+	help
+	  Enable mitigation for Transient Scheduler Attacks. TSA is a hardware
+	  security vulnerability on AMD CPUs which can lead to forwarding of
+	  invalid info to subsequent instructions and thus can affect their
+	  timing and thereby cause a leakage.
 endif
 
 config ARCH_HAS_ADD_PAGES

arch/x86/include/asm/cpufeatures.h

@@ -415,6 +415,7 @@
 #define X86_FEATURE_SEV_ES		(19*32+ 3) /* AMD Secure Encrypted Virtualization - Encrypted State */
 #define X86_FEATURE_SME_COHERENT	(19*32+10) /* "" AMD hardware-enforced cache coherency */
 
+#define X86_FEATURE_VERW_CLEAR		(20*32+ 5) /* "" The memory form of VERW mitigates TSA */
 #define X86_FEATURE_AUTOIBRS		(20*32+ 8) /* "" Automatic IBRS */
 #define X86_FEATURE_SBPB		(20*32+27) /* "" Selective Branch Prediction Barrier */
 #define X86_FEATURE_IBPB_BRTYPE		(20*32+28) /* "" MSR_PRED_CMD[IBPB] flushes all branch type predictions */
@@ -430,6 +431,8 @@
 #define X86_FEATURE_BHI_CTRL		(21*32+ 2) /* "" BHI_DIS_S HW control available */
 #define X86_FEATURE_CLEAR_BHB_HW	(21*32+ 3) /* "" BHI_DIS_S HW control enabled */
 #define X86_FEATURE_CLEAR_BHB_LOOP_ON_VMEXIT (21*32+ 4) /* "" Clear branch history at vmexit using SW loop */
+#define X86_FEATURE_TSA_SQ_NO		(21*32+ 6) /* "" AMD CPU not vulnerable to TSA-SQ */
+#define X86_FEATURE_TSA_L1_NO		(21*32+ 7) /* "" AMD CPU not vulnerable to TSA-L1 */
 #define X86_FEATURE_INDIRECT_THUNK_ITS	(21*32 + 9) /* "" Use thunk for indirect branches in lower half of cacheline */
 
 /*
@@ -491,5 +494,6 @@
 #define X86_BUG_BHI			X86_BUG(1) /* CPU is affected by Branch History Injection */
 #define X86_BUG_ITS			X86_BUG(2) /* CPU is affected by Indirect Target Selection */
 #define X86_BUG_ITS_NATIVE_ONLY		X86_BUG(3) /* CPU is affected by ITS, VMX is not affected */
+#define X86_BUG_TSA                     X86_BUG(4) /* CPU is affected by Transient Scheduler Attacks */
 
 #endif /* _ASM_X86_CPUFEATURES_H */

arch/x86/include/asm/processor.h

@@ -930,11 +930,13 @@ extern u16 amd_get_nb_id(int cpu);
 extern u32 amd_get_nodes_per_socket(void);
 extern void amd_clear_divider(void);
 extern bool cpu_has_ibpb_brtype_microcode(void);
+extern bool amd_check_tsa_microcode(void);
 #else
 static inline u16 amd_get_nb_id(int cpu)		{ return 0; }
 static inline u32 amd_get_nodes_per_socket(void)	{ return 0; }
 static inline void amd_clear_divider(void)		{ }
 static inline bool cpu_has_ibpb_brtype_microcode(void)	{ return false; }
+static inline bool amd_check_tsa_microcode(void)	{ return false; }
 #endif
 
 static inline uint32_t hypervisor_cpuid_base(const char *sig, uint32_t leaves)

arch/x86/kernel/cpu/amd.c

@@ -566,6 +566,14 @@ static void early_init_amd_mc(struct cpuinfo_x86 *c)
 #endif
 }
 
+static void tsa_set_cpuid_flags(struct cpuinfo_x86 *c)
+{
+	if (!cpu_has(c, X86_FEATURE_HYPERVISOR) && c->x86 != 0x19) {
+		setup_force_cpu_cap(X86_FEATURE_TSA_SQ_NO);
+		setup_force_cpu_cap(X86_FEATURE_TSA_L1_NO);
+	}
+}
+
 static void bsp_init_amd(struct cpuinfo_x86 *c)
 {
 
@@ -740,6 +748,8 @@ static void early_detect_mem_encrypt(struct cpuinfo_x86 *c)
 		setup_clear_cpu_cap(X86_FEATURE_SEV);
 		setup_clear_cpu_cap(X86_FEATURE_SEV_ES);
 	}
+
+	tsa_set_cpuid_flags(c);
 }
 
 static void early_init_amd(struct cpuinfo_x86 *c)
@@ -1401,6 +1411,38 @@ void amd_check_microcode(void)
 	on_each_cpu(zenbleed_check_cpu, NULL, 1);
 }
 
+bool amd_check_tsa_microcode(void)
+{
+	struct cpuinfo_x86 *c = &boot_cpu_data;
+	bool ret = false;
+
+	if (cpu_has(c, X86_FEATURE_ZEN3)) {
+		if (c->x86_model == 0x1) {
+			if (c->x86_stepping == 0x1)
+				ret = c->microcode >= 0x0a0011d7;
+			else if (c->x86_stepping == 0x2)
+				ret = c->microcode >= 0x0a00123b;
+			} else if (c->x86_model == 0x30) {
+				ret = c->microcode >= 0x0a30010d;
+			}
+	} else if (cpu_has(c, X86_FEATURE_ZEN4)) {
+		if (c->x86_model == 0x11) {
+			if (c->x86_stepping == 0x1)
+				ret = c->microcode >= 0x0a10114c;
+			else if (c->x86_stepping == 0x2)
+				ret = c->microcode >= 0x0a10124c;
+		} else if (c->x86_model == 0xa0) {
+			if (c->x86_stepping == 0x2)
+				ret = c->microcode >= 0x0aa00216;
+		}
+	}
+
+	if (ret)
+		setup_force_cpu_cap(X86_FEATURE_VERW_CLEAR);
+
+	return ret;
+}
+
 /*
  * Issue a DIV 0/1 insn to clear any division data from previous DIV
  * operations.

arch/x86/kernel/cpu/bugs.c

@@ -154,6 +154,7 @@ static void srbds_select_mitigation(void);
 static void srso_select_mitigation(void);
 static void gds_select_mitigation(void);
 static void its_select_mitigation(void);
+static void tsa_select_mitigation(void);
 
 static enum ssb_mitigation ssb_mode = SPEC_STORE_BYPASS_NONE;
 
@@ -335,6 +336,8 @@ void __init cpu_select_mitigations(void)
 	srso_select_mitigation();
 	gds_select_mitigation();
 	its_select_mitigation();
+
+	tsa_select_mitigation();
 }
 
 void x86_spec_ctrl_set(enum spec_ctrl_set_context context)
@@ -3216,6 +3219,92 @@ static void __init srso_select_mitigation(void)
 		x86_pred_cmd = PRED_CMD_SBPB;
 }
 
+#undef pr_fmt
+#define pr_fmt(fmt)	"Transient Scheduler Attacks: " fmt
+
+enum tsa_mitigations {
+	TSA_MITIGATION_NONE,
+	TSA_MITIGATION_UCODE_NEEDED,
+	TSA_MITIGATION_USER_KERNEL,
+	TSA_MITIGATION_VM,
+	TSA_MITIGATION_FULL,
+};
+
+static const char * const tsa_strings[] = {
+	[TSA_MITIGATION_NONE]		= "Vulnerable",
+	[TSA_MITIGATION_UCODE_NEEDED]	= "Vulnerable: No microcode",
+	[TSA_MITIGATION_USER_KERNEL]	= "Mitigation: user/kernel boundary",
+	[TSA_MITIGATION_VM]		= "Mitigation: VM only",
+	[TSA_MITIGATION_FULL]		= "Mitigation: enabled",
+};
+
+static enum tsa_mitigations tsa_mitigation __ro_after_init =
+	IS_ENABLED(CONFIG_MITIGATION_TSA) ? TSA_MITIGATION_FULL : TSA_MITIGATION_NONE;
+
+static int __init tsa_parse_cmdline(char *str)
+{
+	if (!str)
+		return -EINVAL;
+
+	if (!strcmp(str, "off"))
+		tsa_mitigation = TSA_MITIGATION_NONE;
+	else if (!strcmp(str, "on"))
+		tsa_mitigation = TSA_MITIGATION_FULL;
+	else if (!strcmp(str, "user"))
+		tsa_mitigation = TSA_MITIGATION_USER_KERNEL;
+	else if (!strcmp(str, "vm"))
+		tsa_mitigation = TSA_MITIGATION_VM;
+	else
+		pr_err("Ignoring unknown tsa=%s option.\n", str);
+
+	return 0;
+}
+early_param("tsa", tsa_parse_cmdline);
+
+static void __init tsa_select_mitigation(void)
+{
+	bool has_microcode = false;
+
+	if (tsa_mitigation == TSA_MITIGATION_NONE)
+		return;
+
+	if (cpu_mitigations_off() || !boot_cpu_has_bug(X86_BUG_TSA)) {
+		tsa_mitigation = TSA_MITIGATION_NONE;
+		return;
+	}
+
+	if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
+		if (boot_cpu_has(X86_FEATURE_VERW_CLEAR)) {
+			has_microcode = true;
+		}
+	} else {
+		has_microcode = amd_check_tsa_microcode();
+	}
+
+	if (!has_microcode) {
+		tsa_mitigation = TSA_MITIGATION_UCODE_NEEDED;
+		goto out;
+	}
+
+	switch (tsa_mitigation) {
+	case TSA_MITIGATION_USER_KERNEL:
+		static_branch_enable(&cpu_buf_idle_clear);
+		break;
+	case TSA_MITIGATION_VM:
+		static_branch_enable(&cpu_buf_vm_clear);
+		break;
+	case TSA_MITIGATION_FULL:
+		static_branch_enable(&cpu_buf_idle_clear);
+		static_branch_enable(&cpu_buf_vm_clear);
+		break;
+	default:
+		break;
+	}
+
+out:
+	pr_info("%s\n", tsa_strings[tsa_mitigation]);
+}
+
 #undef pr_fmt
 #define pr_fmt(fmt) fmt
 
@@ -3447,6 +3536,11 @@ static ssize_t gds_show_state(char *buf)
 	return sysfs_emit(buf, "%s\n", gds_strings[gds_mitigation]);
 }
 
+static ssize_t tsa_show_state(char *buf)
+{
+	return sysfs_emit(buf, "%s\n", tsa_strings[tsa_mitigation]);
+}
+
 static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr,
 			       char *buf, unsigned int bug)
 {
@@ -3505,6 +3599,9 @@ static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr
 	case X86_BUG_ITS:
 		return its_show_state(buf);
 
+	case X86_BUG_TSA:
+		return tsa_show_state(buf);
+
 	default:
 		break;
 	}
@@ -3584,4 +3681,9 @@ ssize_t cpu_show_indirect_target_selection(struct device *dev, struct device_att
 {
 	return cpu_show_common(dev, attr, buf, X86_BUG_ITS);
 }
+
+ssize_t cpu_show_tsa(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	return cpu_show_common(dev, attr, buf, X86_BUG_TSA);
+}
 #endif

arch/x86/kernel/cpu/common.c

@@ -1273,6 +1273,8 @@ static const struct x86_cpu_id cpu_vuln_whitelist[] = {
 #define ITS		BIT(8)
 /* CPU is affected by Indirect Target Selection, but guest-host isolation is not affected */
 #define ITS_NATIVE_ONLY	BIT(9)
+/* CPU is affected by Transient Scheduler Attacks */
+#define TSA		BIT(10)
 
 static const struct x86_cpu_id cpu_vuln_blacklist[] = {
 	VULNBL_INTEL_STEPPINGS(IVYBRIDGE,	X86_STEPPING_ANY,		SRBDS),
@@ -1311,7 +1313,7 @@ static const struct x86_cpu_id cpu_vuln_blacklist[] = {
 	VULNBL_AMD(0x16, RETBLEED),
 	VULNBL_AMD(0x17, RETBLEED | SMT_RSB | SRSO),
 	VULNBL_HYGON(0x18, RETBLEED | SMT_RSB),
-	VULNBL_AMD(0x19, SRSO),
+	VULNBL_AMD(0x19, SRSO | TSA),
 	{}
 };
 
@@ -1523,6 +1525,12 @@ void cpu_set_bug_bits(struct cpuinfo_x86 *c)
 			setup_force_cpu_bug(X86_BUG_ITS_NATIVE_ONLY);
 	}
 
+	if (!cpu_has(c, X86_FEATURE_TSA_SQ_NO) ||
+	    !cpu_has(c, X86_FEATURE_TSA_L1_NO)) {
+		if (cpu_matches(cpu_vuln_blacklist, TSA))
+			setup_force_cpu_bug(X86_BUG_TSA);
+	}
+
 	if (cpu_matches(cpu_vuln_whitelist, NO_MELTDOWN))
 		return;
 

arch/x86/kernel/cpu/scattered.c

@@ -42,6 +42,9 @@ static const struct cpuid_bit cpuid_bits[] = {
 	{ X86_FEATURE_CPB,		CPUID_EDX,  9, 0x80000007, 0 },
 	{ X86_FEATURE_PROC_FEEDBACK,    CPUID_EDX, 11, 0x80000007, 0 },
 	{ X86_FEATURE_MBA,		CPUID_EBX,  6, 0x80000008, 0 },
+	{ X86_FEATURE_TSA_SQ_NO,	CPUID_ECX,  1, 0x80000021, 0 },
+	{ X86_FEATURE_TSA_L1_NO,	CPUID_ECX,  2, 0x80000021, 0 },
+
 	{ 0, 0, 0, 0, 0 }
 };
 

arch/x86/kvm/svm/svm.c

@@ -3368,6 +3368,8 @@ static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva)
 
 static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu)
 {
+	if (static_branch_likely(&cpu_buf_vm_clear))
+		x86_clear_cpu_buffers();
 }
 
 static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)

drivers/base/cpu.c

@@ -606,6 +606,12 @@ ssize_t __weak cpu_show_indirect_target_selection(struct device *dev,
 	return sysfs_emit(buf, "Not affected\n");
 }
 
+ssize_t __weak cpu_show_tsa(struct device *dev,
+			    struct device_attribute *attr, char *buf)
+{
+        return sysfs_emit(buf, "Not affected\n");
+}
+
 static DEVICE_ATTR(meltdown, 0444, cpu_show_meltdown, NULL);
 static DEVICE_ATTR(spectre_v1, 0444, cpu_show_spectre_v1, NULL);
 static DEVICE_ATTR(spectre_v2, 0444, cpu_show_spectre_v2, NULL);
@@ -620,6 +626,7 @@ static DEVICE_ATTR(retbleed, 0444, cpu_show_retbleed, NULL);
 static DEVICE_ATTR(spec_rstack_overflow, 0444, cpu_show_spec_rstack_overflow, NULL);
 static DEVICE_ATTR(gather_data_sampling, 0444, cpu_show_gds, NULL);
 static DEVICE_ATTR(indirect_target_selection, 0444, cpu_show_indirect_target_selection, NULL);
+static DEVICE_ATTR(tsa, 0444, cpu_show_tsa, NULL);
 
 static struct attribute *cpu_root_vulnerabilities_attrs[] = {
 	&dev_attr_meltdown.attr,
@@ -636,6 +643,7 @@ static struct attribute *cpu_root_vulnerabilities_attrs[] = {
 	&dev_attr_spec_rstack_overflow.attr,
 	&dev_attr_gather_data_sampling.attr,
 	&dev_attr_indirect_target_selection.attr,
+	&dev_attr_tsa.attr,
 	NULL
 };
 

include/linux/cpu.h

@@ -74,6 +74,8 @@ extern ssize_t cpu_show_spec_rstack_overflow(struct device *dev,
 					     struct device_attribute *attr, char *buf);
 extern ssize_t cpu_show_indirect_target_selection(struct device *dev,
 						  struct device_attribute *attr, char *buf);
+extern ssize_t cpu_show_tsa(struct device *dev,
+			    struct device_attribute *attr, char *buf);
 
 extern __printf(4, 5)
 struct device *cpu_device_create(struct device *parent, void *drvdata,