[linux-6.6.y] Add support for ZXPAUSE instruction

arch/x86/include/asm/cpufeature.h

@@ -34,6 +34,7 @@ enum cpuid_leafs
 	CPUID_8000_001F_EAX,
 	CPUID_8000_0021_EAX,
 	CPUID_LNX_5,
+	CPUID_C000_0006_EAX,
 	NR_CPUID_WORDS,
 };
 
@@ -94,8 +95,9 @@ extern const char * const x86_bug_flags[NBUGINTS*32];
 	   CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 19, feature_bit) ||	\
 	   CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 20, feature_bit) ||	\
 	   CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 21, feature_bit) ||	\
+	   CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 22, feature_bit) ||    \
 	   REQUIRED_MASK_CHECK					  ||	\
-	   BUILD_BUG_ON_ZERO(NCAPINTS != 22))
+	   BUILD_BUG_ON_ZERO(NCAPINTS != 23))
 
 #define DISABLED_MASK_BIT_SET(feature_bit)				\
 	 ( CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  0, feature_bit) ||	\
@@ -120,8 +122,9 @@ extern const char * const x86_bug_flags[NBUGINTS*32];
 	   CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 19, feature_bit) ||	\
 	   CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 20, feature_bit) ||	\
 	   CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 21, feature_bit) ||	\
+	   CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 22, feature_bit) ||	\
 	   DISABLED_MASK_CHECK					  ||	\
-	   BUILD_BUG_ON_ZERO(NCAPINTS != 22))
+	   BUILD_BUG_ON_ZERO(NCAPINTS != 23))
 
 #define cpu_has(c, bit)							\
 	(__builtin_constant_p(bit) && REQUIRED_MASK_BIT_SET(bit) ? 1 :	\

arch/x86/include/asm/cpufeatures.h

@@ -13,7 +13,7 @@
 /*
  * Defines x86 CPU feature bits
  */
-#define NCAPINTS			22	   /* N 32-bit words worth of info */
+#define NCAPINTS			23	   /* N 32-bit words worth of info */
 #define NBUGINTS			2	   /* N 32-bit bug flags */
 
 /*
@@ -470,6 +470,9 @@
 #define X86_FEATURE_CLEAR_BHB_HW	(21*32+ 3) /* "" BHI_DIS_S HW control enabled */
 #define X86_FEATURE_CLEAR_BHB_LOOP_ON_VMEXIT (21*32+ 4) /* "" Clear branch history at vmexit using SW loop */
 
+/* VIA/Cyrix/Centaur-defined CPU features, CPUID level 0xC0000006, word 22 */
+#define X86_FEATURE_ZXPAUSE            (22*32+ 0) /* ZHAOXIN ZXPAUSE */
+
 /*
  * BUG word(s)
  */

arch/x86/include/asm/delay.h

@@ -7,6 +7,7 @@
 
 void __init use_tsc_delay(void);
 void __init use_tpause_delay(void);
+void __init use_zxpause_delay(void);
 void use_mwaitx_delay(void);
 
 #endif /* _ASM_X86_DELAY_H */

arch/x86/include/asm/disabled-features.h

@@ -144,6 +144,7 @@
 #define DISABLED_MASK19	0
 #define DISABLED_MASK20	0
 #define DISABLED_MASK21	0
-#define DISABLED_MASK_CHECK BUILD_BUG_ON_ZERO(NCAPINTS != 22)
+#define DISABLED_MASK22	0
+#define DISABLED_MASK_CHECK BUILD_BUG_ON_ZERO(NCAPINTS != 23)
 
 #endif /* _ASM_X86_DISABLED_FEATURES_H */

arch/x86/include/asm/msr-index.h

@@ -75,12 +75,23 @@
 #define MSR_IA32_UMWAIT_CONTROL			0xe1
 #define MSR_IA32_UMWAIT_CONTROL_C02_DISABLE	BIT(0)
 #define MSR_IA32_UMWAIT_CONTROL_RESERVED	BIT(1)
+
+#define MSR_ZX_PAUSE_CONTROL			0x187f
+#define MSR_ZX_PAUSE_CONTROL_C02_DISABLE	BIT(0)
+#define MSR_ZX_PAUSE_CONTROL_RESERVED		BIT(1)
+
 /*
  * The time field is bit[31:2], but representing a 32bit value with
  * bit[1:0] zero.
  */
 #define MSR_IA32_UMWAIT_CONTROL_TIME_MASK	(~0x03U)
 
+/*
+ * The time field is bit[31:2], but representing a 32bit value with
+ * bit[1:0] zero.
+ */
+#define MSR_ZX_PAUSE_CONTROL_TIME_MASK		(~0x03U)
+
 /* Abbreviated from Intel SDM name IA32_CORE_CAPABILITIES */
 #define MSR_IA32_CORE_CAPS			  0x000000cf
 #define MSR_IA32_CORE_CAPS_INTEGRITY_CAPS_BIT	  2
@@ -768,6 +779,13 @@
 #define MSR_VIA_RNG			0x0000110b
 #define MSR_VIA_BCR2			0x00001147
 
+/*
+ * Zhaoxin extend VMCS capabilities:
+ *     bit 0: exec-cntl3 VMCS field.
+ */
+#define MSR_ZX_EXT_VMCS_CAPS                   0x1675
+#define MSR_ZX_VMCS_EXEC_CTL3                  BIT(0)
+
 /* Transmeta defined MSRs */
 #define MSR_TMTA_LONGRUN_CTRL		0x80868010
 #define MSR_TMTA_LONGRUN_FLAGS		0x80868011

arch/x86/include/asm/mwait.h

@@ -26,6 +26,8 @@
 #define TPAUSE_C01_STATE		1
 #define TPAUSE_C02_STATE		0
 
+#define ZXPAUSE_C01_STATE		1
+
 static __always_inline void __monitor(const void *eax, unsigned long ecx,
 			     unsigned long edx)
 {
@@ -148,4 +150,23 @@ static inline void __tpause(u32 ecx, u32 edx, u32 eax)
 	#endif
 }
 
+/*
+ * Caller can specify whether to enter C0.1 (low latency, less
+ * power saving) or C0.2 state (saves more power, but longer wakeup
+ * latency). This may be overridden by the ZX_PAUSE_CONTROL MSR
+ * which can force requests for C0.2 to be downgraded to C0.1.
+ */
+static inline void __zxpause(u32 ecx, u32 edx, u32 eax)
+{
+	/* "zxpause %ecx, %edx, %eax;" */
+	#ifdef CONFIG_AS_ZXPAUSE
+	asm volatile("zxpause %%ecx\n"
+		     :
+		     : "c"(ecx), "d"(edx), "a"(eax));
+	#else
+	asm volatile(".byte 0xf2, 0x0f, 0xa6, 0xd0\t\n"
+		     :
+		     : "c"(ecx), "d"(edx), "a"(eax));
+	#endif
+ }
 #endif /* _ASM_X86_MWAIT_H */

arch/x86/include/asm/required-features.h

@@ -100,6 +100,7 @@
 #define REQUIRED_MASK19	0
 #define REQUIRED_MASK20	0
 #define REQUIRED_MASK21	0
-#define REQUIRED_MASK_CHECK BUILD_BUG_ON_ZERO(NCAPINTS != 22)
+#define REQUIRED_MASK22	0
+#define REQUIRED_MASK_CHECK BUILD_BUG_ON_ZERO(NCAPINTS != 23)
 
 #endif /* _ASM_X86_REQUIRED_FEATURES_H */

arch/x86/include/asm/vmx.h

@@ -84,6 +84,12 @@
  */
 #define TERTIARY_EXEC_IPI_VIRT			VMCS_CONTROL_BIT(IPI_VIRT)
 
+/*
+ * Definitions of Zhaoxin Tertiary Processor-Based VM-Execution Controls.
+ */
+#define ZX_TERTIARY_EXEC_GUEST_ZXPAUSE         VMCS_CONTROL_BIT(GUEST_ZXPAUSE)
+
+
 #define PIN_BASED_EXT_INTR_MASK                 VMCS_CONTROL_BIT(INTR_EXITING)
 #define PIN_BASED_NMI_EXITING                   VMCS_CONTROL_BIT(NMI_EXITING)
 #define PIN_BASED_VIRTUAL_NMIS                  VMCS_CONTROL_BIT(VIRTUAL_NMIS)
@@ -235,6 +241,7 @@ enum vmcs_field {
 	TERTIARY_VM_EXEC_CONTROL_HIGH	= 0x00002035,
 	PID_POINTER_TABLE		= 0x00002042,
 	PID_POINTER_TABLE_HIGH		= 0x00002043,
+	ZXPAUSE_VMEXIT_TSC              = 0x00002200,
 	GUEST_PHYSICAL_ADDRESS          = 0x00002400,
 	GUEST_PHYSICAL_ADDRESS_HIGH     = 0x00002401,
 	VMCS_LINK_POINTER               = 0x00002800,
@@ -284,6 +291,7 @@ enum vmcs_field {
 	PLE_GAP                         = 0x00004020,
 	PLE_WINDOW                      = 0x00004022,
 	NOTIFY_WINDOW                   = 0x00004024,
+	ZX_TERTIARY_VM_EXEC_CONTROL     = 0x00004200,
 	VM_INSTRUCTION_ERROR            = 0x00004400,
 	VM_EXIT_REASON                  = 0x00004402,
 	VM_EXIT_INTR_INFO               = 0x00004404,

arch/x86/include/asm/vmxfeatures.h

@@ -5,7 +5,7 @@
 /*
  * Defines VMX CPU feature bits
  */
-#define NVMXINTS			5 /* N 32-bit words worth of info */
+#define NVMXINTS			6 /* N 32-bit words worth of info */
 
 /*
  * Note: If the comment begins with a quoted string, that string is used
@@ -89,4 +89,7 @@
 
 /* Tertiary Processor-Based VM-Execution Controls, word 3 */
 #define VMX_FEATURE_IPI_VIRT		( 3*32+  4) /* Enable IPI virtualization */
+
+/* Zhaoxin Tertiary Processor-Based VM-Execution Controls, word 3 */
+#define VMX_FEATURE_GUEST_ZXPAUSE      ( 5*32+  0) /* zxpause instruction in guest mode */
 #endif /* _ASM_X86_VMXFEATURES_H */

arch/x86/kernel/cpu/Makefile

@@ -25,6 +25,7 @@ obj-y			+= bugs.o
 obj-y			+= aperfmperf.o
 obj-y			+= cpuid-deps.o
 obj-y			+= umwait.o
+obj-y                   += zxpause.o
 
 obj-$(CONFIG_PROC_FS)	+= proc.o
 obj-y += capflags.o powerflags.o

arch/x86/kernel/cpu/centaur.c

@@ -109,6 +109,9 @@ static void early_init_centaur(struct cpuinfo_x86 *c)
 		set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC);
 		set_cpu_cap(c, X86_FEATURE_NONSTOP_TSC);
 	}
+
+	if (cpuid_eax(0xC0000000) >= 0xC0000006)
+		c->x86_capability[CPUID_C000_0006_EAX] = cpuid_eax(0xC0000006);
 }
 
 static void init_centaur(struct cpuinfo_x86 *c)

arch/x86/kernel/cpu/feat_ctl.c

@@ -17,6 +17,7 @@ enum vmx_feature_leafs {
 	SECONDARY_CTLS,
 	TERTIARY_CTLS_LOW,
 	TERTIARY_CTLS_HIGH,
+	ZX_TERTIARY_CTLS,
 	NR_VMX_FEATURE_WORDS,
 };
 
@@ -97,6 +98,13 @@ static void init_vmx_capabilities(struct cpuinfo_x86 *c)
 		set_cpu_cap(c, X86_FEATURE_EPT_AD);
 	if (c->vmx_capability[MISC_FEATURES] & VMX_F(VPID))
 		set_cpu_cap(c, X86_FEATURE_VPID);
+	/*
+	 * Initialize Zhaoxin Tertiary Exec Control feature flags.
+	 */
+	rdmsr_safe(MSR_ZX_EXT_VMCS_CAPS, &supported, &ign);
+	if (supported & MSR_ZX_VMCS_EXEC_CTL3)
+		c->vmx_capability[ZX_TERTIARY_CTLS] |= VMX_F(GUEST_ZXPAUSE);
+
 }
 #endif /* CONFIG_X86_VMX_FEATURE_NAMES */
 

arch/x86/kernel/cpu/zhaoxin.c

@@ -65,6 +65,7 @@ static void early_init_zhaoxin(struct cpuinfo_x86 *c)
 		set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC);
 		set_cpu_cap(c, X86_FEATURE_NONSTOP_TSC);
 	}
+
 }
 
 static void init_zhaoxin(struct cpuinfo_x86 *c)