cacheline demote to improve cache performance

After the JITTed code is generated, cacheline demote would give a hint to hardware to push out the cache line that contains the linear address. This gets nearly 1% performance gain on our workload. Signed-off-by: Xue,Wang <[email protected]> Signed-off-by: Tao,Su <[email protected]> Signed-off-by: Hu,chen <[email protected]>
wxue1 · May 12, 2023 · c69b476 · c69b476
1 parent 964f494
commit c69b476
Show file tree

Hide file tree

Showing 2 changed files with 47 additions and 0 deletions.
diff --git a/Zend/zend_cpuinfo.h b/Zend/zend_cpuinfo.h
@@ -220,4 +220,15 @@ static inline int zend_cpu_supports_pclmul(void) {
 }
 #endif
 
+/* __builtin_cpu_supports has cldemote from gcc11 */
+#if PHP_HAVE_BUILTIN_CPU_SUPPORTS && defined(__GNUC__) && (ZEND_GCC_VERSION >= 11000)
+ZEND_NO_SANITIZE_ADDRESS
+static inline int zend_cpu_supports_cldemote(void) {
+#if PHP_HAVE_BUILTIN_CPU_INIT
+	__builtin_cpu_init();
+#endif
+	return __builtin_cpu_supports("cldemote");
+}
+#endif
+
 #endif
diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c
@@ -137,6 +137,31 @@ static zend_jit_trace_info *zend_jit_get_current_trace_info(void);
 static uint32_t zend_jit_trace_find_exit_point(const void* addr);
 #endif
 
+#if ZEND_JIT_TARGET_X86 && defined(__linux__)
+# if defined(__GNUC__) && (ZEND_GCC_VERSION >= 11000)
+# define ZEND_JIT_SUPPORT_CLDEMOTE 1
+# else
+# define ZEND_JIT_SUPPORT_CLDEMOTE 0
+# endif
+#endif
+
+#if ZEND_JIT_SUPPORT_CLDEMOTE
+#include <immintrin.h>
+#pragma GCC push_options
+#pragma GCC target("cldemote")
+// check cldemote by CPUID when JIT startup
+static int cpu_support_cldemote = 0;
+static inline void shared_cacheline_demote(uintptr_t start, size_t size) {
+    uintptr_t cache_line_base = start & ~0x3F;
+    do {
+        _cldemote((void *)cache_line_base);
+        // next cacheline start size
+        cache_line_base += 64;
+    } while (cache_line_base < start + size);
+}
+#pragma GCC pop_options
+#endif
+
 static int zend_jit_assign_to_variable(dasm_State    **Dst,
                                        const zend_op  *opline,
                                        zend_jit_addr   var_use_addr,
@@ -972,6 +997,13 @@ static void *dasm_link_and_encode(dasm_State             **dasm_state,
 	/* flush the hardware I-cache */
 	JIT_CACHE_FLUSH(entry, entry + size);
 
+	/* hint to the hardware to push out the cache line that contains the linear address */
+#if ZEND_JIT_SUPPORT_CLDEMOTE
+	if (cpu_support_cldemote && JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
+		shared_cacheline_demote((uintptr_t)entry, size);
+	}
+#endif
+
 	if (trace_num) {
 		zend_jit_trace_add_code(entry, dasm_getpclabel(dasm_state, 1));
 	}
@@ -4931,6 +4963,10 @@ ZEND_EXT_API int zend_jit_startup(void *buf, size_t size, bool reattached)
 	}
 #endif
 
+#if ZEND_JIT_SUPPORT_CLDEMOTE
+	cpu_support_cldemote = zend_cpu_supports_cldemote();
+#endif
+
 	dasm_buf = buf;
 	dasm_size = size;