cputime optimize for arm64 (#3247)

* cputime optimize for arm64
Update src/butil/time.cpp
modify  using __attribute__((constructor)) and merge init_invariant_cpu_freq to inside function read_invariant_cpu_frequency

* Add the macro switch BUTIL_USE_CPU_FREQUENCY while preserving the original default behavior for ARM64

* 1、Activation options: CMake -DWITH_CPU_FREQUENCY=ON, Bazel --define BUTIL_USE_CPU_FREQUENCY=true, script argument --with-cpu-frequency.
2、Disabled by default for consistent legacy behavior.

* fix compile failed

---------

Co-authored-by: seekdwh <dongweihao@huawei.com>

BUILD.bazel

@@ -60,7 +60,10 @@ DEFINES = [
  }) + select({
      "//bazel/config:brpc_with_no_pthread_mutex_hook": ["NO_PTHREAD_MUTEX_HOOK"],
      "//conditions:default": [],
- })
+ }) + select({
+    "//bazel/config:brpc_with_cpu_frequency": ["BUTIL_USE_CPU_FREQUENCY=1"],
+    "//conditions:default": ["BUTIL_USE_CPU_FREQUENCY=0"],
+})
 
 LINKOPTS = [
     "-pthread",
@@ -116,6 +119,14 @@ genrule(
               "//conditions:default": "0",
           }) +
           """
+#ifdef BUTIL_USE_CPU_FREQUENCY
+#undef BUTIL_USE_CPU_FREQUENCY
+#endif
+#define BUTIL_USE_CPU_FREQUENCY """ + select({
+              "//bazel/config:brpc_with_cpu_frequency": "1",
+              "//conditions:default": "0",
+          }) +
+          """
 #endif  // BUTIL_CONFIG_H
 EOF
     """,

CMakeLists.txt

@@ -30,6 +30,7 @@ option(WITH_RDMA "With RDMA" OFF)
 option(WITH_UBRING "With UB" OFF)
 option(WITH_DEBUG_BTHREAD_SCHE_SAFETY "With debugging bthread sche safety" OFF)
 option(WITH_DEBUG_LOCK "With debugging lock" OFF)
+option(WITH_CPU_FREQUENCY "Use CPU frequency for cpuwide_time" OFF)
 option(WITH_ASAN "With AddressSanitizer" OFF)
 option(BUILD_UNIT_TESTS "Whether to build unit tests" OFF)
 option(BUILD_FUZZ_TESTS "Whether to build fuzz tests" OFF)
@@ -76,6 +77,12 @@ if(WITH_GLOG)
     set(BRPC_WITH_GLOG 1)
 endif()
 
+set(WITH_CPU_FREQUENCY_VAL "0")
+if(WITH_CPU_FREQUENCY)
+    set(WITH_CPU_FREQUENCY_VAL "1")
+    set(BUTIL_USE_CPU_FREQUENCY 1)
+endif()
+
 if(WITH_DEBUG_SYMBOLS)
     list(APPEND BRPC_COMMON_COMPILE_OPTIONS -g)
 endif()
@@ -156,6 +163,7 @@ list(APPEND BRPC_COMMON_DEFINITIONS
     BRPC_WITH_UBRING=${WITH_UBRING_VAL}
     BRPC_DEBUG_BTHREAD_SCHE_SAFETY=${WITH_DEBUG_BTHREAD_SCHE_SAFETY_VAL}
     BRPC_DEBUG_LOCK=${WITH_DEBUG_LOCK_VAL}
+    BUTIL_USE_CPU_FREQUENCY=${WITH_CPU_FREQUENCY_VAL}
     BTHREAD_USE_FAST_PTHREAD_MUTEX
     __const__=__unused__
     _GNU_SOURCE
@@ -166,6 +174,7 @@ list(APPEND BRPC_COMMON_DEFINITIONS
     __STDC_CONSTANT_MACROS
     __STRICT_ANSI__
 )
+
 if(NOT DEBUG)
     list(APPEND BRPC_COMMON_DEFINITIONS NDEBUG)
 endif()

bazel/config/BUILD.bazel

@@ -36,6 +36,12 @@ config_setting(
     define_values = {"BRPC_WITH_GLOG": "true"},
 )
 
+config_setting(
+    name = "brpc_with_cpu_frequency",
+    define_values = {"BUTIL_USE_CPU_FREQUENCY": "true"},
+    visibility = ["//visibility:public"],
+)
+
 selects.config_setting_group(
     name = "brpc_with_mesalink",
     match_any = [

config.h.in

@@ -21,4 +21,9 @@
 #endif
 #cmakedefine BRPC_WITH_GLOG @WITH_GLOG_VAL@
 
+#ifdef BUTIL_USE_CPU_FREQUENCY
+#undef BUTIL_USE_CPU_FREQUENCY
+#endif
+#cmakedefine BUTIL_USE_CPU_FREQUENCY @WITH_CPU_FREQUENCY_VAL@
+
 #endif  // BUTIL_CONFIG_H

config_brpc.sh

@@ -54,7 +54,7 @@ else
     LDD=ldd
 fi
 
-TEMP=`getopt -o v: --long headers:,libs:,cc:,cxx:,with-glog,with-thrift,with-rdma,with-mesalink,with-bthread-tracer,with-debug-bthread-sche-safety,with-debug-lock,with-asan,with-riscv-zvbc,with-riscv-zbc,nodebugsymbols,werror -n 'config_brpc' -- "$@"`
+TEMP=`getopt -o v: --long headers:,libs:,cc:,cxx:,with-glog,with-thrift,with-rdma,with-mesalink,with-bthread-tracer,with-debug-bthread-sche-safety,with-debug-lock,with-asan,with-riscv-zvbc,with-riscv-zbc,with-cpu-frequency,nodebugsymbols,werror -n 'config_brpc' -- "$@"`
 WITH_GLOG=0
 WITH_THRIFT=0
 WITH_RDMA=0
@@ -67,6 +67,7 @@ BRPC_DEBUG_BTHREAD_SCHE_SAFETY=0
 DEBUGSYMBOLS=-g
 WERROR=
 BRPC_DEBUG_LOCK=0
+WITH_CPU_FREQUENCY=0
 
 if [ $? != 0 ] ; then >&2 $ECHO "Terminating..."; exit 1 ; fi
 
@@ -93,6 +94,7 @@ while true; do
         --with-bthread-tracer) WITH_BTHREAD_TRACER=1; shift 1 ;;
         --with-debug-bthread-sche-safety ) BRPC_DEBUG_BTHREAD_SCHE_SAFETY=1; shift 1 ;;
         --with-debug-lock ) BRPC_DEBUG_LOCK=1; shift 1 ;;
+        --with-cpu-frequency ) WITH_CPU_FREQUENCY=1; shift 1 ;;
         --with-asan) WITH_ASAN=1; shift 1 ;;
         --with-riscv-zvbc) WITH_RISCV_ZVBC=1; shift 1 ;;
         --with-riscv-zbc) WITH_RISCV_ZBC=1; shift 1 ;;
@@ -479,7 +481,7 @@ append_to_output "STATIC_LINKINGS=$STATIC_LINKINGS"
 append_to_output "DYNAMIC_LINKINGS=$DYNAMIC_LINKINGS"
 
 # CPP means C PreProcessing, not C PlusPlus
-CPPFLAGS="${CPPFLAGS} -DBRPC_WITH_GLOG=$WITH_GLOG -DBRPC_DEBUG_BTHREAD_SCHE_SAFETY=$BRPC_DEBUG_BTHREAD_SCHE_SAFETY -DBRPC_DEBUG_LOCK=$BRPC_DEBUG_LOCK"
+CPPFLAGS="${CPPFLAGS} -DBRPC_WITH_GLOG=$WITH_GLOG -DBRPC_DEBUG_BTHREAD_SCHE_SAFETY=$BRPC_DEBUG_BTHREAD_SCHE_SAFETY -DBRPC_DEBUG_LOCK=$BRPC_DEBUG_LOCK -DBUTIL_USE_CPU_FREQUENCY=$WITH_CPU_FREQUENCY"
 
 # Avoid over-optimizations of TLS variables by GCC>=4.8
 # See: https://github.com/apache/brpc/issues/1693
@@ -649,6 +651,11 @@ cat << EOF > src/butil/config.h
 #endif
 #define BRPC_WITH_GLOG $WITH_GLOG
 
+#ifdef BUTIL_USE_CPU_FREQUENCY
+#undef BUTIL_USE_CPU_FREQUENCY
+#endif
+#define BUTIL_USE_CPU_FREQUENCY $WITH_CPU_FREQUENCY
+
 #endif  // BUTIL_CONFIG_H
 EOF
 

example/BUILD.bazel

@@ -31,6 +31,9 @@ COPTS = [
 ] + select({
     "//bazel/config:brpc_with_glog": ["-DBRPC_WITH_GLOG=1"],
     "//conditions:default": ["-DBRPC_WITH_GLOG=0"],
+}) + select({
+    "//bazel/config:brpc_with_cpu_frequency": ["-DBUTIL_USE_CPU_FREQUENCY=1"],
+    "//conditions:default": ["-DBUTIL_USE_CPU_FREQUENCY=0"],
 }) + select({
     "//bazel/config:brpc_with_rdma": ["-DBRPC_WITH_RDMA=1"],
     "//conditions:default": [""],

src/butil/time.cpp

@@ -143,16 +143,22 @@ int64_t read_cpu_frequency(bool* invariant_tsc) {
 }
 
 // Return value must be >= 0
-int64_t read_invariant_cpu_frequency() {
+static int64_t read_invariant_cpu_frequency() {
     bool invariant_tsc = false;
-    const int64_t freq = read_cpu_frequency(&invariant_tsc);
+    int64_t freq = -1;
+#if defined(__aarch64__) && BUTIL_USE_CPU_FREQUENCY
+    __asm__ __volatile__("mrs %0, CNTFRQ_EL0" : "=r"(freq));
+#else
+    freq = read_cpu_frequency(&invariant_tsc);
     if (!invariant_tsc || freq < 0) {
         return 0;
     }
+#endif
+
     return freq;
 }
 
-int64_t invariant_cpu_freq = -1;
+int64_t invariant_cpu_freq = read_invariant_cpu_frequency();
 }  // namespace detail
 
 }  // namespace butil

src/butil/time.h

@@ -265,10 +265,12 @@ inline uint64_t clock_cycles() {
   #error "unsupported arch"
 #endif
 }
-extern int64_t read_invariant_cpu_frequency();
+
 // Be positive iff:
 // 1 Intel x86_64 CPU (multiple cores) supporting constant_tsc and
 // nonstop_tsc(check flags in /proc/cpuinfo)
+// 2 aarch64, where the counter frequency is read from CNTFRQ_EL0.
+// Only computed when BUTIL_USE_CPU_FREQUENCY is enabled.
 extern int64_t invariant_cpu_freq;
 }  // namespace detail
 
@@ -279,7 +281,7 @@ extern int64_t invariant_cpu_freq;
 // note: Inlining shortens time cost per-call for 15ns in a loop of many
 //       calls to this function.
 inline int64_t cpuwide_time_ns() {
-#if !defined(BAIDU_INTERNAL)
+#if !defined(BAIDU_INTERNAL) && !BUTIL_USE_CPU_FREQUENCY
     // nearly impossible to get the correct invariant cpu frequency on
     // different CPU and machines. CPU-ID rarely works and frequencies
     // in "model name" and "cpu Mhz" are both unreliable.
@@ -298,16 +300,11 @@ inline int64_t cpuwide_time_ns() {
         const uint64_t remain = tsc % cpu_freq;
         // TODO: should be OK until CPU's frequency exceeds 16GHz.
         return remain * 1000000000L / cpu_freq + sec * 1000000000L;
-    } else if (!cpu_freq) {
+    } else {
         // Lack of necessary features, return system-wide monotonic time instead.
         return monotonic_time_ns();
-    } else {
-        // Use a thread-unsafe method(OK to us) to initialize the freq
-        // to save a "if" test comparing to using a local static variable
-        detail::invariant_cpu_freq = detail::read_invariant_cpu_frequency();
-        return cpuwide_time_ns();
     }
-#endif // defined(BAIDU_INTERNAL)
+#endif // defined(BAIDU_INTERNAL) || BUTIL_USE_CPU_FREQUENCY
 }
 
 // Get cpu clock time of the current thread in nanoseconds without the time spent in blocking I/O operations.