Merge pull request #6620 from hamb3r/perf/precise-sleep-until-qpc-fastpath

perf(libc): add steady_clock fast path in PreciseSleepUntil

core/libs/prx/libc/include/PreciseWait.hpp

@@ -51,6 +51,19 @@ inline HANDLE PreciseWaitThreadTimer() {
     return HostThreadLocal<Timer, Timer>().handle;
 }
 
+inline std::uint64_t PreciseQpcFrequency() {
+    static std::atomic<std::uint64_t> cachedFreq{0};
+    std::uint64_t freq = cachedFreq.load(std::memory_order_relaxed);
+    if (freq == 0) [[unlikely]] {
+        LARGE_INTEGER f;
+        if (QueryPerformanceFrequency(&f) && f.QuadPart > 0) {
+            freq = static_cast<std::uint64_t>(f.QuadPart);
+            cachedFreq.store(freq, std::memory_order_relaxed);
+        }
+    }
+    return freq;
+}
+
 inline void PreciseSleepFromCounterWithTimer(
     LARGE_INTEGER counter,
     unsigned long long hundredNanos,
@@ -62,21 +75,12 @@ inline void PreciseSleepFromCounterWithTimer(
     }
     constexpr unsigned long long kSpinTailHundredNanos = 2500;
     constexpr unsigned long long kMinTimerSpinSleepHundredNanos = 10000;
-    static std::atomic<std::uint64_t> cachedFreq{0};
-    std::uint64_t freq = cachedFreq.load(std::memory_order_relaxed);
+    const std::uint64_t freq = PreciseQpcFrequency();
     std::uint64_t deltaTicks = hundredNanos;
     std::uint64_t pauseThresholdTicks = 2ULL;
     if (freq != 10000000ULL) [[unlikely]] {
-        if (freq == 0) {
-            LARGE_INTEGER f;
-            QueryPerformanceFrequency(&f);
-            freq = static_cast<std::uint64_t>(f.QuadPart);
-            cachedFreq.store(freq, std::memory_order_relaxed);
-        }
-        if (freq != 10000000ULL) {
-            deltaTicks = PreciseHundredNanosToQpcTicks(hundredNanos, freq);
-            pauseThresholdTicks = PrecisePauseThresholdTicks(freq);
-        }
+        deltaTicks = PreciseHundredNanosToQpcTicks(hundredNanos, freq);
+        pauseThresholdTicks = PrecisePauseThresholdTicks(freq);
     }
     const std::uint64_t deadlineTicks = static_cast<std::uint64_t>(counter.QuadPart) + deltaTicks;
     const bool useSpinTail = allowSpinTail || hundredNanos >= kMinTimerSpinSleepHundredNanos;
@@ -129,7 +133,11 @@ inline void PreciseSpinPause() {
 
 inline std::uint64_t PreciseMonotonicNanos() {
     struct timespec ts;
+#if defined(__APPLE__) && defined(_LIBCPP_VERSION) && defined(CLOCK_MONOTONIC_RAW)
+    clock_gettime(CLOCK_MONOTONIC_RAW, &ts);
+#else
     clock_gettime(CLOCK_MONOTONIC, &ts);
+#endif
     return static_cast<std::uint64_t>(ts.tv_sec) * 1000000000ULL + static_cast<std::uint64_t>(ts.tv_nsec);
 }
 
@@ -214,6 +222,29 @@ inline void PreciseSleepUs(unsigned long long micros) {
 template <class TClock, class TDuration>
 inline void PreciseSleepUntil(const std::chrono::time_point<TClock, TDuration>& deadline) {
 #ifdef _WIN32
+    if constexpr (std::is_same_v<TClock, std::chrono::steady_clock>) {
+        if (PreciseQpcFrequency() == 10000000ULL) [[likely]] {
+            const auto rawDeadline = deadline.time_since_epoch().count();
+            if (rawDeadline <= 0) [[unlikely]] {
+                return;
+            }
+            using HundredNanos = std::chrono::duration<std::int64_t, std::ratio<1, 10000000>>;
+            std::uint64_t targetHns = 0;
+            if constexpr (std::is_same_v<typename TDuration::period, std::nano>) {
+                targetHns = (static_cast<std::uint64_t>(rawDeadline) + 99ULL) / 100ULL;
+            } else {
+                targetHns = static_cast<std::uint64_t>(
+                    std::chrono::ceil<HundredNanos>(deadline.time_since_epoch()).count());
+            }
+            LARGE_INTEGER counter;
+            QueryPerformanceCounter(&counter);
+            const std::uint64_t nowTicks = static_cast<std::uint64_t>(counter.QuadPart);
+            if (nowTicks < targetHns) {
+                PreciseSleepFromCounter(counter, targetHns - nowTicks);
+            }
+            return;
+        }
+    }
     LARGE_INTEGER counter;
     QueryPerformanceCounter(&counter);
     const auto now = TClock::now();
@@ -229,6 +260,27 @@ inline void PreciseSleepUntil(const std::chrono::time_point<TClock, TDuration>&
         }
     }
 #elif !defined(__EMSCRIPTEN__)
+#if (defined(__linux__) && (defined(__GLIBCXX__) || (defined(_LIBCPP_VERSION) && !defined(_LIBCPP_HAS_TIMESPEC_GET)))) || \
+    (defined(__APPLE__) && defined(_LIBCPP_VERSION) && defined(CLOCK_MONOTONIC_RAW))
+    if constexpr (std::is_same_v<TClock, std::chrono::steady_clock>) {
+        const auto rawDeadline = deadline.time_since_epoch().count();
+        if (rawDeadline <= 0) [[unlikely]] {
+            return;
+        }
+        std::uint64_t targetNanos = 0;
+        if constexpr (std::is_same_v<typename TDuration::period, std::nano>) {
+            targetNanos = static_cast<std::uint64_t>(rawDeadline);
+        } else {
+            targetNanos = static_cast<std::uint64_t>(
+                std::chrono::ceil<std::chrono::nanoseconds>(deadline.time_since_epoch()).count());
+        }
+        const std::uint64_t startNanos = PreciseMonotonicNanos();
+        if (startNanos < targetNanos) {
+            PreciseSleepFromNanos(startNanos, targetNanos - startNanos);
+        }
+        return;
+    }
+#endif
     const std::uint64_t startNanos = PreciseMonotonicNanos();
     const auto now = TClock::now();
     if (now < deadline) {

core/libs/tests/PreciseWait.cpp

@@ -144,6 +144,15 @@ int main() {
         return 1;
     }
 
+    PreciseSleepUntil(std::chrono::steady_clock::time_point{});
+    const auto coarseDeadline = std::chrono::ceil<std::chrono::microseconds>(
+        std::chrono::steady_clock::now() + std::chrono::microseconds(25));
+    PreciseSleepUntil(coarseDeadline);
+    if (std::chrono::steady_clock::now() < coarseDeadline) {
+        std::cerr << "PreciseSleepUntil returned before coarse steady_clock deadline\n";
+        return 1;
+    }
+
 #ifdef _WIN32
     for (const HANDLE badHandle : {static_cast<HANDLE>(nullptr), GetCurrentProcess()}) {
         const auto start = std::chrono::steady_clock::now();