diff --git a/core/libs/prx/libc/include/PreciseWait.hpp b/core/libs/prx/libc/include/PreciseWait.hpp index e1829564db..b187039495 100644 --- a/core/libs/prx/libc/include/PreciseWait.hpp +++ b/core/libs/prx/libc/include/PreciseWait.hpp @@ -51,6 +51,19 @@ inline HANDLE PreciseWaitThreadTimer() { return HostThreadLocal().handle; } +inline std::uint64_t PreciseQpcFrequency() { + static std::atomic cachedFreq{0}; + std::uint64_t freq = cachedFreq.load(std::memory_order_relaxed); + if (freq == 0) [[unlikely]] { + LARGE_INTEGER f; + if (QueryPerformanceFrequency(&f) && f.QuadPart > 0) { + freq = static_cast(f.QuadPart); + cachedFreq.store(freq, std::memory_order_relaxed); + } + } + return freq; +} + inline void PreciseSleepFromCounterWithTimer( LARGE_INTEGER counter, unsigned long long hundredNanos, @@ -62,21 +75,12 @@ inline void PreciseSleepFromCounterWithTimer( } constexpr unsigned long long kSpinTailHundredNanos = 2500; constexpr unsigned long long kMinTimerSpinSleepHundredNanos = 10000; - static std::atomic cachedFreq{0}; - std::uint64_t freq = cachedFreq.load(std::memory_order_relaxed); + const std::uint64_t freq = PreciseQpcFrequency(); std::uint64_t deltaTicks = hundredNanos; std::uint64_t pauseThresholdTicks = 2ULL; if (freq != 10000000ULL) [[unlikely]] { - if (freq == 0) { - LARGE_INTEGER f; - QueryPerformanceFrequency(&f); - freq = static_cast(f.QuadPart); - cachedFreq.store(freq, std::memory_order_relaxed); - } - if (freq != 10000000ULL) { - deltaTicks = PreciseHundredNanosToQpcTicks(hundredNanos, freq); - pauseThresholdTicks = PrecisePauseThresholdTicks(freq); - } + deltaTicks = PreciseHundredNanosToQpcTicks(hundredNanos, freq); + pauseThresholdTicks = PrecisePauseThresholdTicks(freq); } const std::uint64_t deadlineTicks = static_cast(counter.QuadPart) + deltaTicks; const bool useSpinTail = allowSpinTail || hundredNanos >= kMinTimerSpinSleepHundredNanos; @@ -129,7 +133,11 @@ inline void PreciseSpinPause() { inline std::uint64_t PreciseMonotonicNanos() { struct timespec ts; +#if defined(__APPLE__) && defined(_LIBCPP_VERSION) && defined(CLOCK_MONOTONIC_RAW) + clock_gettime(CLOCK_MONOTONIC_RAW, &ts); +#else clock_gettime(CLOCK_MONOTONIC, &ts); +#endif return static_cast(ts.tv_sec) * 1000000000ULL + static_cast(ts.tv_nsec); } @@ -214,6 +222,29 @@ inline void PreciseSleepUs(unsigned long long micros) { template inline void PreciseSleepUntil(const std::chrono::time_point& deadline) { #ifdef _WIN32 + if constexpr (std::is_same_v) { + if (PreciseQpcFrequency() == 10000000ULL) [[likely]] { + const auto rawDeadline = deadline.time_since_epoch().count(); + if (rawDeadline <= 0) [[unlikely]] { + return; + } + using HundredNanos = std::chrono::duration>; + std::uint64_t targetHns = 0; + if constexpr (std::is_same_v) { + targetHns = (static_cast(rawDeadline) + 99ULL) / 100ULL; + } else { + targetHns = static_cast( + std::chrono::ceil(deadline.time_since_epoch()).count()); + } + LARGE_INTEGER counter; + QueryPerformanceCounter(&counter); + const std::uint64_t nowTicks = static_cast(counter.QuadPart); + if (nowTicks < targetHns) { + PreciseSleepFromCounter(counter, targetHns - nowTicks); + } + return; + } + } LARGE_INTEGER counter; QueryPerformanceCounter(&counter); const auto now = TClock::now(); @@ -229,6 +260,27 @@ inline void PreciseSleepUntil(const std::chrono::time_point& } } #elif !defined(__EMSCRIPTEN__) +#if (defined(__linux__) && (defined(__GLIBCXX__) || (defined(_LIBCPP_VERSION) && !defined(_LIBCPP_HAS_TIMESPEC_GET)))) || \ + (defined(__APPLE__) && defined(_LIBCPP_VERSION) && defined(CLOCK_MONOTONIC_RAW)) + if constexpr (std::is_same_v) { + const auto rawDeadline = deadline.time_since_epoch().count(); + if (rawDeadline <= 0) [[unlikely]] { + return; + } + std::uint64_t targetNanos = 0; + if constexpr (std::is_same_v) { + targetNanos = static_cast(rawDeadline); + } else { + targetNanos = static_cast( + std::chrono::ceil(deadline.time_since_epoch()).count()); + } + const std::uint64_t startNanos = PreciseMonotonicNanos(); + if (startNanos < targetNanos) { + PreciseSleepFromNanos(startNanos, targetNanos - startNanos); + } + return; + } +#endif const std::uint64_t startNanos = PreciseMonotonicNanos(); const auto now = TClock::now(); if (now < deadline) { diff --git a/core/libs/tests/PreciseWait.cpp b/core/libs/tests/PreciseWait.cpp index cf7d7d6979..d9120b173c 100644 --- a/core/libs/tests/PreciseWait.cpp +++ b/core/libs/tests/PreciseWait.cpp @@ -144,6 +144,15 @@ int main() { return 1; } + PreciseSleepUntil(std::chrono::steady_clock::time_point{}); + const auto coarseDeadline = std::chrono::ceil( + std::chrono::steady_clock::now() + std::chrono::microseconds(25)); + PreciseSleepUntil(coarseDeadline); + if (std::chrono::steady_clock::now() < coarseDeadline) { + std::cerr << "PreciseSleepUntil returned before coarse steady_clock deadline\n"; + return 1; + } + #ifdef _WIN32 for (const HANDLE badHandle : {static_cast(nullptr), GetCurrentProcess()}) { const auto start = std::chrono::steady_clock::now();