#ifndef CORE_LIBS_PRX_LIBC_INCLUDE_PRECISEWAIT_HPP #define CORE_LIBS_PRX_LIBC_INCLUDE_PRECISEWAIT_HPP #include #include #include #include #include inline constexpr std::uint64_t PreciseHundredNanosToQpcTicks(unsigned long long hundredNanos, std::uint64_t freq) { if (freq == 10000000ULL) [[likely]] { return hundredNanos; } const std::uint64_t wholeSeconds = hundredNanos / 10000000ULL; const std::uint64_t remHundredNanos = hundredNanos % 10000000ULL; return wholeSeconds * freq + (remHundredNanos * freq + 9999999ULL) / 10000000ULL; } inline constexpr std::uint64_t PrecisePauseThresholdTicks(std::uint64_t freq) { return PreciseHundredNanosToQpcTicks(2ULL, freq); } #ifdef _WIN32 #include #include "prx/libc/include/HostThreadLocal.hpp" inline void PreciseSleepWaitableTimerWithHandle(HANDLE handle, unsigned long long sleepHundredNanos) { bool slept = false; if (handle != nullptr && handle != INVALID_HANDLE_VALUE) [[likely]] { LARGE_INTEGER due; due.QuadPart = -static_cast(sleepHundredNanos); if (SetWaitableTimer(handle, &due, 0, nullptr, nullptr, FALSE) != FALSE) [[likely]] { slept = (WaitForSingleObject(handle, INFINITE) == WAIT_OBJECT_0); } } if (!slept) [[unlikely]] { std::this_thread::sleep_for(std::chrono::nanoseconds(sleepHundredNanos * 100ULL)); } } inline HANDLE PreciseWaitThreadTimer() { struct Timer { HANDLE handle = CreateWaitableTimerExW(nullptr, nullptr, CREATE_WAITABLE_TIMER_HIGH_RESOLUTION, TIMER_ALL_ACCESS); ~Timer() { if (handle != nullptr) { CloseHandle(handle); } } }; return HostThreadLocal().handle; } inline void PreciseSleepFromCounterWithTimer( LARGE_INTEGER counter, unsigned long long hundredNanos, HANDLE timerHandle, bool useCustomTimer, bool allowSpinTail = true) { if (hundredNanos == 0) { return; } constexpr unsigned long long kSpinTailHundredNanos = 2500; constexpr unsigned long long kMinTimerSpinSleepHundredNanos = 10000; static std::atomic cachedFreq{0}; std::uint64_t freq = cachedFreq.load(std::memory_order_relaxed); std::uint64_t deltaTicks = hundredNanos; std::uint64_t pauseThresholdTicks = 2ULL; if (freq != 10000000ULL) [[unlikely]] { if (freq == 0) { LARGE_INTEGER f; QueryPerformanceFrequency(&f); freq = static_cast(f.QuadPart); cachedFreq.store(freq, std::memory_order_relaxed); } if (freq != 10000000ULL) { deltaTicks = PreciseHundredNanosToQpcTicks(hundredNanos, freq); pauseThresholdTicks = PrecisePauseThresholdTicks(freq); } } const std::uint64_t deadlineTicks = static_cast(counter.QuadPart) + deltaTicks; const bool useSpinTail = allowSpinTail || hundredNanos >= kMinTimerSpinSleepHundredNanos; if (!useSpinTail || hundredNanos > kSpinTailHundredNanos) [[unlikely]] { const HANDLE handle = useCustomTimer ? timerHandle : PreciseWaitThreadTimer(); const unsigned long long timerHundredNanos = useSpinTail ? (hundredNanos - kSpinTailHundredNanos) : hundredNanos; PreciseSleepWaitableTimerWithHandle(handle, timerHundredNanos); } for (;;) { QueryPerformanceCounter(&counter); const std::uint64_t nowTicks = static_cast(counter.QuadPart); if (nowTicks >= deadlineTicks) { break; } if (deadlineTicks - nowTicks > pauseThresholdTicks) { YieldProcessor(); } } } inline void PreciseSleepFromCounter(LARGE_INTEGER counter, unsigned long long hundredNanos) { PreciseSleepFromCounterWithTimer(counter, hundredNanos, nullptr, false, true); } inline void PreciseSleepHundredNanos(unsigned long long hundredNanos) { if (hundredNanos == 0) { return; } LARGE_INTEGER counter; QueryPerformanceCounter(&counter); PreciseSleepFromCounterWithTimer(counter, hundredNanos, nullptr, false, false); } #elif !defined(__EMSCRIPTEN__) #include #include #ifdef __linux__ #include #endif #if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86) #include #endif inline void PreciseSpinPause() { #if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86) _mm_pause(); #else std::atomic_signal_fence(std::memory_order_seq_cst); #endif } inline std::uint64_t PreciseMonotonicNanos() { struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); return static_cast(ts.tv_sec) * 1000000000ULL + static_cast(ts.tv_nsec); } #ifdef __linux__ inline bool PreciseLinuxLowTimerSlack() { static thread_local const bool lowSlack = [] { const long current = prctl(PR_GET_TIMERSLACK, 0UL, 0UL, 0UL, 0UL); return current == 1L || prctl(PR_SET_TIMERSLACK, 1UL, 0UL, 0UL, 0UL) == 0; }(); return lowSlack; } #endif inline void PreciseSleepFromNanosWithSlack(std::uint64_t startNanos, unsigned long long nanos, bool allowLowSlack) { if (nanos == 0) { return; } const std::uint64_t deadlineNanos = startNanos + nanos; #ifdef __linux__ const bool lowSlack = allowLowSlack && PreciseLinuxLowTimerSlack(); const unsigned long long spinTailNanos = lowSlack ? 15000ULL : 100000ULL; if (nanos > spinTailNanos) [[unlikely]] { const std::uint64_t wakeNanos = deadlineNanos - spinTailNanos; struct timespec req; req.tv_sec = static_cast(wakeNanos / 1000000000ULL); req.tv_nsec = static_cast(wakeNanos % 1000000000ULL); while (clock_nanosleep(CLOCK_MONOTONIC, TIMER_ABSTIME, &req, nullptr) == EINTR) { } } #else (void)allowLowSlack; constexpr unsigned long long kSpinTailNanos = 100000ULL; if (nanos > kSpinTailNanos) [[unlikely]] { const unsigned long long sleepNanos = nanos - kSpinTailNanos; struct timespec req; req.tv_sec = static_cast(sleepNanos / 1000000000ULL); req.tv_nsec = static_cast(sleepNanos % 1000000000ULL); while (nanosleep(&req, &req) == -1 && errno == EINTR) { } } #endif for (;;) { const std::uint64_t nowNanos = PreciseMonotonicNanos(); if (nowNanos >= deadlineNanos) { break; } if (deadlineNanos - nowNanos > 200ULL) { PreciseSpinPause(); } } } inline void PreciseSleepFromNanos(std::uint64_t startNanos, unsigned long long nanos) { PreciseSleepFromNanosWithSlack(startNanos, nanos, true); } inline void PreciseSleepNanos(unsigned long long nanos) { if (nanos == 0) { return; } PreciseSleepFromNanos(PreciseMonotonicNanos(), nanos); } #endif inline constexpr unsigned long long kSubTickWaitUs = 2000; inline void PreciseSleepUs(unsigned long long micros) { if (micros == 0) { return; } #ifdef _WIN32 PreciseSleepHundredNanos(micros * 10ULL); return; #elif !defined(__EMSCRIPTEN__) PreciseSleepNanos(micros * 1000ULL); return; #else std::this_thread::sleep_for(std::chrono::microseconds(micros)); #endif } template inline void PreciseSleepUntil(const std::chrono::time_point& deadline) { #ifdef _WIN32 LARGE_INTEGER counter; QueryPerformanceCounter(&counter); const auto now = TClock::now(); if (now < deadline) { const auto remainingNs = std::chrono::duration_cast(deadline - now).count(); if (remainingNs > 0) { PreciseSleepFromCounter(counter, (static_cast(remainingNs) + 99ULL) / 100ULL); } } for (auto current = TClock::now(); current < deadline; current = TClock::now()) { if (deadline - current > std::chrono::nanoseconds(200)) { YieldProcessor(); } } #elif !defined(__EMSCRIPTEN__) const std::uint64_t startNanos = PreciseMonotonicNanos(); const auto now = TClock::now(); if (now < deadline) { const auto remainingNs = std::chrono::duration_cast(deadline - now).count(); if (remainingNs > 0) { PreciseSleepFromNanos(startNanos, static_cast(remainingNs)); } } for (auto current = TClock::now(); current < deadline; current = TClock::now()) { if (deadline - current > std::chrono::nanoseconds(200)) { PreciseSpinPause(); } } #else for (auto now = TClock::now(); now < deadline; now = TClock::now()) { PreciseSleepUs(static_cast(std::chrono::ceil(deadline - now).count())); } #endif } #endif