time_stamp_count.hpp

// Copyright Take Vos 2019-2020.
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt)
 
#pragma once
 
#include "architecture.hpp"
#include "cast.hpp"
#include "int_carry.hpp"
#include "thread.hpp"
#include <atomic>
#include <array>
#include <cstdint>
 
#if TT_OPERATING_SYSTEM == TT_OS_WINDOWS
#include <intrin.h>
#elif TT_OPERATING_SYSTEM == TT_OS_LINUX
#include <x86intrin.h>
#endif
 
namespace tt {
 
class time_stamp_count {
public:
    struct inplace {};
    struct inplace_with_cpu_id {};
    struct inplace_with_thread_id {};
 
    constexpr time_stamp_count() noexcept : _count(0), _aux(0), _thread_id(0) {}
 
    constexpr time_stamp_count(uint64_t count, uint32_t aux) noexcept : _count(count), _aux(aux), _thread_id(0) {}
 
    explicit time_stamp_count(time_stamp_count::inplace) noexcept : _aux(0), _thread_id(0)  
    {
        if constexpr (processor::current == processor::x64) {
            uint32_t tmp;
            _count = __rdtscp(&tmp);
        } else {
            tt_not_implemented();
        }
    }
 
    explicit time_stamp_count(time_stamp_count::inplace_with_cpu_id) noexcept : _thread_id(0)
    {
        if constexpr (processor::current == processor::x64) {
            _count = __rdtscp(&_aux);
        } else {
            tt_not_implemented();
        }
    }
 
    explicit time_stamp_count(time_stamp_count::inplace_with_thread_id) noexcept
    {
        if constexpr (processor::current == processor::x64) {
            constexpr uint64_t NT_TIB_CurrentThreadID = 0x48;
 
            _count = __rdtscp(&_aux);
            _thread_id = __readgsdword(NT_TIB_CurrentThreadID);
        } else {
            tt_not_implemented();
        }
    }
 
    [[nodiscard]] static time_stamp_count now() noexcept
    {
        return time_stamp_count{time_stamp_count::inplace_with_cpu_id{}};
    }
 
    [[nodiscard]] ssize_t cpu_id() const noexcept
    {
        if (_aux_is_cpu_id.load(std::memory_order::relaxed)) {
            // On Linux the upper bits are used for a node-id.
            return _aux & 0xfff;
        } else {
            return cpu_id_fallback();
        }
    }
 
    [[nodiscard]] constexpr uint32_t thread_id() const noexcept
    {
        return _thread_id;
    }
 
    [[nodiscard]] constexpr uint64_t count() const noexcept
    {
        return _count;
    }
 
    [[nodiscard]] static std::chrono::nanoseconds duration_from_count(uint64_t count) noexcept
    {
        using namespace std::literals::chrono_literals;
 
        ttlet [lo, hi] = mul_carry(count, _period.load(std::memory_order::relaxed));
        return 1ns * static_cast<int64_t>((hi << 32) | (lo >> 32));
    }
 
    [[nodiscard]] std::chrono::nanoseconds time_since_epoch() const noexcept
    {
        return duration_from_count(_count);
    }
 
    constexpr time_stamp_count &operator+=(uint64_t rhs) noexcept
    {
        _count += rhs;
        return *this;
    }
 
    [[nodiscard]] constexpr time_stamp_count operator+(uint64_t rhs) const noexcept
    {
        auto tmp = *this;
        tmp += rhs;
        return tmp;
    }
 
    [[nodiscard]] static uint64_t measure_frequency(std::chrono::milliseconds duration) noexcept;
 
    static void set_frequency(uint64_t frequency) noexcept
    {
        auto period = (uint64_t{1'000'000'000} << 32) / frequency;
        _period.store(period, std::memory_order_relaxed);
    }
 
    static void start_subsystem() noexcept;
 
private:
    uint64_t _count;
 
    uint32_t _aux;
 
    uint32_t _thread_id;
 
    inline static std::atomic<uint64_t> _period = 0;
 
    inline static std::atomic<bool> _aux_is_cpu_id = false;
 
    inline static std::atomic<size_t> _num_aux_values = 0;
 
    inline static std::array<uint32_t, maximum_num_cpus> _aux_values;
 
    inline static std::array<size_t, maximum_num_cpus> _cpu_ids;
 
    [[nodiscard]] ssize_t cpu_id_fallback() const noexcept;
 
    static void populate_aux_values() noexcept;
    static void configure_frequency() noexcept;
};
 
} // namespace tt