char_converter.hpp

Go to the documentation of this file.

// Copyright Take Vos 2022.
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt)

 
#pragma once
 
#include "../utility/utility.hpp"
#include "../macros.hpp"
#include <string>
#include <string_view>
#if defined(HI_HAS_SSE2)
#include <emmintrin.h>
#endif
 
 
 
namespace hi { inline namespace v1 {

template<fixed_string Encoding>
struct char_map;

template<fixed_string From, fixed_string To>
struct char_converter {
public:
    using from_encoder_type = char_map<From>;
    using to_encoder_type = char_map<To>;
    using from_char_type = from_encoder_type::char_type;
    using to_char_type = to_encoder_type::char_type;
    using from_string_type = std::basic_string<from_char_type>;
    using to_string_type = std::basic_string<to_char_type>;

    template<typename OutRange, typename InRange>
    [[nodiscard]] constexpr OutRange convert(InRange&& src) const noexcept
    {
        using std::cbegin;
        using std::cend;
        using std::begin;
        using std::end;
 
        hilet[size, valid] = _size(cbegin(src), cend(src));
 
        auto r = OutRange{};
        if constexpr (From == To and std::is_same_v<InRange, OutRange>) {
            if (valid) {
                r = std::forward<InRange>(src);
                // If and identity conversion is requested and the src is valid, then shortcut by return the src.
                return r;
            }
        }
 
        if (size == 0) {
            return r;
        }
 
        r.resize(size);
        if (From == To and valid) {
            hi_axiom(size != 0);
 
            using std::size;
            std::memcpy(std::addressof(*begin(r)), std::addressof(*cbegin(src)), size(src) * sizeof(from_char_type));
        } else {
            _convert(cbegin(src), cend(src), begin(r));
        }
        return r;
    }

    template<typename OutRange, typename It, typename EndIt>
    [[nodiscard]] constexpr OutRange convert(It first, EndIt last) const noexcept
    {
        using std::begin;
 
        hilet[size, valid] = _size(first, last);
        auto r = OutRange{};
        if (size == 0) {
            return r;
        }
 
        r.resize(size);
        if (From == To and valid) {
            hi_axiom(size != 0);
 
            std::memcpy(std::addressof(*begin(r)), std::addressof(*first), std::distance(first, last) * sizeof(from_char_type));
        } else {
            _convert(first, last, begin(r));
        }
        return r;
    }

    template<typename OutRange = std::basic_string<to_char_type>>
    [[nodiscard]] OutRange read(void const *ptr, size_t size, std::endian endian = std::endian::native) noexcept
    {
        hi_assert_not_null(ptr);
 
        hilet num_chars = size / sizeof(from_char_type);
 
        endian = from_encoder_type{}.guess_endian(ptr, size, endian);
        if (endian == std::endian::native) {
            if (floor(ptr, sizeof(from_char_type)) == ptr) {
                return convert<OutRange>(
                    reinterpret_cast<from_char_type const *>(ptr), reinterpret_cast<from_char_type const *>(ptr) + num_chars);
            } else {
                auto tmp = std::basic_string<from_char_type>{};
                tmp.resize(num_chars);
                std::memcpy(std::addressof(*tmp.begin()), ptr, num_chars * sizeof(from_char_type));
                return convert<OutRange>(std::move(tmp));
            }
        } else {
            auto tmp = std::basic_string<from_char_type>{};
            tmp.resize(num_chars);
            std::memcpy(std::addressof(*tmp.begin()), ptr, num_chars * sizeof(from_char_type));
            for (auto& c : tmp) {
                c = std::byteswap(c);
            }
            return convert<OutRange>(std::move(tmp));
        }
    }

    template<typename InRange>
    [[nodiscard]] constexpr to_string_type operator()(InRange&& src) const noexcept
    {
        return convert<to_string_type>(std::forward<InRange>(src));
    }
 
private:
#if defined(HI_HAS_SSE2)
    using chunk16_type = __m128i;
#else
    using chunk16_type = void;
#endif
 
    constexpr static bool _has_read_ascii_chunk16 = true;
    constexpr static bool _has_write_ascii_chunk16 = true;
 
    template<typename It, typename EndIt>
    [[nodiscard]] constexpr void _size_ascii(It& it, EndIt last, size_t& count) const noexcept
    {
        if (not std::is_constant_evaluated()) {
#if defined(HI_HAS_SSE2)
            if constexpr (_has_read_ascii_chunk16 and _has_write_ascii_chunk16) {
                while (std::distance(it, last) >= 16) {
                    hilet chunk = from_encoder_type{}.read_ascii_chunk16(it);
                    hilet ascii_mask = _mm_movemask_epi8(chunk);
                    if (ascii_mask) {
                        // This chunk contains non-ASCII characters.
                        auto partial_count = std::countr_zero(truncate<uint16_t>(ascii_mask));
                        it += partial_count;
                        count += partial_count;
                        break;
                    }
                    it += 16;
                    count += 16;
                }
            }
#endif
        }
    }
 
    template<typename SrcIt, typename SrcEndIt, typename DstIt>
    void _convert_ascii(SrcIt& src, SrcEndIt src_last, DstIt& dst) const noexcept
    {
        if (not std::is_constant_evaluated()) {
#if defined(HI_HAS_SSE2)
            if constexpr (_has_read_ascii_chunk16 and _has_write_ascii_chunk16) {
                while (std::distance(src, src_last) >= 16) {
                    hilet chunk = from_encoder_type{}.read_ascii_chunk16(src);
                    hilet ascii_mask = _mm_movemask_epi8(chunk);
                    if (ascii_mask) {
                        // This chunk contains non-ASCII characters.
                        break;
                    }
                    // The complete chunk only contains ASCII characters.
                    to_encoder_type{}.write_ascii_chunk16(chunk, dst);
                    src += 16;
                    dst += 16;
                }
            }
#endif
        }
    }
 
    template<typename It, typename EndIt>
    [[nodiscard]] constexpr std::pair<size_t, bool> _size(It it, EndIt last) const noexcept
    {
        auto count = 0_uz;
        auto valid = true;
        while (true) {
            // This loop toggles between converting chunks of ASCII characters and converting
            // a single non-ASCII character.
            _size_ascii(it, last, count);
 
            if (it == last) {
                break;
            }
 
            hilet[code_point, read_valid] = from_encoder_type{}.read(it, last);
            valid &= read_valid;
 
            hilet[write_count, write_valid] = to_encoder_type{}.size(code_point);
            count += write_count;
            valid &= write_valid;
        }
 
        return {count, valid};
    }
 
    template<typename SrcIt, typename SrcEndIt, typename DstIt>
    void _convert(SrcIt src, SrcEndIt src_last, DstIt dst) const noexcept
    {
        while (true) {
            // This loop toggles between converting chunks of ASCII characters and converting
            // a single non-ASCII character.
            _convert_ascii(src, src_last, dst);
 
            if (src == src_last) {
                break;
            }
 
            hilet[code_point, from_valid] = from_encoder_type{}.read(src, src_last);
            to_encoder_type{}.write(code_point, dst);
        }
    }
};
 
}} // namespace hi::v1