utf8.hpp

Go to the documentation of this file.

 
#ifndef JOIN_DATA_UTF8_HPP
#define JOIN_DATA_UTF8_HPP
 
// libjoin.
#include <join/utils.hpp>
 
// C++.
#include <algorithm>
 
// C.
#include <cstring>
#include <cstddef>
#include <cstdint>
 
namespace join
{
    namespace utf8
    {
        namespace details
        {
            struct alignas (64) SequenceTable
            {
                uint8_t data[256]; 
                uint8_t lower[8];  
                uint8_t span[8];   
                constexpr SequenceTable ()
                : data{}
                , lower{0x80, 0xA0, 0x80, 0x90, 0x80}
                , span{0x3F, 0x1F, 0x1F, 0x2F, 0x0F}
                {
                    for (unsigned i = 0xC2; i < 0xE0; ++i)
                    {
                        data[i] = 2;
                    }
                    for (unsigned i = 0xE1; i < 0xF0; ++i)
                    {
                        data[i] = 3;
                    }
                    for (unsigned i = 0xF1; i < 0xF4; ++i)
                    {
                        data[i] = 4;
                    }
                    data[0xE0] = 3 | (1 << 3);
                    data[0xED] = 3 | (2 << 3);
                    data[0xF0] = 4 | (3 << 3);
                    data[0xF4] = 4 | (4 << 3);
                }
            };
 
            constexpr SequenceTable sequenceLookup{};
 
            struct alignas (64) OrderTable
            {
                uint8_t data[256]; 
                constexpr OrderTable ()
                : data{}
                {
                    for (unsigned i = 0; i < 0xEE; ++i)
                    {
                        data[i] = static_cast<uint8_t> (i);
                    }
                    for (unsigned i = 0xF0; i < 0x100; ++i)
                    {
                        data[i] = static_cast<uint8_t> (i - 2);
                    }
                    data[0xEE] = 0xFE;
                    data[0xEF] = 0xFF;
                }
            };
 
            constexpr OrderTable orderLookup{};
 
            constexpr uint64_t highBits = 0x8080808080808080ULL;
 
            inline uint32_t sequenceLength (const uint8_t* cur, const uint8_t* end) noexcept
            {
                uint32_t entry = sequenceLookup.data[cur[0]];
                uint32_t n = entry & 7;
 
                if (JOIN_UNLIKELY (n == 0 || static_cast<size_t> (end - cur) < n))
                {
                    return 0;
                }
 
                uint32_t range = entry >> 3;
                uint32_t bad = static_cast<uint8_t> (cur[1] - sequenceLookup.lower[range]) > sequenceLookup.span[range];
 
                if (JOIN_LIKELY (end - cur >= 4))
                {
                    bad |= (n >= 3 ? (cur[2] & 0xC0) ^ 0x80 : 0) | (n == 4 ? (cur[3] & 0xC0) ^ 0x80 : 0);
                }
                else
                {
                    for (uint32_t i = 2; i < n; ++i)
                    {
                        bad |= (cur[i] & 0xC0) ^ 0x80;
                    }
                }
 
                return bad ? 0 : n;
            }
 
            inline int validateMixed (const uint8_t* cur, const uint8_t* end) noexcept
            {
                while (cur < end)
                {
                    if (JOIN_LIKELY (end - cur >= 8))
                    {
                        uint64_t word;
                        std::memcpy (&word, cur, 8);
                        if (JOIN_LIKELY ((word & highBits) == 0))
                        {
                            cur += 8;
                            continue;
                        }
                    }
 
                    if (*cur < 0x80)
                    {
                        ++cur;
                        continue;
                    }
 
                    do
                    {
                        uint32_t n = sequenceLength (cur, end);
                        if (JOIN_UNLIKELY (n == 0))
                        {
                            return -1;
                        }
                        cur += n;
                    }
                    while (cur < end && *cur >= 0x80);
                }
 
                return 0;
            }
        }
 
        inline int decode (const char*& cur, const char* end, uint32_t& codepoint) noexcept
        {
            const uint8_t* p = reinterpret_cast<const uint8_t*> (cur);
 
            if (JOIN_LIKELY (p[0] < 0x80))
            {
                codepoint = p[0];
                ++cur;
                return 0;
            }
 
            uint32_t n = details::sequenceLength (p, reinterpret_cast<const uint8_t*> (end));
            if (JOIN_UNLIKELY (n == 0))
            {
                return -1;
            }
 
            uint32_t cp = p[0] & (0xFF >> (n + 1));
            for (uint32_t i = 1; i < n; ++i)
            {
                cp = (cp << 6) | (p[i] & 0x3F);
            }
 
            codepoint = cp;
            cur += n;
            return 0;
        }
 
        inline size_t encode (uint32_t codepoint, char* out) noexcept
        {
            if (JOIN_LIKELY (codepoint < 0x80))
            {
                out[0] = static_cast<char> (codepoint);
                return 1;
            }
 
            if (codepoint < 0x800)
            {
                out[0] = static_cast<char> (0xC0 | (codepoint >> 6));
                out[1] = static_cast<char> (0x80 | (codepoint & 0x3F));
                return 2;
            }
 
            if (codepoint < 0x10000)
            {
                out[0] = static_cast<char> (0xE0 | (codepoint >> 12));
                out[1] = static_cast<char> (0x80 | ((codepoint >> 6) & 0x3F));
                out[2] = static_cast<char> (0x80 | (codepoint & 0x3F));
                return 3;
            }
 
            out[0] = static_cast<char> (0xF0 | (codepoint >> 18));
            out[1] = static_cast<char> (0x80 | ((codepoint >> 12) & 0x3F));
            out[2] = static_cast<char> (0x80 | ((codepoint >> 6) & 0x3F));
            out[3] = static_cast<char> (0x80 | (codepoint & 0x3F));
            return 4;
        }
 
        inline int validate (const char* first, const char* last) noexcept
        {
            const uint8_t* cur = reinterpret_cast<const uint8_t*> (first);
            const uint8_t* end = reinterpret_cast<const uint8_t*> (last);
            uint64_t word;
 
            if (end - cur < 8)
            {
                uint32_t acc = 0;
                for (const uint8_t* p = cur; p < end; ++p)
                {
                    acc |= *p;
                }
                return JOIN_LIKELY (acc < 0x80) ? 0 : details::validateMixed (cur, end);
            }
 
            while (end - cur >= 8)
            {
                std::memcpy (&word, cur, 8);
                if (JOIN_UNLIKELY (word & details::highBits))
                {
                    return details::validateMixed (cur, end);
                }
                cur += 8;
            }
 
            std::memcpy (&word, end - 8, 8);
            return JOIN_LIKELY ((word & details::highBits) == 0) ? 0 : details::validateMixed (cur, end);
        }
 
        inline int compareUtf16Order (const char* first1, const char* last1, const char* first2,
                                      const char* last2) noexcept
        {
            auto mis = std::mismatch (first1, last1, first2, last2);
 
            if (mis.first == last1)
            {
                return (mis.second == last2) ? 0 : -1;
            }
 
            if (mis.second == last2)
            {
                return 1;
            }
 
            return static_cast<int> (details::orderLookup.data[static_cast<uint8_t> (*mis.first)]) -
                   static_cast<int> (details::orderLookup.data[static_cast<uint8_t> (*mis.second)]);
        }
    }
}
 
#endif