proactor_uring_impl.hpp

Go to the documentation of this file.

 
// =========================================================================
//   CLASS     : BasicProactor
//   METHOD    : BasicProactor
// =========================================================================
template <typename IoPolicy>
join::BasicProactor<IoPolicy>::BasicProactor ()
: _commands (_queueSize)
, _wakeup (is_default_v<IoPolicy> ? eventfd (0, EFD_NONBLOCK | EFD_CLOEXEC) : -1)
{
    static_assert (has_spin_v<IoPolicy> || !has_sqpoll_v<IoPolicy>, "spin required for sq poll policy");
 
    io_uring_params params{};
    params.flags = IoPolicy::flags;
 
    if constexpr (has_cq_entries_v<IoPolicy>)
    {
        params.flags |= IORING_SETUP_CQSIZE;
        params.cq_entries = IoPolicy::cqEntries;
    }
 
    if constexpr (has_sq_thread_idle_v<IoPolicy>)
    {
        params.sq_thread_idle = IoPolicy::sqThreadIdle;
    }
 
    if constexpr (has_sq_thread_cpu_v<IoPolicy>)
    {
        params.flags |= IORING_SETUP_SQ_AFF;
        params.sq_thread_cpu = IoPolicy::sqThreadCpu;
    }
 
    if (io_uring_queue_init_params (IoPolicy::sqEntries, &_ring, &params) < 0)
    {
        // LCOV_EXCL_START
        ::close (_wakeup);
        throw std::system_error (errno, std::system_category (), "io_uring_queue_init failed");
        // LCOV_EXCL_STOP
    }
 
    if constexpr (is_default_v<IoPolicy>)
    {
        _wakeupOp = IoOperation::makeRead (_wakeup, &_wakeupBuf, sizeof (_wakeupBuf), nullptr);
    }
 
    _pendingOps.reserve (IoPolicy::sqEntries);
}
 
// =========================================================================
//   CLASS     : BasicProactor
//   METHOD    : ~BasicProactor
// =========================================================================
template <typename IoPolicy>
join::BasicProactor<IoPolicy>::~BasicProactor () noexcept
{
    stop (true);
 
    io_uring_queue_exit (&_ring);
 
    if (_wakeup != -1)
    {
        ::close (_wakeup);
    }
}
 
// =========================================================================
//   CLASS     : BasicProactor
//   METHOD    : flush
// =========================================================================
template <typename IoPolicy>
int join::BasicProactor<IoPolicy>::flush (bool sync) noexcept
{
    if (isProactorThread ())
    {
        if (JOIN_UNLIKELY (io_uring_submit (&_ring) < 0))
        {
            // LCOV_EXCL_START
            lastError = std::error_code (errno, std::system_category ());
            return -1;
            // LCOV_EXCL_STOP
        }
        return 0;
    }
 
    std::atomic<bool> done{false}, *pdone = nullptr;
    std::error_code errc, *perrc = nullptr;
 
    if (JOIN_UNLIKELY (sync))
    {
        pdone = &done;
        perrc = &errc;
    }
 
    if (JOIN_UNLIKELY (writeCommand ({CommandType::Flush, nullptr, false, pdone, perrc, nullptr}) == -1))
    {
        return -1;  // LCOV_EXCL_LINE
    }
 
    if (JOIN_UNLIKELY (sync))
    {
        Backoff backoff;
        while (!done.load (std::memory_order_acquire))
        {
            backoff ();
        }
 
        if (JOIN_UNLIKELY (errc))
        {
            lastError = errc;
            return -1;
        }
    }
 
    return 0;
}
 
// =========================================================================
//   CLASS     : BasicProactor
//   METHOD    : run
// =========================================================================
template <typename IoPolicy>
void join::BasicProactor<IoPolicy>::run ()
{
    _threadId.store (Thread::currentId (), std::memory_order_release);
 
    const int slack = prctl (PR_GET_TIMERSLACK, 0, 0, 0, 0);
    prctl (PR_SET_TIMERSLACK, 1UL, 0, 0, 0);
    _running.store (true, std::memory_order_release);
    eventLoop ();
    prctl (PR_SET_TIMERSLACK, static_cast<unsigned long> (slack), 0, 0, 0);
 
    _threadId.store (Thread::invalidId, std::memory_order_release);
}
 
// =========================================================================
//   CLASS     : BasicProactor
//   METHOD    : stop
// =========================================================================
template <typename IoPolicy>
void join::BasicProactor<IoPolicy>::stop (bool sync) noexcept
{
    if (isProactorThread ())
    {
        _running.store (false, std::memory_order_release);
        cancelAllOperations ();
        return;
    }
 
    if (JOIN_UNLIKELY (!isRunning ()))
    {
        return;
    }
 
    writeCommand ({CommandType::Stop, nullptr, sync, nullptr, nullptr, nullptr});
 
    if (JOIN_LIKELY (sync))
    {
        waitStopped ();
    }
}
 
// =========================================================================
//   CLASS     : BasicProactor
//   METHOD    : waitStopped
// =========================================================================
template <typename IoPolicy>
void join::BasicProactor<IoPolicy>::waitStopped () const noexcept
{
    Backoff backoff;
 
    while (_threadId.load (std::memory_order_acquire) != Thread::invalidId)
    {
        backoff ();
    }
}
 
// =========================================================================
//   CLASS     : BasicProactor
//   METHOD    : registerFixedBuffers
// =========================================================================
template <typename IoPolicy>
template <size_t Count, size_t... Sizes>
int join::BasicProactor<IoPolicy>::registerFixedBuffers (LocalMem::SlabAllocator<Count, Sizes...>& arena) noexcept
{
    return registerFixedBuffers (arena, std::make_index_sequence<sizeof...(Sizes)>{});
}
 
// =========================================================================
//   CLASS     : BasicProactor
//   METHOD    : unregisterFixedBuffers
// =========================================================================
template <typename IoPolicy>
int join::BasicProactor<IoPolicy>::unregisterFixedBuffers () noexcept
{
    int ret = io_uring_unregister_buffers (&_ring);
    if (JOIN_UNLIKELY (ret < 0))
    {
        lastError = std::error_code (-ret, std::system_category ());
        return -1;
    }
 
    return 0;
}
 
// =========================================================================
//   CLASS     : BasicProactor
//   METHOD    : registerBufferRing
// =========================================================================
template <typename IoPolicy>
template <size_t Count, size_t Size>
int join::BasicProactor<IoPolicy>::registerBufferRing (uint16_t group, LocalMem::SlabAllocator<Count, Size>& arena)
{
    int result = 0;
    std::error_code errc;
 
    InvokeHandler fn = [this, group, &arena, &result, &errc] () {
        if (JOIN_UNLIKELY (_bufferRings.count (group)))
        {
            lastError = make_error_code (Errc::InUse);
            errc = lastError;
            result = -1;
            return;
        }
 
        auto it = _bufferRings.emplace (group, &_ring).first;
        if (JOIN_UNLIKELY (it->second.registerBuffer (group, arena) == -1))
        {
            errc = lastError;
            _bufferRings.erase (it);
            result = -1;
            return;
        }
    };
 
    if (JOIN_UNLIKELY (!isRunning ()))
    {
        lastError = make_error_code (Errc::OperationFailed);
        return -1;
    }
 
    if (JOIN_UNLIKELY (invoke (&fn) == -1))
    {
        return -1;  // LCOV_EXCL_LINE
    }
 
    if (JOIN_UNLIKELY (result == -1))
    {
        lastError = errc;
        return -1;
    }
 
    return 0;
}
 
// =========================================================================
//   CLASS     : BasicProactor
//   METHOD    : registerBufferRing
// =========================================================================
template <typename IoPolicy>
template <size_t Count, size_t Size>
int join::BasicProactor<IoPolicy>::registerBufferRing (LocalMem::SlabAllocator<Count, Size>& arena)
{
    int result = 0;
    std::error_code errc;
 
    InvokeHandler fn = [this, &arena, &result, &errc] () {
        uint32_t group = 0;
 
        while ((group <= UINT16_MAX) && _bufferRings.count (static_cast<uint16_t> (group)))
        {
            ++group;
        }
 
        if (JOIN_UNLIKELY (group > UINT16_MAX))
        {
            // LCOV_EXCL_START
            lastError = make_error_code (Errc::OutOfMemory);
            errc = lastError;
            result = -1;
            return;
            // LCOV_EXCL_STOP
        }
 
        auto it = _bufferRings.emplace (static_cast<uint16_t> (group), &_ring).first;
        if (JOIN_UNLIKELY (it->second.registerBuffer (static_cast<uint16_t> (group), arena) == -1))
        {
            errc = lastError;
            _bufferRings.erase (it);
            result = -1;
            return;
        }
 
        result = static_cast<int> (group);
    };
 
    if (JOIN_UNLIKELY (!isRunning ()))
    {
        lastError = make_error_code (Errc::OperationFailed);
        return -1;
    }
 
    if (JOIN_UNLIKELY (invoke (&fn) == -1))
    {
        return -1;  // LCOV_EXCL_LINE
    }
 
    if (JOIN_UNLIKELY (result == -1))
    {
        lastError = errc;
    }
 
    return result;
}
 
// =========================================================================
//   CLASS     : BasicProactor
//   METHOD    : unregisterBufferRing
// =========================================================================
template <typename IoPolicy>
int join::BasicProactor<IoPolicy>::unregisterBufferRing (uint16_t group)
{
    int result = 0;
    std::error_code errc;
 
    InvokeHandler fn = [this, group, &result, &errc] () {
        auto it = _bufferRings.find (group);
        if (JOIN_UNLIKELY (it == _bufferRings.end ()))
        {
            lastError = make_error_code (Errc::NotFound);
            errc = lastError;
            result = -1;
            return;
        }
 
        if (JOIN_UNLIKELY (it->second.armed ()))
        {
            lastError = make_error_code (Errc::InUse);
            errc = lastError;
            result = -1;
            return;
        }
 
        if (JOIN_UNLIKELY (it->second.unregisterBuffer () == -1))
        {
            // LCOV_EXCL_START
            errc = lastError;
            result = -1;
            return;
            // LCOV_EXCL_STOP
        }
 
        _bufferRings.erase (it);
    };
 
    if (JOIN_UNLIKELY (!isRunning ()))
    {
        lastError = make_error_code (Errc::OperationFailed);
        return -1;
    }
 
    if (JOIN_UNLIKELY (invoke (&fn) == -1))
    {
        return -1;  // LCOV_EXCL_LINE
    }
 
    if (JOIN_UNLIKELY (result == -1))
    {
        lastError = errc;
        return -1;
    }
 
    return 0;
}
 
#ifdef JOIN_HAS_NUMA
// =========================================================================
//   CLASS     : BasicProactor
//   METHOD    : mbind
// =========================================================================
template <typename IoPolicy>
int join::BasicProactor<IoPolicy>::mbind (int numa) const noexcept
{
    if (_wheel.mbind (numa) == -1)
    {
        return -1;  // LCOV_EXCL_LINE
    }
 
    return _commands.mbind (numa);
}
#endif
 
// =========================================================================
//   CLASS     : BasicProactor
//   METHOD    : mlock
// =========================================================================
template <typename IoPolicy>
int join::BasicProactor<IoPolicy>::mlock () const noexcept
{
    if (_wheel.mlock () == -1)
    {
        return -1;  // LCOV_EXCL_LINE
    }
 
    return _commands.mlock ();
}
 
// =========================================================================
//   CLASS     : BasicProactor
//   METHOD    : isRunning
// =========================================================================
template <typename IoPolicy>
bool join::BasicProactor<IoPolicy>::isRunning () const noexcept
{
    return _threadId.load (std::memory_order_acquire) != Thread::invalidId;
}
 
// =========================================================================
//   CLASS     : BasicProactor
//   METHOD    : isProactorThread
// =========================================================================
template <typename IoPolicy>
bool join::BasicProactor<IoPolicy>::isProactorThread () const noexcept
{
    return _threadId.load (std::memory_order_acquire) == Thread::currentId ();
}
 
// =========================================================================
//   CLASS     : BasicProactor
//   METHOD    : wheel
// =========================================================================
template <typename IoPolicy>
typename join::BasicProactor<IoPolicy>::Wheel& join::BasicProactor<IoPolicy>::wheel () noexcept
{
    return _wheel;
}
 
// =========================================================================
//   CLASS     : BasicProactor
//   METHOD    : writeCommand
// =========================================================================
template <typename IoPolicy>
int join::BasicProactor<IoPolicy>::writeCommand (const Command& cmd) noexcept
{
    if (JOIN_UNLIKELY (_commands.push (cmd) == -1))
    {
        return -1;  // LCOV_EXCL_LINE
    }
 
    return wakeup ();
}
 
// =========================================================================
//   CLASS     : BasicProactor
//   METHOD    : wakeup
// =========================================================================
template <typename IoPolicy>
int join::BasicProactor<IoPolicy>::wakeup () noexcept
{
    if constexpr (is_default_v<IoPolicy>)
    {
        // pairs with the fence in the event loop: an acquire load would let the push above be
        // reordered after it, the loop would sleep and the wakeup would be lost.
        std::atomic_thread_fence (std::memory_order_seq_cst);
 
        WakeupState state = _wakeupState.load (std::memory_order_relaxed);
        if (state != WakeupState::Sleeping)
        {
            return 0;
        }
 
        if (_wakeupState.compare_exchange_strong (state, WakeupState::Waking, std::memory_order_seq_cst))
        {
            uint64_t value = 1;
            if (JOIN_UNLIKELY (::write (_wakeup, &value, sizeof (uint64_t)) == -1))
            {
                _wakeupState.store (WakeupState::Sleeping, std::memory_order_seq_cst);  // LCOV_EXCL_LINE
            }
        }
    }
 
    return 0;
}
 
// =========================================================================
//   CLASS     : BasicProactor
//   METHOD    : readCommands
// =========================================================================
template <typename IoPolicy>
void join::BasicProactor<IoPolicy>::readCommands () noexcept
{
    Command cmd;
    while (_commands.tryPop (cmd) == 0)
    {
        processCommand (cmd);
    }
}
 
// =========================================================================
//   CLASS     : BasicProactor
//   METHOD    : processCommand
// =========================================================================
template <typename IoPolicy>
void join::BasicProactor<IoPolicy>::processCommand (const Command& cmd) noexcept
{
    int err = 0;
 
    switch (cmd.type)
    {
        case CommandType::Submit:
            err = submitOperation (*cmd.op, cmd.flush);
            if (JOIN_UNLIKELY ((err == -1) && (cmd.done == nullptr) &&
                               (cmd.op->state.load (std::memory_order_relaxed) == IoOperation::State::Idle)))
            {
                dispatchOperation (*cmd.op, -lastError.default_error_condition ().value (), false);
            }
            break;
 
        case CommandType::Cancel:
            err = cancelOperation (*cmd.op, cmd.flush);
            break;
 
        case CommandType::Invoke:
            err = invokeFunction (cmd.fn);
            break;
 
        case CommandType::Stop:
            _running.store (false, std::memory_order_release);
            cancelAllOperations ();
            if (JOIN_UNLIKELY (cmd.flush))
            {
                io_uring_submit (&_ring);
            }
            break;
 
        case CommandType::Flush:
            if (JOIN_UNLIKELY (io_uring_submit (&_ring) < 0))
            {
                // LCOV_EXCL_START
                lastError = std::error_code (errno, std::system_category ());
                err = -1;
                // LCOV_EXCL_STOP
            }
            break;
 
        default:    // LCOV_EXCL_LINE
            break;  // LCOV_EXCL_LINE
    }
 
    if (JOIN_UNLIKELY (cmd.done))
    {
        if (cmd.errc && (err != 0))
        {
            *cmd.errc = lastError;
        }
        cmd.done->store (true, std::memory_order_release);
    }
}
 
// =========================================================================
//   CLASS     : BasicProactor
//   METHOD    : submitOperation
// =========================================================================
template <typename IoPolicy>
int join::BasicProactor<IoPolicy>::submitOperation (IoOperation& op, bool flush) noexcept
{
    if (JOIN_UNLIKELY (!_running.load (std::memory_order_acquire)))
    {
        lastError = std::make_error_code (std::errc::operation_canceled);
        return -1;
    }
 
    Backoff backoff;
    while (JOIN_UNLIKELY (op.state.load (std::memory_order_acquire) == IoOperation::State::Suspended))
    {
        backoff ();
    }
 
    IoOperation::State expected = IoOperation::State::Idle;
    if (!op.state.compare_exchange_strong (expected, IoOperation::State::Submitted, std::memory_order_acquire,
                                           std::memory_order_relaxed) &&
        (expected == IoOperation::State::Busy))
    {
        op.state.store (IoOperation::State::Submitted, std::memory_order_release);
    }
 
    if (JOIN_UNLIKELY (op.fd () < 0))
    {
        resetOperation (op);
        lastError = std::make_error_code (std::errc::bad_file_descriptor);
        return -1;
    }
 
    if (JOIN_UNLIKELY ((op.index < _pendingOps.size ()) && (_pendingOps[op.index] == &op)))
    {
        lastError = make_error_code (std::errc::device_or_resource_busy);
        return -1;
    }
 
    IoRingBuffer* ring = nullptr;
 
    if (JOIN_UNLIKELY (op.multishot && ((op.code == static_cast<uint8_t> (IoOperation::Opcode::RecvMsg)) ||
                                        (op.code == static_cast<uint8_t> (IoOperation::Opcode::Recv)))))
    {
        auto it = _bufferRings.find (op.group);
        if (JOIN_UNLIKELY (it == _bufferRings.end ()))
        {
            resetOperation (op);
            lastError = make_error_code (Errc::NotFound);
            return -1;
        }
 
        ring = &it->second;
    }
 
    io_uring_sqe* sqe = getSqe ();
    if (JOIN_UNLIKELY (sqe == nullptr))
    {
        // LCOV_EXCL_START
        resetOperation (op);
        lastError = make_error_code (std::errc::no_buffer_space);
        return -1;
        // LCOV_EXCL_STOP
    }
 
    prepareSqe (sqe, op);
    op.index = static_cast<uint32_t> (_pendingOps.size ());
    _pendingOps.push_back (&op);
 
    if (JOIN_UNLIKELY (ring != nullptr))
    {
        op.ring = ring;
        ring->bind ();
    }
 
    if (JOIN_UNLIKELY (flush))
    {
        io_uring_submit (&_ring);
    }
 
    return 0;
}
 
// =========================================================================
//   CLASS     : BasicProactor
//   METHOD    : cancelOperation
// =========================================================================
template <typename IoPolicy>
int join::BasicProactor<IoPolicy>::cancelOperation (IoOperation& op, bool flush) noexcept
{
    if (JOIN_UNLIKELY (op.fd () < 0))
    {
        lastError = std::make_error_code (std::errc::bad_file_descriptor);
        return -1;
    }
 
    if (JOIN_UNLIKELY (op.state.load (std::memory_order_acquire) != IoOperation::State::Submitted))
    {
        lastError = make_error_code (Errc::OperationFailed);
        return -1;
    }
 
    if (JOIN_UNLIKELY (op.index >= _pendingOps.size () || _pendingOps[op.index] != &op))
    {
        lastError = make_error_code (Errc::InvalidParam);
        return -1;
    }
 
    io_uring_sqe* sqe = getSqe ();
    if (JOIN_UNLIKELY (sqe == nullptr))
    {
        // LCOV_EXCL_START
        lastError = make_error_code (Errc::OperationFailed);
        return -1;
        // LCOV_EXCL_STOP
    }
 
    io_uring_prep_cancel (sqe, &op, 0);
    io_uring_sqe_set_data (sqe, nullptr);
 
    if (JOIN_UNLIKELY (flush))
    {
        io_uring_submit (&_ring);
    }
 
    return 0;
}
 
// =========================================================================
//   CLASS     : BasicProactor
//   METHOD    : cancelAllOperations
// =========================================================================
template <typename IoPolicy>
void join::BasicProactor<IoPolicy>::cancelAllOperations () noexcept
{
    for (IoOperation* op : _pendingOps)
    {
        Backoff backoff;
        while (op->state.load (std::memory_order_acquire) == IoOperation::State::Suspended)
        {
            backoff ();  // LCOV_EXCL_LINE
        }
        cancelOperation (*op, false);
    }
}
 
// =========================================================================
//   CLASS     : BasicProactor
//   METHOD    : endOperation
// =========================================================================
template <typename IoPolicy>
void join::BasicProactor<IoPolicy>::endOperation (IoOperation& op, int result, bool cancelled) noexcept
{
    if (JOIN_LIKELY (op.index < _pendingOps.size () && _pendingOps[op.index] == &op))
    {
        IoOperation* last = _pendingOps.back ();
        _pendingOps[op.index] = last;
        last->index = op.index;
        _pendingOps.pop_back ();
    }
 
    dispatchOperation (op, result, cancelled);
}
 
// =========================================================================
//   CLASS     : BasicProactor
//   METHOD    : registerFixedBuffers
// =========================================================================
template <typename IoPolicy>
template <size_t Count, size_t... Sizes, size_t... Is>
int join::BasicProactor<IoPolicy>::registerFixedBuffers (LocalMem::SlabAllocator<Count, Sizes...>& arena,
                                                         std::index_sequence<Is...>) noexcept
{
    iovec iovecs[] = {iovec{arena.template getPtr<Is> (0), Count * Sizes}...};
 
    int ret = io_uring_register_buffers (&_ring, iovecs, sizeof...(Sizes));
    if (JOIN_UNLIKELY (ret < 0))
    {
        lastError = std::error_code (-ret, std::system_category ());
        return -1;
    }
 
    return 0;
}
 
// =========================================================================
//   CLASS     : BasicProactor
//   METHOD    : getSqe
// =========================================================================
template <typename IoPolicy>
io_uring_sqe* join::BasicProactor<IoPolicy>::getSqe () noexcept
{
    io_uring_sqe* sqe = io_uring_get_sqe (&_ring);
    if (JOIN_UNLIKELY (sqe == nullptr))
    {
        io_uring_submit (&_ring);
        sqe = io_uring_get_sqe (&_ring);
    }
 
    return sqe;
}
 
// =========================================================================
//   CLASS     : BasicProactor
//   METHOD    : prepareSqe
// =========================================================================
template <typename IoPolicy>
void join::BasicProactor<IoPolicy>::prepareSqe (io_uring_sqe* sqe, IoOperation& op) noexcept
{
    switch (static_cast<IoOperation::Opcode> (op.code))
    {
        case IoOperation::Opcode::Poll:
            if (op.multishot)
            {
                io_uring_prep_poll_multishot (sqe, op.data.poll.fd, op.data.poll.events);
            }
            else
            {
                io_uring_prep_poll_add (sqe, op.data.poll.fd, op.data.poll.events);
            }
            break;
 
        case IoOperation::Opcode::Accept:
            if (op.multishot)
            {
                io_uring_prep_multishot_accept (sqe, op.data.accept.fd, op.data.accept.addr, op.data.accept.addrlen,
                                                op.data.accept.flags);
            }
            else
            {
                io_uring_prep_accept (sqe, op.data.accept.fd, op.data.accept.addr, op.data.accept.addrlen,
                                      op.data.accept.flags);
            }
            break;
 
        case IoOperation::Opcode::Connect:
            io_uring_prep_connect (sqe, op.data.connect.fd, op.data.connect.addr, op.data.connect.addrlen);
            break;
 
        case IoOperation::Opcode::Read:
            io_uring_prep_read (sqe, op.data.rw.fd, op.data.rw.buf, op.data.rw.len, 0);
            break;
 
        case IoOperation::Opcode::Write:
            io_uring_prep_write (sqe, op.data.rw.fd, op.data.rw.buf, op.data.rw.len, 0);
            break;
 
        case IoOperation::Opcode::ReadFixed:
            io_uring_prep_read_fixed (sqe, op.data.rw.fd, op.data.rw.buf, op.data.rw.len, 0, op.data.rw.index);
            break;
 
        case IoOperation::Opcode::WriteFixed:
            io_uring_prep_write_fixed (sqe, op.data.rw.fd, op.data.rw.buf, op.data.rw.len, 0, op.data.rw.index);
            break;
 
        case IoOperation::Opcode::RecvMsg:
            if (op.multishot)
            {
                op.data.msg.msg->msg_namelen = op.data.msg.namelen;
                op.data.msg.msg->msg_controllen = op.data.msg.controllen;
                op.data.msg.msg->msg_iovlen = 0;
                io_uring_prep_recvmsg_multishot (sqe, op.data.msg.fd, op.data.msg.msg, op.data.msg.flags);
                sqe->flags |= IOSQE_BUFFER_SELECT;
                sqe->buf_group = op.group;
            }
            else
            {
                io_uring_prep_recvmsg (sqe, op.data.msg.fd, op.data.msg.msg, op.data.msg.flags);
            }
            break;
 
        case IoOperation::Opcode::SendMsg:
            io_uring_prep_sendmsg (sqe, op.data.msg.fd, op.data.msg.msg, op.data.msg.flags);
            break;
 
        case IoOperation::Opcode::Recv:
            if (op.multishot)
            {
                io_uring_prep_recv_multishot (sqe, op.data.stream.fd, nullptr, 0, op.data.stream.flags);
                sqe->flags |= IOSQE_BUFFER_SELECT;
                sqe->buf_group = op.group;
            }
            else
            {
                io_uring_prep_recv (sqe, op.data.stream.fd, op.data.stream.buf, op.data.stream.len,
                                    op.data.stream.flags);
            }
            break;
 
        case IoOperation::Opcode::Send:
            io_uring_prep_send (sqe, op.data.stream.fd, op.data.stream.buf, op.data.stream.len, op.data.stream.flags);
            break;
 
        default:                      // LCOV_EXCL_LINE
            io_uring_prep_nop (sqe);  // LCOV_EXCL_LINE
    }
 
    io_uring_sqe_set_data (sqe, &op);
 
    if (JOIN_UNLIKELY (op.linked))
    {
        sqe->flags |= IOSQE_IO_LINK;
    }
}
 
// =========================================================================
//   CLASS     : BasicProactor
//   METHOD    : dispatchCqe
// =========================================================================
template <typename IoPolicy>
inline void join::BasicProactor<IoPolicy>::dispatchCqe (io_uring_cqe* cqe) noexcept
{
    IoOperation* op = static_cast<IoOperation*> (io_uring_cqe_get_data (cqe));
 
    if constexpr (is_default_v<IoPolicy>)
    {
        if (JOIN_UNLIKELY (op == &_wakeupOp))
        {
            endOperation (*op, cqe->res, false);
            if (JOIN_LIKELY (_running.load (std::memory_order_acquire)))
            {
                submitOperation (_wakeupOp, true);
            }
            return;
        }
    }
 
    if (JOIN_UNLIKELY (op == nullptr))
    {
        return;
    }
 
    if (JOIN_UNLIKELY (op->state == IoOperation::State::Idle))
    {
        return;  // LCOV_EXCL_LINE
    }
 
    IoOperation::State current = op->state.load (std::memory_order_acquire);
    Backoff backoff;
 
    while (JOIN_UNLIKELY (current == IoOperation::State::Suspended))
    {
        backoff ();
        current = op->state.load (std::memory_order_acquire);
    }
 
    if (JOIN_UNLIKELY (current == IoOperation::State::Idle))
    {
        return;  // LCOV_EXCL_LINE
    }
 
    int result = cqe->res;
    IoRingBuffer* br = nullptr;
    uint16_t bid = 0;
 
    if ((cqe->flags & IORING_CQE_F_BUFFER) != 0)
    {
        bid = static_cast<uint16_t> (cqe->flags >> IORING_CQE_BUFFER_SHIFT);
        br = op->ring;
 
        if (op->code == static_cast<uint8_t> (IoOperation::Opcode::RecvMsg))
        {
            msghdr reserved = {};
            reserved.msg_namelen = op->data.msg.namelen;
            reserved.msg_controllen = op->data.msg.controllen;
 
            io_uring_recvmsg_out* out = io_uring_recvmsg_validate (br->get (bid), result, &reserved);
            if (JOIN_UNLIKELY (out == nullptr))
            {
                result = -EFAULT;  // LCOV_EXCL_LINE
            }
            else
            {
                uint32_t payloadlen = io_uring_recvmsg_payload_length (out, result, &reserved);
 
                op->data.msg.msg->msg_name = io_uring_recvmsg_name (out);
                op->data.msg.msg->msg_control = io_uring_recvmsg_cmsg_firsthdr (out, &reserved);
                op->data.msg.msg->msg_iov->iov_base = io_uring_recvmsg_payload (out, &reserved);
                op->data.msg.msg->msg_iov->iov_len = payloadlen;
                op->data.msg.msg->msg_iovlen = 1;
                op->data.msg.msg->msg_namelen = out->namelen;
                op->data.msg.msg->msg_controllen = out->controllen;
                op->data.msg.msg->msg_flags = static_cast<int> (out->flags);
 
                result = static_cast<int> (payloadlen);
            }
        }
        else
        {
            op->data.stream.buf = br->get (bid);
        }
    }
 
    if ((cqe->flags & IORING_CQE_F_MORE) != 0)
    {
        op->more = true;
        notifyOperation (*op, result, false);
    }
    else
    {
        endOperation (*op, result, (result == -ECANCELED));
    }
 
    if (br != nullptr)
    {
        br->recycle (bid);
    }
}
 
// =========================================================================
//   CLASS     : BasicProactor
//   METHOD    : eventLoop
// =========================================================================
template <typename IoPolicy>
void join::BasicProactor<IoPolicy>::eventLoop () noexcept
{
    eventLoop (std::bool_constant<has_spin_v<IoPolicy>>{}, std::bool_constant<has_sqpoll_v<IoPolicy>>{});
}
 
// =========================================================================
//   CLASS     : BasicProactor
//   METHOD    : eventLoop
// =========================================================================
template <typename IoPolicy>
void join::BasicProactor<IoPolicy>::eventLoop (std::false_type, std::false_type) noexcept
{
    if (JOIN_LIKELY (_running.load (std::memory_order_acquire)))
    {
        submitOperation (_wakeupOp, true);
    }
 
    Backoff backoff;
    bool running;
 
    while ((running = _running.load (std::memory_order_acquire)) || !_pendingOps.empty ())
    {
        WheelPolicy::advance (_wheel);
        readCommands ();
        io_uring_submit (&_ring);
 
        io_uring_cqe* cqe = nullptr;
        if (io_uring_peek_cqe (&_ring, &cqe) == 0)
        {
            backoff.reset ();
 
            do
            {
                dispatchCqe (cqe);
                io_uring_cqe_seen (&_ring, cqe);
            }
            while (io_uring_peek_cqe (&_ring, &cqe) == 0);
 
            continue;
        }
 
        if (!running || !backoff.spinExhausted ())
        {
            backoff ();
            continue;
        }
 
        _wakeupState.store (WakeupState::Sleeping, std::memory_order_relaxed);
        std::atomic_thread_fence (std::memory_order_seq_cst);
        readCommands ();
        WheelPolicy::advance (_wheel);
 
        io_uring_submit (&_ring);
        if (JOIN_LIKELY (_running.load (std::memory_order_acquire)))
        {
            if (io_uring_peek_cqe (&_ring, &cqe) != 0)
            {
                __kernel_timespec ts;
                io_uring_wait_cqe_timeout (&_ring, &cqe, WheelPolicy::timeout (_wheel, ts));
            }
        }
 
        _wakeupState.store (WakeupState::Spinning, std::memory_order_seq_cst);
    }
}
 
// =========================================================================
//   CLASS     : BasicProactor
//   METHOD    : eventLoop
// =========================================================================
template <typename IoPolicy>
void join::BasicProactor<IoPolicy>::eventLoop (std::true_type, std::false_type) noexcept
{
    Backoff backoff{std::chrono::nanoseconds (spin_ns_v<IoPolicy>), std::chrono::nanoseconds (yield_ns_v<IoPolicy>),
                    std::chrono::nanoseconds (sleep_ns_v<IoPolicy>)};
    bool running;
 
    while ((running = _running.load (std::memory_order_acquire)) || !_pendingOps.empty ())
    {
        WheelPolicy::advance (_wheel);
 
        if (JOIN_LIKELY (running))
        {
            readCommands ();
        }
        else
        {
            io_uring_submit (&_ring);  // LCOV_EXCL_LINE
        }
 
        io_uring_cqe* cqe = nullptr;
        if (JOIN_UNLIKELY (io_uring_peek_cqe (&_ring, &cqe) != 0))
        {
            backoff ();
            continue;
        }
 
        do
        {
            dispatchCqe (cqe);
            io_uring_cqe_seen (&_ring, cqe);
        }
        while (io_uring_peek_cqe (&_ring, &cqe) == 0);
 
        backoff.reset ();
    }
}
 
// =========================================================================
//   CLASS     : BasicProactor
//   METHOD    : eventLoop
// =========================================================================
template <typename IoPolicy>
void join::BasicProactor<IoPolicy>::eventLoop (std::true_type, std::true_type) noexcept
{
    Backoff backoff{std::chrono::nanoseconds (spin_ns_v<IoPolicy>), std::chrono::nanoseconds (yield_ns_v<IoPolicy>),
                    std::chrono::nanoseconds (sleep_ns_v<IoPolicy>)};
    bool running;
 
    while ((running = _running.load (std::memory_order_acquire)) || !_pendingOps.empty ())
    {
        WheelPolicy::advance (_wheel);
 
        if (JOIN_LIKELY (running))
        {
            readCommands ();
            // publishes the SQ tail and, in SQPOLL mode, only enters the kernel
            // if the poll thread went to sleep (IORING_SQ_NEED_WAKEUP).
            io_uring_submit (&_ring);
        }
        else
        {
            io_uring_submit (&_ring);  // LCOV_EXCL_LINE
        }
 
        io_uring_cqe* cqe = nullptr;
        if (JOIN_UNLIKELY (io_uring_peek_cqe (&_ring, &cqe) != 0))
        {
            backoff ();
            continue;
        }
 
        do
        {
            dispatchCqe (cqe);
            io_uring_cqe_seen (&_ring, cqe);
        }
        while (io_uring_peek_cqe (&_ring, &cqe) == 0);
 
        backoff.reset ();
    }
}