2021SC@SDUSC BRPC源码分析（七） SOCKET

2021SC@SDUSC BRPC源码分析（七） SOCKET

Socket.cpp代码分析

int Socket::HandleEpollOut(SocketId id) {
    SocketUniquePtr s;
    if (Socket::AddressFailedAsWell(id, &s) < 0) {
        // Ignore recycled sockets
        return -1;
    }

    EpollOutRequest* req = dynamic_cast<EpollOutRequest*>(s->user());
    if (req != NULL) {
        return s->HandleEpollOutRequest(0, req);
    }
    
    s->_epollout_butex->fetch_add(1, butil::memory_order_relaxed);
    bthread::butex_wake_except(s->_epollout_butex, 0);  
    return 0;
}

Socket在被添加到epoll之前可能已经被’ SetFailed’了，这些Socket在’ SetFailed’里面没有信号，因此使用’ AddressFailedAsWell’来发送信号，以防止死锁。

---

int Socket::HandleEpollOutRequest(int error_code, EpollOutRequest* req) {
    if (SetFailed() != 0) {
        return -1;
    }
    GetGlobalEventDispatcher(req->fd).RemoveEpollOut(id(), req->fd, false);
    return req->on_epollout_event(req->fd, error_code, req->data);
}

• 有且只有一个线程成功地调用’ SetFailed’ 的 ’ Socket’。
• 当引用值为0时，销毁req。
• 调用用户的callback。
• 在EpollOutRequest的析构函数中，将计时器删除。

---

int Socket::KeepWriteIfConnected(int fd, int err, void* data) {
    WriteRequest* req = static_cast<WriteRequest*>(data);
    Socket* s = req->socket;
    if (err == 0 && s->ssl_state() == SSL_CONNECTING) {
        bthread_t th;
        google::protobuf::Closure* thrd_func = brpc::NewCallback(
            Socket::CheckConnectedAndKeepWrite, fd, err, data);
        if ((err = bthread_start_background(&th, &BTHREAD_ATTR_NORMAL,
                                            RunClosure, thrd_func)) == 0) {
            return 0;
        } else {
            PLOG(ERROR) << "Fail to start bthread";
        }
    }
    CheckConnectedAndKeepWrite(fd, err, data);
    return 0;
}

• 在新的bthread中运行ssl connect以避免阻塞当前bthread(并且阻塞EventDispatcher)
• 发生非零’ err’输出"Fail to start bthread"
• 检查连接状况

---

	req->data.swap(*data);
    req->next = WriteRequest::UNCONNECTED;
    req->id_wait = opt.id_wait;
    req->set_pipelined_count_and_user_message(
        opt.pipelined_count, DUMMY_USER_MESSAGE, opt.with_auth);
    return StartWrite(req, opt);

将’ req->next’设置为UNCONNECTED，使得KeepWrite线程将一直等待，直到它指向一个有效的WriteRequest或NULL。

---

int Socket::StartWrite(WriteRequest* req, const WriteOptions& opt) {
    WriteRequest* const prev_head =
        _write_head.exchange(req, butil::memory_order_release);
    if (prev_head != NULL) {
        req->next = prev_head;
        return 0;
    }

    int saved_errno = 0;
    bthread_t th;
    SocketUniquePtr ptr_for_keep_write;
    ssize_t nw = 0;

    req->next = NULL;
    
    int ret = ConnectIfNot(opt.abstime, req);
    if (ret < 0) {
        saved_errno = errno;
        SetFailed(errno, "Fail to connect %s directly: %m", description().c_str());
        goto FAIL_TO_WRITE;
    } else if (ret == 1) {
        return 0;
    }

    req->Setup(this);
    
    if (ssl_state() != SSL_OFF) {
        goto KEEPWRITE_IN_BACKGROUND;
    }
   
    if (_conn) {
        butil::IOBuf* data_arr[1] = { &req->data };
        nw = _conn->CutMessageIntoFileDescriptor(fd(), data_arr, 1);
    } else {
        nw = req->data.cut_into_file_descriptor(fd());
    }
    if (nw < 0) {
        if (errno != EAGAIN && errno != EOVERCROWDED) {
            saved_errno = errno;
            PLOG_IF(WARNING, errno != EPIPE) << "Fail to write into " << *this;
            SetFailed(saved_errno, "Fail to write into %s: %s", 
                      description().c_str(), berror(saved_errno));
            goto FAIL_TO_WRITE;
        }
    } else {
        AddOutputBytes(nw);
    }
    if (IsWriteComplete(req, true, NULL)) {
        ReturnSuccessfulWriteRequest(req);
        return 0;
    }

KEEPWRITE_IN_BACKGROUND:
    ReAddress(&ptr_for_keep_write);
    req->socket = ptr_for_keep_write.release();
    if (bthread_start_background(&th, &BTHREAD_ATTR_NORMAL,
                                 KeepWrite, req) != 0) {
        LOG(FATAL) << "Fail to start KeepWrite";
        KeepWrite(req);
    }
    return 0;

FAIL_TO_WRITE:
    ReleaseAllFailedWriteRequests(req);
    errno = saved_errno;
    return -1;
}

• 释放fence确保得到请求的线程获取*req。
• 当对fd进行数据写入时。KeepWrite线程可能会一直请求Spin协议，直到旁边的req->next变为非未连接状态。这个过程不是无锁。
• 设置写入的权限。
• 测试是否连接到remote_side()。
• 进行连接并且CallbackKeepWriteIfConnected，KeepWriteIfConnected之后将会被’ req’调用
• Setup()在Connect之后调用，而Connect可能会调用app_connect。
• 写入SSL可能会阻塞当前bthread，SSL在后台写入。
• 在调用线程中写入一次。如果写入没有完成，在KeepWrite线程中继续。
• 如果nw < 0，RTMP可能返回EOVERCROWDED。
• EPIPE在池连接并且请求备份。
• 在ReturnFailedWriteRequest(ReturnFailedWriteRequest调用的on_reset函数并且在id对象中callback)之前的’ SetFailed’，这样可以立即知道这个Socket在on_reset函数中Callback失败。

---

 while (true) {
        int rc = SSL_do_handshake(_ssl_session);
        if (rc == 1) {
            _ssl_state = SSL_CONNECTED;
            AddBIOBuffer(_ssl_session, fd, FLAGS_ssl_bio_buffer_size);
            return 0;
        }

        int ssl_error = SSL_get_error(_ssl_session, rc);
        switch (ssl_error) {
        case SSL_ERROR_WANT_READ:
#if defined(OS_LINUX)
            if (bthread_fd_wait(fd, EPOLLIN) != 0) {
#elif defined(OS_MACOSX)
            if (bthread_fd_wait(fd, EVFILT_READ) != 0) {
#endif
                return -1;
            }

        case SSL_ERROR_WANT_WRITE:
#if defined(OS_LINUX)
            if (bthread_fd_wait(fd, EPOLLOUT) != 0) {
#elif defined(OS_MACOSX)
            if (bthread_fd_wait(fd, EVFILT_WRITE) != 0) {
#endif
                return -1;
            }

循环直到SSL握手完成。对于SSL_ERROR_WANT_READ/WRITE，使用bthread_fd_wait作为轮询机制而不是EventDispatcher，EventDispatcher可能会混淆原始事件处理代码。

SocketPool（池）

在prev rev中，SocketPool可以被分片到多个subsocketpool中，以减少线程争用。分片键与pthread-id混合，以便更好地保持数据局部性。

inline void SocketPool::ReturnSocket(Socket* sock) {
    const int connection_pool_size = FLAGS_max_connection_pool_size;

    if (_numfree.fetch_add(1, butil::memory_order_relaxed) <
        connection_pool_size) {
        const SocketId sid = sock->id();
        BAIDU_SCOPED_LOCK(_mutex);
        _pool.push_back(sid);
    } else {
        _numfree.fetch_sub(1, butil::memory_order_relaxed);
        sock->SetFailed(EUNUSED, "Close unused pooled socket");
    }
    _numinflight.fetch_sub(1, butil::memory_order_relaxed);
}

保存可以在任何时候重新加载的gflag。检查池是否已满，满了就取消添加并关闭池中的Socket。

---

int Socket::ReturnToPool() {
    SharedPart* sp = _shared_part.exchange(NULL, butil::memory_order_acquire);
    if (sp == NULL) {
        LOG(ERROR) << "_shared_part is NULL";
        SetFailed(EINVAL, "_shared_part is NULL");
        return -1;
    }
    SocketPool* pool = sp->socket_pool.load(butil::memory_order_consume);
    if (pool == NULL) {
        LOG(ERROR) << "_shared_part->socket_pool is NULL";
        SetFailed(EINVAL, "_shared_part->socket_pool is NULL");
        sp->RemoveRefManually();
        return -1;
    }
    _connection_type_for_progressive_read = CONNECTION_TYPE_UNKNOWN;
    _controller_released_socket.store(false, butil::memory_order_relaxed);
    pool->ReturnSocket(this);
    sp->RemoveRefManually();
    return 0;
}

• 返回sp和pool为NULL时的错误。
• sp和pool在返回池之前被重置。
• sp在返回pool后被释放，因为sp拥有sp独立的pool。