wazuh-remoted 接收数据进程

接收远端消息处理主函数

/* Handle remote connections */
void HandleRemote(int uid)
{
    const int position = logr.position;
    int recv_timeout;    //timeout in seconds waiting for a client reply
    int send_timeout;
    char * str_protocol = NULL;

    recv_timeout = getDefine_Int("remoted", "recv_timeout", 1, 60);
    send_timeout = getDefine_Int("remoted", "send_timeout", 1, 60);

    tcp_keepidle = getDefine_Int("remoted", "tcp_keepidle", 1, 7200);
    tcp_keepintvl = getDefine_Int("remoted", "tcp_keepintvl", 1, 100);
    tcp_keepcnt = getDefine_Int("remoted", "tcp_keepcnt", 1, 50);

    /* If syslog connection and allowips is not defined, exit */
    if (logr.conn[position] == SYSLOG_CONN) {
        if (logr.allowips == NULL) {
            minfo(NO_SYSLOG);
            exit(0);
        } else {
            os_ip **tmp_ips;

            tmp_ips = logr.allowips;
            while (*tmp_ips) {
                minfo("Remote syslog allowed from: '%s'", (*tmp_ips)->ip);
                tmp_ips++;
            }
        }
    }

    // Set resource limit for file descriptors

    {
        nofile = getDefine_Int("remoted", "rlimit_nofile", 1024, 1048576);
        struct rlimit rlimit = { nofile, nofile };

        if (setrlimit(RLIMIT_NOFILE, &rlimit) < 0) {
            merror("Could not set resource limit for file descriptors to %d: %s (%d)", (int)nofile, strerror(errno), errno);
        }
    }

    /* If TCP is enabled then bind the TCP socket */
    if (logr.proto[position] & REMOTED_NET_PROTOCOL_TCP) {

        logr.tcp_sock = OS_Bindporttcp(logr.port[position], logr.lip[position], logr.ipv6[position]);

        if (logr.tcp_sock < 0) {
            merror_exit(BIND_ERROR, logr.port[position], errno, strerror(errno));
        }
        else if (logr.conn[position] == SECURE_CONN) {

            if (OS_SetKeepalive(logr.tcp_sock) < 0) {
                merror("OS_SetKeepalive failed with error '%s'", strerror(errno));
            }
#ifndef CLIENT
            else {
                OS_SetKeepalive_Options(logr.tcp_sock, tcp_keepidle, tcp_keepintvl, tcp_keepcnt);
            }
#endif
            if (OS_SetRecvTimeout(logr.tcp_sock, recv_timeout, 0) < 0) {
                merror("OS_SetRecvTimeout failed with error '%s'", strerror(errno));
            }
            if (OS_SetSendTimeout(logr.tcp_sock, send_timeout) < 0) {
                merror("OS_SetSendTimeout failed with error '%s'", strerror(errno));
            }
        }
    }
    /* If UDP is enabled then bind the UDP socket */
    if (logr.proto[position] & REMOTED_NET_PROTOCOL_UDP) {
        /* Using UDP. Fast, unreliable... perfect */
        logr.udp_sock = OS_Bindportudp(logr.port[position], logr.lip[position], logr.ipv6[position]);

        if (logr.udp_sock < 0) {
            merror_exit(BIND_ERROR, logr.port[position], errno, strerror(errno));
        }
    }


    /* Revoke privileges */
    if (Privsep_SetUser(uid) < 0) {
        merror_exit(SETUID_ERROR, USER, errno, strerror(errno));
    }

    /* Create PID */
    if (CreatePID(ARGV0, getpid()) < 0) {
        merror_exit(PID_ERROR);
    }

    /* Start up message */
    // If TCP is enabled
    if (logr.proto[position] & REMOTED_NET_PROTOCOL_TCP) {
        wm_strcat(&str_protocol, REMOTED_NET_PROTOCOL_TCP_STR, WM_STRCAT_NO_SEPARATOR);
    }
    // If UDP is enabled
    if (logr.proto[position] & REMOTED_NET_PROTOCOL_UDP) {
        wm_strcat(&str_protocol, REMOTED_NET_PROTOCOL_UDP_STR, (str_protocol == NULL) ? WM_STRCAT_NO_SEPARATOR : ',');
    }

    /* This should never happen */
    if (str_protocol == NULL) {
        merror_exit(REMOTED_NET_PROTOCOL_NOT_SET);
    }

    minfo(STARTUP_MSG " Listening on port %d/%s (%s).",
        (int)getpid(),
        logr.port[position],
        str_protocol,
        logr.conn[position] == SECURE_CONN ? "secure" : "syslog");
    os_free(str_protocol);

    /* If secure connection, deal with it */
    if (logr.conn[position] == SECURE_CONN) {
        HandleSecure();
    }
    else if (logr.proto[position] == REMOTED_NET_PROTOCOL_TCP) {
        HandleSyslogTCP();
    }
    else { /* If not, deal with syslog */
        HandleSyslog();
    }
}

配置获取，是通过ossec.conf配置文件来的，默认情况下是tcp、连接方式是secure

  <remote>
    <connection>secure</connection>
    <port>1514</port>
    <protocol>tcp</protocol>
    <queue_size>131072</queue_size>
  </remote>

    /* Return 0 if not configured */
    if (RemotedConfig(cfg, &logr) < 0) {
        merror_exit(CONFIG_ERROR, cfg);
    }

 由于配置中是secure 所有我们看HandleSecure()，这里我们可以看到函数前半部分创建很多线程，接收消息的是从while (1) {开始一直到该函数结束，通过epoll来监听网络事件、然后通过nb_recv接收tcp数据。

/* Handle secure connections */
void HandleSecure()
{
    const int protocol = logr.proto[logr.position];
    int sock_client;
    int n_events = 0;
    char buffer[OS_MAXSTR + 1];
    int recv_b;
    struct sockaddr_in peer_info;
    memset(&peer_info, 0, sizeof(struct sockaddr_in));
    wnotify_t * notify = NULL;

    /* Initialize manager */
    manager_init();

    // Initialize messag equeue
    rem_msginit(logr.queue_size);

    /* Initialize the agent key table mutex */
    key_lock_init();

    /* Create shared file updating thread */
    w_create_thread(update_shared_files, NULL);

    /* Create Active Response forwarder thread */
    w_create_thread(AR_Forward, NULL);

    /* Create Security configuration assessment forwarder thread */
    w_create_thread(SCFGA_Forward, NULL);

    // Create Request listener thread
    w_create_thread(req_main, NULL);

    // Create State writer thread
    w_create_thread(rem_state_main, NULL);

    key_request_queue = queue_init(1024);

    // Create key request thread
    w_create_thread(w_key_request_thread, NULL);

    /* Create wait_for_msgs threads */

    {
        int i;
        sender_pool = getDefine_Int("remoted", "sender_pool", 1, 64);

        mdebug2("Creating %d sender threads.", sender_pool);

        for (i = 0; i < sender_pool; i++) {
            w_create_thread(wait_for_msgs, NULL);
        }
    }

    // Reset all the agents' connection status in Wazuh DB
    // The master will disconnect and alert the agents on its own DB. Thus, synchronization is not required.
    if (OS_SUCCESS != wdb_reset_agents_connection("synced", NULL))
        mwarn("Unable to reset the agents' connection status. Possible incorrect statuses until the agents get connected to the manager.");

    // Create message handler thread pool
    {
        int worker_pool = getDefine_Int("remoted", "worker_pool", 1, 16);
        // Initialize FD list and counter.
        global_counter = 0;
        rem_initList(FD_LIST_INIT_VALUE);
        while (worker_pool > 0) {
            w_create_thread(rem_handler_main, NULL);
            worker_pool--;
        }
    }

    /* Connect to the message queue
     * Exit if it fails.
     */
    if ((logr.m_queue = StartMQ(DEFAULTQUEUE, WRITE, INFINITE_OPENQ_ATTEMPTS)) < 0) {
        merror_exit(QUEUE_FATAL, DEFAULTQUEUE);
    }

    /* Read authentication keys */
    minfo(ENC_READ);
    OS_ReadKeys(&keys, W_ENCRYPTION_KEY, 0);
    OS_StartCounter(&keys);

    // Key reloader thread
    w_create_thread(rem_keyupdate_main, NULL);

    // fp closer thread
    w_create_thread(close_fp_main, &keys);

    /* Set up peer size */
    logr.peer_size = sizeof(peer_info);

    /* Initialize some variables */
    memset(buffer, '\0', OS_MAXSTR + 1);

    /* Events watcher is started (is used to monitor sockets events) */
    if (notify = wnotify_init(MAX_EVENTS), !notify) {
        merror_exit("wnotify_init(): %s (%d)", strerror(errno), errno);
    }

    /* If TCP is set on the config, then the corresponding sockets is added to the watching list  */
    if (protocol & REMOTED_NET_PROTOCOL_TCP) {
        if (wnotify_add(notify, logr.tcp_sock) < 0) {
            merror_exit("wnotify_add(%d): %s (%d)", logr.tcp_sock, strerror(errno), errno);
        }
    }

    /* If UDP is set on the config, then the corresponding sockets is added to the watching list  */
    if (protocol & REMOTED_NET_PROTOCOL_UDP) {
        if (wnotify_add(notify, logr.udp_sock) < 0) {
            merror_exit("wnotify_add(%d): %s (%d)", logr.udp_sock, strerror(errno), errno);
        }
    }

    while (1) {

        /* It waits for a socket event */
        if (n_events = wnotify_wait(notify, EPOLL_MILLIS), n_events < 0) {
            if (errno != EINTR) {
                merror("Waiting for connection: %s (%d)", strerror(errno), errno);
                sleep(1);
            }

            continue;
        }

        int i;
        for (i = 0; i < n_events; i++) {
            // Returns the fd of the socket that recived a message
            int fd = wnotify_get(notify, i);

            // In case of failure or unexpected file descriptor
            if (fd <= 0) {
                merror("Unexpected file descriptor: %d, %s (%d)", fd, strerror(errno), errno);
                continue;
            }
            // If a new TCP connection was received and TCP is enabled
            else if ((fd == logr.tcp_sock) && (protocol & REMOTED_NET_PROTOCOL_TCP)) {
                sock_client = accept(logr.tcp_sock, (struct sockaddr *) &peer_info, &logr.peer_size);
                if (sock_client < 0) {
                    switch (errno) {
                    case ECONNABORTED:
                        mdebug1(ACCEPT_ERROR, strerror(errno), errno);
                        break;
                    default:
                        merror(ACCEPT_ERROR, strerror(errno), errno);
                    }

                    continue;
                }

                nb_open(&netbuffer, sock_client, &peer_info);
                rem_inc_tcp();
                mdebug1("New TCP connection at %s [%d]", inet_ntoa(peer_info.sin_addr), sock_client);

                if (wnotify_add(notify, sock_client) < 0) {
                    merror("wnotify_add(%d, %d): %s (%d)", notify->fd, sock_client, strerror(errno), errno);
                    _close_sock(&keys, sock_client);
                }
            }
            // If a new UDP connection was received and UDP is enabled
            else if (fd == logr.udp_sock && protocol & REMOTED_NET_PROTOCOL_UDP) {
                recv_b = recvfrom(logr.udp_sock, buffer, OS_MAXSTR, 0, (struct sockaddr *) &peer_info, &logr.peer_size);

                /* Nothing received */
                if (recv_b <= 0) {
                    continue;
                } else {
                    rem_msgpush(buffer, recv_b, &peer_info, USING_UDP_NO_CLIENT_SOCKET);
                    rem_add_recv((unsigned long) recv_b);
                }
            }
            // If a message was received through a TCP client and tcp is enabled
            else if (protocol & REMOTED_NET_PROTOCOL_TCP) {
                sock_client = fd;

                switch (recv_b = nb_recv(&netbuffer, sock_client), recv_b) {
                case -2:
                    mwarn("Too big message size from %s [%d].", inet_ntoa(peer_info.sin_addr), sock_client);
                    _close_sock(&keys, sock_client);
                    continue;

                case -1:
                    switch (errno) {
                    case ECONNRESET:
                    case ENOTCONN:
                    case EAGAIN:
#if EAGAIN != EWOULDBLOCK
                    case EWOULDBLOCK:
#endif
#if ETIMEDOUT
                    case ETIMEDOUT:
#endif
                        mdebug2("TCP peer [%d] at %s: %s (%d)", sock_client,
                                inet_ntoa(peer_info.sin_addr), strerror(errno), errno);
                        break;
                    default:
                        merror("TCP peer [%d] at %s: %s (%d)", sock_client,
                                inet_ntoa(peer_info.sin_addr), strerror(errno), errno);
                    }
                    fallthrough;
                case 0:
                    _close_sock(&keys, sock_client);
                    continue;

                default:
                    rem_add_recv((unsigned long) recv_b);
                }
            }
        }
    }

    manager_free();
}

tcp消息接收函数，接收到数据之后通过字节序转换，然后rem_msgpush(sockbuf->data + cur_offset, cur_len, &sockbuf->peer_info, sock);往队列（双向循环链表）中存

/*
 * Receive available data from the network and push as many message as possible
 * Returns -2 on data corruption at application layer (header).
 * Returns -1 on system call error: recv().
 * Returns 0 if no data was available in the socket.
 * Returns the number of bytes received on success.
*/
int nb_recv(netbuffer_t * buffer, int sock) {
    sockbuffer_t * sockbuf = &buffer->buffers[sock];
    unsigned long data_ext = sockbuf->data_len + receive_chunk;
    long recv_len;
    unsigned long i;
    unsigned long cur_offset;
    uint32_t cur_len;

    w_mutex_lock(&mutex);

    // Extend data buffer

    if (data_ext > sockbuf->data_size) {
        os_realloc(sockbuf->data, data_ext, sockbuf->data);
        sockbuf->data_size = data_ext;
    }

    // Receive and append

    recv_len = recv(sock, sockbuf->data + sockbuf->data_len, receive_chunk, 0);

    if (recv_len <= 0) {
        goto end;
    }

    sockbuf->data_len += recv_len;

    // Dispatch as most messages as possible

    for (i = 0; i + sizeof(uint32_t) <= sockbuf->data_len; i = cur_offset + cur_len) {
        cur_len = wnet_order(*(uint32_t *)(sockbuf->data + i));

        if (cur_len > OS_MAXSTR) {
            recv_len = -2;
            goto end;
        }

        cur_offset = i + sizeof(uint32_t);

        if (cur_offset + cur_len > sockbuf->data_len) {
            break;
        }

        rem_msgpush(sockbuf->data + cur_offset, cur_len, &sockbuf->peer_info, sock);
    }

    // Move remaining data to data start

    if (i > 0) {
        if (i < sockbuf->data_len) {
            memcpy(sockbuf->data, sockbuf->data + i, sockbuf->data_len - i);
        }

        sockbuf->data_len -= i;

        switch (buffer_relax) {
        case 0:
            // Do not deallocate memory.
            break;

        case 1:
            // Shrink memory to fit the current buffer or the receive chunk.
            sockbuf->data_size = sockbuf->data_len > receive_chunk ? sockbuf->data_len : receive_chunk;
            os_realloc(sockbuf->data, sockbuf->data_size, sockbuf->data);
            break;

        default:
            // Full memory deallocation.
            sockbuf->data_size = sockbuf->data_len;

            if (sockbuf->data_size) {
                os_realloc(sockbuf->data, sockbuf->data_size, sockbuf->data);
            } else {
                os_free(sockbuf->data);
            }
        }
    }

end:

    w_mutex_unlock(&mutex);
    return recv_len;
}

消息出队列rem_msgpop();然后处理，释放内存

// Message handler thread
void * rem_handler_main(__attribute__((unused)) void * args) {
    message_t * message;
    char buffer[OS_MAXSTR + 1] = "";
    int wdb_sock = -1;
    mdebug1("Message handler thread started.");

    while (1) {
        message = rem_msgpop();
        if (message->sock == USING_UDP_NO_CLIENT_SOCKET || message->counter > rem_getCounter(message->sock)) {
            memcpy(buffer, message->buffer, message->size);
            HandleSecureMessage(buffer, message->size, &message->addr, message->sock, &wdb_sock);
        } else {
            rem_inc_dequeued();
        }
        rem_msgfree(message);
    }

    return NULL;
}

 HandleSecureMessage获取有效agentid，keys的管理用读写锁，并且key是被建立成一个红黑树进行管理。对消息进行解码，解压缩，新老格式兼容处理。解析完消息之后通过SendMSG(logr.m_queue, tmp_msg, srcmsg, SECURE_MQ) < 0) ，这里的m_queue是通过unix domain socket实现的，提供给其他进程读，通过SendMSG函数最后一个参数来标记发送的消息类型。

STATIC void HandleSecureMessage(char *buffer, int recv_b, struct sockaddr_in *peer_info, int sock_client, int *wdb_sock) {
    int agentid;
    const int protocol = (sock_client == USING_UDP_NO_CLIENT_SOCKET) ? REMOTED_NET_PROTOCOL_UDP : REMOTED_NET_PROTOCOL_TCP;
    char cleartext_msg[OS_MAXSTR + 1];
    char srcmsg[OS_FLSIZE + 1];
    char srcip[IPSIZE + 1] = {0};
    char agname[KEYSIZE + 1] = {0};
    char *tmp_msg;
    size_t msg_length;
    char ip_found = 0;
    int r;

    /* Set the source IP */
    inet_ntop(peer_info->sin_family, &peer_info->sin_addr, srcip, IPSIZE);

    /* Initialize some variables */
    memset(cleartext_msg, '\0', OS_MAXSTR + 1);
    memset(srcmsg, '\0', OS_FLSIZE + 1);
    tmp_msg = NULL;

    /* Get a valid agent id */
    ...


    /* Decrypt the message */
    ...


    /* Check if it is a control message */
    if (IsValidHeader(tmp_msg)) {

        /* We need to save the peerinfo if it is a control msg */

        keys.keyentries[agentid]->net_protocol = protocol;

        memcpy(&keys.keyentries[agentid]->peer_info, peer_info, logr.peer_size);
        keyentry * key = OS_DupKeyEntry(keys.keyentries[agentid]);
        r = (protocol == REMOTED_NET_PROTOCOL_TCP) ? OS_AddSocket(&keys, agentid, sock_client) : REMOTED_USING_UDP;
        keys.keyentries[agentid]->rcvd = time(0);

        switch (r) {
        case OS_ADDSOCKET_ERROR:
            merror("Couldn't add TCP socket to keystore.");
            break;
        case OS_ADDSOCKET_KEY_UPDATED:
            mdebug2("TCP socket %d already in keystore. Updating...", sock_client);
            break;
        case OS_ADDSOCKET_KEY_ADDED:
            mdebug2("TCP socket %d added to keystore.", sock_client);
            break;
        case REMOTED_USING_UDP:
            keys.keyentries[agentid]->sock = USING_UDP_NO_CLIENT_SOCKET;
            break;
        default:
            ;
        }

        key_unlock();

        // The critical section for readers closes within this function
        save_controlmsg(key, tmp_msg, msg_length - 3, wdb_sock);
        rem_inc_ctrl_msg();

        OS_FreeKey(key);
        return;
    }

    /* Generate srcmsg */

    snprintf(srcmsg, OS_FLSIZE, "[%s] (%s) %s", keys.keyentries[agentid]->id,
             keys.keyentries[agentid]->name, keys.keyentries[agentid]->ip->ip);

    key_unlock();

    /* If we can't send the message, try to connect to the
     * socket again. If it not exit.
     */
    if (SendMSG(logr.m_queue, tmp_msg, srcmsg,
                SECURE_MQ) < 0) {
        merror(QUEUE_ERROR, DEFAULTQUEUE, strerror(errno));

        // Try to reconnect infinitely
        logr.m_queue = StartMQ(DEFAULTQUEUE, WRITE, INFINITE_OPENQ_ATTEMPTS);

        minfo("Successfully reconnected to '%s'", DEFAULTQUEUE);

        if (SendMSG(logr.m_queue, tmp_msg, srcmsg, SECURE_MQ) < 0) {
            // Something went wrong sending a message after an immediate reconnection...
            merror(QUEUE_ERROR, DEFAULTQUEUE, strerror(errno));
        }
    } else {
        rem_inc_evt();
    }
}

remoted进程开启debug日志

echo "remoted.debug=2" > /var/ossec/etc/local_internal_options.conf

systemctl restart wazuh-manager

 在/var/ossec/logs/ossec.log中可以看到remoted相关debug日志