最简单流媒体服务器设计

这些年用了不少开源的流媒体服务器，也看过一些以前同事写的，都感觉不是太臃肿就是太复杂，反正就是各种不爽，遂自行实现一个简洁高效的轻量级流媒体服务器骨架，以备不时之需。

 

一、目标

1、最简单连接响应，用线程池批量处理

2、最简单通信方式，以socket传输为例

3、最简单通信加密，数据序列化，用google开源的protobuf

4、最简单数据处理方式，用环形缓冲队列

5、最简单负载均衡，LVS、Tengine、Nginx、Haproxy等，可选的太多，目前还没确定哪个最优

6、最简单分布式部署, 信令服务器和数据服务器组成一个单元，服务单元之间可以是树状或者网状结构

 

二、各模块设计，整合

1、首先开发服务器必定会有大量的客户端来连接，所以用一个线程池来批量处理这些连接是非常合适的。

线程池咱就不设计了，在github上找了一圈，以下这个最好，附上demo

thrdtest.c

#define THREAD 32
#define QUEUE  256

#include <stdio.h>
#include <pthread.h>
#include <unistd.h>
#include <assert.h>

#include "threadpool.h"

int tasks = 0, done = 0;
pthread_mutex_t lock;

void dummy_task(void *arg) {
    usleep(10000);
    pthread_mutex_lock(&lock);
    done++;
    pthread_mutex_unlock(&lock);
}

int main(int argc, char **argv)
{
    threadpool_t *pool;

    pthread_mutex_init(&lock, NULL);

    assert((pool = threadpool_create(THREAD, QUEUE, 0)) != NULL);
    fprintf(stderr, "Pool started with %d threads and "
            "queue size of %d\n", THREAD, QUEUE);

    while(threadpool_add(pool, &dummy_task, NULL, 0) == 0) {
        pthread_mutex_lock(&lock);
        tasks++;
        pthread_mutex_unlock(&lock);
    }

    fprintf(stderr, "Added %d tasks\n", tasks);

    while((tasks / 2) > done) {
        usleep(10000);
    }
    assert(threadpool_destroy(pool, 0) == 0);
    fprintf(stderr, "Did %d tasks\n", done);

    return 0;
}

源码包在这里

https://github.com/mbrossard/threadpool

 

2、通信，数据交互，就以最简单的socket为例，平时做多媒体开发，大多用rtsp/rtmp/gb28181之类的协议，跟服务器架构本身没什么关系，这里只做个骨架，枝叶是无穷尽的。

直接搜 socket programming in C ，出来一堆，都比自己写得好。

Socket Server Example

#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <errno.h>
#include <string.h>
#include <sys/types.h>
#include <time.h> 

int main(int argc, char *argv[])
{
    int listenfd = 0, connfd = 0;
    struct sockaddr_in serv_addr; 

    char sendBuff[1025];
    time_t ticks; 

    listenfd = socket(AF_INET, SOCK_STREAM, 0);
    memset(&serv_addr, '0', sizeof(serv_addr));
    memset(sendBuff, '0', sizeof(sendBuff)); 

    serv_addr.sin_family = AF_INET;
    serv_addr.sin_addr.s_addr = htonl(INADDR_ANY);
    serv_addr.sin_port = htons(5000); 

    bind(listenfd, (struct sockaddr*)&serv_addr, sizeof(serv_addr)); 

    listen(listenfd, 10); 

    while(1)
    {
        connfd = accept(listenfd, (struct sockaddr*)NULL, NULL); 

        ticks = time(NULL);
        snprintf(sendBuff, sizeof(sendBuff), "%.24s\r\n", ctime(&ticks));
        write(connfd, sendBuff, strlen(sendBuff)); 

        close(connfd);
        sleep(1);
     }
}

 

Socket Client Example

#include <sys/socket.h>
#include <sys/types.h>
#include <netinet/in.h>
#include <netdb.h>
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <unistd.h>
#include <errno.h>
#include <arpa/inet.h> 

int main(int argc, char *argv[])
{
    int sockfd = 0, n = 0;
    char recvBuff[1024];
    struct sockaddr_in serv_addr; 

    if(argc != 2)
    {
        printf("\n Usage: %s <ip of server> \n",argv[0]);
        return 1;
    } 

    memset(recvBuff, '0',sizeof(recvBuff));
    if((sockfd = socket(AF_INET, SOCK_STREAM, 0)) < 0)
    {
        printf("\n Error : Could not create socket \n");
        return 1;
    } 

    memset(&serv_addr, '0', sizeof(serv_addr)); 

    serv_addr.sin_family = AF_INET;
    serv_addr.sin_port = htons(5000); 

    if(inet_pton(AF_INET, argv[1], &serv_addr.sin_addr)<=0)
    {
        printf("\n inet_pton error occured\n");
        return 1;
    } 

    if( connect(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0)
    {
       printf("\n Error : Connect Failed \n");
       return 1;
    } 

    while ( (n = read(sockfd, recvBuff, sizeof(recvBuff)-1)) > 0)
    {
        recvBuff[n] = 0;
        if(fputs(recvBuff, stdout) == EOF)
        {
            printf("\n Error : Fputs error\n");
        }
    } 

    if(n < 0)
    {
        printf("\n Read error \n");
    } 

    return 0;
}

以下socket实例都写得不错，都收藏在这里，我不是处女座，还是纠结了半天才选了上面这个。

https://www.thegeekstuff.com/2011/12/c-socket-programming/

http://www.linuxhowtos.org/C_C++/socket.htm

https://www.binarytides.com/socket-programming-c-linux-tutorial

https://www.binarytides.com/winsock-socket-programming-tutorial

 

3、服务器一般都需要处理大量的数据，网络吞吐量需求较高，比如此次设计的初始版本虽然只是一个最简单的视频服务器，但是多几路连接数据量也很大，负荷很重。

用一个ringbuf来处理是比较常规的做法，每个连接可以用上面的线程池添加两个线程来配合处理数据，比如一个负责生产数据，一个消费数据，就是设计模式中典型的生产者和消费者模式

simple.c

#include <assert.h>
#include <stdio.h>
#include "../ringbuffer.h"

int main(void) {
  int i, cnt;
  char buf;
  char buf_arr[50];
  
  /* Create and initialize ring buffer */
  ring_buffer_t ring_buffer;
  ring_buffer_init(&ring_buffer);
  
  /* Add elements to buffer; one at a time */
  for(i = 0; i < 100; i++) {
    ring_buffer_queue(&ring_buffer, i);
  }

  /* Verify size */
  assert(ring_buffer_num_items(&ring_buffer) == 100);

  /* Peek third element */
  cnt = ring_buffer_peek(&ring_buffer, &buf, 3);
  /* Assert byte returned */
  assert(cnt == 1);
  /* Assert contents */
  assert(buf == 3);

  /* Dequeue all elements */
  for(cnt = 0; ring_buffer_dequeue(&ring_buffer, &buf) > 0; cnt++) {
    /* Do something with buf... */
    assert(buf == cnt);
    printf("Read: %d\n", buf);
  }
  printf("\n===============\n");

  /* Add array */
  ring_buffer_queue_arr(&ring_buffer, "Hello, Ring Buffer!", 20);

  /* Is buffer empty? */
  assert(!ring_buffer_is_empty(&ring_buffer));

  /* Dequeue all elements */
  while(ring_buffer_dequeue(&ring_buffer, &buf) > 0) {
    /* Print contents */
    printf("Read: %c\n", buf);
  }
  
  /* Add new array */
  ring_buffer_queue_arr(&ring_buffer, "Hello again, Ring Buffer!", 26);
  
  /* Dequeue array in two parts */
  printf("Read:\n");
  cnt = ring_buffer_dequeue_arr(&ring_buffer, buf_arr, 13);
  printf("%d\n", cnt);
  assert(cnt == 13);
  /* Add \0 termination before printing */
  buf_arr[13] = '\0';
  printf("%s\n", buf_arr);
  /* Dequeue remaining */
  cnt = ring_buffer_dequeue_arr(&ring_buffer, buf_arr, 13);
  assert(cnt == 13);
  printf("%s", buf_arr);  
  

  printf("\n===============\n");

  /* Overfill buffer */
  for(i = 0; i < 1000; i++) {
    ring_buffer_queue(&ring_buffer, (i % 127));
  }
  
  /* Is buffer full? */
  if(ring_buffer_is_full(&ring_buffer)) {
    cnt = ring_buffer_num_items(&ring_buffer);
    printf("Buffer is full and contains %d bytes\n", cnt);
  }
  
  /* Dequeue all elements */
  while(ring_buffer_dequeue(&ring_buffer, &buf) > 0) {
    /* Print contents */
    printf("Read: 0x%02x\n", buf);
  }
  
  return 0;
}

https://github.com/AndersKaloer/Ring-Buffer

 

4、以上三个主要的轮子都有了，先整合成一个完整得服务器程序，后面的模块再在这个工程上添加。

实现从socket客户端接收数据，转发到rtmp服务器上，也就是做了个视频数据接收——>封装——>转发的服务器

调试服务器本身可以把转发rtmp改成写个hello,world这种简单的任务，毕竟是设计服务器，而不是rtmp推流器。

因为我把rtmp推送封装好了，用起来跟写个hello,world一样简单，所以就拿来当个示例了。

main.c

#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <stdbool.h>
#include <string.h>
#include <errno.h>
#include <fcntl.h>
#include <pthread.h>
#include <unistd.h>
#include <assert.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/socket.h>
#include <netinet/in.h>

#include "threadpool.h"
#include "ringbuffer.h"
#include "rtmp_relay.h"

#define THREAD 32
#define QUEUE  256

char* rtmp_url = "rtmp://182.61.45.149:1935/live/movie";

pthread_mutex_t lock;

typedef struct task
{
    ring_buffer_t ring_buffer;
    int connfd;
    int index;
    
}task_t;

void source_task(void *arg) {
    
    task_t* task = (task_t*)arg;

    char buf[1024*1024*4];
    int n;
    int len;
    uint32_t tick;

    while(1){
        if((n=read(task->connfd, buf+len, 1024*50)) > 0){    
            char url[64]; sprintf(url,"%s%d   %d   %d",rtmp_url,task->index,n,tick++);
            write(task->connfd, url, strlen(url));
            len+=n;
            if(len > 1024*50){    
                pthread_mutex_lock(&lock);    
                ring_buffer_queue_arr(&(task->ring_buffer), buf, len);
                pthread_mutex_unlock(&lock);
                len =0;
            }                                                    
        }
        printf("socket idle %d\n",n);
    }        
}

void sink_task(void *arg) {

    task_t* task = (task_t*)arg;
    uint32_t tick;

    char buf[1024*1024*4];
    
    char url[64]; sprintf(url,"%s%d",rtmp_url,task->index);
    
    rtmp_relay_t *rtmp_relay = new rtmp_relay_t();
    rtmp_relay->rtmp_connect(url);

    while(1){
        pthread_mutex_lock(&lock);
        int len = ring_buffer_num_items(&(task->ring_buffer));
        if(ring_buffer_dequeue_arr(&(task->ring_buffer), buf, len) > 0) {
            printf("push %s   %d   %d \n",url, len, tick++);
            rtmp_relay->rtmp_send_buf((uint8_t*)buf,len);
        }
        pthread_mutex_unlock(&lock);
    }

    rtmp_relay->rtmp_close();
    delete rtmp_relay;
}


int main(int argc, char **argv)
{
    //threadpool
    threadpool_t *pool;

    assert((pool = threadpool_create(THREAD, QUEUE, 0)) != NULL);
    fprintf(stderr, "Pool started with %d threads and "
            "queue size of %d\n", THREAD, QUEUE);

    //socket
    int listenfd;
    int connfd;
    int index;
    struct sockaddr_in servaddr;

    listenfd = socket(PF_INET, SOCK_STREAM, 0);

    memset(&servaddr, 0, sizeof(servaddr));
    servaddr.sin_family = AF_INET;
    servaddr.sin_addr.s_addr = htonl(INADDR_ANY);
    servaddr.sin_port = htons(50001);

    bind(listenfd, (struct sockaddr *)&servaddr, sizeof(servaddr));

    listen(listenfd, 32);
    
    while(1){
        connfd = accept(listenfd, (struct sockaddr *)NULL, NULL);

        task_t* task = (task_t*)malloc(sizeof(task_t));
        ring_buffer_init(&(task->ring_buffer));
        task->connfd = connfd;
        task->index = index++;

        if(threadpool_add(pool, &sink_task, task, 0) == 0) {
        }
        if(threadpool_add(pool, &source_task, task, 0) == 0) {
        }
        
        //always hold tasks memery until socket disconnect
        //free(task);    
    }

    close(connfd);
    close(listenfd);

    assert(threadpool_destroy(pool, 0) == 0);

    return 0;
}

服务器设计好了，还需要个配套的客户端来测试

提供一个socket客户端demo，实现循环向服务器发送视频数据流的功能

test_client.c

#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/socket.h>
#include <netinet/in.h>

int main(int argc, char **argv)
{
    int sockfd;
    struct sockaddr_in servaddr;

    sockfd = socket(PF_INET, SOCK_STREAM, 0);

    bzero(&servaddr, sizeof(servaddr));
    servaddr.sin_family = AF_INET;
    servaddr.sin_port = htons(50001);
    servaddr.sin_addr.s_addr = inet_addr("182.61.45.149");

    connect(sockfd, (struct sockaddr *)&servaddr, sizeof(servaddr));
        
    unsigned int buf_size = 1024*1024*8;
    unsigned char *buf = (unsigned char*)malloc(buf_size);
    
    FILE *fp = fopen("./tc10.264", "rb");
    
    int n = 0;
    while(1){

        while(!feof(fp)){
            n=fread(buf,1,1024,fp);    
            write(sockfd, buf, n);
            usleep(10000);

            if((n=read(sockfd, buf, 1024*50)) > 0)
            {
                buf[n] = 0;
                printf("%s\n",buf);
            }
        }
        fseek(fp,0,SEEK_SET);    
    }
        
    free(buf);
    fclose(fp);
    close(sockfd);

    return 1;
}

 

 

4、负载均衡，直接上实例

负载均衡可以通过dns，硬件或者软方式来实现，常见的软件负载均衡有LVS、Nginx、Haproxy，

阿里云负载均衡用的是LVS和Tengine，可以参考一下。

http://www.linuxvirtualserver.org

http://tengine.taobao.org/

阿里云负载均衡

https://help.aliyun.com/product/27537.html?spm=a2c4g.11186623.6.540.3e2336e3Lnu8kI

 

5、分布式部署

...

 

先整理到这里， 2019-07-23 02:24:46，先睡了，明天还上班。

 

 

2019-07-24 23:38:55

我打算看一本书《jinpingmei》，这个项目先晾着......

 

转载于:https://www.cnblogs.com/dong1/p/11235482.html