[多核并行计算]进程间通信（三）套接字

socket套接字不仅可以在不同主机间通信，如TCP/UDP等网络方式通信。也可以再本机为不同进程间通信提供服务。

int socket(int domain, int type, int protocol);

socket函数接口中有3个参数，其中第一个domain参数决定了通信域，如果是本机内通信传入AF_UNIX, AF_LOCAL，如果是跨主机使用ip进行通信则传入AF_INET等。

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <fcntl.h> 
#include <math.h> 
#include <sys/time.h> 
#include <sys/stat.h>
#include <errno.h>
#include <string.h>
#include <math.h>
#include <pthread.h>
#include <sys/wait.h>
#include <sys/mman.h>
#include <sys/resource.h>
#include <sys/select.h>
#include <sys/epoll.h>

#include<sys/socket.h>
#include<sys/un.h>
#define UNIX_DOMAIN "/home/tmp/UNIX.domain"
 
void set_thread_affinity(int cpuid);
int set_sc(int type,int prority);
void rtclock_gettime(unsigned long long *tt);
int allocate_rtcore();
int release_rtcore();
int smp_comm1();

#define PRINT_NS		1
#define SEND_ONLY		0
#define RT_MODE_ENABLE		1
#define MAX_PROCESSER_NUM	1024
#define MSG_SIZE		(1024*8)
struct Process
{
	int     id;
	pid_t	pid;
	
}process;

struct Process *processes;
int nr_process = 4;
int shm_fd;
int epoll_fd;
int myid = 0;
static pthread_barrier_t *barrier;
static char *shm_buffer;
 
#define SHARE_RFIFO_PATH "/home/tmp/_fifo"
#define SHARE_WFIFO_PATH "/home/tmp/_fifo"
#define F 2.2E3
#define COMM_NUM 24
 
int    on_cpu;
double dval = 1.23;
double sendbuf[4096];
double recvbuf[4096];
int sendfd[MAX_PROCESSER_NUM];
int recvfd[MAX_PROCESSER_NUM];
static int testloop = 8000;
static int calcloop = 20;
static int cpu_freq = 0;

/*
   1. all processes use a same share memory
   2. The memory area is divided into several sub-blocks
   3. each process use two sub-blocks, one for read and another for write.

                         (Four process IPC)
                   --------              --------
                  |        | ---------> |        |
                  |process0| <--------- |process1|
                  |        |            |        |
                   --------              --------
                   /| |     |\\      //|   | |\
                    | |       \\    //     | |
                    | |        \\  //      | |
                    | |         \\//       | |
                    | |          *         | |
                    | |         //\\       | |
                    | |        //  \\      | |
                    | |       //    \\     | |
                    | |/    |//      \\|  \| |
                   --------              --------
                  |        |            |        |
                  |process2| ---------> |process3|
                  |        | <--------- |        |
                   --------             --------

                                 shared memory range(4 process)
 -----------------------------------------------------------------------------------------------
|       |       |       |       |       |       |       |       |       |       |       |       |
|  0-1  |  0-2  |  0-3  |  1-0  |  1-2  |  1-3  |  2-0  |  2-1  |  2-3  |  3-0  |  3-1  |  3-2  |
|       |       |       |       |       |       |       |       |       |       |       |       |
`-----------------------------------------------------------------------------------------------
*/

void delay(int loop)
{
	int i,j;
	for(i=0; i<loop;i++)
		for(j=0; j<100000; j++)
			dval = sin(3.1415926 + 0.5*dval);
}
int do_calc(int loop)
{
        int i,j;
        for(i=0; i<loop; i++) {
		for(j=0; j<COMM_NUM; j++) {
			sendbuf[j] += sin(recvbuf[j]*3.1415926 + 0.5*myid);
		}                
        }
}

int process_barrier(char *barrier)
{
	char *p = barrier;
	int i,sum = 0;
	int id = myid;
	unsigned int count=0;
	static int flag = 1;

	p[id] = flag;
	while(sum != nr_process){
		sum = 0;
		for(i=0; i<nr_process; i++) {
			if(p[i] == flag)
				sum++;
		}
		count++;
	}
	flag = flag ? 0 : 1;
	//printf("child%d: barrier num %d\n", myid, count);
	return 0;
}

int do_simulation(int loop_num, int one_loop_num)
{
        int i, count = 0, ret = 0;
        struct timeval tv1, tv2;
	unsigned long long t1,t2,t3,t4,t5,t6;
        unsigned long long us1, us2, ms1, ms2;
        unsigned long long ms, us;
	unsigned long long val, cal, sum_cal, sum_val, avg_cal, avg_val;
	unsigned long long tmax_val=0, tmin_val=100000000,tmax_cal=0,tmin_cal=100000000;
	int skip = 0, warm = 10;

	sum_cal = 0;
	sum_val = 0;
        for(i=1; i<=loop_num; i++) {
		
		ret = process_barrier((char*)barrier);
		if(ret != 0 && ret != PTHREAD_BARRIER_SERIAL_THREAD) {
			printf("process%d pid[%d]: barrier_wait error! %s\n", myid, process.pid, strerror(errno));
			return -1;
		}

		rtclock_gettime(&t1);
		skip = smp_comm1();
		rtclock_gettime(&t2);
                do_calc(one_loop_num);
		rtclock_gettime(&t3);
				
		if(skip)
			continue;
                if(i > warm) {
			val = (t3 - t1)/1000;
			if(val > tmax_val) {
				tmax_val = val;
			}
			if(val < tmin_val) {
				tmin_val = val;
			}

			cal = (t3 - t2)/1000;
			if(cal > tmax_cal) {
				tmax_cal = cal;
			}
			if(cal < tmin_cal) {
				tmin_cal = cal;
			}
			sum_val += val;
			sum_cal += cal;
			count++;
                }
        }
	avg_val = sum_val/(loop_num-warm);
	avg_cal = sum_cal/(loop_num-warm);
#if 0
	tmax_val /= F;tmin_val /= F;avg_val /= F;
	tmax_cal /= F;tmin_cal /= F;avg_cal /= F;
#endif
        printf("CPU %d Jitter %llu mean %llu max %llu min %llu | CALC: J %llu mean %llu max %llu min %llu\n", \
                        myid+1,  tmax_val-avg_val, avg_val, tmax_val, tmin_val,\
			tmax_cal-avg_cal, avg_cal, tmax_cal, tmin_cal);
	
	process_barrier((char*)barrier);
	if(cpu_freq)
		printf("CPU %d Jitter %.6f mean %.6f\n", \
			myid+1, (float)(tmax_val-avg_val)/cpu_freq,(float)(avg_val)/cpu_freq);
	return 0;
}

void *process_func(void *args)
{
	int ret;
	int loop = 1;
	char buf[4096];
	
	set_thread_affinity(myid+1);
	sleep(1);
	set_sc(0,90);
	
#if RT_MODE_ENABLE
	allocate_rtcore();
#endif
	do_simulation(testloop, calcloop);

#if RT_MODE_ENABLE
	release_rtcore();
#endif

err:
	exit(ret);
}

void print_shm()
{
	int i,j,k;
	int idx=0;
	char *p;
	for(i=0; i<nr_process; i++) {
		for(j=0; j<nr_process; j++) {
			if(i == j)
				continue;
			printf("[%d-%d]: ",i,j);
		
			p = shm_buffer+(idx*MSG_SIZE);
			for(k=0; k<MSG_SIZE; k++)
				printf(" %d", p[k]);
			printf("\n");
			idx++;
		}
	}
}

int get_shm_idx(int send_id, int recv_id)
{
	int i,j;
	int idx = 0;
	int tmp;

	for(i=0; i<nr_process; i++) {
		for(j=0; j<nr_process; j++) {
			if(i == j)
				continue;
			if(send_id == i && recv_id == j)
				return idx;
			idx++;
		}
	}
	
	return idx;
}

char* get_shm_offset(int send_id, int recv_id)
{
	int idx = 0;

	idx = get_shm_idx(send_id, recv_id);
	if(idx == nr_process*(nr_process-1))
		printf("child%d: cannot find shm !\n", myid);
	return shm_buffer + idx*MSG_SIZE;
}

int smp_recv()
{
	int i,j,k,n,sum = 0;
	char *p;
	char buf[MSG_SIZE];
	int flag[256] = {0};

	for(i=0; i<nr_process; i++) {
		if(i == myid)
			continue;
		if(read(recvfd[i], (char*)recvbuf, sizeof(recvbuf)) < 0) {
			perror("[smp_recv]: read error");
		}
		sum++;
	}
	if(sum != nr_process-1)
		printf("child%d: smp_recv sum != nr_process-1\n", myid);
	return 0;
}

int smp_send()
{
	int i, j,wlen, rlen;
	char *p, *buf;
	
	wlen = MSG_SIZE;
	rlen = MSG_SIZE;
	for(i=0; i<nr_process; i++) {
		if(i == myid)
			continue;

		if(write(sendfd[i], (char*)sendbuf, MSG_SIZE) < 0)
			perror("[smp_send]: write error");
	}
	return 0;
}

int smp_comm1()
{
	static int i = 0;
	static int n = sizeof(recvbuf)/MSG_SIZE;
	smp_send();
#if SEND_ONLY
	if(++i == n) {
		i = 0;
		smp_recv();
		return 1;
	}
#else
	smp_recv();
#endif
	//printf("[%d] %s %d count=%d\n", process.pid,__func__,__LINE__, ++i);
	return 0;
}

 
int barrier_init()
{
	int oflag = 0;
	pthread_barrierattr_t ba;
	int pshared = PTHREAD_PROCESS_SHARED;
	
	char shm_name[] = "tmp_process_barrier";

	/* Initialize a barrier attributes object */
	if (pthread_barrierattr_init(&ba) != 0) {
		printf("Error at pthread_barrierattr_init()\n");
		return -1;
	}
	/* Set the pshard value to private to shared */
	if (pthread_barrierattr_setpshared(&ba, pshared) != 0) {
		printf("Error at pthread_barrierattr_setpshared()\n");
		return -1;
	}
	/* Create shared object */
	if(myid == 0) {
		shm_unlink(shm_name);
		oflag = O_RDWR | O_CREAT | O_EXCL;
	}else{
		oflag = O_RDWR | O_EXCL;
		delay(10);
		//usleep(100000);
	}
	shm_fd = shm_open(shm_name, oflag, S_IRUSR | S_IWUSR);
	if (shm_fd == -1) {
		perror("Error at shm_open()");
		return -1;
	}
	if (ftruncate(shm_fd, sizeof(pthread_barrier_t)+nr_process) != 0) {
		perror("Error at ftruncate()");
		shm_unlink(shm_name);
		return -1;
	}
	/* Map the shared memory object to my memory */
	barrier = mmap(NULL, sizeof(pthread_barrier_t)+nr_process, PROT_READ | PROT_WRITE, MAP_SHARED, shm_fd, 0);
	if (barrier == MAP_FAILED) {
		perror("Error at first mmap()");
		shm_unlink(shm_name);
		return -1;
	}
	if(myid == 0) {
	/* Initialize a barrier */
	if ((pthread_barrier_init(barrier, &ba, nr_process)) != 0) {
		printf("Error at pthread_barrier_init()\n");
		return -1;
	}
	char *p = (char*)barrier;
	p[myid] = 0;
	memset(p, 0, sizeof(pthread_barrier_t)+nr_process);
	}
	/* Cleanup */
	if ((pthread_barrierattr_destroy(&ba)) != 0) {
		printf("Error at pthread_barrierattr_destroy()\n");
		return -1;
	}
	return 0;	
}

int shm_init()
{
	int shm_fd;
	int oflag;
	int shm_size = (nr_process*(nr_process-1))*MSG_SIZE;
	char shm_name[] = "tmp_process_shm";

	if(myid == 0) {
		shm_unlink(shm_name);
		oflag = O_RDWR | O_CREAT | O_EXCL;
		shm_fd = shm_open(shm_name, oflag, S_IRUSR | S_IWUSR);
		if (shm_fd == -1) {
			perror("Error at shm_open()");
			return -1;
		}
		if (ftruncate(shm_fd, shm_size) != 0) {
			perror("Error at ftruncate()");
			shm_unlink(shm_name);
			return -1;
		}
	}else{
		delay(10);
		oflag = O_RDWR | O_EXCL;
		shm_fd = shm_open(shm_name, oflag, S_IRUSR | S_IWUSR);
		if (shm_fd == -1) {
			perror("Error at shm_open()");
			return -1;
		}

	}
	/* Map the shared memory object to my memory */
	shm_buffer = mmap(NULL, shm_size, PROT_READ | PROT_WRITE, MAP_SHARED, shm_fd, 0);
	if (shm_buffer == MAP_FAILED) {
		perror("Error at first mmap()");
		shm_unlink(shm_name);
		return -1;
	}
	//close(shm_fd);
	if(myid == 0) {
		memset(shm_buffer, 0, shm_size);
	}else {
		delay(40);
	}
	return 0;
}

void *thread_emt(__attribute__((unused)) void *arg)
{
	process_func(arg);
}
 
int socket_init()
{
	struct sockaddr_un clt_addr;
	struct sockaddr_un srv_addr;
	struct sockaddr_un conn_addr;
	socklen_t clt_addr_len;
    
	int listen_fd[MAX_PROCESSER_NUM];
	int conn_fd[MAX_PROCESSER_NUM];
	int i, ret;
	char rpath[512];
	char wpath[512];
	int len;
	for(i=0; i<nr_process; i++) {
		if(i == myid)
			continue;
		listen_fd[i] = socket(AF_UNIX, SOCK_STREAM, 0);
		if(listen_fd[i] < 0) {
			printf("child%d socket create error! %s\n", myid, strerror(errno));
			exit(-1);
		}
		conn_fd[i] = socket(AF_UNIX, SOCK_STREAM, 0);
		if(conn_fd[i] < 0) {
			printf("child%d socket create error! %s\n", myid, strerror(errno));
			exit(-1);
		}
		len = 1;
		if(setsockopt(listen_fd[i], SOL_SOCKET, SO_REUSEADDR, &len, sizeof(len)) < 0) {
			printf("child%d setsockopt SO_REUSEADDR error! %s\n", myid, strerror(errno));
			exit(-1);
		}
		sprintf(rpath, "%s%d_%d", UNIX_DOMAIN, myid, i);
		sprintf(wpath, "%s%d_%d", UNIX_DOMAIN, i, myid);
		strncpy(srv_addr.sun_path, rpath, sizeof(srv_addr.sun_path)-1);
		strncpy(conn_addr.sun_path, wpath, sizeof(conn_addr.sun_path)-1);
		srv_addr.sun_family = AF_UNIX;
		conn_addr.sun_family = AF_UNIX;
		ret = bind(listen_fd[i], (struct sockaddr*)&srv_addr, sizeof(srv_addr));
		if(ret < 0) {
			printf("child%d socket bind error! %s\n", myid, strerror(errno));
			exit(-1);
		}
		ret = listen(listen_fd[i], 1);
		if(ret < 0) {
			printf("child%d socket listen error! %s\n", myid, strerror(errno));
			exit(-1);
		}
	}
	for(i=0; i<nr_process; i++) {
		if(i == myid)
			continue;
		sprintf(rpath, "%s%d_%d", UNIX_DOMAIN, myid, i);
		sprintf(wpath, "%s%d_%d", UNIX_DOMAIN, i, myid);
		strncpy(srv_addr.sun_path, rpath, sizeof(srv_addr.sun_path)-1);
		strncpy(conn_addr.sun_path, wpath, sizeof(conn_addr.sun_path)-1);
		srv_addr.sun_family = AF_UNIX;
		conn_addr.sun_family = AF_UNIX;
		if(i > myid) {
			len = sizeof(clt_addr);
			recvfd[i] = accept(listen_fd[i], (struct sockaddr*)&clt_addr, &len);
			if(recvfd[i] < 0) {
				printf("child%d socket accept error! %s\n", myid, strerror(errno));
				exit(-1);
			}
			usleep(10*1000);
			ret = connect(conn_fd[i], (struct sockaddr*)&conn_addr,sizeof(conn_addr));
			if(ret < 0) {
				printf("child%d connect error! %s\n", myid, strerror(errno));
				exit(-1);
			}
			sendfd[i] = conn_fd[i];
		} else {
			usleep(10*1000);
			ret = connect(conn_fd[i], (struct sockaddr*)&conn_addr,sizeof(conn_addr));
			if(ret < 0) {
				printf("child%d connect error! %s\n", myid, strerror(errno));
				exit(-1);
			}
			sendfd[i] = conn_fd[i];
			len = sizeof(clt_addr);
			recvfd[i] = accept(listen_fd[i], (struct sockaddr*)&clt_addr, &len);
			if(recvfd[i] < 0) {
				printf("child%d socket accept error! %s\n", myid, strerror(errno));
				exit(-1);
			}
		}
	}
	return 0;
}

int main(int argc, char **argv)
{
        int i, ret = 0;
        int rfd, wfd;
	int cpu_num;
	struct Process *p;
	
	if(argc > 1) {
		myid = atoi(argv[1]);
	}
	if(argc > 2) {
		cpu_num = sysconf(_SC_NPROCESSORS_ONLN);
		nr_process = atoi(argv[2]);
		if(nr_process > cpu_num-2) {
			printf("max process is %d!\n", cpu_num-2);
			return -1;
		}
	}
	if(argc > 3) {
		testloop = atoi(argv[3]);
	}
	if(argc > 4) {
		calcloop = atoi(argv[4]);
	}
	if(argc > 5) {
		cpu_freq = atoi(argv[5]) / 1000; /* GHz */
	}
	process.id = myid;
	process.pid = getpid();
	printf("child%d: pid %d, nr_process %d, testloop %d calcloop %d\n",\
		 myid,getpid(), nr_process, testloop, calcloop);

	struct rlimit rlimit_new;
	rlimit_new.rlim_cur = RLIM_INFINITY;
	rlimit_new.rlim_max = RLIM_INFINITY;
	if(setrlimit(RLIMIT_MEMLOCK, &rlimit_new) != 0) {
		printf("setrlimit error ,exit\n");
		exit(1);
	}
	if(mlockall(MCL_CURRENT|MCL_FUTURE) !=0) {
		printf("lock memeory error ,exit\n");
		exit(1);
	}
	if(barrier_init() < 0) {
		printf("barrier_init error ,exit\n");
		exit(1);
	}
	if(socket_init() < 0) {
		printf("socket_init error ,exit\n");
		exit(1);
	}

	memset(sendbuf, myid+1, sizeof(sendbuf));
	memset(recvbuf, myid+2, sizeof(recvbuf));
	ret = process_barrier((char*)barrier);
	if(ret != 0 && ret != PTHREAD_BARRIER_SERIAL_THREAD) {
		printf("process%d pid[%d]: barrier_wait error! %s\n", myid, process.pid, strerror(errno));
		goto err;
	}

	process_func(&process);

	ret = process_barrier((char*)barrier);
	if(ret != 0 && ret != PTHREAD_BARRIER_SERIAL_THREAD) {
		printf("process%d pid[%d]: barrier_wait error! %s\n", myid, process.pid, strerror(errno));
		goto err;
	}
	munlockall();
	
	return 0;
err:
	return -1;
}