Hadoop socket测试

概要：
hadoop上以streaming方式运行map任务，map任务为socke client端，和在外部系统运行的server交互

代码：
server.cpp

#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <errno.h>
#include <string.h>
#include <sys/types.h>
#include <arpa/inet.h>
#include <netinet/in.h>
#include <sys/socket.h>
#include <sys/wait.h>
#include <iostream>
#define SERVPORT 9899    /*服务器监听端口号 */
#define BACKLOG 1000    /* 最大同时连接请求数 */
#define MAXDATASIZE 100    /*每次最大数据传输量 */

using namespace std;

int main()
{
	int sock_fd,client_fd;    /*sock_fd：监听socket；client_fd：数据传输socket */
	int sin_size;
	char buf[MAXDATASIZE];
	struct sockaddr_in my_addr;    /* 本机地址信息 */
	struct sockaddr_in remote_addr;    /* 客户端地址信息 */
	
	if((sock_fd = socket(AF_INET, SOCK_STREAM, 0)) == -1) 
	{
		perror("socket创建出错！");
		exit(1);
	}
	my_addr.sin_family=AF_INET;
	my_addr.sin_port=htons(SERVPORT);
	my_addr.sin_addr.s_addr = INADDR_ANY;
	bzero(&(my_addr.sin_zero),8);
	
	if(bind(sock_fd, (struct sockaddr *)&my_addr, sizeof(struct sockaddr)) == -1) 
	{
		perror("bind出错！");
		exit(1);
	}
	
	if(listen(sock_fd, BACKLOG) == -1) 
	{
		perror("listen出错！");
		exit(1);
	}
	
	while(1) 
	{
		sin_size = sizeof(struct sockaddr_in);
		if((client_fd = accept(sock_fd, (struct sockaddr *)&remote_addr, (socklen_t*)&sin_size)) == -1) 
		{
			perror("accept出错");
			continue;
		}
		printf("received a connection from %s\n", inet_ntoa(remote_addr.sin_addr));
		if(!fork()) 
		{ 
			/* 子进程代码段 */
			if(send(client_fd, "Hello, you are connected!\n", 26, 0) == -1) 
			{
				perror("send出错！");
			}
			
			int recvbytes = 0;
			if((recvbytes=recv(client_fd, buf, MAXDATASIZE, 0)) == -1) 
			{
				perror("recv出错！");
				exit(1);
			}

			cout << buf << endl;
			close(client_fd);
			exit(0);
		}
		close(client_fd);
	}
}

client.cpp

#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <string.h>
#include <netdb.h>
#include <unistd.h>
#include <sys/types.h>
#include <netinet/in.h>
#include <sys/socket.h>
#include <iostream>
#define SERVPORT 9899
#define MAXDATASIZE 100    /*每次最大数据传输量 */
using namespace std;

int main(int argc, char *argv[])
{
	int sock_fd, recvbytes;
	char buf[MAXDATASIZE];
	struct hostent *host;
	struct sockaddr_in serv_addr;
	
	//读取输入
	string input;
	while(cin >> input)
	{
		cout << input << endl;
	}
	
	if(argc< 2) 
	{
		fprintf(stderr,"Please enter the server's hostname!\n");
		exit(1);
	}
	if((host=gethostbyname(argv[1])) == NULL) 
	{
		herror("gethostbyname出错！");
		exit(1);
	}
	
	//初始化socket
	if((sock_fd = socket(AF_INET, SOCK_STREAM, 0)) == -1) 
	{
		perror("socket创建出错！");
		exit(1);
	}
	serv_addr.sin_family=AF_INET;
	serv_addr.sin_port=htons(SERVPORT);
	serv_addr.sin_addr = *((struct in_addr *)host->h_addr);
	bzero(&(serv_addr.sin_zero),8);
	
	//连接
	if(connect(sock_fd, (struct sockaddr *)&serv_addr, sizeof(struct sockaddr)) == -1) 
	{
		perror("connect出错！");
		exit(1);
	}
	
	//接受数据
	if((recvbytes=recv(sock_fd, buf, MAXDATASIZE, 0)) == -1) 
	{
		perror("recv出错！");
		exit(1);
	}
	
	//发送数据
	if(send(sock_fd, "hello server", 13, 0) == -1) 
	{
		perror("send出错！");
	}
	buf[recvbytes] = '\0';
	printf("Received: %s",buf);
	close(sock_fd);
	return 0;
}

编译程序：
$ g++ -o client client.cpp
$ g++ -o server server.cpp

测试：
测试机上运行server
$./server

hadoop上运行任务：
$ hadoop streaming -input /user/test.txt -output /user/result -mapper "./client **domain**" -reducer "cat" -file client-jobconf mapred.reduce.tasks=1 -jobconf mapred.map.tasks=5 -jobconf mapre.job.name="socket_test"

input内容/user/test.txt 为1324
-jobconf mapred.reduce.tasks=1 -jobconf mapred.map.tasks=5 ：mapper任务5个，reduce任务1个

./server输出：
received a connection from 10.*.*.*
hello server
received a connection from 10.*.*.*
hello server
received a connection from 10.*.*.*
hello server
received a connection from 10.*.*.*
hello server
received a connection from 10.*.*.*
hello server

运行结果：
$ hadoop fs -cat /user/result/part-00000
1324
Received: Hello, you are connected!
Received: Hello, you are connected!
Received: Hello, you are connected!
Received: Hello, you are connected!
Received: Hello, you are connected!