海量数据的查询

概述：开发内存服务程序，存储2亿数据的索引，用进程通信管道的管道池进行服务，每次只应付１０个请求、

服务端：

#include<stdio.h>

#include<stdlib.h>

#include<string.h>
#include<process.h>
#include<time.h>
#include<Windows.h>
#include<malloc.h>
#define totalData 201515740//数据总量
#define bufferSize 4096
#define findThread 128
int  pipeThread = 10;//10个管道
char  pipeName[20] = "\\\\.\\Pipe\\cloudpipe";
char dataPath[20] = "D:\\personInfo.txt";
char indexPath[20] = "D:\\info.txt";
char outPutInfo[512] = { 0 };//存储查找到的信息
struct dataIndex  //索引
{
int *pindex;
int length;
}bigDataIndex = { 0 };
struct threadInfo
{
int *pstart;//开始位置
int length;//长度
char findstr[20];//要查找
int id;//线程编号
};
typedef struct info
{
HANDLE hthread;
HANDLE hpipe;
HANDLE hevent;
}PIPE_ST;
PIPE_ST  pipeinst[findThread];
void  generateIndex(char *path)
{
bigDataIndex.length = totalData;
bigDataIndex.pindex = calloc(totalData, sizeof(int));
FILE*pf = fopen(path, "rb");
if (pf == NULL)
{
puts("文件未成功打开，请检查文件路径是否正确\n");
}
else
{
int indexPosition = 0;
for (int i = 0; i < totalData; i++)
{
char data[500] = { 0 };
fgets(data, 500, pf);//获取字符串
bigDataIndex.pindex[i] = indexPosition;
int datalength = strlen(data);
indexPosition += datalength;//索引位置
}
fclose(pf);
}
printf("\n大数据索引生成结束");
FILE *pfw = fopen(indexPath, "wb");
fwrite(bigDataIndex.pindex, sizeof(int), bigDataIndex.length, pfw);
fclose(pfw);
}
void loadMemary()//载入索引文件内存
{
bigDataIndex.length = totalData;
bigDataIndex.pindex = calloc(totalData, sizeof(int));//初始化分配
FILE *pfw = fopen(indexPath, "rb");
fread(bigDataIndex.pindex, sizeof(int), bigDataIndex.length, pfw);
fclose(pfw);
}
void memFind(void *pThread)
{
FILE *pf = fopen(dataPath, "rb");
struct threadInfo *thread_Task = pThread;
for (int i = 0; i < thread_Task->length; i++)
{
char readStr[512] = { 0 };
fgets(readStr, 512, pf);
char *px = strstr(readStr, thread_Task->findstr);
if (px != NULL)
{
printf("服务器查找到%s相应的信息", readStr);
fclose(pf);
strcpy(outPutInfo, readStr);
return;
}
}
fclose(pf);
}
void creaetFindThread(char *str)
{
#define findDataThread 100
struct threadInfo pthread[findDataThread];//创建线程使用信息数组
if (totalData%findDataThread == 0)
{
for (int i = 0; i < findDataThread; i++)
{
pthread[i].id = i;
strcpy(pthread[i].findstr, str);
pthread[i].length = totalData / findDataThread;
pthread[i].pstart = bigDataIndex.pindex + i*(totalData / findDataThread);
_beginthread(memFind, 0, &pthread[i]);
}
}
else
{
for (int i = 0; i < findDataThread - 1; i++)
{
pthread[i].id = i;
strcpy(pthread[i].findstr, str);
pthread[i].length = totalData / (findDataThread - 1);
pthread[i].pstart = bigDataIndex.pindex + i*(totalData / (findDataThread - 1));
_beginthread(memFind, 0, &pthread[i]);
}
int i = findDataThread - 1;
pthread[i].id = i;
strcpy(pthread[i].findstr, str);
pthread[i].length = totalData % (findDataThread - 1);
pthread[i].pstart = bigDataIndex.pindex + i*(totalData / (findDataThread - 1));
_beginthread(memFind, 0, &pthread[i]);
}
}
DWORD WINAPI severThread(void *lp)
{
DWORD nread = 0;
DWORD nwrite = 0;
DWORD dwbyte = 0;
char szbuf[bufferSize] = { 0 };
PIPE_ST curpipe = *(PIPE_ST*)lp;
OVERLAPPED overlap = { 0, 0, 0, 0, curpipe.hevent };
while (1)
{
memset(szbuf, 0, sizeof(szbuf));
ConnectNamedPipe(curpipe.hpipe, &overlap);//链接上，信息写入overlap
WaitForSingleObject(curpipe.hevent, INFINITE);
//检测IO，如果完成就跳出
if (!GetOverlappedResult(curpipe.hpipe, &overlap, &dwbyte, TRUE))
{
break;
}
if (!ReadFile(curpipe.hpipe, szbuf, bufferSize, &nread, NULL))
{
puts("read fail");
break;
}
char scanBuf[15];
sscanf(szbuf, "%s", scanBuf);
memset(szbuf, 0, sizeof(szbuf));
creaetFindThread(scanBuf);
printf("海量数据的检索完毕\n");
sprintf(szbuf, "%s", outPutInfo);
memset(outPutInfo, 0, sizeof(outPutInfo));
WriteFile(curpipe.hpipe, szbuf, strlen(szbuf), &nwrite, NULL);
DisconnectNamedPipe(curpipe.hpipe);
}
return 0;
}
void createPipe()
{
for (int i = 0; i <pipeThread; i++)
{
pipeinst[i].hpipe = CreateNamedPipeA(
pipeName,
PIPE_ACCESS_DUPLEX | FILE_FLAG_OVERLAPPED,
PIPE_TYPE_BYTE | PIPE_READMODE_BYTE | PIPE_WAIT,
10,
0,
0,
1000,
NULL);
if (pipeinst[i].hpipe == INVALID_HANDLE_VALUE)
{
printf("\n管道%d创建失败", i);
return;
}
//创建事件
pipeinst[i].hevent = CreateEventA(NULL, FALSE, FALSE, FALSE);//创建事件
//创建线程
pipeinst[i].hthread = CreateThread(NULL, 0, severThread, &pipeinst[i], 0, NULL);
}
printf("服务器已经开始查找了，但因为数据量比较大，需要时间，请耐心等待\n");
}
int main()
{
//generateIndex(dataPath);
loadMemary();
createPipe();
system("pause");
return 0;

}

客户端：

#define  _CRT_SECURE_NO_WARNINGS
#include<stdio.h>
#include<stdlib.h>
#include<Windows.h>
#include<time.h>
#define SIZE 4096
char  pipeName[128] = "\\\\.\\Pipe\\cloudpipe";
HANDLE m_pipe = NULL;
int main()
{
m_pipe = CreateFileA(pipeName, 
GENERIC_WRITE | GENERIC_READ,
0,
NULL,
OPEN_EXISTING,
FILE_ATTRIBUTE_NORMAL,
NULL);
if (m_pipe == INVALID_HANDLE_VALUE)
{
printf("失败");
return;
}
int nwrite;
int nread;
char winfo[1024] = { 0 };
printf("请输入要查找的信息：");
scanf("%s", winfo);
WriteFile(m_pipe, winfo, strlen(winfo), &nwrite, NULL);
memset(winfo, 0, sizeof(winfo));//清零
ReadFile(m_pipe, winfo, 1024, &nread, NULL);
char  res[100];
sscanf(winfo, "%s", res);

printf("\n%s", res);

return 0;

}