MPI快排

 #include <stdio.h>
 #include <sys/time.h>
 #include <stdlib.h>
 #include <time.h>
 #include <math.h>
 #include <mpi.h>
 #include <unistd.h>
 #include <string.h>

#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <stdbool.h>
#include <stdio.h>
#include <errno.h>
#include <string.h>
#include <libgen.h>
#include <stdlib.h>
#define FILEPATH "/home/mpiuser/myprojects/mydata/sort_round.txt"
 #define length 50000000
 int isFileExisted(const char *pathname){
	if(access(pathname,F_OK) == 0){
		return true;
	}
	return false;
}
 
void makeDirectory(const char *path){
	if((strcmp(path,".") == 0) || (strcmp(path,"/")==0))
		return ;
	if(isFileExisted(path))
		return ;
	else{
		char *duppath = strdup(path);
		const char *dir_name = dirname(duppath);
		makeDirectory(dir_name);
		free(duppath);
	}
	if(mkdir(path,0766) < 0){
		perror("mkdir");
		exit(1);
	}
	return;
}  
 
// int createFile(const char*filename,int mode){
// 	if(creat(filename,mode) < 0){
// 		if(errno == ENOENT){
// 			char *dup_filename=strdup(filename);
// 			char *dir = dirname(dup_filename);
// 			makeDirectory(dir);
// 			free(dup_filename);
// 			creat(filename, mode);
// 		}
// 		return false;
// 	}
// 	return true;
// }

int isFileNull(const char *filename){
	FILE *fp = fopen(filename,"r");
	if(fp == NULL)
		return -1;	
	int val = 0;
	char ch = fgetc(fp);
	if(ch == EOF)
		val = 1;
	fclose(fp);
	return val;
}


 void swap(int *data, int i, int j) {
    int temp = data[i];
    data[i] = data[j];
    data[j] = temp;
 }
 
 int partition(int *data, int start, int end) {
    if (start >= end) return 0;
    int pivotValue = data[start];
    int low = start;
    int high = end - 1;
    while (low < high) {
        while (data[low] <= pivotValue && low < end) low++;
        while (data[high] > pivotValue && high > start) high--;
        if (low < high) swap(data, low, high);
    }
    swap(data, start, high);
    return high;
 }
 
 void quicksort(int *data, int start, int end) {
    if (end-start+1 < 2) return;
    int pivot = partition(data, start, end);
    quicksort(data, start, pivot);
    quicksort(data, pivot+1, end);
 }
 
 void mergesort(int *data, int blocklen,int len){
    int *tmp = (int*)malloc(sizeof(int)*len);
    int left = 0;
    int right = blocklen;
    int i = 0;
    // printf("开始归并排序%d...\n",tmp[0]);
    while(left < blocklen&&right < len){
        // printf("程序已运行到此！\n");
        if(data[left] < data[right]){
            
            tmp[i++] = data[left++];
        }
        else{
            tmp[i++] = data[right++];
        }
        // printf("程序已运行到此！\n");
    }
    
    while(left < blocklen){
        tmp[i++] = data[left++];
    }
    while(right < len){
        tmp[i++] = data[right++];
    }
    for(int j = 0; j < len; j++){
        data[j] = tmp[j];
    }
    // printf("归并排序已完成！\n");
 }

// 该函数用于将一轮归并排序的数据进行备份
 void saveData(int *data,int blocklen,int len,int roundID){
    char openfile[100];
    // sprintf(openfile,"/home/mpiuser/myprojects/mydata/sort_round.txt",roundID);
    sprintf(openfile,"/home/mpiuser/myprojects/mydata/sort_round.txt");
    printf("正在将排序结果保存到%s中\n",openfile);
    FILE *fpWrite = fopen(openfile,"w");
	if(fpWrite==NULL)
	{
		return;
	}
    int k = 0;
    fprintf(fpWrite,"%d ", blocklen);
    for(int i = 0; i < len; i++){
        fprintf(fpWrite,"%d ",data[i]);
        k++;
        // if(i != len){
        //     fprintf(fpWrite," ");
        // }
        // if(i % 1000 == 0)
        //     printf("已添加%d个数据\n",i+1);
    }
    printf("总共保存了%d条数据\n",k);
	fclose(fpWrite);
    printf("已成功保存第%d轮的排序结果\n",roundID);
 }
 int loadLastStatus(int * data,int* blocklen){
    FILE *fpRead = fopen(FILEPATH,"r");
    int i = -1;
    int tmp = 0;
    *blocklen = 0;
    if(fpRead==NULL)
	{
		return 0;
	}
    else
    {
        while(!feof(fpRead)){
        // while(i++ < length){
            // if(i >= length){
            //     break;
            // }
            char c = fgetc(fpRead);
            // printf("%c", c);
            if(c == '\n'|| c == ' '){
                if(i != -1){
                    data[i] = tmp;
                    tmp = 0;
                    // if(i > 0 && data[i] < data[i-1])
                        // printf("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$出现一次断点：%d < %d$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$\n",data[i],data[i-1]);
                }
                i++;
                continue;
            }
            if(i == -1){
                *blocklen *= 10;
                *blocklen += c - '0';
            }
            else{
                tmp *= 10;
                tmp += c - '0';
            }
        }
    }
    fclose(fpRead);
    printf("总共读到了%d条数据\n",i);
    return 0;
 }
 int main(int argc, char *argv[]) {
     MPI_Init(&argc, &argv);
     int rank, size;
     int namelen;
     MPI_Request handle;
    char processor_name[MPI_MAX_PROCESSOR_NAME];
     double startwtime,endwtime;
     MPI_Comm_rank (MPI_COMM_WORLD, &rank);
     MPI_Comm_size (MPI_COMM_WORLD, &size);
     MPI_Get_processor_name(processor_name, &namelen);
    fprintf(stdout, "Process %d of %d is on %s\n", rank, size, processor_name);
    fflush(stdout);
     srand(time(0));
     int *data = (int*)malloc(sizeof(int)*length);
     int i;
    // 按块进行分配
    int blocklen = length/(size-1);
    int isContinue_last = 0; // 标识是否进行上一次的进程排序
    if(rank != size-1){
        char openfile[100];
        // int roundID = 1; // 从第一轮开始，如果有文件，则说明上一轮程序异常退出了，不重新生成数据，继续上次程序进行排序
        // int lastRound = 0; // 记录上一次程序终止的最后轮数
        // for(;roundID < 5; roundID++){
        sprintf(openfile,"/home/mpiuser/myprojects/mydata/sort_round.txt");
        if(isFileExisted(openfile)){
            printf("%s文件已存在(%s)\n",openfile,processor_name);
            isContinue_last = 1;
        }
        else{
            printf("%s文件不存在(%s)\n",openfile,processor_name);
        }
        // }
    }
     MPI_Status status;
     // 主进程接收来自其它进程初始化的数据
     if (rank == 0) {
        // 首先需要检查上一次的程序是否执行成功，每个进程都需要告诉0号进程，自己是否发现上一轮程序异常。
        int readID = -1; // 指定读取上一轮中间数据的进程ID
        int localread = 0; // 表示0号进程是否进行本地读取
        if(isContinue_last == 1){
            localread = 1;
        }
        printf("Processor %d 开始接收其它进程的检查结果，验证上一轮程序是否发生异常\n",rank);
        for(int i = 1; i < size-1; i++){
            int tmp_last = 0;
            MPI_Recv(&tmp_last,1,MPI_INT,i, 0, MPI_COMM_WORLD,&status);
            if(tmp_last > isContinue_last){
                isContinue_last = tmp_last; // 保证只要有一个进程发现上一轮有异常就通知所有进程重启上一轮的任务
                readID = i;
            }
        }
        if(isContinue_last == 1){
            printf("上一轮程序存在异常，需要重启\n");
        }
        else{
            printf("上一轮程序不存在异常，不需要重启\n");
        }
        // 广播所有进程，如果readID为-1，则表示上一轮程序无异常，否则，指定加载上一轮进程的进程号为readID
        if(localread == 1){
            // 如果主进程读取到了上一轮程序异常，其它进程并未读取到，则不需要接收其它进程的数据，0号进程自动加载数据
            if(isContinue_last == 0)
                readID = -1;
            else
                readID = 0;
        }
        printf("Processor %d 广播读取上一轮程序中间结果的进程ID\n",rank);
        for(int i = 1; i < size-1; i++){
            MPI_Send(&readID,1,MPI_INT,i,0,MPI_COMM_WORLD);
        }

        int nextstart = blocklen;
        if(localread == 1){
            printf("Processor %d 开始加载上一轮程序的中间结果\n",rank);
            // 如果主进程读取到了上一轮程序异常，其它进程并未读取到，则不需要接收其它进程的数据，0号进程自动加载数据
            loadLastStatus(data,&blocklen);
            printf("Processor %d 已成功加载上一轮程序的中间结果\n",rank);

            nextstart = blocklen;
            // 将上一次异常程序的执行结果分发给其它进程
            // 先发送blocklen
            for(int i = 0; i < size-1; i++){
                if(i != rank){
                    MPI_Send(&blocklen,1,MPI_INT,i,0,MPI_COMM_WORLD);
                    // MPI_Send(data+nextstart, blocklen, MPI_INT, i, 0, MPI_COMM_WORLD);
                    printf("Processor %d 已成功分发上一轮程序的中间结果给Processor %d \n",rank,i);
                    nextstart += blocklen;
                }
            }

        }
        else if(readID != -1){
            // 从其它主机的进程上读取上一轮的中间结果，恢复运行
            MPI_Recv(&blocklen,1,MPI_INT,readID,0,MPI_COMM_WORLD,&status);
            MPI_Recv(data,length,MPI_INT,readID,0,MPI_COMM_WORLD,&status);
        }
        else{
            // 随机生成待排序的数组
            printf("上一轮程序无异常，%d正在重新生成数据\n",rank);
            for (i=0; i<blocklen; i++)
                data[i] = rand()%length;
        }
        int maxSendNum = (int)log2(size-1);
         startwtime = MPI_Wtime();
        if(isContinue_last == 0){
            quicksort(data, 0, blocklen);
            printf("*********************************\n当前总进程数为：%d\n排序的数据量为：%d\n*********************************\n",size,length);
            // 这里每一个blocken都已经排好序
            for (i=1; i < size-1; i++)
                MPI_Recv(data+i*blocklen, blocklen, MPI_INT, i, 0, MPI_COMM_WORLD, &status);
            printf("局部快排之后：\n");
            for(int i = 0; i < length; i += 500000){
                printf("%d ",data[i]);
            }
            printf("\n");
            // 在此将分发给其它进程做局部归并排序
            // 这里计算最大可发送的次数;
            printf("当前主进程需要循环发送%d次\n",maxSendNum);
            // 告知所有进程，它们需要循环接收多少次
            for(int i = 1; i < size; i++){
                MPI_Send(&maxSendNum, 1, MPI_INT, i, 1, MPI_COMM_WORLD);
            }
        }
        else{
            printf("*********************************\n当前总进程数为：%d\n排序的数据量为：%d\n*********************************\n",size,length);
            printf("读取异常程序的中间数据之后,blocklen = %d：\n",blocklen);
            for(int i = 0; i < length; i++){
                if(i % 50000 == 0)
                    printf("%d ",data[i]);
                if(i > 0 && data[i] < data[i-1])
                        printf("\n$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$出现一次断点：data[%d]=%d < data[%d]=%d$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$\n",i,data[i],i-1,data[i-1]);
            }
            printf("\n");
            // quicksort(data, 0, blocklen);
            maxSendNum -= log2(blocklen/(length/(size-1)));
            printf("当前主进程需要循环发送%d次\n",maxSendNum);
            // 告知所有进程，它们需要循环接收多少次
            for(int i = 1; i < size; i++){
                MPI_Send(&maxSendNum, 1, MPI_INT, i, 1, MPI_COMM_WORLD);
            }
        }

        int t = 0;
        while(maxSendNum--){
            blocklen *= 2;
            nextstart = 0;
            // 计算发送数据的次数
            int sendNUM = length / blocklen;
            printf("当前需要%d个子进程进行归并排序\n",sendNUM);
            // 告知所有进程，它们是否作为子任务的进程
            for(int j = 1; j < size-1; j++){
                MPI_Send(&sendNUM, 1, MPI_INT, j, 2, MPI_COMM_WORLD);
            }
            for(int i = 1; i < size-1; i++){
                printf("主进程正在发送数据给%d进程...\n",i);
                MPI_Send(data+nextstart, blocklen, MPI_INT, i, 3, MPI_COMM_WORLD);
                printf("主进程已发送数据给%d进程！！！\n",i);
                nextstart += blocklen;
                if(nextstart >= length){
                    break;
                }
                // mergesort(data,blocklen,length);
            }
            for (i=0; i<sendNUM; i++)
                MPI_Recv(data+i*blocklen, blocklen, MPI_INT, i+1, 4, MPI_COMM_WORLD, &status);
            printf("-----------------------\n已完成一轮归并，告知进程%d...\n",size-1);
            // 将当前排好序的数据进行溢写当磁盘中备份，并告知监控程序当前轮的排序已结束
            saveData(data, blocklen, length, t+1);
            t++;
            int flag = 1;
            // sleep(60);
            MPI_Send(&flag,1,MPI_INT,size-1,5,MPI_COMM_WORLD);
            printf("进程%d已重新计时，开启下一轮排序...\n-----------------------\n",size-1);
        }
        printf("主进程进行最后的归并排序...\n");
        if(blocklen < length){
            // 循环结束之后记得对剩下的模块进行一次归并
            mergesort(data,blocklen,length);
        }
     }
     else if(rank == size-1){
         printf("*********************************\nProcessor %d 用于计时\n*********************************\n",rank);
         int maxRecvNUM,flag2 = 0;
         MPI_Request request2;
         MPI_Recv(&maxRecvNUM, 1, MPI_INT, 0, 1, MPI_COMM_WORLD, &status);
         double startwtime = MPI_Wtime();
         double endwtime;
         int flag = 0;
         int secondtime = 0;
         while(maxRecvNUM--){
             flag = 0;
             flag2 = 0;
            MPI_Irecv(&flag2, 1, MPI_INT, 0, 5, MPI_COMM_WORLD, &request2);
            while(flag == 0 || flag2 == 0){
                printf("*********************************\n正在等待进程完成当前轮次的归并排序...%d s\n*********************************\n",secondtime);
                sleep(2); // 每隔一秒钟查看一下进程状态
                secondtime+=2;
                MPI_Test(&request2, &flag, &status);
                if(secondtime > 120){
                    printf("执行超时，退出运行！\n");
                    MPI_Abort(MPI_COMM_WORLD,1);
                }
            }
            secondtime = 0;
            endwtime = MPI_Wtime();
         }
     }
     else {
        int readID = -1;
        // 除了计时的进程，每一个进程都要告知0号进程自己是否发现上一轮程序有遗留
        MPI_Send(&isContinue_last,1,MPI_INT,0,0,MPI_COMM_WORLD);
        // 接收是否需要读取上一轮的数据的标识进程
        MPI_Recv(&readID,1,MPI_INT, 0, 0, MPI_COMM_WORLD,&status);
        printf("Processor %d 已接收到需要读取上一轮程序的进程ID为：%d\n",rank,readID);
        if(readID == rank){
            printf("Processor %d 开始加载上一轮程序的中间结果并分发程序状态\n",rank);
            int nextstart = blocklen;
            // 表示需要当前进程进行读取上一轮失败的数据
            loadLastStatus(data,&blocklen); // 加载上一轮的执行状态和中间结果，分发给其它进程
            // // 先发送blocklen
            printf("Processor %d 已成功加载上一轮程序的中间结果\n",rank);
            for(int i = 0; i < size-1; i++){
                if(i != rank){
                    MPI_Send(&blocklen,1,MPI_INT,i,0,MPI_COMM_WORLD);
                    // MPI_Send(data+nextstart, blocklen, MPI_INT, i, 0, MPI_COMM_WORLD);
                    // nextstart += blocklen;
                }
            }
            // MPI_Send(&blocklen,1,MPI_INT,0,0,MPI_COMM_WORLD);
            MPI_Send(data, length, MPI_INT, 0, 0, MPI_COMM_WORLD);
        }
        else if(readID != -1){
            // 先接收blocklen
            printf("Processor %d 正在接收来自Processor %d分发的上一轮异常程序状态\n",rank,readID);
            MPI_Recv(&blocklen,1,MPI_INT,readID,0,MPI_COMM_WORLD,&status);
            // MPI_Recv(&data,blocklen,MPI_INT,readID,0,MPI_COMM_WORLD,&status);
            // printf("Processor %d 已成功接收来自Processor %d分发的上一轮异常程序状态\n",rank,readID);
        }
        else{
            // 随机生成待排序的数组
            printf("上一轮程序无异常，%d正在重新生成数据\n",rank);
            // 随机生成待排序的数组
            for (i=0; i<blocklen; i++)
                data[i] = rand()%length;
            // fprintf(stdout, "Process %d of %d is on %s\n", rank, size, processor_name);
            // fflush(stdout);
            quicksort(data, 0, blocklen);
            MPI_Send(data,  blocklen,  MPI_INT,  0,  0,  MPI_COMM_WORLD);
        }
  
         // 接收需要循环的次数
         int maxRecvNUM,maxRunID;
         MPI_Recv(&maxRecvNUM, 1, MPI_INT, 0, 1, MPI_COMM_WORLD, &status);
         printf("子进程%d得知需要循环%d次\n",rank,maxRecvNUM);
         while(maxRecvNUM--){
             // 判断主进程分配任务的进程ID是否是自己
             MPI_Recv(&maxRunID, 1, MPI_INT, 0, 2, MPI_COMM_WORLD, &status);
             if(maxRunID >= rank){
                 printf("子进程%d得知自己有任务，准备接收数据...\n",rank);
                 MPI_Recv(data, blocklen*2, MPI_INT, 0, 3, MPI_COMM_WORLD, &status);
                 printf("子进程%d已接受数据，开始归并排序...\n",rank);
                 double starttime = MPI_Wtime();
                 mergesort(data,blocklen,blocklen*2);
                 double endtime = MPI_Wtime();
                 blocklen*=2;
                 printf("子进程%d归并排序完毕,用时%f s，将数据发回主进程...\n",rank,endtime-starttime);
                 MPI_Send(data,  blocklen,  MPI_INT,  0,  4,  MPI_COMM_WORLD);
             }
             else{
                 printf("子进程%d已收到通知，但不是自己的任务\n",rank);
             }
         }
     }
     if (rank == 0)
     {
         int jud = 1;
         endwtime = MPI_Wtime();
         for(int i = 0; i < length; i++){
             if(i > 0&&data[i]<data[i-1]){
                 printf("data[%d] = %d < data[%d] = %d排序失败！！！\n",i,data[i],i-1,data[i-1]);
                 jud = 0;
                //  break;
             }
             if(i % 500000 == 0){
                 printf("%d ",data[i]);
             }
            //  printf("%d ",data[i]);
         }
         double  time  =  endwtime - startwtime;
         if(jud == 1){
            printf("\n*********************************\n排序成功！！！\nTime: %f s\n*********************************\n", time);
            remove(FILEPATH);
            printf("已成功删除中间结果文件\n");
         }
     }
     MPI_Finalize();
    
     return 0;
}