最近修改MPI程序,遇到了一些细节问题,在此标记一下,不知是MPI自身缺陷还是我不是很精通MPI, 有些问题还是不太理解,敬请各位专家批评指正
1、MPI_Reduce各进程的数据操作问题
比如说,对复数的操作,看下面程序
if (rank == 0) {
MPI_Reduce(MPI_IN_PLACE, imgcir[0][0],
cub->amx.n*cub->amy.n*cub->az.n,
MPI_FLOAT, MPI_SUM, 0, MPI_COMM_WORLD); //(1)
MPI_Reduce(MPI_IN_PLACE, imgcii[0][0],
cub->amx.n*cub->amy.n*cub->az.n,
MPI_FLOAT, MPI_SUM, 0, MPI_COMM_WORLD); //(2)
#ifdef _OPENMP
#pragma omp parallel for schedule(dynamic) \
private(iz,imy,imx)
#endif
YOOP(cip->ci[iz][imy][imx]=sf_cmplx(imgcir[iz][imy][imx],
imgcii[iz][imy][imx]);
);
sf_complexwrite(cip->ci[0][0],cub->amx.n*cub->amy.n*cub->az.n,imag);
} else {
MPI_Reduce(imgcir[0][0], imgcir[0][0],
cub->amx.n*cub->amy.n*cub->az.n,
MPI_FLOAT, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(imgcii[0][0], imgcii[0][0],
cub->amx.n*cub->amy.n*cub->az.n,
MPI_FLOAT, MPI_SUM, 0, MPI_COMM_WORLD);
}// else rank
其中imgcir、imgcii分别是复数的实部、虚部,将实部虚部分别进行加和,非主进程将数据加和并发送到非主进程,如果将主进程中的两个加和位置互换,则最终求和结果就是错误的,不是很明白MPI内部是怎么进行计算的。
2、在intel openmp线程级循环中调用MPI_Send\Recv函数出现错误(堵塞)
看如下程序
for (ie=0; ie<(int)ceilf((float)(cub->ae.n)/(float)(size)); ie++) {
// ....
// ....
// ....
// ....
#ifdef _OPENMP
#pragma omp parallel for schedule(static) \
private(ompith,iw,w,imx,imy,iz)
#endif
for (iw=0; iw<cub->aw.n; iw++) {
// ....
// ....
// ....
// ....
if (rank == 0) {
for (iw=0; iw<cub->aw.n; iw++) {
sf_seek(bws, sizeof(sf_complex)*cub->amx.n*
cub->amy.n*
cub->az.n*
(ie*cub->aw.n+iw),
SEEK_SET);
sf_complexread(wtr[0][0], cub->amx.n*cub->amy.n*cub->az.n, bws);
sf_seek(wflr, sizeof(sf_complex)*cub->amx.n*
cub->amy.n*
cub->az.n*
(ie*size*cub->aw.n+iw),
SEEK_SET);
sf_complexwrite(wtr[0][0],cub->amx.n*cub->amy.n*cub->az.n, wflr);
}
for (irank=1; irank<size; irank++) {
for (iw=0; iw<cub->aw.n; iw++) {
if (ie*size+irank < cub->ae.n) {
sf_seek(wflr, sizeof(sf_complex)*cub->amx.n*
cub->amy.n*
cub->az.n*
((ie*size+irank)*cub->aw.n+iw), SEEK_SET);
MPI_Recv(wtr[0][0], cub->amx.n*cub->amy.n*cub->az.n, MPI_FLOAT_COMPLEX,
irank, (ie*size+irank)*cub->aw.n+iw, MPI_COMM_WORLD, &status);
sf_complexwrite(wtr[0][0],cub->amx.n*cub->amy.n*cub->az.n, wflr);
}
}//for
}//for
} else {
if (ie*size+rank < cub->ae.n) {
for (iw=0; iw<cub->aw.n; iw++) {
sf_seek(bws, sizeof(sf_complex)*cub->amx.n*
cub->amy.n*
cub->az.n*(ie*cub->aw.n+iw), SEEK_SET);
sf_complexread(wtr[0][0], cub->amx.n*cub->amy.n*cub->az.n, bws);
MPI_Send(wtr[0][0], cub->amx.n*cub->amy.n*cub->az.n, MPI_FLOAT_COMPLEX,
0, (ie*size+rank)*cub->aw.n+iw, MPI_COMM_WORLD);
}//for
}//if
}
}//for
}
理论上,将openmp循环中每个线程生成的数据利用MPI_Send进行发送应该是没有问题的,即openmp和MPI的联合并行不是搭配的完美。查看了MPI的函数,他是完美支持Pthread的,对于openmp,MPI的兼容性可能还有些问题。
有了解的朋友麻烦解答一下