linux多线程非递归遍历文件夹模拟处理文件

最新推荐文章于 2023-06-26 14:59:27 发布

FerrisYu

最新推荐文章于 2023-06-26 14:59:27 发布

阅读量1.7k

点赞数 1

分类专栏： C++ 文章标签：多线程递归遍历 linux 文件夹

本文链接：https://blog.csdn.net/yufei_email/article/details/42624551

版权

C++ 专栏收录该内容

22 篇文章 0 订阅

订阅专栏

#include <unistd.h>
#include <cstdio>
#include <dirent.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <iostream>
#include <deque>
#include <list>
#include <pthread.h>
using namespace std;

bool g_bIsListDirEnd = false;
pthread_mutex_t g_mutex;

void FormatDir(string& strDir)
{
	if (strDir.at(strDir.length() -1) != '\\' &&
		strDir.at(strDir.length() -1) != '/')
	{
		strDir += '/';
	}
}

void* thrd_func(void *arg)
{
	list<string>* pList = (list<string>*)arg;
	if (!arg)
	{
		cout << "arg is null" << endl;
		pthread_exit((void *)1);
	}
	int nCount = 0;

	while(true)
	{
		string strFileName;
		pthread_mutex_lock(&g_mutex);
		if (pList->size() > 0)
		{
			strFileName = pList->front();
			pList->pop_front();
			++nCount;
		}
		pthread_mutex_unlock(&g_mutex);

		if (!strFileName.empty())
		{
			usleep(10000);
			if (nCount % 1000 == 0)
			{
				cout << strFileName << " " << nCount << endl;
			}
		}
		else
		{
			usleep(1);
		}

		pthread_mutex_lock(&g_mutex);
		int nSize = pList->size();
		pthread_mutex_unlock(&g_mutex);
		
		if (g_bIsListDirEnd && nSize == 0)
		{
			break;
		}
	}
	cout << "thread deals total files:" << nCount << endl;
}

void ListDir(const string& strTopDir)
{
	deque<string> deqDirs;
	deqDirs.push_back(strTopDir);

	list<string> listFiles;

	const int nThreadNum = 4;
	pthread_t tid[nThreadNum];
	for(int i = 0; i < nThreadNum; ++i)
	{
		if (pthread_create(&tid[i], NULL, thrd_func, &listFiles) != 0)
		{
			cout << "create thread failed:" << i << endl;
			return;
		}
	}

	int nCountDirs = 0;
	int nCountFiles = 0;

	while(deqDirs.size() > 0)
	{
		string strDir = deqDirs.front();
		deqDirs.pop_front();

		if (strDir.empty())
		{
			continue;
		}

		DIR *dir;
		if(!(dir = opendir(strDir.c_str())))
		{
			cout << "open dir failed:" << strDir << endl;
			continue;
		}

		FormatDir(strDir);

		struct dirent *file;
		while((file = readdir(dir)) != NULL)
		{
			if (file->d_name[0] == '.')
			{
				continue;
			}

			string strFullPath = strDir + file->d_name;
			struct stat stFile; 
			if(stat(strFullPath.c_str(), &stFile) >= 0 && S_ISDIR(stFile.st_mode))
			{
				if (strFullPath.find("/proc") == string::npos && 
					strFullPath.find("/sys") == string::npos &&
					strFullPath.find("/dev") == string::npos)
				{
					deqDirs.push_back(strFullPath);
					++nCountDirs;
				}
			}
			else
			{
				while(true)
				{
					pthread_mutex_lock(&g_mutex);
					int nSize = listFiles.size();
					pthread_mutex_unlock(&g_mutex);
					if (nSize < 5000)
					{
						break;
					}
					cout << "wait files dealed" << endl;
					usleep(1000000);				
				}
 				pthread_mutex_lock(&g_mutex);
 				listFiles.push_back(strFullPath);
 				pthread_mutex_unlock(&g_mutex);
				++nCountFiles;
			}	
		}
		closedir(dir);
	}
	cout << "list dir end" << endl;
	g_bIsListDirEnd = true;

	for(int i = 0; i < nThreadNum; ++i)
	{
		void *tret;
		pthread_join(tid[i],&tret);
	}

	cout << "dirs:" << nCountDirs << "  files:" << nCountFiles << "  total:" << nCountDirs + nCountFiles << endl;
}

int main(int argc, char* argv[])
{
	if (argc != 2)
	{
		printf("argc error!\n");
		return -1;
	}

	pthread_mutex_init(&g_mutex,NULL);
	
	struct timeval tv1;
	gettimeofday(&tv1, NULL);

	ListDir(argv[1]);

	struct timeval tv2;
	gettimeofday(&tv2, NULL);
	cout << "cost time:" << (tv2.tv_sec - tv1.tv_sec)*1000 + (tv2.tv_usec - tv1.tv_usec)/1000 << "ms" << endl;
	pthread_mutex_destroy(&g_mutex);

	return 0;
}