#include <unistd.h>
#include <cstdio>
#include <dirent.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <iostream>
#include <deque>
#include <list>
#include <pthread.h>
using namespace std;
bool g_bIsListDirEnd = false;
pthread_mutex_t g_mutex;
void FormatDir(string& strDir)
{
if (strDir.at(strDir.length() -1) != '\\' &&
strDir.at(strDir.length() -1) != '/')
{
strDir += '/';
}
}
void* thrd_func(void *arg)
{
list<string>* pList = (list<string>*)arg;
if (!arg)
{
cout << "arg is null" << endl;
pthread_exit((void *)1);
}
int nCount = 0;
while(true)
{
string strFileName;
pthread_mutex_lock(&g_mutex);
if (pList->size() > 0)
{
strFileName = pList->front();
pList->pop_front();
++nCount;
}
pthread_mutex_unlock(&g_mutex);
if (!strFileName.empty())
{
usleep(10000);
if (nCount % 1000 == 0)
{
cout << strFileName << " " << nCount << endl;
}
}
else
{
usleep(1);
}
pthread_mutex_lock(&g_mutex);
int nSize = pList->size();
pthread_mutex_unlock(&g_mutex);
if (g_bIsListDirEnd && nSize == 0)
{
break;
}
}
cout << "thread deals total files:" << nCount << endl;
}
void ListDir(const string& strTopDir)
{
deque<string> deqDirs;
deqDirs.push_back(strTopDir);
list<string> listFiles;
const int nThreadNum = 4;
pthread_t tid[nThreadNum];
for(int i = 0; i < nThreadNum; ++i)
{
if (pthread_create(&tid[i], NULL, thrd_func, &listFiles) != 0)
{
cout << "create thread failed:" << i << endl;
return;
}
}
int nCountDirs = 0;
int nCountFiles = 0;
while(deqDirs.size() > 0)
{
string strDir = deqDirs.front();
deqDirs.pop_front();
if (strDir.empty())
{
continue;
}
DIR *dir;
if(!(dir = opendir(strDir.c_str())))
{
cout << "open dir failed:" << strDir << endl;
continue;
}
FormatDir(strDir);
struct dirent *file;
while((file = readdir(dir)) != NULL)
{
if (file->d_name[0] == '.')
{
continue;
}
string strFullPath = strDir + file->d_name;
struct stat stFile;
if(stat(strFullPath.c_str(), &stFile) >= 0 && S_ISDIR(stFile.st_mode))
{
if (strFullPath.find("/proc") == string::npos &&
strFullPath.find("/sys") == string::npos &&
strFullPath.find("/dev") == string::npos)
{
deqDirs.push_back(strFullPath);
++nCountDirs;
}
}
else
{
while(true)
{
pthread_mutex_lock(&g_mutex);
int nSize = listFiles.size();
pthread_mutex_unlock(&g_mutex);
if (nSize < 5000)
{
break;
}
cout << "wait files dealed" << endl;
usleep(1000000);
}
pthread_mutex_lock(&g_mutex);
listFiles.push_back(strFullPath);
pthread_mutex_unlock(&g_mutex);
++nCountFiles;
}
}
closedir(dir);
}
cout << "list dir end" << endl;
g_bIsListDirEnd = true;
for(int i = 0; i < nThreadNum; ++i)
{
void *tret;
pthread_join(tid[i],&tret);
}
cout << "dirs:" << nCountDirs << " files:" << nCountFiles << " total:" << nCountDirs + nCountFiles << endl;
}
int main(int argc, char* argv[])
{
if (argc != 2)
{
printf("argc error!\n");
return -1;
}
pthread_mutex_init(&g_mutex,NULL);
struct timeval tv1;
gettimeofday(&tv1, NULL);
ListDir(argv[1]);
struct timeval tv2;
gettimeofday(&tv2, NULL);
cout << "cost time:" << (tv2.tv_sec - tv1.tv_sec)*1000 + (tv2.tv_usec - tv1.tv_usec)/1000 << "ms" << endl;
pthread_mutex_destroy(&g_mutex);
return 0;
}
linux多线程非递归遍历文件夹模拟处理文件
最新推荐文章于 2023-06-26 14:59:27 发布