google的数据后端处理大家可能听说过,所使用的并行机制是map-reduce, 也就是用map去收集整理一个数据集,之后用reduce对数据集进行处理,得出想要的结果。原理很简单,但是就是这个简单的原理构成了google搜索的强大的基石,另外google的map-reduece是分布式的,呵呵
这段时间研究qt4.6,它对于并行处理这一块做了非常不错的封装,使用很简单,不用考虑线程同步处理等问题,下面就用这个最著名的统计单词个数的例子说明,代码真实太简单了
#include <QList> #include <QMap> #include <QTextStream> #include <QString> #include <QStringList> #include <QDir> #include <QTime> #include <QApplication> #include <QDebug> #include <qtconcurrentmap.h> #ifndef QT_NO_CONCURRENT using namespace QtConcurrent; /* Utility function that recursivily searches for files. */ QStringList findFiles(const QString &startDir, QStringList filters) { QStringList names; QDir dir(startDir); foreach (QString file, dir.entryList(filters, QDir::Files)) names += startDir + "/" + file; foreach (QString subdir, dir.entryList(QDir::AllDirs | QDir::NoDotAndDotDot)) names += findFiles(startDir + "/" + subdir, filters); return names; } typedef QMap<QString, int> WordCount; /* Single threaded word counter function. */ WordCount singleThreadedWordCount(QStringList files) { WordCount wordCount; foreach (QString file, files) { QFile f(file); f.open(QIODevice::ReadOnly); QTextStream textStream(&f); while (textStream.atEnd() == false) foreach(QString word, textStream.readLine().split(" ")) wordCount[word] += 1; } return wordCount; } // countWords counts the words in a single file. This function is // called in parallel by several threads and must be thread // safe. WordCount countWords(const QString &file) { QFile f(file); f.open(QIODevice::ReadOnly); QTextStream textStream(&f); WordCount wordCount; while (textStream.atEnd() == false) foreach (QString word, textStream.readLine().split(" ")) wordCount[word] += 1; return wordCount; } // reduce adds the results from map to the final // result. This functor will only be called by one thread // at a time. void reduce(WordCount &result, const WordCount &w) { QMapIterator<QString, int> i(w); while (i.hasNext()) { i.next(); result[i.key()] += i.value(); } } int main(int argc, char** argv) { QApplication app(argc, argv); qDebug() << "finding files..."; // 首先得到要统计目录下的所有文件 QStringList files = findFiles("../../", QStringList() << "*.cpp" << "*.h"); qDebug() << files.count() << "files"; qDebug() << "warmup"; { QTime time; time.start(); WordCount total = singleThreadedWordCount(files); } qDebug() << "warmup done"; int singleThreadTime = 0; { QTime time; time.start(); // 单线程统计,好与mapreduce这种机制实现的作对比 WordCount total = singleThreadedWordCount(files); singleThreadTime = time.elapsed(); // 打印出所耗费时间 qDebug() << "single thread" << singleThreadTime; } int mapReduceTime = 0; { QTime time; time.start(); // mapreduce方式进行统计 WordCount total = mappedReduced(files, countWords, reduce); mapReduceTime = time.elapsed(); qDebug() << "MapReduce" << mapReduceTime; } // 输出mapreduce方式比单线程处理方式要快的倍数 qDebug() << "MapReduce speedup x" << ((double)singleThreadTime - (double)mapReduceTime) / (double)mapReduceTime + 1; } #else int main() { qDebug() << "Qt Concurrent is not yet supported on this platform"; } #endif
通过这个测试用例,在统计文件很少的情况下,或者单核cpu的情况下,基本性能相同,但是如果统计文件一多,那么速度提升是很大的,自己的双核机器上统计300多个文件的加速比是1.3613,还是很不错的。
转载于:https://blog.51cto.com/dxwang/523564