点云读取速度比较——QTextStream、C++文件流、C++文件映射

    最近研究了一下CC的点云读取类，速度简直快到不行。

    后来发现CC就是简单使用了QTextStream进行读取。

    笔者之前研究过文件映射进行点云读取，速度也是非常快。内存映射之所以能达到这么高的速度是因为系统直接把整块硬盘内存直接交由程序处理，省去了数据交换过程。

    那么文件映射和QTextStream究竟谁快呢？

    笔者准备了一个1000W多的XYZ格式的点云，点云内部有6列，分别是XYZRGB，分别用三种方式读写，然后记录了时间。

    下面先上程序：

    首先是文件映射的方式读取，笔者就把读取XYZ格式的点云：

//PTX format:
//All the row is point cloud
//field width : unknown
int qScarlet_GLCloudEngin_ES2::ReadPointCloudFrom_XYZ()
{
    HANDLE hSrcFile = CreateFileA(m_FullPath.c_str(), GENERIC_READ, 0, NULL, OPEN_ALWAYS, 0, NULL);
    if (hSrcFile == INVALID_HANDLE_VALUE) return 0;
    LARGE_INTEGER tInt2;
    GetFileSizeEx(hSrcFile, &tInt2);
    __int64 dwRemainSize = tInt2.QuadPart;
    __int64 dwFileSize = dwRemainSize;
    HANDLE hSrcFileMapping = CreateFileMapping(hSrcFile, NULL, PAGE_READONLY, tInt2.HighPart, tInt2.LowPart, NULL);
    if (hSrcFileMapping == INVALID_HANDLE_VALUE)
    {
        return 0;
    }
    SYSTEM_INFO SysInfo;
    GetSystemInfo(&SysInfo);
    DWORD dwGran = SysInfo.dwAllocationGranularity;
    const int BUFFERBLOCKSIZE = dwGran * 1024;

    const int XYZ_FC = 3;
    const int XYZI_FC = 4;
    const int XYZRGB_FC = 6;
    const int XYZIRGB_FC = 7;
    bool AlreadySetFiledCount = false;//是否已经设置了数据宽度
    int usefulFiledCount = 0;

    int totalRows = 0;//文件总行数：
    int FieldIndex = 0;//每一个小数字的填充位置
    int   FieldCount = 0;//每一行中整数字位置，用来判定数据列数究竟是XYZARGB。
    double arrXYZ_RGB[XYZIRGB_FC];
    char  strLine[1024] = { 0 };

    double time = (double)cv::getTickCount();
    vector<CloudVertex> PointVec;//就怕这玩意溢出啊
    vector<CloudVertex>().swap(PointVec);//先进行清空
    while (dwRemainSize > 0)
    {
        DWORD dwBlock = dwRemainSize < BUFFERBLOCKSIZE ? dwRemainSize : BUFFERBLOCKSIZE;
        __int64 qwFileOffset = dwFileSize - dwRemainSize;
        PBYTE pSrc = (PBYTE)MapViewOfFile(hSrcFileMapping, FILE_MAP_READ, (DWORD)(qwFileOffset >> 32), (DWORD)(qwFileOffset & 0xFFFFFFFF), dwBlock);
        PBYTE pSrcBak = pSrc;
        for (int i = 0; i < dwBlock; i++)
        {
            //这样的处理方式有一个很大的缺点
            //当整个文件的最后一行不是空一行的话，整个数据会少一行。
            //但是一般默认情况下整个数据的最后一行是有一个换行的
            if (*pSrc == '\n')
            {
                //整行读完了====================================================
                if (FieldIndex != 0)//先处理一次字段。
                {
                    strLine[FieldIndex] = '\0';//在末尾处加上符号。
                    arrXYZ_RGB[FieldCount++] = atof(strLine);
                    FieldIndex = 0;
                }
                if (AlreadySetFiledCount == false)//如果没有设置数据宽度
                {
                    //有用的字段大于3个吧
                    if (FieldCount == XYZ_FC || FieldCount == XYZI_FC || FieldCount == XYZRGB_FC || FieldCount == XYZIRGB_FC)
                    {
                        m_FieldCount = FieldCount;//数据宽度
                        usefulFiledCount = FieldCount;
                    }
                    else
                    {
                        m_FieldCount = XYZ_FC;//数据宽度默认为3
                        usefulFiledCount = XYZ_FC;//如果没有可用的数据类型，就默认为3列
                    }
                    AlreadySetFiledCount = true;//已经设置了列宽
                }

                CloudVertex myVerPoint;
                if (usefulFiledCount == XYZ_FC)
                {
                    myVerPoint.pos.setX(arrXYZ_RGB[0]);
                    myVerPoint.pos.setY(arrXYZ_RGB[1]);
                    myVerPoint.pos.setZ(arrXYZ_RGB[2]);
                    myVerPoint.col.setX(1.0);
                    myVerPoint.col.setY(1.0);
                    myVerPoint.col.setZ(1.0);
                }
                if (usefulFiledCount == XYZI_FC)
                {
                    myVerPoint.pos.setX(arrXYZ_RGB[0]);
                    myVerPoint.pos.setY(arrXYZ_RGB[1]);
                    myVerPoint.pos.setZ(arrXYZ_RGB[2]);
                    myVerPoint.col.setX(1.0);
                    myVerPoint.col.setY(1.0);
                    myVerPoint.col.setZ(1.0);
                }
                if (usefulFiledCount == XYZRGB_FC)
                {
                    myVerPoint.pos.setX(arrXYZ_RGB[0]);
                    myVerPoint.pos.setY(arrXYZ_RGB[1]);
                    myVerPoint.pos.setZ(arrXYZ_RGB[2]);
                    myVerPoint.col.setX(arrXYZ_RGB[3]/255.0);
                    myVerPoint.col.setY(arrXYZ_RGB[4]/255.0);
                    myVerPoint.col.setZ(arrXYZ_RGB[5]/255.0);
                }
                if (usefulFiledCount == XYZIRGB_FC)
                {
                    myVerPoint.pos.setX(arrXYZ_RGB[0]);
                    myVerPoint.pos.setY(arrXYZ_RGB[1]);
                    myVerPoint.pos.setZ(arrXYZ_RGB[2]);
                    myVerPoint.col.setX(arrXYZ_RGB[4]/255.0);
                    myVerPoint.col.setY(arrXYZ_RGB[5]/255.0);
                    myVerPoint.col.setZ(arrXYZ_RGB[6]/255.0);
                }
                PointVec.push_back(myVerPoint);
                totalRows++;
                FieldCount = 0;//字段位置清零
                memset(strLine, 0, sizeof(strLine));//数字字符数组清空
            }
            else if ((*pSrc >= '0' && *pSrc <= '9') || *pSrc == '.' || *pSrc == '-' || *pSrc == 'e' || *pSrc == '+')
            {
                //空格 或Tab
                strLine[FieldIndex++] = *pSrc;
            }
            else
            {
                //此时为行内分割===关键是连续几次无用字符==============================
                if (FieldIndex != 0)
                {
                    //一个字段处理完毕
                    strLine[FieldIndex] = '\0';
                    arrXYZ_RGB[FieldCount++] = atof(strLine);
                    FieldIndex = 0;
                }
            }
            pSrc++;
        }
        UnmapViewOfFile(pSrcBak);
        dwRemainSize -= dwBlock;
    }
    CloseHandle(hSrcFileMapping);
    CloseHandle(hSrcFile);

    m_PointNum = PointVec.size();
    m_PointCloud = new CloudVertex[m_PointNum];
    for (size_t i = 0; i != PointVec.size(); i++)
    {
        *(m_PointCloud + i) = PointVec[i];
    }
    vector<CloudVertex>().swap(PointVec);//清空
    time = ((double)cv::getTickCount() - time) / cv::getTickFrequency();
    std::cout << "time cost:" << time << endl;
    std::cout << "total row:" << totalRows << endl;
    return totalRows;
}

    由于不同文件点云的列数不确定，我们需要自适应进行判断所以才会有这么多判断。

    接下来是QTextStream的读取方式：

//The Reading Speed of the QT Stream
double QTTextStreamTest(QString FilePath)
{
    double start=0,end=0;
    start = getTickCount();
    //other useful variables
    unsigned linesRead = 0;
    unsigned pointsRead = 0;
    char separator = ' ';
    QFile file(FilePath);
    if (!file.open(QFile::ReadOnly))
        return -1; //we clear already initialized data
    QTextStream stream(&file);
    QString currentLine = stream.readLine();
    QVector<QVector3D> CloudVec;
    while (!currentLine.isNull())
    {
        ++linesRead;
        if (currentLine.startsWith("//"))
        {
            currentLine = stream.readLine();
            continue;
        }
        if (currentLine.size() == 0)
        {
            currentLine = stream.readLine();
            continue;
        }
        //we split current line
        QStringList parts = currentLine.split(separator,QString::SkipEmptyParts);
        int nParts = parts.size();//for simple test we dont have other part
        QVector3D P;
        P.setX(parts[0].toDouble());
        P.setY(parts[1].toDouble());
        P.setZ(parts[2].toDouble());
        CloudVec.push_back(P);
        ++pointsRead;

        //read next line
        currentLine = stream.readLine();
    }
    file.close();
    end = getTickCount();
    qDebug()<<"Read Cloud Size:"<<CloudVec.size();
    double timeCost =(end - start)/getTickFrequency();
    return timeCost;//Return time cost
}

    最后是文件流读取方式：

struct CloudStruct
{
    double x=0,y=0,z=0,r=0,g=0,b=0;
};
double CFileStreamTest(QString FilePath)
{
    std::string stdFilePath = FilePath.toStdString();
    ifstream CloudReader(stdFilePath);
    if(!CloudReader)
    {
        return -1;
    }
    double start=0,end=0;
    start = getTickCount();

    QVector<CloudStruct> CloudVec;
    CloudStruct Temp;
    while(CloudReader>>Temp.x)
    {
        CloudReader>>Temp.y;
        CloudReader>>Temp.z;
        CloudReader>>Temp.r;
        CloudReader>>Temp.g;
        CloudReader>>Temp.b;
        CloudVec.push_back(Temp);
    }
    CloudReader.close();
    end = getTickCount();
    qDebug()<<"Read Cloud Size:"<<CloudVec.size();
    double timeCost =(end - start)/getTickFrequency();
    return timeCost;
}

具体调用：

void qscarlet_opencvMaster::on_actionQTReadingSpeedTest_triggered()
{
    ui->tabWidget->setCurrentIndex(4);//change to Index 4
    QFileDialog *CloudfileDialog = new QFileDialog(this);
    CloudfileDialog->setFileMode(QFileDialog::ExistingFiles);//可以多选
    //fileDialog->setFileMode(QFileDialog::Directory);//变成了选择文件夹的对话框
    CloudfileDialog->setWindowTitle(tr("Please Select a Cloud"));
    CloudfileDialog->setNameFilter(tr("Cloud Files (*.txt *.ptx *.pts *.ply *.asc *.xyz)"));
    CloudfileDialog->setDirectory(".");
    CloudfileDialog->setViewMode(QFileDialog::Detail);//List
    if(CloudfileDialog->exec() == QDialog::Accepted)
    {
        ui->ViewTabWidget->setCurrentIndex(2);//切换到3DView
        for(int i=0;i<CloudfileDialog->selectedFiles().size();i++)
        {
            QString path =CloudfileDialog->selectedFiles()[i];
            int64 start=0,end=0;
            start = getTickCount();
            m_GLWidget_ES2->AddCloudFromPath(path);
            end = getTickCount();
            cout << "C++ File Map Time Cost: " << (end - start)/getTickFrequency()<<"s"<< endl;
            SetCurrentFile(path);

            double ReadingTime = QTTextStreamTest(path);
            cout << "QT Text Stream Time Cost: " << ReadingTime<<"s"<< endl;

            ReadingTime = CFileStreamTest(path);
            cout << "C++ file Stream Time Cost: " << ReadingTime<<"s"<< endl;
        }
    }
    else
    {
        //QMessageBox::information(NULL, tr("Path"), tr("You didn't select any files."));
    }


}

最终的结果对比：

C++ File Map Time Cost: 14.5303s

Read Cloud Size: 10545264

QT Text Stream Time Cost: 16.3132s

Read Cloud Size: 10545264

C++ file Stream Time Cost: 179.773s

    可见速度最快的还是文件映射，其次是QTextStream，最慢的是C++文件流。

    然而在编程难度上，QTextStream要远远小于文件映射，而且基于QT的子类可以加入QProgress等进度条，这一点文件映射是做不到的。

    就到此为止了。