文本文件从UTF8转码为GBK的C语言程序源代码

      本程序使用C语言实现UTF8到GBK的文本文件的转码，适用于大批量的文件编码转换。读文件和写文件均使用了缓存方式，在普通PC上测试，转码平均速度约11MB/S。

      基本逻辑是建立一个64KB的UTF8到GBK字符集的映射数组，读取源文件，并在映射数组中查找每个字符对应的GBK编码，输出到新的文件。

      程序中UTF-16BE_FIXED.txt和GBK_FIXED.txt是编码一一映射的对照文件，可以根据需要自行建立，这两个文件的读取时间可以忽略不计。

      调用方法：utf2gbk.exe 源文件名 目标文件名。

      读写缓存均设置为32KB，参照以下测试结果确定，实际转码效率的决定于磁盘IO。

      测试结果：

     

读缓存（BYTE）	写缓存 （BYTE）	时间（S）	文件大小 （BYTE）	平均速度(MB)
4096	4096	8.590011	95764200	10.63187
4096	8192	8.335358	95764200	10.95668
4096	16384	8.339024	95764200	10.95186
4096	32768	8.455411	95764200	10.80111
4096	65536	8.17109	95764200	11.17695
4096	131072	8.071299	95764200	11.31514
8192	4096	8.325038	95764200	10.97026
8192	8192	8.530569	95764200	10.70595
8192	16384	8.280397	95764200	11.02941
8192	32768	8.378514	95764200	10.90025
8192	65536	8.062362	95764200	11.32768
8192	131072	7.995	95764200	11.42312
16384	4096	8.326182	95764200	10.96876
16384	8192	8.286298	95764200	11.02155
16384	16384	9.060944	95764200	10.07929
16384	32768	8.31521	95764200	10.98323
16384	65536	8.099703	95764200	11.27546
16384	131072	7.904147	95764200	11.55442
32768	4096	7.941703	95764200	11.49978
32768	8192	7.979317	95764200	11.44557
32768	16384	7.859548	95764200	11.61999
32768	32768	7.698843	95764200	11.86254
32768	65536	9.86077	95764200	9.261737
32768	131072	8.409786	95764200	10.85971
65536	4096	9.594587	95764200	9.518686
65536	8192	9.589273	95764200	9.523961
65536	16384	9.461487	95764200	9.65259
65536	32768	9.380491	95764200	9.735936
65536	65536	8.962597	95764200	10.18989
65536	131072	8.751248	95764200	10.43598
131072	4096	8.316328	95764200	10.98175
131072	8192	8.284002	95764200	11.02461
131072	16384	8.336569	95764200	10.95509
131072	32768	8.548768	95764200	10.68316
131072	65536	8.768317	95764200	10.41567
131072	131072	8.627917	95764200	10.58516

 

C程序源码：

#include <stdio.h> #define CLOCKS_PER_SEC ((clock_t)1000) #define READ_BUFFER 32768 #define WRITE_BUFFER 32768 // for win 32bit with buffer void main(int argc, char* argv[]) { char* srcFile; char* tarFile; unsigned int word[65536] = {0}; unsigned int nCount = 0; FILE* inFile = 0; FILE* outFile = 0; FILE* utfFile = 0; FILE* gbkFile = 0; srcFile = argv[1]; tarFile = argv[2]; inFile = fopen(srcFile,"rb"); outFile = fopen(tarFile,"wb"); utfFile = fopen("UTF-16BE_FIXED.txt","rb"); gbkFile = fopen("GBK_FIXED.txt","rb"); // 建立UTF-16BE到GBK映射 if(utfFile){ unsigned int utfBuffer[1] = {0}; unsigned int gbkBuffer[1] = {0}; unsigned int index = 0; unsigned int value = 0; unsigned int ch = 0; unsigned int cl = 0; while(!feof(utfFile)){ nCount = fread(utfBuffer, sizeof(unsigned char),1,utfFile ); if(nCount>0){ ch = utfBuffer[0]<< 8; nCount = fread(utfBuffer, sizeof(unsigned char),1,utfFile ); cl = utfBuffer[0]; index = ch + cl; nCount = fread(gbkBuffer, sizeof(unsigned char),2,gbkFile ); value = gbkBuffer[0]; word[index] = value; } } fclose( utfFile ); fclose( gbkFile ); } // 转码 if(inFile) { unsigned char buffer[READ_BUFFER] = {0}; unsigned char inBuffer[1] = {0}; unsigned char outBuffer[WRITE_BUFFER] = {0}; unsigned int b; unsigned int integer; unsigned int len; unsigned int bufferIndex = 0; unsigned int bw; unsigned int i = 0; while(!feof(inFile)){ nCount = fread(buffer, sizeof(unsigned char),READ_BUFFER,inFile ); if(nCount>0){ for(i=0; i<READ_BUFFER;){ b = buffer[i]; if(b==0){ break; }else{ if ((b & 0xFF) < 0x80) { len = 1; integer = b; buffer[i] = 0; i++; } else if (((b & 0xFF) < 0xDF) && ((b & 0xFF) >= 0x80)) { len = 2; integer = b & 0x1F; integer = integer << 6; if(i+1>=READ_BUFFER){ nCount = fread(inBuffer, sizeof(unsigned char),1,inFile ); if(nCount > 0){ b = inBuffer[0]; } } else{ b = buffer[i+1]; } integer += b & 0x3F; buffer[i] = 0; i++; buffer[i] = 0; i++; } else if (((b & 0xFF) <= 0xEF) && ((b & 0xFF) >= 0xDF)) { len = 2; integer = b & 0x0F; integer = integer << 6; if(i+1>=READ_BUFFER){ // 需要读2位 nCount = fread(inBuffer, sizeof(unsigned char),1,inFile ); if(nCount>0){ b = inBuffer[0]; integer += b & 0x3F; integer = integer << 6; nCount = fread(inBuffer, sizeof(unsigned char),1,inFile ); if(nCount>0){ b = inBuffer[0]; integer += b & 0x3F; } } buffer[i] = 0; i++; }else if (i+2>=READ_BUFFER){ // 需要读1位 b = buffer[i+1]; integer += b & 0x3F; integer = integer << 6; nCount = fread(inBuffer, sizeof(unsigned char),1,inFile ); if(nCount>0){ b = inBuffer[0]; integer += b & 0x3F; } buffer[i] = 0; i++; buffer[i] = 0; i++; }else{ b = buffer[i+1]; integer += b & 0x3F; integer = integer << 6; b = buffer[i+2]; integer += b & 0x3F; buffer[i] = 0; i++; buffer[i] = 0; i++; buffer[i] = 0; i++; } } else { len = 1; integer = 0; buffer[i] = 0; i++; } if(word[integer]!=0){ integer = word[integer]; } if(bufferIndex < WRITE_BUFFER){ if(1==len){ outBuffer[bufferIndex] = integer; bufferIndex++; if(bufferIndex>=WRITE_BUFFER){ fwrite(&outBuffer,sizeof(unsigned char),bufferIndex,outFile); bufferIndex = 0; } }else if(2==len){ bw = integer & 0xFF; outBuffer[bufferIndex] = bw; bufferIndex++; if(bufferIndex>=WRITE_BUFFER){ fwrite(&outBuffer,sizeof(unsigned char),bufferIndex,outFile); bufferIndex = 0; } bw = (integer>>8) & 0xFF; outBuffer[bufferIndex] = bw; bufferIndex++; if(bufferIndex>=WRITE_BUFFER){ fwrite(&outBuffer,sizeof(unsigned char),bufferIndex,outFile); bufferIndex = 0; } } } else{ fwrite(&outBuffer,sizeof(unsigned char),bufferIndex,outFile); bufferIndex = 0; } } } } } if(bufferIndex>0){ fwrite(&outBuffer,sizeof(unsigned char),bufferIndex,outFile); bufferIndex = 0; } fclose( outFile ); fclose( inFile ); } }

注：代码中涉及的GBK_FIXED.txt、UTF-16BE_FIXED.txt为GBK和UTF-16BE字符的一一映射文件。

以下为下载链接

GBK_FIXED.txt

UTF-16BE_FIXED.txt