（C/C++）UTF8字符串中字的切分

2019独角兽企业重金招聘Python工程师标准>>>

UTF-8 采用变长度字节来表示字符，理论上最多可以到 6 个字节长度。UTF-8 编码兼容了 ASC II(0-127)， 也就是说 UTF-8 对于 ASC II 字符的编码是和 ASC II 一样的。对于超过一个字节长度的字符，才用以下编码规范：
左边第一个字节1的个数表示这个字符编码字节的位数，例如两位字节字符编码样式为为：110xxxxx 10xxxxxx； 三位字节字符的编码样式为：1110xxxx 10xxxxxx 10xxxxxx.；以此类推，六位字节字符的编码样式为：1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx。 xxx 的值由字符编码的二进制表示的位填入。
1字节：0xxxxxxx
2字节：110xxxxx 10xxxxxx
3字节：1110xxxx 10xxxxxx 10xxxxxx
4字节：11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
5字节：111110xx 10xxxxxx 10xxxxxx 10xxxxxx

1 // / 2 # include < string .h > 3 # include < vector > 4 5 // / 6 using namespace std; 7 8 // / 9 void fnReadCharactersUTF8( const char * pszSentence, vector < string >& vec ) 10 { 11 int iLen; 12 iLen = strlen( pszSentence ); 13 14 const char * p; 15 16 p = pszSentence; 17 18 unsigned char * q; 19 20 char szCharacter[ 101 ]; 21 int iChar; 22 23 int iNumChars; 24 iNumChars = 0 ; 25 26 vec.clear(); 27 28 string strCharacter; 29 30 while ( p != NULL && strlen( p ) > 0 ) 31 { 32 q = ( unsigned char * ) p; 33 if ( q[ 0 ] < 0x80 ) 34 { 35 // p[ 0 ] must be an ASCII character 36 iChar = 0 ; 37 szCharacter[iChar ++ ] = p[ 0 ]; 38 p ++ ; 39 q = ( unsigned char * ) p; 40 while ( p != NULL && q[ 0 ] < 0x80 ) 41 { 42 szCharacter[iChar ++ ] = p[ 0 ]; 43 p ++ ; 44 q = ( unsigned char * ) p; 45 } 46 szCharacter[iChar] = ' \0 ' ; 47 48 vec.push_back( string ( szCharacter ) ); 49 50 iNumChars ++ ; 51 } 52 else if ( q[ 0 ] < 0xC0 ) 53 { 54 // invalid char between 0x80 and 0xC0 55 p ++ ; 56 } 57 else if ( q[ 0 ] < 0xE0 ) 58 { 59 // two chars 60 szCharacter[ 0 ] = p[ 0 ]; 61 szCharacter[ 1 ] = p[ 1 ]; 62 szCharacter[ 2 ] = ' \0 ' ; 63 p = p + 2 ; 64 65 strCharacter = string ( szCharacter ); 66 vec.push_back( strCharacter ); 67 68 iNumChars ++ ; 69 } 70 else if ( q[ 0 ] < 0xF0 ) 71 { 72 // three chars 73 szCharacter[ 0 ] = p[ 0 ]; 74 szCharacter[ 1 ] = p[ 1 ]; 75 szCharacter[ 2 ] = p[ 2 ]; 76 szCharacter[ 3 ] = ' \0 ' ; 77 p = p + 3 ; 78 79 strCharacter = string ( szCharacter ); 80 vec.push_back( strCharacter ); 81 82 // printf( "%s ", strCharacter.c_str( ) ); 83 84 iNumChars ++ ; 85 } 86 else if ( q[ 0 ] < 0xF8 ) 87 { 88 // four chars 89 p += 4 ; 90 } 91 else if ( q[ 0 ] < 0xFC ) 92 { 93 // five chars 94 p += 5 ; 95 } 96 else if ( q[ 0 ] < 0xFE ) 97 { 98 // 6 chars 99 p += 5 ; 100 } 101 else 102 { 103 // >=0xFE 104 p ++ ; 105 } 106 } 107 } 108 109 / / FILE END / // 110

转载于:https://my.oschina.net/jacobin/blog/178813