linux&windows utf8和gbk编码识别&互相转换

linux默认是utf8编码，Windows默认是gbk编码，linux系统下可用locale命令查看系统编码。

linux下使用iconv命令转换文件编码

iconv -f 源编码 -t 目标编码 1.txt > 2.txt

如 gbk转utf8

iconv -f gbk -t utf8 1.txt > 2.txt

2.txt是转换完的文件，如覆盖源文件，去掉>2,txt即可

utf8和gbk编码判定

转自csdn博客http://m.blog.csdn.net/thedarkfairytale/article/details/73457200

utf8编码

bool is_str_utf8(const char* str)
{
    unsigned int nBytes = 0;//UFT8可用1-6个字节编码,ASCII用一个字节  
    unsigned char chr = *str;
    bool bAllAscii = true;

    for (unsigned int i = 0; str[i] != '\0'; ++i){
        chr = *(str + i);
        //判断是否ASCII编码,如果不是,说明有可能是UTF8,ASCII用7位编码,最高位标记为0,0xxxxxxx 
        if (nBytes == 0 && (chr & 0x80) != 0){
            bAllAscii = false;
        }

        if (nBytes == 0) {
            //如果不是ASCII码,应该是多字节符,计算字节数  
            if (chr >= 0x80) {

                if (chr >= 0xFC && chr <= 0xFD){
                    nBytes = 6;
                }
                else if (chr >= 0xF8){
                    nBytes = 5;
                }
                else if (chr >= 0xF0){
                    nBytes = 4;
                }
                else if (chr >= 0xE0){
                    nBytes = 3;
                }
                else if (chr >= 0xC0){
                    nBytes = 2;
                }
                else{
                    return false;
                }

                nBytes--;
            }
        }
        else{
            //多字节符的非首字节,应为 10xxxxxx 
            if ((chr & 0xC0) != 0x80){
                return false;
            }
            //减到为零为止
            nBytes--;
        }
    }

    //违返UTF8编码规则 
    if (nBytes != 0)  {
        return false;
    }

    if (bAllAscii){ //如果全部都是ASCII, 也是UTF8
        return true;
    }

    return true;
}

gbk编码

bool is_str_gbk(const char* str)
{
    unsigned int nBytes = 0;//GBK可用1-2个字节编码,中文两个 ,英文一个 
    unsigned char chr = *str;
    bool bAllAscii = true; //如果全部都是ASCII,  

    for (unsigned int i = 0; str[i] != '\0'; ++i){
        chr = *(str + i);
        if ((chr & 0x80) != 0 && nBytes == 0){// 判断是否ASCII编码,如果不是,说明有可能是GBK
            bAllAscii = false;
        }

        if (nBytes == 0) {
            if (chr >= 0x80) {
                if (chr >= 0x81 && chr <= 0xFE){
                    nBytes = +2;
                }
                else{
                    return false;
                }

                nBytes--;
            }
        }
        else{
            if (chr < 0x40 || chr>0xFE){
                return false;
            }
            nBytes--;
        }//else end
    }

    if (nBytes != 0)  {     //违反规则 
        return false;
    }

    if (bAllAscii){ //如果全部都是ASCII, 也是GBK
        return true;
    }

    return true;
}

使用boost库进行编码转换

boost大法好

#include <boost/locale.hpp>
boost/locale/encoding.hpp中有这样的函数
 template<typename CharType>
std::string from_utf(CharType const *text,std::string const &charset,method_type how=default_method)
{
    CharType const *text_end = text;
    while(*text_end) 
        text_end++;
    return from_utf(text,text_end,charset,how);
}
template<typename CharType>
std::basic_string<CharType> to_utf(char const *text,std::string const &charset,method_type how=default_method)
{
    char const *text_end = text;
    while(*text_end) 
        text_end++;
    return to_utf<CharType>(text,text_end,charset,how);
}
返回值是个string
utf转gbk可以
std::string gbkstr =  boost::locale::conv::from_utf<char>(strUtf8.c_str(), "gb2312");//strUtf8是源utf8字符串
同理gbk转utf8可以
std::string utf8str = boost::locale::conv::to_utf<char>(strANSI.c_str(), "gb2312");//strANSI是源gbk字符串

linux utf8和gbk编码互转

utf8转gbk
#include <iconv.h>
char * UTF8toANSI(const std::string &from)
{
    char *inbuf=const_cast<char*>(from.c_str());
    size_t inlen = strlen(inbuf);
    size_t outlen = inlen *4;
    char *outbuf = (char *)malloc(inlen * 4 );
    bzero( outbuf, inlen * 4);
    char *in = inbuf;
    char *out = outbuf;
    iconv_t cd=iconv_open("GBK","UTF-8");
    iconv(cd,&in,&inlen,&out,&outlen);
    iconv_close(cd);
    return outbuf;
}

Windows utf8和gbk编码互转