Java爬虫内容为ASCII码_java爬虫爬取网页内容前，对网页内容的编码格式进行判断的方式...

packagecom.mobivans.encoding;importinfo.monitorenter.cpdetector.io.ASCIIDetector;importinfo.monitorenter.cpdetector.io.ByteOrderMarkDetector;importinfo.monitorenter.cpdetector.io.CodepageDetectorProxy;importinfo.monitorenter.cpdetector.io.JChardetFacade;importinfo.monitorenter.cpdetector.io.ParsingDetector;importinfo.monitorenter.cpdetector.io.UnicodeDetector;importjava.io.ByteArrayInputStream;importjava.io.IOException;importjava.io.InputStream;importjava.net.MalformedURLException;importjava.net.URL;importjava.net.URLConnection;importjava.nio.charset.Charset;importjava.util.List;importjava.util.Map;importorg.apache.commons.io.IOUtils;public classPageEncoding {/**测试用例

*@paramargs*/

public static voidmain(String[] args) {//String charset = getEncodingByHeader("http://blog.csdn.net/liuzhenwen/article/details/4060922");//String charset = getEncodingByMeta("http://blog.csdn.net/liuzhenwen/article/details/4060922");

String charset = getEncodingByContentStream("http://blog.csdn.net/liuzhenwen/article/details/5930910");

System.out.println(charset);

}/*** 从header中获取页面编码

*@paramstrUrl

*@return

*/

public staticString getEncodingByHeader(String strUrl){

String charset= null;try{

URLConnection urlConn= newURL(strUrl).openConnection();//获取链接的header

Map> headerFields =urlConn.getHeaderFields();//判断headers中是否存在Content-Type

if(headerFields.containsKey("Content-Type")){//拿到header 中的 Content-Type ：[text/html; charset=utf-8]

List attrs = headerFields.get("Content-Type");

String[] as= attrs.get(0).split(";");for(String att : as) {if(att.contains("charset")){//System.out.println(att.split("=")[1]);

charset = att.split("=")[1];

}

}

}returncharset;

}catch(MalformedURLException e) {

e.printStackTrace();returncharset;

}catch(IOException e) {

e.printStackTrace();returncharset;

}

}/*** 从meta中获取页面编码

*@paramstrUrl

*@return

*/

public staticString getEncodingByMeta(String strUrl){

String charset= null;try{

URLConnection urlConn= newURL(strUrl).openConnection();//避免被拒绝

urlConn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36");//将html读取成行,放入list

List lines =IOUtils.readLines(urlConn.getInputStream());for(String line : lines) {if(line.contains("http-equiv") && line.contains("charset")){//System.out.println(line);

String tmp = line.split(";")[1];

charset= tmp.substring(tmp.indexOf("=")+1, tmp.indexOf("\""));

}else{continue;

}

}returncharset;

}catch(MalformedURLException e) {

e.printStackTrace();returncharset;

}catch(IOException e) {

e.printStackTrace();returncharset;

}

}/*** 根据网页内容获取页面编码

* case : 适用于可以直接读取网页的情况(例外情况:一些博客网站禁止不带User-Agent信息的访问请求)

*@paramurl

*@return

*/

public staticString getEncodingByContentUrl(String url) {

CodepageDetectorProxy cdp=CodepageDetectorProxy.getInstance();

cdp.add(JChardetFacade.getInstance());//依赖jar包 ：antlr.jar & chardet.jar

cdp.add(ASCIIDetector.getInstance());

cdp.add(UnicodeDetector.getInstance());

cdp.add(new ParsingDetector(false));

cdp.add(newByteOrderMarkDetector());

Charset charset= null;try{

charset= cdp.detectCodepage(newURL(url));

}catch(MalformedURLException e) {

e.printStackTrace();

}catch(IOException e) {

e.printStackTrace();

}

System.out.println(charset);return charset == null ? null: charset.name().toLowerCase();

}/*** 根据网页内容获取页面编码

* case : 适用于不可以直接读取网页的情况,通过将该网页转换为支持mark的输入流,然后解析编码

*@paramstrUrl

*@return

*/

public staticString getEncodingByContentStream(String strUrl) {

Charset charset= null;try{

URLConnection urlConn= newURL(strUrl).openConnection();//打开链接,加上User-Agent,避免被拒绝

urlConn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36");//解析页面内容

CodepageDetectorProxy cdp =CodepageDetectorProxy.getInstance();

cdp.add(JChardetFacade.getInstance());//依赖jar包 ：antlr.jar & chardet.jar

cdp.add(ASCIIDetector.getInstance());

cdp.add(UnicodeDetector.getInstance());

cdp.add(new ParsingDetector(false));

cdp.add(newByteOrderMarkDetector());

InputStream in=urlConn.getInputStream();

ByteArrayInputStream bais= newByteArrayInputStream(IOUtils.toByteArray(in));//detectCodepage(InputStream in, int length) 只支持可以mark的InputStream

charset = cdp.detectCodepage(bais, 2147483647);

}catch(MalformedURLException e) {

e.printStackTrace();

}catch(IOException e) {

e.printStackTrace();

}return charset == null ? null: charset.name().toLowerCase();

}

}