httpClient抓取页面的简易使用

本次用的httpClient版本是4.5.2，maven坐标如下所示：

 <dependency>

<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.2</version>

    </dependency>

此版本的httpClient中DefaultHttpClient()已经过时，应使用CloseableHttpClient hclient = HttpClients.createDefault();创建对象。

4.5版本文档地址：http://hc.apache.org/httpcomponents-client-4.5.x/tutorial/html/index.html（点击官网download—>httpclient4.5->tutorial即可找到此网页）

例一：ResponseHandler方式

public static void httpHandler1(String urlstr){
CloseableHttpClient hclient = HttpClients.createDefault();
HttpGet hGet = new HttpGet(urlstr);

ResponseHandler<String> rhandler = new ResponseHandler<String>() {
@Override
public String handleResponse( HttpResponse response) throws ClientProtocolException, IOException {
int status = response.getStatusLine().getStatusCode();
if(status>=200 && status<300){
HttpEntity hentity = response.getEntity();
return hentity != null ? EntityUtils.toString(hentity):null;
}else{
throw new ClientProtocolException("error code:"+status);
}
}
};
String responseBody = "";
try {
responseBody = hclient.execute(hGet, rhandler);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
System.out.println(responseBody);
try {
hclient.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}

例二：EntityUtils.toString(hentity)直接取得连接返回的内容

public static void hc2(String urlstr){
CloseableHttpClient hc = HttpClients.createDefault();
HttpGet hg = new HttpGet(urlstr);
CloseableHttpResponse hr = null;
InputStream stream = null ;
try {
hr = hc.execute(hg);
HttpEntity hentity = hr.getEntity();
System.out.println(EntityUtils.toString(hentity));
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}finally{
try {
if(stream != null){
stream.close();
}
if(hr != null){
hr.close();
}
hc.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}

}


}

例三：instream = entity.getContent();取得连接返回的内容

public static void one(String urlstr) throws ClientProtocolException, IOException{
CloseableHttpClient httpclient = HttpClients.createDefault();
        try {
            HttpGet httpget = new HttpGet(urlstr);


            System.out.println("Executing request " + httpget.getRequestLine());
            CloseableHttpResponse response = httpclient.execute(httpget);
            try {
                //System.out.println("----------------------------------------");
                //System.out.println(response.getStatusLine()); 
                HttpEntity entity = response.getEntity();       
                StringBuffer sbr = new StringBuffer();
                if (entity != null) {
                    InputStream instream = entity.getContent();
                    BufferedReader br = new BufferedReader(new InputStreamReader(instream, "utf-8"));
                   // byte [] tmp = new byte[1024];
                    try {
                    //int l;
                    String line;
                        while( (line = br.readLine()) != null){
                        sbr.append(line+"\r\n");
                        };
                      
                    } catch (IOException ex) {
                        throw ex;
                    } finally {
                        instream.close();
                    }
                    System.out.println("---------------begin-----------");
                    System.out.println(sbr);
                    System.out.println("---------------end-------------");
                }
            } finally {
                response.close();
            }
        } finally {
            httpclient.close();
        }
}
另外，可使用java.net包内的类来连接url获取内容

eg：

public static void urltest(String urlstr) throws IOException{
URL url = new URL(urlstr);
HttpURLConnection hucon = (HttpURLConnection)url.openConnection();
hucon.setRequestMethod("GET");
hucon.connect();
InputStream is = hucon.getInputStream();
BufferedReader br = new BufferedReader(new InputStreamReader(is));

StringBuffer str = new StringBuffer();
String tmp = "";
while((tmp = br.readLine()) != null){
str.append(tmp+"\r\n");
}
System.out.println(str);
is.close();
}