java方式httpclient/htmlunit爬虫入门案例

@Test
public void test1() throws Exception{

    CloseableHttpClient httpClient = HttpClients.createDefault();
    
    HttpGet httpGet = new HttpGet("http://www.baidu.com");
    CloseableHttpResponse response = httpClient.execute(httpGet);
    if (response != null){
        HttpEntity entity =  response.getEntity();  
        String result = EntityUtils.toString(entity, "UTF-8");
        System.out.println("网页内容:"+result);
    }
    if (response != null){
        response.close();
    }
    if (httpClient != null){
        httpClient.close();
    }


}

@Test
public void test2() throws  Exception{
    final WebClient client = new WebClient();

    client.getOptions().setJavaScriptEnabled(true); 
    client.getOptions().setUseInsecureSSL(true);
    client.getOptions().setCssEnabled(false);
    client.getOptions().setThrowExceptionOnScriptError(false);
    client.getOptions().setThrowExceptionOnFailingStatusCode(false);
    client.setAjaxController(new NicelyResynchronizingAjaxController());
    HtmlPage page =(HtmlPage)  client.getPage("http://www.baidu.com");
    System.out.print(page.asText());
}
@Test
public void test3() throws  Exception{
//htmlunit不是专门做爬虫的，推荐用selenium+phantomjs无头浏览器去做，通过xpath的方式爬虫更加灵活
    final WebClient client = new WebClient();

    client.getOptions().setJavaScriptEnabled(true); S
    client.getOptions().setUseInsecureSSL(true);
    client.getOptions().setCssEnabled(false);
    client.getOptions().setThrowExceptionOnScriptError(false);
    client.getOptions().setThrowExceptionOnFailingStatusCode(false);

    HtmlPage page =(HtmlPage)  client.getPage("http://www.baidu.com");

    final HtmlForm form = page.getFormByName("f");

    final HtmlSubmitInput button = form.getInputByValue("百度一下");

    final HtmlTextInput textField = form.getInputByName("wd");

    textField.setValueAttribute("山炮");
  
    final HtmlPage nextPage = button.click();

    String result = nextPage.asText();

    System.out.print(result);



}