@Test public void test1() throws Exception{ CloseableHttpClient httpClient = HttpClients.createDefault(); HttpGet httpGet = new HttpGet("http://www.baidu.com"); CloseableHttpResponse response = httpClient.execute(httpGet); if (response != null){ HttpEntity entity = response.getEntity(); String result = EntityUtils.toString(entity, "UTF-8"); System.out.println("网页内容:"+result); } if (response != null){ response.close(); } if (httpClient != null){ httpClient.close(); } } @Test public void test2() throws Exception{ final WebClient client = new WebClient(); client.getOptions().setJavaScriptEnabled(true); client.getOptions().setUseInsecureSSL(true); client.getOptions().setCssEnabled(false); client.getOptions().setThrowExceptionOnScriptError(false); client.getOptions().setThrowExceptionOnFailingStatusCode(false); client.setAjaxController(new NicelyResynchronizingAjaxController()); HtmlPage page =(HtmlPage) client.getPage("http://www.baidu.com"); System.out.print(page.asText()); } @Test public void test3() throws Exception{ //htmlunit不是专门做爬虫的,推荐用selenium+phantomjs无头浏览器去做,通过xpath的方式爬虫更加灵活 final WebClient client = new WebClient(); client.getOptions().setJavaScriptEnabled(true); S client.getOptions().setUseInsecureSSL(true); client.getOptions().setCssEnabled(false); client.getOptions().setThrowExceptionOnScriptError(false); client.getOptions().setThrowExceptionOnFailingStatusCode(false); HtmlPage page =(HtmlPage) client.getPage("http://www.baidu.com"); final HtmlForm form = page.getFormByName("f"); final HtmlSubmitInput button = form.getInputByValue("百度一下"); final HtmlTextInput textField = form.getInputByName("wd"); textField.setValueAttribute("山炮"); final HtmlPage nextPage = button.click(); String result = nextPage.asText(); System.out.print(result); }
java方式httpclient/htmlunit爬虫入门案例
最新推荐文章于 2024-01-06 02:42:47 发布