抓取网页的两种方式.可直接使用

1: CloseableHttpClient 

/*
	 * 爬取网页信息
	 */
	private static String pickData(String url) {
		CloseableHttpClient httpclient = HttpClients.createDefault();
		try {
			HttpGet httpget = new HttpGet(url);
			CloseableHttpResponse response = httpclient.execute(httpget);
			try {
				// 获取响应实体
				HttpEntity entity = response.getEntity();
				// 打印响应状态
				if (entity != null) {
					return EntityUtils.toString(entity);
				}
			} finally {
				response.close();
			}
		} catch (ClientProtocolException e) {
			e.printStackTrace();
		} catch (ParseException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		} finally {
			// 关闭连接,释放资源
			try {
				httpclient.close();
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
		return null;
	}

2:模拟浏览器:

	/*
	 * 爬取网页信息
	 */
	public static String sendGet(String url) throws Exception {
		String result = "";
		URLConnection conn = null;
		InputStream inputStream = null;
		BufferedReader in = null;

		URL realURL = new URL(url);
		conn = realURL.openConnection();
		conn.setRequestProperty("accept", "*/*");
		conn.setRequestProperty("connection", "Keep-Alive");
		conn.setRequestProperty("user-agent",
				"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 Safari/537.36");
		Map<String, List<String>> map = conn.getHeaderFields();
		for (String s : map.keySet()) {
			System.out.println(s + "-->" + map.get(s));
		}
		inputStream = conn.getInputStream();
		String line;
		// 读取操作
		in = new BufferedReader(new InputStreamReader(inputStream, "utf-8"));
		while ((line = in.readLine()) != null) {
			result += "\n" + line;
		}

		in.close();

		System.out.println(result);
		return result;
	}