java抓取网页数据，登录之后抓取数据。

最近做了一个从网络上抓取数据的一个小程序。主要关于信贷方面，收集的一些黑名单网站，从该网站上抓取到自己系统中。

也找了一些资料，觉得没有一个很好的，全面的例子。因此在这里做个笔记提醒自己。

首先需要一个jsoup的jar包，我用的1.6.0。。下载地址为：http://pan.baidu.com/s/1mgqOuHa

1，获取网页内容（核心代码，技术有限没封装）。

2，登录之后抓取网页数据（如何在请求中携带cookie）。

3，获取网站的ajax请求方法（返回json）。

以上这三点我就用一个类全部包含（比较糙望见谅，直接copy代码过去，应该就可以用）

一，这个类分别有这上面的1,2,3三中方法，直接main方法可以进行测试

package com.minxinloan.black.web.utils;

import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File; import java.io.FileOutputStream; import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; import java.io.PrintWriter; import java.net.HttpURLConnection; import java.net.URL; import java.net.URLConnection; import java.net.URLEncoder; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.StringTokenizer; import net.sf.json.JSONArray; import net.sf.json.JSONObject; import org.jsoup.Connection; import org.jsoup.Connection.Method; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; public class CookieUtil { public final static String CONTENT_TYPE = "Content-Type"; public static void main(String[] args) { //String loginURL = "http://www.p2peye.com/member.php?mod=logging&action=login&loginsubmit=yes&loginhash=Lsc66&username=puqiuxiaomao&password=a1234567"; String listURL = "http://www.p2peye.com/blacklist.php?p=2"; String logURL = "http://www.p2peye.com/member.php"; //********************************需要登录的************************************************* try { Connection.Response res = Jsoup.connect(logURL) .data("mod","logging" ,"action","login" ,"loginsubmit","yes" ,"loginhash","Lsc66" ,"username","puqiuxiaomao" ,"password","a1234567") .method(Method.POST) .execute(); //这儿的SESSIONID需要根据要登录的目标网站设置的session Cookie名字而定 Connection con=Jsoup.connect(listURL); //设置访问形式（电脑访问，手机访问）：直接百度都参数设置 con.header("User-Agent", "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)"); //把登录信息的cookies保存如map对象里面 Map <String,String> map=res.cookies(); Iterator<Entry<String,String>> it =map.entrySet().iterator(); while(it.hasNext()){ Entry<String,String> en= it.next(); //把登录的信息放入请求里面 con =con.cookie(en.getKey(), en.getValue()); } //再次获取Document对象。 Document objectDoc = con.get(); Elements elements = objectDoc.getAllElements();//获取这个连接返回页面的源码内容（不是源码跟源码差不多） for (Element element : elements) { //element是迭代出来的标签：如：<div><span></span></div> Elements elements2= element.getAllElements();// for (Element element2 : elements2) { element2.text(); element2.attr("href");//获取标签属性。element2代表a标签：href代表属性 element2.text();//获取标签文本  } } //********************************不需要登录的*************************************************  String URL = "http://www.p2peye.com/blacklist.php?p=2"; Document conTemp = Jsoup.connect(URL).get(); Elements elementsTemps = conTemp.getAllElements(); for (Element elementsTemp : elementsTemps) { elementsTemp.text(); elementsTemp.attr("href");//获取标签属性。element2代表a标签：href代表属性 elementsTemp.text();//获取标签文本  } //********************************ajax方法获取内容。。。*************************************************。 HttpURLConnection connection = null; BufferedReader reader = null; try { StringBuffer sb = new StringBuffer(); URL getUrl = new URL(URL); connection = (HttpURLConnection)getUrl.openConnection(); reader = new BufferedReader(new InputStreamReader( connection.getInputStream(),"utf-8")); String lines; while ((lines = reader.readLine()) != null) { sb.append(lines); }; List<Map<String, Object>> list = parseJSON2List(sb.toString());//json转换成list } catch (Exception e) { } finally{ if(reader!=null) try { reader.close(); } catch (IOException e) { } // 断开连接  connection.disconnect(); } } catch (IOException e) { // TODO Auto-generated catch block  e.printStackTrace(); } } public static Map<String, Object> parseJSON2Map(String jsonStr){ Map<String, Object> map = new HashMap<String, Object>(); //最外层解析 JSONObject json = JSONObject.fromObject(jsonStr); for(Object k : json.keySet()){ Object v = json.get(k); //如果内层还是数组的话，继续解析 if(v instanceof JSONArray){ List<Map<String, Object>> list = new ArrayList<Map<String,Object>>(); Iterator<JSONObject> it = ((JSONArray)v).iterator(); while(it.hasNext()){ JSONObject json2 = it.next(); list.add(parseJSON2Map(json2.toString())); } map.put(k.toString(), list); } else { map.put(k.toString(), v); } } return map; } public static List<Map<String, Object>> parseJSON2List(String jsonStr){ JSONArray jsonArr = JSONArray.fromObject(jsonStr); List<Map<String, Object>> list = new ArrayList<Map<String,Object>>(); Iterator<JSONObject> it = jsonArr.iterator(); while(it.hasNext()){ JSONObject json2 = it.next(); list.add(parseJSON2Map(json2.toString())); } return list; } }

二，这个是获取验证码的类，可以研究下。（但你要要分析出网站的验证码的请求地址）

package com.minxinloan.black.web.utils;

import java.io.BufferedReader;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream; import java.io.FileWriter; import java.io.InputStream; import java.io.InputStreamReader; import java.io.PrintWriter; import java.net.HttpURLConnection; import java.net.URL; import java.net.URLConnection; import java.nio.charset.Charset; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.StringTokenizer; public class Utils {//解析验证码的 public static Content getRandom(String method, String sUrl,// 要解析的url Map<String, String> paramMap, // 存放用户名和密码的map Map<String, String> requestHeaderMap,// 存放COOKIE的map boolean isOnlyReturnHeader, String path) { Content content = null; HttpURLConnection httpUrlConnection = null; InputStream in = null; try { URL url = new URL(sUrl); boolean isPost = "POST".equals(method); if (method == null || (!"GET".equalsIgnoreCase(method) && !"POST" .equalsIgnoreCase(method))) { method = "POST"; } URL resolvedURL = url; URLConnection urlConnection = resolvedURL.openConnection(); httpUrlConnection = (HttpURLConnection) urlConnection; httpUrlConnection.setRequestMethod(method); httpUrlConnection.setRequestProperty("Accept-Language", "zh-cn,zh;q=0.5"); // Do not follow redirects, We will handle redirects ourself httpUrlConnection.setInstanceFollowRedirects(false); httpUrlConnection.setDoOutput(true); httpUrlConnection.setDoInput(true); httpUrlConnection.setConnectTimeout(5000); httpUrlConnection.setReadTimeout(5000); httpUrlConnection.setUseCaches(false); httpUrlConnection.setDefaultUseCaches(false); httpUrlConnection.connect(); int responseCode = httpUrlConnection.getResponseCode(); if (responseCode == HttpURLConnection.HTTP_OK || responseCode == HttpURLConnection.HTTP_CREATED) { byte[] bytes = new byte[0]; if (!isOnlyReturnHeader) { DataInputStream ins = new DataInputStream( httpUrlConnection.getInputStream()); // 验证码的位置 DataOutputStream out = new DataOutputStream( new FileOutputStream(path + "/code.bmp")); byte[] buffer = new byte[4096]; int count = 0; while ((count = ins.read(buffer)) > 0) { out.write(buffer, 0, count); } out.close(); ins.close(); } String encoding = null; if (encoding == null) { encoding = getEncodingFromContentType(httpUrlConnection .getHeaderField("")); } content = new Content(sUrl, new String(bytes, encoding), httpUrlConnection.getHeaderFields()); } } catch (Exception e) { return null; } finally { if (httpUrlConnection != null) { httpUrlConnection.disconnect(); } } return content; } public static String getEncodingFromContentType(String contentType) { String encoding = null; if (contentType == null) { return null; } StringTokenizer tok = new StringTokenizer(contentType, ";"); if (tok.hasMoreTokens()) { tok.nextToken(); while (tok.hasMoreTokens()) { String assignment = tok.nextToken().trim(); int eqIdx = assignment.indexOf('='); if (eqIdx != -1) { String varName = assignment.substring(0, eqIdx).trim(); if ("charset".equalsIgnoreCase(varName)) { String varValue = assignment.substring(eqIdx + 1) .trim(); if (varValue.startsWith("\"") && varValue.endsWith("\"")) { // substring works on indices varValue = varValue.substring(1, varValue.length() - 1); } if (Charset.isSupported(varValue)) { encoding = varValue; } } } } } if (encoding == null) { return "UTF-8"; } return encoding; } // 这个是输出 public static boolean inFile(String content, String path) { PrintWriter out = null; File file = new File(path); try { if (!file.exists()) { file.createNewFile(); } out = new PrintWriter(new FileWriter(file)); out.write(content); out.flush(); return true; } catch (Exception e) { e.printStackTrace(); } finally { out.close(); } return false; } public static String getHtmlReadLine(String httpurl) { String CurrentLine = ""; String TotalString = ""; InputStream urlStream; String content = ""; try { URL url = new URL(httpurl); HttpURLConnection connection = (HttpURLConnection) url .openConnection(); connection.connect(); System.out.println(connection.getResponseCode()); urlStream = connection.getInputStream(); BufferedReader reader = new BufferedReader( new InputStreamReader(urlStream, "utf-8")); while ((CurrentLine = reader.readLine()) != null) { TotalString += CurrentLine + "\n"; } content = TotalString; } catch (Exception e) { } return content; } } class Content { private String url; private String body; private Map<String, List<String>> m_mHeaders = new HashMap<String, List<String>>(); public Content(String url, String body, Map<String, List<String>> headers) { this.url = url; this.body = body; this.m_mHeaders = headers; } public String getUrl() { return url; } public String getBody() { return body; } public Map<String, List<String>> getHeaders() { return m_mHeaders; } }

 

转载于:https://www.cnblogs.com/puqiuxiaomao/p/4037918.html