java多线程数据采集_java多线程网络数据采集 1

java采集网页数据。获取html文本节点

第一种： 采用HttpURLConnection

packagecom.yjf.util;

importjava.io.BufferedReader;

importjava.io.IOException;

importjava.io.InputStream;

importjava.io.InputStreamReader;

importjava.net.HttpURLConnection;

importjava.net.URL;

publicclassHttpWebUtil {

publicstaticString GetWebContent(String urlString,finalString charset,inttimeout)throwsIOException {

if(urlString ==null|| urlString.length() ==0) {

return"";

}

urlString = (urlString.startsWith("http://") || urlString.startsWith("https://")) ? urlString : ("http://"+ urlString).intern();

URL url =newURL(urlString);

HttpURLConnection conn = (HttpURLConnection) url.openConnection();

conn.setDoOutput(true);

conn.setRequestProperty("Pragma","no-cache");

conn.setRequestProperty("Cache-Control","no-cache");

inttemp = Integer.parseInt(Math.round(Math.random()*(UserAgent.length-1))+"");

conn.setRequestProperty(

"User-Agent",

UserAgent[temp]);// 模拟手机系统

conn.setRequestProperty("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,**任意，就是tomcat/conf/web里面定义那些

conn.setConnectTimeout(timeout);

try{

if(conn.getResponseCode() != HttpURLConnection.HTTP_OK) {

return"";

}

}catch(Exception e) {

try{

System.out.println(e.getMessage());

}catch(Exception e2) {

e2.printStackTrace();

}

return"";

}

InputStream input = conn.getInputStream();

BufferedReader reader =newBufferedReader(newInputStreamReader(input,

charset));

String line =null;

StringBuffer sb =newStringBuffer("");

while((line = reader.readLine()) !=null) {

sb.append(line).append("\r\n");

}

if(reader !=null) {

reader.close();

}

if(conn !=null) {

conn.disconnect();

}

returnsb.toString();

}

publicstaticString[] UserAgent = {

"Mozilla/5.0 (Linux; U; Android 2.2; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.2",

"Mozilla/5.0 (iPad; U; CPU OS 3_2_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4 Mobile/7B500 Safari/531.21.11",

"Mozilla/5.0 (SymbianOS/9.4; Series60/5.0 NokiaN97-1/20.0.019; Profile/MIDP-2.1 Configuration/CLDC-1.1) AppleWebKit/525 (KHTML, like Gecko) BrowserNG/7.1.18121",

"Nokia5700AP23.01/SymbianOS/9.1 Series60/3.0",

"UCWEB7.0.2.37/28/998",

"NOKIA5700/UCWEB7.0.2.37/28/977",

"Openwave/UCWEB7.0.2.37/28/978",

"Mozilla/4.0 (compatible; MSIE 6.0; ) Opera/UCWEB7.0.2.37/28/989"

};

}

package com.yjf.util;

import java.io.BufferedReader;

import java.io.IOException;

import java.io.InputStream;

import java.io.InputStreamReader;

import java.net.HttpURLConnection;

import java.net.URL;

public class HttpWebUtil {

public static String GetWebContent(String urlString, final String charset, int timeout) throws IOException {

if (urlString == null || urlString.length() == 0) {

return "";

}

urlString = (urlString.startsWith("http://") || urlString.startsWith("https://")) ? urlString : ("http://" + urlString).intern();

URL url = new URL(urlString);

HttpURLConnection conn = (HttpURLConnection) url.openConnection();

conn.setDoOutput(true);

conn.setRequestProperty("Pragma", "no-cache");

conn.setRequestProperty("Cache-Control", "no-cache");

int temp = Integer.parseInt(Math.round(Math.random()*(UserAgent.length-1))+"");

conn.setRequestProperty(

"User-Agent",

UserAgent[temp]); // 模拟手机系统

conn.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,**任意，就是tomcat/conf/web里面定义那些

conn.setConnectTimeout(timeout);

try {

if (conn.getResponseCode() != HttpURLConnection.HTTP_OK) {

return "";

}

} catch (Exception e) {

try {

System.out.println(e.getMessage());

} catch (Exception e2) {

e2.printStackTrace();

}

return "";

}

InputStream input = conn.getInputStream();

BufferedReader reader = new BufferedReader(new InputStreamReader(input,

charset));

String line = null;

StringBuffer sb = new StringBuffer("");

while ((line = reader.readLine()) != null) {

sb.append(line).append("\r\n");

}

if (reader != null) {

reader.close();

}

if (conn != null) {

conn.disconnect();

}

return sb.toString();

}

public static String[] UserAgent = {

"Mozilla/5.0 (Linux; U; Android 2.2; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.2",

"Mozilla/5.0 (iPad; U; CPU OS 3_2_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4 Mobile/7B500 Safari/531.21.11",

"Mozilla/5.0 (SymbianOS/9.4; Series60/5.0 NokiaN97-1/20.0.019; Profile/MIDP-2.1 Configuration/CLDC-1.1) AppleWebKit/525 (KHTML, like Gecko) BrowserNG/7.1.18121",

"Nokia5700AP23.01/SymbianOS/9.1 Series60/3.0",

"UCWEB7.0.2.37/28/998",

"NOKIA5700/UCWEB7.0.2.37/28/977",

"Openwave/UCWEB7.0.2.37/28/978",

"Mozilla/4.0 (compatible; MSIE 6.0; ) Opera/UCWEB7.0.2.37/28/989"

};

}

第二种：采用jar插件Jsoup.jar

org.jsoup.Jsoup

//站点入口列表

publicstaticList getSiteUrlList1(List list,String listurl){

if(list==null|| list.size()<=0){

list =newArrayList();

}

try{

Document docdata = Jsoup.connect(listurl).timeout(10000).get();

String hb =".m_book li a";

String page =".page_list .page_up";

Elements ele = docdata.select(hb);

for(Element el : ele) {

list.add(el.attr("href"));

}

if(docdata.select(page)!=null&& docdata.select(page).first()!=null){

String url ="http://www.xxxxx.com/site_map/"+docdata.select(page).first().attr("href");

getSiteUrlList1(list, url);

}

}catch(Exception e) {

e.printStackTrace();

}

returnlist;

}

//站点入口列表

public static List getSiteUrlList1(List list,String listurl){

if(list==null || list.size()<=0){

list = new ArrayList();

}

try {

Document docdata = Jsoup.connect(listurl).timeout(10000).get();

String hb = ".m_book li a";

String page = ".page_list .page_up";

Elements ele = docdata.select(hb);

for (Element el : ele) {

list.add(el.attr("href"));

}

if(docdata.select(page)!=null && docdata.select(page).first()!=null){

String url = "http://www.xxxxx.com/site_map/"+docdata.select(page).first().attr("href");

getSiteUrlList1(list, url);

}

} catch (Exception e) {

e.printStackTrace();

}

return list;

}

第三种：http模拟器

可以请求表单数据和 重定向

http://blog.csdn.net/yjflinchong/article/details/8004706

以上先描述第一步抓取数据。后期加入多线程网络数据采集完整介绍。