java爬虫开发_java开发爬虫Deno

java开发爬虫Deno

身为一个程序员不会两三手爬虫怎么能在行业里立足啊，这是开发中自己写的一个java爬虫的Demo，供大家参考。

java爬虫的开发依赖于jsoup.jar

直接上代码

public static void main(String[] args) {

HttpClient client = new DefaultHttpClient();

HttpGet httpGet = new HttpGet();

Map map = new HashMap();

try {

//百度百科999感冒灵连接

String url = "http://baike.baidu.com/link?url=c95Y4QJym_d_wFKGmcibRTI_KIyj-X_tOjnlOGJS9qekgO1tmWaWnrn7QyAjqvZX8At7LbI1XIa69IBZWejiCXDVM0jkSBMnVZKKu4jeg-ef4TJkKCXEXWcGJ8DRGTuHxW4qWB3pNNU7Y0KdrbNvGK";

// get请求获取页面信息

String bb = doget(url);

Document doc;

//用jsoup接收页面信息

doc = Jsoup.parse(bb);

// 选择所有div的class为para的标签

Elements news = doc.select("div[class=para]");

for (Element result : news) {

//获取标签的内容并打印

String str=result.text();

System.out.println(str);

}

} catch (Exception e) {

e.printStackTrace();

}

}

public static String doget(String path) {

InputStream is = null;

ByteArrayOutputStream baos = null;

try {

// 伪造referer 绕过防盗链设置

URL url = new URL(path.trim());

HttpURLConnection conn = (HttpURLConnection) url.openConnection();

if (200 == conn.getResponseCode()) {

byte[] buff = new byte[4096];

int count;

ByteArrayOutputStream out = new ByteArrayOutputStream(4096);

InputStream in = conn.getInputStream();

while ((count = in.read(buff)) != -1) {

out.write(buff, 0, count);

}

conn.disconnect();

return out.toString("UTF-8");

}

} catch (Exception e) {

e.printStackTrace();

} finally {

if (baos != null) {

try {

baos.close();

} catch (IOException e) {

e.printStackTrace();

}

}

if (is != null) {

try {

is.close();

} catch (IOException e) {

e.printStackTrace();

}

}

}

return null;

}

转载注明引用