java 网络爬虫demo_Java实现网络爬虫入门Demo

需求：

抓取一个网页(比如www.lianhehuishang.com)中的url地址，并存到F:\spider_url.txt文件中。

程序：

package com.zheng;

import java.io.BufferedReader;

import java.io.FileWriter;

import java.io.IOException;

import java.io.InputStreamReader;

import java.io.PrintWriter;

import java.net.MalformedURLException;

import java.net.URL;

import java.net.URLConnection;

import java.util.regex.Matcher;

import java.util.regex.Pattern;

public class WebSpider {

public static void main(String[] args) {

URL url = null;

URLConnection urlconn = null;

BufferedReader br = null;

PrintWriter pw = null;

String regex = "http://[\\w+\\.?/?]+\\.[A-Za-z]+";

Pattern p = Pattern.compile(regex);

try {

url = new URL("http://www.lianhehuishang.com/");

urlconn = url.openConnection();

pw = new PrintWriter(new FileWriter("f:\\spider_url.txt"), true);

br = new BufferedReader(new InputStreamReader(

urlconn.getInputStream()));

String buf = null;

while ((buf = br.readLine()) != null) {

Matcher buf_m = p.matcher(buf);

while (buf_m.find()) {

pw.println(buf_m.group());

}

}

System.out.println("获取成功！");

} catch (MalformedURLException e) {

e.printStackTrace();

} catch (IOException e) {

e.printStackTrace();

} finally {

try {

br.close();

} catch (IOException e) {

e.printStackTrace();

}

pw.close();

}

}

}

运行结果：

打开F:\spider_url.txt