利用正则表达式抽取网页信息

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;


public class ObtainNews {


    /**
     * 
     * @param htmlurl
     * @return 读取一个网页全部内容
     * @throws IOException
     */
    public String getHtml(final String htmlurl) throws IOException {
        URL url;
        String temp;
        final StringBuffer sb = new StringBuffer();
        try {
            url = new URL(htmlurl);
            final BufferedReader in = new BufferedReader(new InputStreamReader(
                url.openStream(), "utf-8")); // 读取网页全部内容
            while ((temp = in .readLine()) != null) {
                sb.append(temp);
            } in .close();
        } catch (final MalformedURLException me) {
            System.out.println("你输入的URL格式有问题！请仔细输入");
            me.getMessage();
            throw me;
        } catch (final IOException e) {
            e.printStackTrace();
            throw e;
        }
        return sb.toString();
    }


    /**
     * 获取 标题/正文/发布时间/发布者：null/来源站点：reuters/记者/分类频道/专题标识：null/图片/视频
     */


    public String getTitle(final String s) {
        String regex;
        String title = "";
        final List < String > list = new ArrayList < String > ();
        regex = "<h1>.*?</h1>";
        final Pattern pa = Pattern.compile(regex, Pattern.CANON_EQ);
        final Matcher ma = pa.matcher(s);
        while (ma.find()) {
            list.add(ma.group());
        }
        for (int i = 0; i < list.size(); i++) {
            title = title + list.get(i);
        }
        return title;
    }

    public String getContent(final String s) {
        String regex;
        String content = "";
        final List < String > list = new ArrayList < String > ();
        regex = "<span id=\"midArticle_start\"></span>.*?</span></span>";
        final Pattern pa = Pattern.compile(regex, Pattern.CANON_EQ);
        final Matcher ma = pa.matcher(s);
        while (ma.find()) {
            list.add(ma.group());
        }
        for (int i = 0; i < list.size(); i++) {
            content = content + list.get(i);
        }
        return content;
    }

    public String getTime(final String s) {
        String regex;
        String time = "";
        final List < String > list = new ArrayList < String > ();
        regex = "<span class=\"timestamp\">.*?</span>        </p>";
        final Pattern pa = Pattern.compile(regex, Pattern.CANON_EQ);
        final Matcher ma = pa.matcher(s);
        while (ma.find()) {
            list.add(ma.group());
        }
        for (int i = 0; i < list.size(); i++) {
            time = time + list.get(i);
        }
        return time;
    }

    public String getReporter(final String s) {
        String regex;
        String reporter = "";
        final List < String > list = new ArrayList < String > ();
        regex = "<p class=\"byline\">.*?</p>        <p>";
        final Pattern pa = Pattern.compile(regex, Pattern.CANON_EQ);
        final Matcher ma = pa.matcher(s);
        while (ma.find()) {
            list.add(ma.group());
        }
        for (int i = 0; i < list.size(); i++) {
            reporter = reporter + " " + list.get(i);
        }
        return reporter;
    }

    public String getChannel(final String s) {
        String regex;
        String channel = "";
        final List < String > list = new ArrayList < String > ();
        regex = "<div class=\"actionButton\">.*?</a></div>";
        final Pattern pa = Pattern.compile(regex, Pattern.CANON_EQ);
        final Matcher ma = pa.matcher(s);
        while (ma.find()) {
            list.add(ma.group());
        }
        for (int i = 0; i < list.size(); i++) {
            channel = channel + " " + list.get(i);
        }
        return channel;
    }

    public String getImgsrc(final String s) {
        String regex;
        String imgsrc = "";
        final List < String > list = new ArrayList < String > ();
        regex = "<img        src=\".*?\"        border";
        final Pattern pa = Pattern.compile(regex, Pattern.CANON_EQ);
        final Matcher ma = pa.matcher(s);
        while (ma.find()) {
            list.add(ma.group());
        }
        for (int i = 0; i < list.size(); i++) {
            imgsrc = imgsrc + list.get(i);
        }
        return imgsrc;
    }

    public String getVideosrc(final String s) {
        String regex;
        String videosrc = "";
        final List < String > list = new ArrayList < String > ();
        regex = "<div class=\"photo\">.*?<img";
        final Pattern pa = Pattern.compile(regex, Pattern.CANON_EQ);
        final Matcher ma = pa.matcher(s);
        while (ma.find()) {
            list.add(ma.group());
        }
        for (int i = 0; i < list.size(); i++) {
            videosrc = videosrc + list.get(i);
        }
        return videosrc;
    }

    /**
     * 
     * @param s
     * @return 去除标签
     */

    public String outTag(final String s) {
        return s.replaceAll("<.*?>", "");
    }


    /**
     * 
     * @param s
     * @return 获取内容
     */


    public HashMap < String, String > getFromWeb(final String s) {

        final HashMap < String, String > hm = new HashMap < String, String > ();
        String html = "";
        System.out.println("\n开始读取网页(" + s + ")");
        try {
            html = getHtml(s);
        } catch (final Exception e) {
            e.getMessage();
        }
        System.out.println(html);
        System.out.println("分析(" + s + ")结果\n");
        String title = outTag(getTitle(html));
        String content = outTag(getContent(html));
        String time = outTag(getTime(html));
        String reporter = outTag(getReporter(html)).replaceAll("By ", "");
        String channel = outTag(getChannel(html));
        String imgsrc = getImgsrc(html)
            .replaceAll("<img        src=\"", "").replaceAll("\"        border", "").replaceAll(" ", "");
        String videosrc = getVideosrc(html)
            .replaceAll("<div class=\"photo\"><a href=\"", "").replaceAll("\"><img", "");
        hm.put("title", title);
        hm.put("content", content);
        hm.put("time", time);
        hm.put("reporter", reporter);
        hm.put("channel", channel);
        hm.put("imgsrc", imgsrc);
        hm.put("videosrc", videosrc);
        return hm;


    }


    /**
     * 
     * @param args
     *            测试网页www.reuters.com
     */


    public static void main(final String args[]) {

        String url = "";
        final List < String > list = new ArrayList < String > ();
        System.out.print("输入新闻页面网址，换行输入run\n");
        final BufferedReader br = new BufferedReader(new InputStreamReader(
            System.in));
        //http://www.reuters.com/article/2014/04/04/us-usa-cia-interrogation-idUSBREA321UC20140404
        //http://www.reuters.com/article/2014/04/04/us-congress-justice-highspeed-idUSBREA3310O20140404
        try {
            while (!(url = br.readLine()).equals("run")) {
                list.add(url);
            }
        } catch (final Exception e) {
            e.getMessage();
        }
        final ObtainNews on = new ObtainNews();
        HashMap < String, String > hm = new HashMap < String, String > ();
        for (int i = 0; i < list.size(); i++) {
            hm = on.getFromWeb(list.get(i));
            String title = hm.get("title");
            String content = hm.get("content");
            String time = hm.get("time");
            String publisher = null;
            String site = "reuters";
            String reporter = hm.get("reporter");
            if (reporter == "") reporter = null;
            String channel = hm.get("channel");
            if (channel == "") channel = null;
            String subject = null;
            String imgsrc = hm.get("imgsrc");
            if (imgsrc == "") imgsrc = null;
            String videosrc = hm.get("videosrc");
            if (videosrc == "") videosrc = null;
            else {
                videosrc = "http://www.reuters.com" + videosrc;
                videosrc = videosrc.replaceAll(" ", "");
            }
            String str = list.get(i) + "\t" + title + "\t" + content + "\t" + time + "\t" + publisher + "\t" +
                site + "\t" + reporter + "\t" + channel + "\t" + subject + "\t" + imgsrc + "\t" + videosrc + "\n";

            System.out.println("URL： " + list.get(i));
            System.out.println("标题： " + title);
            System.out.println("正文： " + content);
            System.out.println("发布时间： " + time);
            System.out.println("发布者：" + publisher);
            System.out.println("来源站点：" + site);
            System.out.println("记者：" + reporter);
            System.out.println("分类频道：" + channel);
            System.out.println("主题：" + subject);
            System.out.println("图片链接：" + imgsrc);
            System.out.println("视频链接：" + videosrc);
            System.out.println(str);

            try {
                FileOutputStream fos = new FileOutputStream("D://News.txt", true);
                OutputStreamWriter osw = new OutputStreamWriter(fos, "utf-8");
                BufferedWriter bw = new BufferedWriter(osw);

                bw.write(str);
                bw.flush();
                bw.close();
            } catch (FileNotFoundException e1) {
                // TODO Auto-generated catch block
                e1.printStackTrace();
            } catch (UnsupportedEncodingException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }

        }
    }
}