java读取（正则表达式分析）网页内容

  由于原来的“插入代码”复制方法要把转义符过滤掉，所以下面用文本粘贴：

package com.xiaofeng.picup;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 *
 * @author 晓峰2007.1.18 抓取雅虎知识堂的文章标题及内容（测试） 手动输入网址抓取，可进一步自动抓取整个知识堂的全部内容
 *
 */
public class WebContent
{
 /**
  * 读取一个网页全部内容
  */
 public String getOneHtml(final String htmlurl) throws IOException
 {
  URL url;
  String temp;
  final StringBuffer sb = new StringBuffer();
  try
  {
   url = new URL(htmlurl);
   final BufferedReader in = new BufferedReader(new InputStreamReader(url.openStream(), "utf-8"));// 读取网页全部内容
   while ((temp = in.readLine()) != null)
   {
    sb.append(temp);
   }
   in.close();
  }
  catch (final MalformedURLException me)
  {
   System.out.println("你输入的URL格式有问题！请仔细输入");
   me.getMessage();
   throw me;
  }
  catch (final IOException e)
  {
   e.printStackTrace();
   throw e;
  }
  return sb.toString();
 }

 /**
  *
  * @param s
  * @return 获得网页标题
  */
 public String getTitle(final String s)
 {
  String regex;
  String title = "";
  final List<String> list = new ArrayList<String>();
  regex = "<title>.*?</title>";
  final Pattern pa = Pattern.compile(regex, Pattern.CANON_EQ);
  final Matcher ma = pa.matcher(s);
  while (ma.find())
  {
   list.add(ma.group());
  }
  for (int i = 0; i < list.size(); i++)
  {
   title = title + list.get(i);
  }
  return outTag(title);
 }

 /**
  *
  * @param s
  * @return 获得链接
  */
 public List<String> getLink(final String s)
 {
  String regex;
  final List<String> list = new ArrayList<String>();
  regex = "<a[^>]*href=(/"([^/"]*)/"|/'([^/']*)/'|([^//s>]*))[^>]*>(.*?)</a>";
  final Pattern pa = Pattern.compile(regex, Pattern.DOTALL);
  final Matcher ma = pa.matcher(s);
  while (ma.find())
  {
   list.add(ma.group());
  }
  return list;
 }

 /**
  *
  * @param s
  * @return 获得脚本代码
  */
 public List<String> getScript(final String s)
 {
  String regex;
  final List<String> list = new ArrayList<String>();
  regex = "<script.*?</script>";
  final Pattern pa = Pattern.compile(regex, Pattern.DOTALL);
  final Matcher ma = pa.matcher(s);
  while (ma.find())
  {
   list.add(ma.group());
  }
  return list;
 }

 /**
  *
  * @param s
  * @return 获得CSS
  */
 public List<String> getCSS(final String s)
 {
  String regex;
  final List<String> list = new ArrayList<String>();
  regex = "<style.*?</style>";
  final Pattern pa = Pattern.compile(regex, Pattern.DOTALL);
  final Matcher ma = pa.matcher(s);
  while (ma.find())
  {
   list.add(ma.group());
  }
  return list;
 }

 /**
  *
  * @param s
  * @return 去掉标记
  */
 public String outTag(final String s)
 {
  return s.replaceAll("<.*?>", "");
 }

 /**
  *
  * @param s
  * @return 获取雅虎知识堂文章标题及内容
  */
 public HashMap<String, String> getFromYahoo(final String s)
 {
  final HashMap<String, String> hm = new HashMap<String, String>();
  final StringBuffer sb = new StringBuffer();
  String html = "";
  System.out.println("/n------------------开始读取网页(" + s + ")--------------------");
  try
  {
   html = getOneHtml(s);
  }
  catch (final Exception e)
  {
   e.getMessage();
  }
  // System.out.println(html);
  System.out.println("------------------读取网页(" + s + ")结束--------------------/n");
  System.out.println("------------------分析(" + s + ")结果如下--------------------/n");
  String title = outTag(getTitle(html));
  title = title.replaceAll("_雅虎知识堂", "");
  // Pattern pa=Pattern.compile("<div
  // class=/"original/">(.*?)((/r/n)*)(.*?)((/r/n)*)(.*?)</div>",Pattern.DOTALL);
  final Pattern pa = Pattern.compile("<div class=/"original/">(.*?)</p></div>", Pattern.DOTALL);
  final Matcher ma = pa.matcher(html);
  while (ma.find())
  {
   sb.append(ma.group());
  }
  String temp = sb.toString();
  temp = temp.replaceAll("(<br>)+?", "/n");// 转化换行
  temp = temp.replaceAll("<p><em>.*?</em></p>", "");// 去图片注释
  hm.put("title", title);
  hm.put("original", outTag(temp));
  return hm;

 }

 /**
  *
  * @param args
  *            测试一组网页，针对雅虎知识堂
  */
 public static void main(final String args[])
 {
  String url = "";
  final List<String> list = new ArrayList<String>();
  System.out.print("输入URL，一行一个，输入结束后输入 go 程序开始运行:   /n");
  /*
   * http://ks.cn.yahoo.com/question/1307121201133.html
   * http://ks.cn.yahoo.com/question/1307121101907.html
   * http://ks.cn.yahoo.com/question/1307121101907_2.html
   * http://ks.cn.yahoo.com/question/1307121101907_3.html
   * http://ks.cn.yahoo.com/question/1307121101907_4.html
   * http://ks.cn.yahoo.com/question/1307121101907_5.html
   * http://ks.cn.yahoo.com/question/1307121101907_6.html
   * http://ks.cn.yahoo.com/question/1307121101907_7.html
   * http://ks.cn.yahoo.com/question/1307121101907_8.html
   */
  final BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
  try
  {
   while (!(url = br.readLine()).equals("go"))
   {
    list.add(url);
   }
  }
  catch (final Exception e)
  {
   e.getMessage();
  }
  final WebContent wc = new WebContent();
  HashMap<String, String> hm = new HashMap<String, String>();
  for (int i = 0; i < list.size(); i++)
  {
   hm = wc.getFromYahoo(list.get(i));
   System.out.println("标题： " + hm.get("title"));
   System.out.println("内容： /n" + hm.get("original"));
  }
  /*
   * String htmlurl[] = {
   * "http://ks.cn.yahoo.com/question/1307121201133.html",
   * "http://ks.cn.yahoo.com/question/1307121101907.html",
   * "http://ks.cn.yahoo.com/question/1307121101907_2.html",
   * "http://ks.cn.yahoo.com/question/1307121101907_3.html",
   * "http://ks.cn.yahoo.com/question/1307121101907_4.html",
   * "http://ks.cn.yahoo.com/question/1307121101907_5.html",
   * "http://ks.cn.yahoo.com/question/1307121101907_6.html",
   * "http://ks.cn.yahoo.com/question/1307121101907_7.html",
   * "http://ks.cn.yahoo.com/question/1307121101907_8.html" }; WebContent
   * wc = new WebContent(); HashMap<String, String> hm = new HashMap<String,
   * String>(); for (int i = 0; i < htmlurl.length; i++) { hm =
   * wc.getFromYahoo(htmlurl[i]); System.out.println("标题： " +
   * hm.get("title")); System.out.println("内容： /n" + hm.get("original")); }
   */
  /*
   * String html=""; String link=""; String sscript=""; String content="";
   * System.out.println(htmlurl+" 开始读取网页内容：");
   * html=wc.getOneHtml(htmlurl); System.out.println(htmlurl+"
   * 读取完毕开始分析……"); html=html.replaceAll("(<script.*?)((/r/n)*)(.*?)((/r/n)*)(.*?)(</script>)","
   * ");//去除脚本 html=html.replaceAll("(<style.*?)((/r/n)*)(.*?)((/r/n)*)(.*?)(</style>)","
   * ");//去掉CSS html=html.replaceAll("<title>.*?</title>"," ");//除去页面标题
   * html=html.replaceAll("<a[^>]*href=(/"([^/"]*)/"|/'([^/']*)/'|([^//s>]*))[^>]*>(.*?)</a>","
   * ");//去掉链接 html=html.replaceAll("(//s){2,}?"," ");//除去多余空格
   * html=wc.outTag(html);//多余标记 System.out.println(html);
   */

  /*
   * String s[]=html.split(" +"); for(int i=0;i<s.length;i++){
   * content=(content.length()>s[i].length())?content:s[i]; }
   * System.out.println(content);
   */

  // System.out.println(htmlurl+"网页内容结束");
  /*
   * System.out.println(htmlurl+"网页脚本开始："); List
   * script=wc.getScript(html); for(int i=0;i<script.size();i++){
   * System.out.println(script.get(i)); }
   * System.out.println(htmlurl+"网页脚本结束：");
   *
   * System.out.println(htmlurl+"CSS开始："); List css=wc.getCSS(html);
   * for(int i=0;i<css.size();i++){ System.out.println(css.get(i)); }
   * System.out.println(htmlurl+"CSS结束：");
   *
   * System.out.println(htmlurl+"全部链接内容开始："); List list=wc.getLink(html);
   * for(int i=0;i<list.size();i++){ link=list.get(i).toString(); }
   * System.out.println(htmlurl+"全部链接内容结束：");
   *
   * System.out.println("内容"); System.out.println(wc.outTag(html));
   */
 }
}

package  com.xiaofeng.picup;

import  java.io.BufferedReader;
import  java.io.IOException;
import  java.io.InputStreamReader;
import  java.net.MalformedURLException;
import  java.net.URL;
import  java.util.ArrayList;
import  java.util.HashMap;
import  java.util.List;
import  java.util.regex.Matcher;
import  java.util.regex.Pattern;

/** */ /**
 * 
 * @author 晓峰2007.1.18 抓取雅虎知识堂的文章标题及内容（测试） 手动输入网址抓取，可进一步自动抓取整个知识堂的全部内容
 * 
 */
public   class  WebContent  ... {
    /** *//**
     * 读取一个网页全部内容
     */
    public String getOneHtml(String htmlurl) throws IOException...{
        URL url;
        String temp;
        StringBuffer sb = new StringBuffer();
        try ...{
            url = new URL(htmlurl);
            BufferedReader in = new BufferedReader(new InputStreamReader(url
                    .openStream(), "utf-8"));// 读取网页全部内容
            while ((temp = in.readLine()) != null) ...{
                sb.append(temp);
            }
            in.close();
        }catch(MalformedURLException me)...{
            System.out.println("你输入的URL格式有问题！请仔细输入");
            me.getMessage();
            throw me;
        }catch (IOException e) ...{
            e.printStackTrace();
            throw e;
        }
        return sb.toString();
    }

    /** *//**
     * 
     * @param s
     * @return 获得网页标题
     */
    public String getTitle(String s) ...{
        String regex;
        String title = "";
        List<String> list = new ArrayList<String>();
        regex = "<title>.*?</title>";
        Pattern pa = Pattern.compile(regex, Pattern.CANON_EQ);
        Matcher ma = pa.matcher(s);
        while (ma.find()) ...{
            list.add(ma.group());
        }
        for (int i = 0; i < list.size(); i++) ...{
            title = title + list.get(i);
        }
        return outTag(title);
    }

    /** *//**
     * 
     * @param s
     * @return 获得链接
     */
    public List<String> getLink(String s) ...{
        String regex;
        List<String> list = new ArrayList<String>();
        regex = "<a[^>]*href=("([^"]*)"|'([^']*)'|([^/s>]*))[^>]*>(.*?)</a>";
        Pattern pa = Pattern.compile(regex, Pattern.DOTALL);
        Matcher ma = pa.matcher(s);
        while (ma.find()) ...{
            list.add(ma.group());
        }
        return list;
    }

    /** *//**
     * 
     * @param s
     * @return 获得脚本代码
     */
    public List<String> getScript(String s) ...{
        String regex;
        List<String> list = new ArrayList<String>();
        regex = "<script.*?</script>";
        Pattern pa = Pattern.compile(regex, Pattern.DOTALL);
        Matcher ma = pa.matcher(s);
        while (ma.find()) ...{
            list.add(ma.group());
        }
        return list;
    }

    /** *//**
     * 
     * @param s
     * @return 获得CSS
     */
    public List<String> getCSS(String s) ...{
        String regex;
        List<String> list = new ArrayList<String>();
        regex = "<style.*?</style>";
        Pattern pa = Pattern.compile(regex, Pattern.DOTALL);
        Matcher ma = pa.matcher(s);
        while (ma.find()) ...{
            list.add(ma.group());
        }
        return list;
    }

    /** *//**
     * 
     * @param s
     * @return 去掉标记
     */
    public String outTag(String s) ...{
        return s.replaceAll("<.*?>", "");
    }

    /** *//**
     * 
     * @param s
     * @return 获取雅虎知识堂文章标题及内容
     */
    public HashMap<String, String> getFromYahoo(String s) ...{
        HashMap<String, String> hm = new HashMap<String, String>();
        StringBuffer sb = new StringBuffer();
        String html="";
        System.out.println(" ------------------开始读取网页(" + s
                + ")--------------------");
        try...{
             html= getOneHtml(s);
        }catch(Exception e)...{
            e.getMessage();
        }
        // System.out.println(html);
        System.out.println("------------------读取网页(" + s
                + ")结束-------------------- ");
        System.out.println("------------------分析(" + s
                + ")结果如下-------------------- ");
        String title = outTag(getTitle(html));
        title = title.replaceAll("_雅虎知识堂", "");
        // Pattern pa=Pattern.compile("<div
        // class="original">(.*?)(( )*)(.*?)(( )*)(.*?)</div>",Pattern.DOTALL);
        Pattern pa = Pattern.compile("<div class="original">(.*?)</p></div>",
                Pattern.DOTALL);
        Matcher ma = pa.matcher(html);
        while (ma.find()) ...{
            sb.append(ma.group());
        }
        String temp = sb.toString();
        temp = temp.replaceAll("(<br>)+?", " ");// 转化换行
        temp = temp.replaceAll("<p><em>.*?</em></p>", "");// 去图片注释
        hm.put("title", title);
        hm.put("original", outTag(temp));
        return hm;

    }

    /** *//**
     * 
     * @param args
     *            测试一组网页，针对雅虎知识堂
     */
    public static void main(String args[]) ...{
        String url="";
        List<String> list = new ArrayList<String>();
        System.out.print("输入URL，一行一个，输入结束后输入 go 程序开始运行:    ");
        /**//*
http://ks.cn.yahoo.com/question/1307121201133.html
http://ks.cn.yahoo.com/question/1307121101907.html
http://ks.cn.yahoo.com/question/1307121101907_2.html
http://ks.cn.yahoo.com/question/1307121101907_3.html
http://ks.cn.yahoo.com/question/1307121101907_4.html
http://ks.cn.yahoo.com/question/1307121101907_5.html
http://ks.cn.yahoo.com/question/1307121101907_6.html
http://ks.cn.yahoo.com/question/1307121101907_7.html
http://ks.cn.yahoo.com/question/1307121101907_8.html
         */
        BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
        try ...{
            while(!(url=br.readLine()).equals("go"))...{
                list.add(url);
            }
        } catch (Exception e) ...{
            e.getMessage();
        }
        WebContent wc = new WebContent();
        HashMap<String, String> hm = new HashMap<String, String>();
        for (int i = 0; i < list.size(); i++) ...{
            hm = wc.getFromYahoo(list.get(i));
            System.out.println("标题： " + hm.get("title"));
            System.out.println("内容：  " + hm.get("original"));
        }
    /**//*
        String htmlurl[] = {
                "http://ks.cn.yahoo.com/question/1307121201133.html",
                "http://ks.cn.yahoo.com/question/1307121101907.html",
                "http://ks.cn.yahoo.com/question/1307121101907_2.html",
                "http://ks.cn.yahoo.com/question/1307121101907_3.html",
                "http://ks.cn.yahoo.com/question/1307121101907_4.html",
                "http://ks.cn.yahoo.com/question/1307121101907_5.html",
                "http://ks.cn.yahoo.com/question/1307121101907_6.html",
                "http://ks.cn.yahoo.com/question/1307121101907_7.html",
                "http://ks.cn.yahoo.com/question/1307121101907_8.html" };
        WebContent wc = new WebContent();
        HashMap<String, String> hm = new HashMap<String, String>();
        for (int i = 0; i < htmlurl.length; i++) {
            hm = wc.getFromYahoo(htmlurl[i]);
            System.out.println("标题： " + hm.get("title"));
            System.out.println("内容：  " + hm.get("original"));
        }
    */
        /**//*
         * String html=""; String link=""; String sscript=""; String content="";
         * System.out.println(htmlurl+" 开始读取网页内容：");
         * html=wc.getOneHtml(htmlurl); System.out.println(htmlurl+"
         * 读取完毕开始分析……"); html=html.replaceAll("(<script.*?)(( )*)(.*?)(( )*)(.*?)(</script>)","
         * ");//去除脚本 html=html.replaceAll("(<style.*?)(( )*)(.*?)(( )*)(.*?)(</style>)","
         * ");//去掉CSS html=html.replaceAll("<title>.*?</title>"," ");//除去页面标题
         * html=html.replaceAll("<a[^>]*href=("([^"]*)"|'([^']*)'|([^/s>]*))[^>]*>(.*?)</a>","
         * ");//去掉链接 html=html.replaceAll("(/s){2,}?"," ");//除去多余空格
         * html=wc.outTag(html);//多余标记 System.out.println(html);
         */

        /**//*
         * String s[]=html.split(" +"); for(int i=0;i<s.length;i++){
         * content=(content.length()>s[i].length())?content:s[i]; }
         * System.out.println(content);
         */

        // System.out.println(htmlurl+"网页内容结束");
        /**//*
         * System.out.println(htmlurl+"网页脚本开始："); List
         * script=wc.getScript(html); for(int i=0;i<script.size();i++){
         * System.out.println(script.get(i)); }
         * System.out.println(htmlurl+"网页脚本结束：");
         * 
         * System.out.println(htmlurl+"CSS开始："); List css=wc.getCSS(html);
         * for(int i=0;i<css.size();i++){ System.out.println(css.get(i)); }
         * System.out.println(htmlurl+"CSS结束：");
         * 
         * System.out.println(htmlurl+"全部链接内容开始："); List list=wc.getLink(html);
         * for(int i=0;i<list.size();i++){ link=list.get(i).toString(); }
         * System.out.println(htmlurl+"全部链接内容结束：");
         * 
         * System.out.println("内容"); System.out.println(wc.outTag(html));
         */
    }
}