java实现截取网页中所包含的网址

最新推荐文章于 2024-09-07 23:30:07 发布

夏天一柠檬

最新推荐文章于 2024-09-07 23:30:07 发布

阅读量919

点赞数 1

文章标签： java url io流

本文链接：https://blog.csdn.net/liushadiyu/article/details/73608954

版权

以网易为例,截取网易中包含的所有网址,并把网址输出到本地txt文件


import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.HashSet;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class URlduqu {

    public static void main(String[] args) {
        //截取网址的规范
        Pattern guize = Pattern.compile("<a href=.*</a>");
        //去掉无用网址的规范
        Pattern wang = Pattern.compile("http.*");
        try {

            URL url = new URL("http://www.163.com");
            URLConnection conn = url.openConnection();
            java.io.InputStream is = conn.getInputStream();
            BufferedReader br = new BufferedReader(new InputStreamReader(is,"gb2312"));
            //未去重的网址存放文件
            File f = new File("F:"+File.separator+"网址1.txt");
            PrintWriter pw = new PrintWriter(f);
            //去重后的网址存放文件
            File f2 = new File("F:"+File.separator+"网址3.txt");
            PrintWriter aaa = new PrintWriter(f2);
            String str = null;
            int n =0;//标记网址的个数(数组长度)
            String[] wangzhi = new String[2000];//所有网址放入这个数组
            String[] zhi;//去重后的网址数组
            while((str=br.readLine())!=null){
                Matcher m = guize.matcher(str);

                if(m.find()){
                    //截取网址
                    str = str.substring(str.indexOf("href=\"")+6);
                    str = str.substring(0,str.indexOf("\""));
                    //判断网址是否符合wang设置的规则,如果符合规则就打印到网址.txt中
                    if(wang.matcher(str).matches()){
                        pw.println(str);
                        wangzhi[n]=str;
                        n++;
                    }
                }
            }
            //去重操作
            Set set = new HashSet();
            for (int i = 0; i < n; i++) {
                set.add(wangzhi[i]);
            }
            zhi = (String[]) set.toArray(new String[0]);

            for(String a:zhi){
                aaa.println(a);
            }

            aaa.close();
            pw.close();
            br.close();
        } catch (MalformedURLException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }

    }

}