Java爬取网页数据HTML,CSS,JS

最近接触了下java的爬虫,文本信息爬完了,就想看看图片怎么爬,于是就研究了一下,案例爬学校的官网

pom依赖

<!-- https://mvnrepository.com/artifact/org.jsoup/jsoup -->
        <dependency>
            <groupId>org.jsoup</groupId>
            <artifactId>jsoup</artifactId>
            <version>1.11.3</version>
        </dependency>

        <!-- 文件下载 -->
        <dependency>
            <groupId>commons-io</groupId>
            <artifactId>commons-io</artifactId>
            <version>2.5</version>
        </dependency>

        <!-- https://mvnrepository.com/artifact/org.apache.httpcomponents/httpclient -->
        <dependency>
            <groupId>org.apache.httpcomponents</groupId>
            <artifactId>httpclient</artifactId>
            <version>4.5.5</version>
        </dependency>

设置配置信息:

public class test {
    // 地址
    private static final String URL = "http://www.ktbdqn.com/";
    // 编码
    private static final String ECODING = "GBK";
    // 获取img标签正则
    private static final String IMGURL_REG = "<img.*src\\s*=\\s*(.*?)[^>]*?>";
    //获取link标签正则
    private static final String LINKURL_REG = "<link.*href\\s*=\\s*(.*?)[^>]*?>";
    // 获取Img的src路径的正则
    private static final String IMGSRC_REG = "(?x)(src|SRC|background|BACKGROUND)=('|\")/?(([\\w-]+/)*([\\w-]+\\.(jpg|JPG|png|PNG|gif|GIF)))('|\")";
    // 获取Link的href路径的正则
    private static final String LINKSRC_REG = "(?x)(href|HREF)=('|\")/?(([\\w-]+/)*([\\w-]+\\.(css|CSS|([\\w-]+/)*([\\w-]+\\.(css|CSS|([\\w-]+/)*([\\w-]+\\.(css|CSS)))))))('|\")";
    // css本地保存路径
    private static final String SAVE_CSS_PATH = "d:\\cskt\\";
    // img本地保存路径
    private static final String SAVE_PATH = "d:\\";

这里注意有些人获取的网页会乱码,根据爬取的网页编码和自己的编码保持一致

图片下载

public static void Download(List<String> listImgSrc) {
        int count = 0;
        try {
            for (int i = 0; i < listImgSrc.size(); i++) {
                String url = listImgSrc.get(i);
                String imageName = url.substring(url.lastIndexOf("/") + 1, url.length());
                URL uri = new URL(url);
                // 打开连接
                URLConnection con = uri.openConnection();
                //设置请求超时为5s
                con.setConnectTimeout(5 * 1000);
                // 输入流
                InputStream is = con.getInputStream();
                // 1K的数据缓冲
                byte[] bs = new byte[1024];
                // 读取到的数据长度
                int len;
                // 输出的文件流
                String src = listImgSrc.get(i).substring(URL.length());
                int index = src.lastIndexOf('/');
                String fileName = src.substring(0, index + 1);
                File sf = new File(SAVE_PATH + fileName);
                if (!sf.exists()) {
                    sf.mkdirs();
                }
                OutputStream os = new FileOutputStream(sf.getPath() + "\\" + imageName);
                System.out.println(++count + ".开始下载:" + url);
                // 开始读取
                while ((len = is.read(bs)) != -1) {
                    os.write(bs, 0, len);
                }
                // 完毕,关闭所有链接
                os.close();
                is.close();
                System.out.println(imageName + ":--下载完成");
                System.out.println();
            }
        } catch (Exception e) {
            System.out.println("下载失败");
        }
    }

样式下载

public static void DownCss(List<String> listCssSrc) {
        int count = 0;
        try {
            for (int i = 0; i < listCssSrc.size(); i++) {
                String url = listCssSrc.get(i);
                String imageName = url.substring(url.lastIndexOf("/") + 1, url.length());
                URL uri = new URL(url);
                // 打开连接
                URLConnection con = uri.openConnection();
                //设置请求超时为5s
                con.setConnectTimeout(5 * 1000);
                // 输入流
                InputStream is = con.getInputStream();
                // 1K的数据缓冲
                byte[] bs = new byte[1024];
                // 读取到的数据长度
                int len;
                // 输出的文件流
                String src = listCssSrc.get(i).substring(URL.length());
                int index = src.lastIndexOf('/');
                String fileName = src.substring(0, index + 1);
                File sf = new File(SAVE_CSS_PATH + fileName);
                if (!sf.exists()) {
                    sf.mkdirs();
                }
                OutputStream os = new FileOutputStream(sf.getPath() + "\\" + imageName);
                System.out.println(++count + ".开始下载:" + url);
                // 开始读取
                while ((len = is.read(bs)) != -1) {
                    os.write(bs, 0, len);
                }
                // 完毕,关闭所有链接
                os.close();
                is.close();
                System.out.println(imageName + ":--下载完成");
                System.out.println();
            }
        } catch (Exception e) {
            System.out.println("下载失败");
        }
    }

得到网页中的图片地址

private static List<String> getImgStr(String htmlStr) {
        List<String> pics = new ArrayList<String>();
        String img = "";
        Pattern p_image;
        Matcher m_image;
        p_image = Pattern.compile(IMGURL_REG, Pattern.CASE_INSENSITIVE);
        m_image = p_image.matcher(htmlStr);
        while (m_image.find()) {
            // 得到<img />数据
            img = m_image.group();
            // 匹配<img>中的src数据
            Matcher m = Pattern.compile(IMGSRC_REG).matcher(img);
            while (m.find()) {
                String imgUrl = m.group(3);
                if (!imgUrl.contains("http://") && !imgUrl.contains("https://")) {//没有这两个头
                    imgUrl = URL + imgUrl;
                }
                pics.add(imgUrl);
            }
        }
        return pics;
    }

获取网页中的样式地址

private static List<String> getCssStr(String htmlStr) {
        List<String> csss = new ArrayList<String>();
        String css = "";
        Pattern p_css;
        Matcher m_css;
        p_css = Pattern.compile(LINKURL_REG, Pattern.CASE_INSENSITIVE);
        m_css = p_css.matcher(htmlStr);
        while (m_css.find()) {
            // 得到<css />数据
            css = m_css.group();
            // 匹配<css>中的src数据
            Matcher m = Pattern.compile(LINKSRC_REG).matcher(css);
            while (m.find()) {
                String cssUrl = m.group(3);
                if (!cssUrl.contains("http://") && !cssUrl.contains("https://")) {//没有这两个头
                    cssUrl = URL + cssUrl;
                }
                csss.add(cssUrl);
            }
        }
        return csss;
    }

网页源码的爬取

public static String getHtmlResourceByUrl(String url, String encoding) {
        URL urlObj = null;
        URLConnection uc = null;
        InputStreamReader isr = null;
        BufferedReader reader = null;
        StringBuffer buffer = new StringBuffer();
        // 建立网络连接
        try {
            urlObj = new URL(url);
            // 打开网络连接
            uc = urlObj.openConnection();
            // 建立文件输入流
            isr = new InputStreamReader(uc.getInputStream(), encoding);
            // 建立缓存导入 将网页源代码下载下来
            reader = new BufferedReader(isr);
            // 临时
            String temp = null;
            while ((temp = reader.readLine()) != null) {// 一次读一行 只要不为空就说明没读完继续读
                // System.out.println(temp+"\n");
                buffer.append(temp + "\n");
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            // 关流
            if (isr != null) {
                try {
                    isr.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
        return buffer.toString();
    }

main方法

public static void main(String[] args) throws ClientProtocolException, IOException {
        //保存网页源码
        getJobInfo(URL, ECODING);
        //获得html文本内容
        String HTML = test.getHtmlResourceByUrl(URL, ECODING);
        //获取图片src的url地址
        List<String> imgSrc = test.getImgStr(HTML);
        //下载图片
        test.Download(imgSrc);
        //获取样式href的url地址
        List<String> cssSrc = test.getCssStr(HTML);
        //下载css样式
        test.DownCss(cssSrc);
    }
相关推荐
©️2020 CSDN 皮肤主题: 大白 设计师:CSDN官方博客 返回首页