java爬虫Deno

最新推荐文章于 2024-07-04 16:04:39 发布

anningx

最新推荐文章于 2024-07-04 16:04:39 发布

阅读量221

点赞数

分类专栏： java 文章标签： java爬虫 jsoup

本文链接：https://blog.csdn.net/qq_39739458/article/details/80361370

版权

java 专栏收录该内容

2 篇文章 0 订阅

订阅专栏

java爬虫的开发依赖于jsoup.jar

直接上代码

public static void main(String[] args) {
        HttpClient client = new DefaultHttpClient();
        HttpGet httpGet = new HttpGet();
        Map map = new HashMap();
        try {
            //百度百科999感冒灵连接
            String url = "http://baike.baidu.com/link?url=c95Y4QJym_d_wFKGmcibRTI_KIyj-X_tOjnlOGJS9qekgO1tmWaWnrn7QyAjqvZX8At7LbI1XIa69IBZWejiCXDVM0jkSBMnVZKKu4jeg-ef4TJkKCXEXWcGJ8DRGTuHxW4qWB3pNNU7Y0KdrbNvGK";
            // get请求获取页面信息
            String bb = doget(url);
            Document doc;
            //用jsoup接收页面信息
            doc = Jsoup.parse(bb);
            // 选择所有div的class为para的标签
            Elements news = doc.select("div[class=para]");
            for (Element result : news) {
                //获取标签的内容并打印
                String str=result.text();
                System.out.println(str);
            }
             
             
        } catch (Exception e) {
            e.printStackTrace();
        }
 
    }
  
    public static String doget(String path) {
        InputStream is = null;
        ByteArrayOutputStream baos = null;
        try {
            // 伪造referer 绕过防盗链设置
            URL url = new URL(path.trim());
            HttpURLConnection conn = (HttpURLConnection) url.openConnection();
              
            if (200 == conn.getResponseCode()) {
                byte[] buff = new byte[4096];
                int count;
                ByteArrayOutputStream out = new ByteArrayOutputStream(4096);
                InputStream in = conn.getInputStream();
 
                while ((count = in.read(buff)) != -1) {
                    out.write(buff, 0, count);
                }
                conn.disconnect();
                return out.toString("UTF-8");
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            if (baos != null) {
                try {
                    baos.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
            if (is != null) {
                try {
                    is.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
 
        }
 
        return null;
    }