用java获取虎扑新闻数据

我采用多线程进行任务抓取  也可以采用单线程

public void begin() {

                //获取虎扑新闻的数据，网址格式为：https://voice.hupu.com/nba/第几页   
            new Thread(new Runnable() {
                @Override
                public void run() {
                    try{
 
                    Document doc = Jsoup.connect("https://voice.hupu.com/nba/").get();
                    Elements titleLinks = doc.select("div.list-hd");    //解析来获取每条新闻的标题与链接地址
          
                    Elements timeLinks = doc.select("div.otherInfo");   //解析来获取每条新闻的时间与来源
                    
                    //for循环遍历获取到每条新闻的四个数据并封装到News实体类中
                    for(int j = 0;j < titleLinks.size();j++){
                        String title = titleLinks.get(j).select("a").text();
                        String uri = titleLinks.get(j).select("a").attr("href");
                        Document doc1 = Jsoup.connect(uri).get();
                        Elements desLinks=doc1.select("div.artical-main-content");
                        String desc="";
                       if(desLinks.size()!=0){
                           Elements e=desLinks.select("p");
                           for(int z = 0;z < e.size();z++){
                               desc+=e.get(z).text();
                           }
                       }else{
                           desc="0";
                       }
                        
                        String time = timeLinks.get(j).select("span.other-left").select("a").text();
                        News news = new News(title,uri,desc,time);
                        String sql ="insert into bll_pageinfo values (null,?,?,?,?)";
                        String[] parameters = { news.getNewsTitle(), news.getNewsUrl(), news.getDesc(), news.getNewsTime()};
                        DbAction.executeUpdate(sql,parameters );
                        System.out.println("当前的线程为："+Thread.currentThread().getName());
                    }
                }
                
            }catch (Exception e){
               e.printStackTrace();
            }
       }
   }).start();

 }    

   

获取的数据保存到数据库  自己数据库可以配置字段跟实体类对应

我的数据库是mysql，小伙伴也可以根据自己的来选择

public class DbAction {

public static int executeUpdate(String sql, String[] parameters) {
        int result = 0;
        Connection conn = null;
        PreparedStatement pstmt = null;
        try {
            conn = getConnection();
            pstmt = conn.prepareStatement(sql);
            for (int i = 0; i < parameters.length; i++) {
                pstmt.setString(i + 1, parameters[i]);
            }
            result = pstmt.executeUpdate();
        } catch (SQLException err) {
            err.printStackTrace();
            free(null, pstmt, conn);
        } finally {
            free(null, pstmt, conn);
        }

        return result;
    }       
 }   
        
        

实体类封装抓取的数据

public class News {
    
        private String newsTitle;   //新闻标题
        private String newsUrl;     //新闻链接地址
        private String desc;        //新闻概要
        private String newsTime;    //新闻时间与来源

        public News(String newsTitle, String newsUrl, String desc, String newsTime) {
            this.newsTitle = newsTitle;
            this.newsUrl = newsUrl;
            this.desc = desc;
            this.newsTime = newsTime;
        }

        public String getDesc() {
            return desc;
        }

        public void setDesc(String desc) {
            this.desc = desc;
        }

        public String getNewsTime() {
            return newsTime;
        }

        public void setNewsTime(String newsTime) {
            this.newsTime = newsTime;
        }

        public String getNewsTitle() {
            return newsTitle;
        }

        public void setNewsTitle(String newsTitle) {
            this.newsTitle = newsTitle;
        }

        public String getNewsUrl() {
            return newsUrl;
        }

        public void setNewsUrl(String newsUrl) {
            this.newsUrl = newsUrl;
        }
    }

  个人亲测有效

需要的jar包

本人JDK使用的1.8