Java 爬虫实战二

一、项目需要

• 对某类网站的数据进行数据采集
• 对主要数据采集点每天定时开启任务
• 对部分数据采用周六日开启任务采集
• 将数据写入数据库
• 对已经采集对信息进行更新处理

二、采用技术

•  项目框架Spring Boot
•  爬虫工具：webmagic
•  数据库：Mysql
•  定时器：@Scheduled 开启
•  cron在心解析地址：http://cron.qqe2.com/
•  网页分析：Goole Chrome
•  语句解析采用：$CSS节点定位、 XPach解析
   

三、任务时序图

 

四、代码

1.定时任务类

    @Scheduled(cron = "1 35 21 ? * 5")
    public void wagicCity()
    {
        String url = magicUrlConfig.get**City();
        Spider.create(jobPosProcessor)
                .addUrl(url)
                .thread(3)
                .run();
    }

    @Scheduled(cron = "1 35 2 ? * 6")
    public void wagicJobAll()
    {
        String url = magicUrlConfig.get*****All();
        Spider.create(jobPosProcessor)
                .addUrl(url)
                .thread(3)
                .run();
    }

    @Scheduled(cron = "1 35 23 * * ?")
    public void wagicHost*******()
    {
        String url = magicUrlConfig.get**home();
        Spider.create(jobPosProcessor)
                .addUrl(url)
                .thread(3)
                .run();
    }

2、爬取配置类

@Component
public class JobPosProcessor implements PageProcessor {
    @Autowired
    CityService cityService;
    @Autowired
    JobService jobService;
    private Site site = Site.me()
            .setRetryTimes(3)
            .setSleepTime(8000)
            .addHeader("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Safari/537.36 Firefox/73.0")
            .addCookie("ganji_uuid","7134539899767277506921")
            .addCookie("ganji_xuuid","73e05d39-b634-4ee9-fc5b-e8d81257f0e5.1570941235445");
    @Override
    public  Site getSite()
    {
        return  site;
    }

    @Override
    public  void  process(Page page)
    {
     
        if (page.getUrl().regex("http://www.****.com/all").match()){
            List<String> jobCities = cityService.selectCityUrl();
            if (jobCities.size()>0){
               page.addTargetRequests(jobCities);
            }

        }
        //查询城市列表
        if (page.getUrl().regex("http://www.****.com/index.htm").match()){
            List<JobCity> jobCities = new ArrayList<>();
            Parse.getJobCities(page, jobCities);
            cityService.insertCities(jobCities);
        }

        if(page.getUrl().regex("http://www.*****.com/jzhi/").match()){
            Html html = page.getHtml();
            Selectable selectable = html.xpath("//dl[@class=\"choose clearfix\"]");
            List<String> urls  =selectable.links().all();
            //过滤掉不需要的链接
            urls.removeIf(s -> !s.contains("jzhi"));
            urls.removeIf(s -> s.contains("www."));
            page.addTargetRequests(urls);
        }
       
        if (page.getUrl().regex("[a-zA-z]+://(\\b((?![wW]{3})\\w)+\\b)\\.****.com/jianzhi/").match()){
            Html html = page.getHtml();
            Selectable selectable = html.xpath("//*[@class=\"f-label-new clearfix\"]");
            List<String> cls  =selectable.links().all();
            page.addTargetRequests(cls);
        }
        
        if (page.getUrl().regex("[a-zA-z]+://(\\b((?![wW]{3})\\w)+\\b)\\.*****.com/jzbiaoqian/*").match()){
            List<String> joburls = new ArrayList<>();
            Parse.getPartJobs(page,joburls);
            page.addTargetRequests(joburls);
        }
        //查询兼职职位 详情
        if(page.getUrl().regex("htt(p|ps)://(\\b((?![wW]{3})\\w)+\\b)\\.*****.com/(\\b((?!jzbiaoqian)\\w)+\\b)/\\d+x.htm").match()){
            List<JobDetail> jobDetailList = new ArrayList<>();
            Parse.getPartJobDetail(page,jobDetailList);
            jobService.insertJobs(jobDetailList);
            System.out.println(jobDetailList);
        }
    }
}

3.解析类

public static void getJobCities(Page page, List<JobCity> jobCities){
        if(jobCities == null){
            jobCities = new ArrayList<>();
        }
        Html html = page.getHtml();
        Selectable selectable = html.xpath("/html/body/div[1]/div[3]/dl");
        Selectable dds   = selectable.xpath("/dl/dd");
        List<Selectable> nodes = dds.nodes();
        for (Selectable node : nodes) {
            int index = nodes.indexOf(node) + 1;
            String group = selectable.xpath("/dl/dt["+index+"]/text()").get();
            Selectable anodes = node.xpath("/dd/a");
            for (Selectable an: anodes.nodes()){
                JobCity jobCity = new JobCity();
                String url   = an.xpath("/a/@href").get()+"jzhi/";
                String name   = an.xpath("/a/text()").get();
                jobCity.setClsification(group);
                jobCity.setUrl(url);
                jobCity.setName(name);
                jobCities.add(jobCity);
            }

        }
    }

    public static void getPartJobs(Page page, List<String> cls){
        if (cls == null){
            cls = new ArrayList<>();
        }
        Html html = page.getHtml();
        Selectable selectable = html.xpath("//*[@id=\"list-job-id\"]/div[5]/dl");
        for (Selectable node : selectable.nodes())
        {
            String url = node.xpath("//dt/a/@post_url").get();
            cls.add(url);
        }
        System.out.println(cls);
    }

    public static void getPartJobDetail(Page page, List<JobDetail> jobDetailList){
        if (jobDetailList == null){
            jobDetailList = new ArrayList<>();
        }
        JobDetail job = new JobDetail();
        Html html = page.getHtml();
        Selectable selectable = html.$("ul[class='clearfix pos-relat']");

        job.setJobName(selectable.xpath("/ul/li[1]/em/a/text()").get());
        job.setJobType(selectable.xpath("/ul/li[1]/em/span/text()").get());
        job.setJobPay(selectable.xpath("/ul/li[2]/em/text()").get());
        job.setEdu(selectable.xpath("/ul/li[3]/em/text()").get());
        job.setWorkbg(selectable.xpath("/ul/li[4]/em/text()").get());
        job.setAge(selectable.xpath("/ul/li[5]/em/text()").get());
        job.setJobCount(selectable.xpath("/ul/li[6]/em/text()").get());
        job.setJobLabel(selectable.xpath("/ul/li[9]/em/span/em/text()").get());
        job.setWorkTime(selectable.xpath("/ul/li[10]/em/text()").get());
        Selectable selheader = html.$("div[class='d-c-left-hear']");
        job.setJobTitle(selheader.xpath("/div/h1/text()").toString());
        String update = selheader.xpath("/div/p/span[1]/text()").toString();
        String[]  strs=update.split("：");
        job.setUpdateDate(strs[strs.length-1]);

        Selectable describenode = html.$("div[class='deta-Corp']");
        String describe = describenode.xpath("/div/text()").get();
        job.setJobDesc(describe);

        job.setPhone(DomainFormat.getDomainForUrl(page.getUrl().toString()) +html.xpath("//*[@id=\"isShowPhoneBotton\"]/img/@src").get());

        Selectable contactnode = html.$("dl[class='detail-contact']");
        job.setContacts(contactnode.xpath("/dl/dd[2]/text()").get());
        job.setAddress(contactnode.xpath("/dl/dd[3]/text()").get());
        jobDetailList.add(job);
    }

五、总结

在节点定位上使用$CSS定位比较好，否则因为因为页面的动态布局变动会出现爬取数据 的情况

解析采用XPath提取