项目需求基础城市数据爬取

简要说明：运用到的框架是webmagic mongonDB做数据库落地数据
maven 依赖jar以及版本 
<dependency>
    <groupId>us.codecraft</groupId>
    <artifactId>webmagic-core</artifactId>
    <version>0.7.3</version>
</dependency>
<dependency>
    <groupId>us.codecraft</groupId>
    <artifactId>webmagic-extension</artifactId>
    <version>0.7.3</version>
</dependency>

入库实体类：

package com.huoli.trip.operation.core.service;

import com.sleepycat.persist.model.Entity;
import lombok.*;
import org.springframework.data.mongodb.core.mapping.Document;
import org.springframework.data.mongodb.core.mapping.Field;

import java.io.Serializable;

@AllArgsConstructor
@NoArgsConstructor
@Setter
@Getter
@Builder
@Entity
@Document(collection = "trip_test")
public class AddressDb implements Serializable {

    // 省市区ID
    @Field("code")
    private String code;

    // 省市区名称
    @Field("name")
    private String name;

    // 等级类型
    @Field("level")
    private Short level;

}

爬取逻辑：

package com.huoli.trip.operation.core.service;


import com.xiaoleilu.hutool.util.CharsetUtil;
import com.xiaoleilu.hutool.util.CollectionUtil;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;

import java.util.List;

@Component
public class CityProcessor implements PageProcessor {

    @Autowired
    private MongoTemplate mongoTemplate;

    private static final String TJSJ_CITY_BASE_URL  = "http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2019/";
    private static final String TJSJ_CITY_WEB_URL = "^http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2019/[A-Za-z]+.html$";

    // 获取市信息
    private static final String PROVINCE_URL = "^http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2019/\\d{1,2}.html$";
    // 获取区县信息
    private static final String CITY_URL = "^http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2019/\\d{1,2}/\\d{1,4}.html$";

    private Site site = Site.me()
            .setDomain("http://www.stats.gov.cn")
            .setSleepTime(5000)
            .setRetryTimes(3)
            .setCharset(CharsetUtil.GBK)
            .setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36");

    public void process(Page page) {
        List<String> urlList;
        String baseUrl;
        if (page.getUrl().regex(TJSJ_CITY_WEB_URL).match()) {
            // 获取省份code列表
            List<String> provinceCodes = page.getHtml().regex("<td><a href=\\\"(.{1,30}).html\\\">.*?<br></a></td>").all();
            // 获取省份信息列表
            List<String> provinceNames = page.getHtml().regex("<td><a href=\\\".*?.html\\\">(.{1,30})<br></a></td>").all();
            AddressDb addressDb;
            for (int i = 0, n = provinceCodes.size(); i < n; i++) {
                addressDb = AddressDb.builder()
                        .code(provinceCodes.get(i)+"0000")
                        .name(provinceNames.get(i))
                        .level((short) 1).build();
                mongoTemplate.insert(addressDb);
            }
            urlList = page.getHtml().xpath("//*[@class=\"provincetr\"]/td/a/@href").all();
            if (CollectionUtil.isNotEmpty(urlList)) {
                urlList.stream().forEach( str -> page.addTargetRequest(TJSJ_CITY_BASE_URL + str));
            }
        } else if (page.getUrl().regex(PROVINCE_URL).match()) {
            // 获取数据库中省份Code和省份信息
            String code = page.getUrl().regex("http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2019/(.{2,}).html").toString()+"0000";
            Query query = new Query(new Criteria().and("code").is(code));
            AddressDb provinceInfo = mongoTemplate.findOne(query, AddressDb.class);
            if (provinceInfo != null) {
                AddressDb addressDb;
                // 获取城市信息
                List<String> cityInfos = page.getHtml().regex("<td><a href=\\\".*?.html\\\">(.{1,30})</a></td>").all();
                for (int i = 0, n = cityInfos.size(); i < n; i++) {
                    addressDb = AddressDb.builder()
                            .code(cityInfos.get(i).substring(0, 6))
                            .name(cityInfos.get(++i))
                            .level((short) 2).build();
                    mongoTemplate.insert(addressDb);
                }
            }
            urlList = page.getHtml().xpath("//*[@class=\"citytr\"]/td/a/@href").all();
            if (CollectionUtil.isNotEmpty(urlList)) {
                urlList.stream().distinct().forEach( str -> page.addTargetRequest(TJSJ_CITY_BASE_URL + str));
            }
        } else if (page.getUrl().regex(CITY_URL).match()) {
            // 获取数据库中城市Code和信息
            String code = page.getUrl().regex("http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2019/\\d{1,}/(.{2,}).html").toString() + "00";
            Query query = new Query(new Criteria().and("code").is(code));
            AddressDb cityInfo = mongoTemplate.findOne(query, AddressDb.class);
            // 获取区县信息
            List<String> countyInfos = page.getHtml().regex("<td><a href=\\\".*?.html\\\">(.{1,30})</a></td>").all();
            if (cityInfo != null) {
                AddressDb addressDb;
                for (int i = 0, n = countyInfos.size(); i < n; i++) {
                    addressDb = AddressDb.builder()
                            .code(countyInfos.get(i).substring(0, 6))
                            .name(countyInfos.get(++i))
                            .level((short) 3).build();
                    mongoTemplate.insert(addressDb);
                }
            }
            baseUrl = page.getUrl().regex("^http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2019/\\d{1,2}/").toString();
            urlList = page.getHtml().xpath("//*[@class=\"countytr\"]/td/a/@href").all();
            if (CollectionUtil.isNotEmpty(urlList)) {
                urlList.stream().distinct().forEach( str -> page.addTargetRequest(baseUrl + str));
            }
        }
    }
    public Site getSite() {
        return site;
    }
}

控制器入口：

package com.huoli.trip.operation.core.controller;

import com.huoli.trip.operation.core.service.CityProcessor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.downloader.HttpClientDownloader;

@RestController
public class NovelController {
    @Autowired
    private CityProcessor cityProcessor;

    @RequestMapping("getCityData")
    public void crawelCity(String url) {
        HttpClientDownloader httpClientDownloader = new HttpClientDownloader();
        Spider.create(cityProcessor).addUrl("http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2019/index.html").setDownloader(httpClientDownloader).thread(10).run();

    }
}

上图为落地数据document结构，根据源数据表，设计mongo项目中业务需求需要的表结构

    @Test
    public void testCity(){
        CityRequest cityRequest = new CityRequest();
        cityRequest.setCode("440100".substring(0,4));
        List<AreaPO> areaPOS = cityDao.queryAreaList(cityRequest);
        System.out.println(areaPOS);
//        CityRequest cityRequest = new CityRequest();
//        Criteria criteria = Criteria.where("country").exists(true);
//        MatchOperation matchOperation = Aggregation.match(criteria);
//        ArrayList<MatchOperation> matchOperations = Lists.newArrayList(matchOperation);
//        Aggregation aggregation = Aggregation.newAggregation(matchOperations);
//        AggregationResults<ProvincePO> output = mongoTemplate.aggregate(aggregation, Constants.TRIP_BASE_PROVINCE, ProvincePO.class);
//
//        output.getMappedResults().forEach(t->{
//            t.getProvincePOS().forEach(cityPO ->{
//                System.out.println(cityPO.getId()+" 输出了 "+cityPO.getName());
//            });
//        });
        /**
         * 源
         */
//        List<TripTest> all = mongoTemplate.findAll(TripTest.class);

        /**
         * 区
         */
//        Map<String, List<TripTest>> cityMap = all.stream().filter(tripTest -> !tripTest.getId().substring(2).equals("0000")&&(tripTest.getName().contains("区")||tripTest.getName().contains("县"))).collect(Collectors.groupingBy(tripTest -> tripTest.getId().substring(0, 4)));
//        Set<Map.Entry<String, List<TripTest>>> entries = cityMap.entrySet();
//        for (Map.Entry<String, List<TripTest>> map: entries){
//            List<AreaPO> areaPOS = new ArrayList<>();
//            List<TripTest> value = map.getValue();
//            value.forEach(tripTest -> {
//                AreaPO areaPO = new AreaPO();
//                areaPO.setId(tripTest.getId());
//                areaPO.setLevel(4);
//                areaPO.setName(tripTest.getName().replaceAll("\\u00A0","").trim());
//                areaPO.setCityId(map.getKey());
//                areaPOS.add(areaPO);
//            });
//            AreaPO areaPO = new AreaPO();
//            areaPO.setCityId(map.getKey());
//            areaPO.setAreaPOS(areaPOS);
//            mongoTemplate.insert(areaPO);
//        }
//        Query query = new Query();
//        query.addCriteria(Criteria.where("level").is(1));
//        Update update = new Update();
//        update.set("areas",areaPOS);
//        mongoTemplate.upsert(query, update, BaseCityPO.class, Constants.TRIP_BASE_CITY);
        /**
         * 城市
         */
//        Map<String, List<TripTest>> cityMap = all.stream().filter(tripTest -> tripTest.getName().contains("市")).collect(Collectors.groupingBy(tripTest -> tripTest.getId().substring(0, 2)));
//        Set<Map.Entry<String, List<TripTest>>> entries = cityMap.entrySet();
//
//        for (Map.Entry<String, List<TripTest>> map: entries){
//            List<BaseCityPO> cities = new ArrayList<>();
//            List<TripTest> value = map.getValue();
//            value.forEach(tripTest -> {
//                BaseCityPO city = new BaseCityPO();
//                city.setId(tripTest.getId());
//                city.setLevel(3);
//                city.setName(tripTest.getName().replaceAll("\\u00A0","").trim());
//                city.setProvinceId(map.getKey());
//                cities.add(city);
//            });
//            BaseCityPO cityPO = new BaseCityPO();
//            cityPO.setProvinceId(map.getKey());
//            cityPO.setCityPOS(cities);
//            mongoTemplate.insert(cityPO);
//        }
        //更新操作
//        Query query = new Query();
//        query.addCriteria(Criteria.where("level").is(1));
//        Update update = new Update();
//        update.set("citys",cities);
//        mongoTemplate.upsert(query, update, BaseCityPO.class, Constants.TRIP_BASE_CITY);
        /**
         * 省
         */
//        String country = "中国";
//        all.forEach(tripTest -> {
//            if(tripTest.getName().contains("省")){
//                ProvincePO province = new ProvincePO();
//                province.setId(tripTest.getId());
//                province.setLevel(2);
//                province.setName(tripTest.getName());
//                province.setCountry(country);
//                mongoTemplate.insert(province);
//            }
//        });
    }