第十章 广告检索系统——加载全量索引（一）

此博客用于个人学习，来源于网上，对知识点进行一个整理。

1. 导出表数据定义：

接下来来实现加载全量索引的服务，加载全量索引就需要我们将数据库内广告主投放的数据导出到文件里面。一般情况下，这种功能是定义在索引相关操作的服务中，但是这份索引数据是针对于所有服务的，于是我们将其定义在 ad-common 中，其他服务都可以调用。

1.1 定义文件的存储位置：

首先，需要确定的就是文件存储到哪个地方，于是先定义一个类，在其中确定好文件的存储路径和各个索引对应的存储文件名称。

public class DConstant {

    public static final String DATA_ROOT_DIR = "/Users/imooc/mysql_data/";

    //各个表数据的存储文件名
    public static final String AD_PLAN = "ad_plan.data";
    public static final String AD_UNIT = "ad_unit.data";
    public static final String AD_CREATIVE = "ad_creative.data";
    public static final String AD_CREATIVE_UNIT = "ad_creative_unit.data";
    public static final String AD_UNIT_IT = "ad_unit_it.data";
    public static final String AD_UNIT_DISTRICT = "ad_unit_district.data";
    public static final String AD_UNIT_KEYWORD = "ad_unit_keyword.data";
}

1.2 定义存储的索引表对象：

索引表对象的属性与一开始定义的索引对象的属性相同。

1）广告计划索引表数据：

@Data
@AllArgsConstructor
@NoArgsConstructor
public class AdPlanTable {

    private Long planId;
    private Long userId;
    private Integer planStatus;
    private Date startDate;
    private Date endDate;
}

2）广告单元索引及其限制表数据：

@Data
@NoArgsConstructor
@AllArgsConstructor
public class AdUnitTable {

    private Long unitId;
    private Integer unitStatus;
    private Integer positionType;
    private Long planId;
}

@Data
@NoArgsConstructor
@AllArgsConstructor
public class UnitKeywordTable {

    private Long unitId;
    private String keyword;
}

@Data
@NoArgsConstructor
@AllArgsConstructor
public class UnitItTable {

    private Long unitId;
    private String itTag;
}

@Data
@NoArgsConstructor
@AllArgsConstructor
public class UnitDistrictTable {

    private Long unitId;
    private String province;
    private String city;
}

3）广告创意索引与关联表对象：

@Data
@NoArgsConstructor
@AllArgsConstructor
public class CreativeTable {

    private Long adId;
    private String name;
    private Integer type;
    private Integer materialType;
    private Integer height;
    private Integer width;
    private Integer auditStatus;
    private String adUrl;
}

@Data
@NoArgsConstructor
@AllArgsConstructor
public class CreativeUnitTable {

    private Long adId;
    private Long unitId;
}

2. 表数据导出文件功能实现：

表数据对象构建完毕后，就是实现表数据导出文件功能了。考虑到需要调用到索引的各种方法，于是将其定义在 ad-sponsor 的 test 文件夹中。

2.1 启动类：

@SpringBootApplication
public class Application {

    public static void main(String[] args) {
        SpringApplication.run(Application.class,args);
    }
}

2.2 配置文件：

server:
  port: 7000
  servlet:
    context-path: /ad-sponsor #Controller 的前缀

spring:
  application:
    name: eureka-client-ad-sponsor
  jpa:
    show-sql: true #是否打印 SQL 语句
    hibernate:
      ddl-auto: none #手动创建表，主动控制权会更高
    properties:
      hibernate.format_sql: true #是否对 SQL 语句进行格式化
    open-in-view: false #在懒加载的时候，如果找不到一些 bean 的配置会导致错误，所以设置为 false
  datasource:
    url: jdbc:mysql://127.0.0.1:3306/imooc_ad_data?autoReconnect=true
    username: root
    password: root
    tomcat:
      max-active: 4 #最大连接数
      min-idle: 2 #最小空闲连接数
      initial-size: 2 #初始化连接

eureka:
  client:
    service-url:
      defaultZone: http://server1:8000/eureka/

2.3 功能实现：

写入文件的逻辑都是一致的：先是遍历获取到的索引值，将其写入之前定义的索引表数据中，转换为 JSON 格式后写入文件中。

@Slf4j
@RunWith(SpringRunner.class)
//不需要 web 环境
@SpringBootTest(classes = {Application.class},webEnvironment = SpringBootTest.WebEnvironment.NONE)
public class DumpDataService {

    @Autowired
    private AdPlanRepository planRepository;
    @Autowired
    private AdUnitRepository unitRepository;
    @Autowired
    private CreativeRepository creativeRepository;
    @Autowired
    private CreativeUnitRepository creativeUnitRepository;
    @Autowired
    private AdUnitDistrictRepository districtRepository;
    @Autowired
    private AdUnitItRepository itRepository;
    @Autowired
    private AdUnitKeyWordRepository keyWordRepository;

    @Test
    public void dumpAdTableData(){
        dumpAdPlanTable(
                String.format("%s%s", DConstant.DATA_ROOT_DIR,DConstant.AD_PLAN)
        );
        dumpAdUnitTable(
                String.format("%s%s", DConstant.DATA_ROOT_DIR,DConstant.AD_UNIT)
        );
        dumpAdCreativeTable(
                String.format("%s%s", DConstant.DATA_ROOT_DIR,DConstant.AD_CREATIVE)
        );
        dumpAdCreativeUnitTable(
                String.format("%s%s", DConstant.DATA_ROOT_DIR,DConstant.AD_CREATIVE_UNIT)
        );
        dumpAdUnitDistrictTable(
                String.format("%s%s", DConstant.DATA_ROOT_DIR,DConstant.AD_UNIT_DISTRICT)
        );
        dumpAdUnitItTable(
                String.format("%s%s", DConstant.DATA_ROOT_DIR,DConstant.AD_UNIT_IT)
        );
        dumpAdUnitKeywordTable(
                String.format("%s%s", DConstant.DATA_ROOT_DIR,DConstant.AD_UNIT_KEYWORD)
        );
    }

    private void dumpAdPlanTable(String fileName){
        List<AdPlan> adPlans = planRepository.findAllByPlanStatus(
                CommonStatus.VALID.getStatus()
        );
        //plans 为空，直接返回
        if (CollectionUtils.isEmpty(adPlans)){
            return;
        }
        //遍历 plans ，将数据插入到索引表数据中
        List<AdPlanTable> planTables = new ArrayList<>();
        adPlans.forEach(p -> planTables.add(
                new AdPlanTable(
                        p.getId(),
                        p.getUserId(),
                        p.getPlanStatus(),
                        p.getStartDate(),
                        p.getEndDate()
                )
        ));
        //获取文件路径
        Path path = Paths.get(fileName);
        //遍历表数据，将其转为 JSON 格式的数据并且写入文件中
        try(BufferedWriter writer = Files.newBufferedWriter(path)) {
            for (AdPlanTable planTable : planTables){
                writer.write(JSON.toJSONString(planTable));
                writer.newLine();
            }
            writer.close();
        }catch (IOException ex){
            log.error("dumpAdPlanTable error");
        }
    }

    private void dumpAdUnitTable(String fileName){
        List<AdUnit> adUnits = unitRepository.findAllByUnitStatus(
                CommonStatus.VALID.getStatus()
        );
        if (CollectionUtils.isEmpty(adUnits)){
            return;
        }
        List<AdUnitTable> unitTables = new ArrayList<>();
        adUnits.forEach(u -> unitTables.add(
                new AdUnitTable(
                        u.getId(),
                        u.getUnitStatus(),
                        u.getPositionType(),
                        u.getPlanId()
                )
        ));
        Path path = Paths.get(fileName);
        try(BufferedWriter writer = Files.newBufferedWriter(path)) {
            for (AdUnitTable unitTable : unitTables){
                writer.write(JSON.toJSONString(unitTable));
                writer.newLine();
            }
            writer.close();
        }catch (IOException ex){
            log.error("dumpAdUnitTable error");
        }
    }

    private void dumpAdCreativeTable(String fileName){
        List<Creative> creatives = creativeRepository.findAll();
        if (CollectionUtils.isEmpty(creatives)){
            return;
        }
        List<CreativeTable> creativeTables = new ArrayList<>();
        creatives.forEach(c -> creativeTables.add(
                new CreativeTable(
                        c.getId(),
                        c.getName(),
                        c.getType(),
                        c.getMaterialType(),
                        c.getHeight(),
                        c.getWidth(),
                        c.getAuditStatus(),
                        c.getUrl()
                )
        ));
        Path path = Paths.get(fileName);
        try(BufferedWriter writer = Files.newBufferedWriter(path)) {
            for (CreativeTable creativeTable : creativeTables){
                writer.write(JSON.toJSONString(creativeTable));
                writer.newLine();
            }
            writer.close();
        }catch (IOException ex){
            log.error("dumpAdCreativeTable error");
        }
    }

    private void dumpAdCreativeUnitTable(String fileName){
        List<CreativeUnit> creativeUnits = creativeUnitRepository.findAll();
        if (CollectionUtils.isEmpty(creativeUnits)){
            return;
        }
        List<CreativeUnitTable> creativeUnitTables = new ArrayList<>();
        creativeUnits.forEach(c -> creativeUnitTables.add(
                new CreativeUnitTable(
                        c.getId(),
                        c.getUnitId()
                )
        ));
        Path path = Paths.get(fileName);
        try(BufferedWriter writer = Files.newBufferedWriter(path)) {
            for (CreativeUnitTable creativeUnitTable : creativeUnitTables){
                writer.write(JSON.toJSONString(creativeUnitTable));
                writer.newLine();
            }
            writer.close();
        }catch (IOException ex){
            log.error("dumpAdCreativeUnitTable error");
        }
    }

    private void dumpAdUnitDistrictTable(String fileName){
        List<AdUnitDistrict> unitDistricts = districtRepository.findAll();
        if (CollectionUtils.isEmpty(unitDistricts)){
            return;
        }
        List<UnitDistrictTable> unitDistrictTables = new ArrayList<>();
        unitDistricts.forEach(u -> unitDistrictTables.add(
                new UnitDistrictTable(
                        u.getId(),
                        u.getProvince(),
                        u.getCity()
                )
        ));
        Path path = Paths.get(fileName);
        try(BufferedWriter writer = Files.newBufferedWriter(path)) {
            for (UnitDistrictTable unitDistrictTable : unitDistrictTables){
                writer.write(JSON.toJSONString(unitDistrictTable));
                writer.newLine();
            }
            writer.close();
        }catch (IOException ex){
            log.error("dumpAdUnitDistrictTable error");
        }
    }

    private void dumpAdUnitItTable(String fileName){
        List<AdUnitIt> adUnitIts = itRepository.findAll();
        if (CollectionUtils.isEmpty(adUnitIts)){
            return;
        }
        List<UnitItTable> unitItTables = new ArrayList<>();
        adUnitIts.forEach(u -> unitItTables.add(
                new UnitItTable(
                        u.getId(),
                        u.getItTag()
                )
        ));
        Path path = Paths.get(fileName);
        try(BufferedWriter writer = Files.newBufferedWriter(path)) {
            for (UnitItTable unitItTable : unitItTables){
                writer.write(JSON.toJSONString(unitItTable));
                writer.newLine();
            }
            writer.close();
        }catch (IOException ex){
            log.error("dumpAdUnitItTable error");
        }
    }

    private void dumpAdUnitKeywordTable(String fileName){
        List<AdUnitKeyword> adUnitKeywords = keyWordRepository.findAll();
        if (CollectionUtils.isEmpty(adUnitKeywords)){
            return;
        }
        List<UnitKeywordTable> unitKeywordTables = new ArrayList<>();
        adUnitKeywords.forEach(u -> unitKeywordTables.add(
                new UnitKeywordTable(
                        u.getId(),
                        u.getKeyword()
                )
        ));
        Path path = Paths.get(fileName);
        try(BufferedWriter writer = Files.newBufferedWriter(path)) {
            for (UnitKeywordTable unitKeywordTable : unitKeywordTables){
                writer.write(JSON.toJSONString(unitKeywordTable));
                writer.newLine();
            }
            writer.close();
        }catch (IOException ex){
            log.error("dumpAdUnitKeywordTable error");
        }
    }
}