packagecom.ververica.cdc.connectors.mysql.source.assigners;privatevoidsplitChunksForRemainingTables(){try{//flink任务会从checkpoint中获取已经进行过全量的表,下面的remainingTables为未进行过snapshot的表for(TableId nextTable : remainingTables){// split the given table into chunks (snapshot splits)Collection<MySqlSnapshotSplit> splits = chunkSplitter.generateSplits(nextTable);synchronized(lock){
remainingSplits.addAll(splits);
remainingTables.remove(nextTable);
lock.notify();}}}catch(Exception e){if(uncaughtSplitterException ==null){
uncaughtSplitterException = e;}else{
uncaughtSplitterException.addSuppressed(e);}// Release the potential waiting getNext() callsynchronized(lock){
lock.notify();}}}
主要看generateSplits方法:
/** Generates all snapshot splits (chunks) for the give table path. */publicCollection<MySqlSnapshotSplit>generateSplits(TableId tableId){try(JdbcConnection jdbc =openJdbcConnection(sourceConfig)){LOG.info("Start splitting table {} into chunks...", tableId);long start =System.currentTimeMillis();Table table = mySqlSchema.getTableSchema(jdbc, tableId).getTable();//获取表对应进行chuck的列Column chunkKeyColumn =ChunkUtils.getChunkKeyColumn(table, sourceConfig.getChunkKeyColumn());finalList<ChunkRange> chunks;try{//根据chunkKeyColumn 进行split
chunks =splitTableIntoChunks(jdbc, tableId, chunkKeyColumn);}catch(SQLException e){thrownewFlinkRuntimeException("Failed to split chunks for table "+ tableId, e);}// convert chunks into splitsList<MySqlSnapshotSplit> splits =newArrayList<>();RowType chunkKeyColumnType =ChunkUtils.getChunkKeyColumnType(chunkKeyColumn);for(int i =0; i < chunks.size(); i++){ChunkRange chunk = chunks.get(i);//把每个chunk封装为MySqlSnapshotSplit MySqlSnapshotSplit split =createSnapshotSplit(
jdbc,
tableId,
i,
chunkKeyColumnType,
chunk.getChunkStart(),
chunk.getChunkEnd());
splits.add(split);}long end =System.currentTimeMillis();LOG.info("Split table {} into {} chunks, time cost: {}ms.",
tableId,
splits.size(),
end - start);return splits;}catch(Exception e){thrownewFlinkRuntimeException(String.format("Generate Splits for table %s error", tableId), e);}}
细看getChunkKeyColumn方法,获取ChunkKeyColumn:
publicstaticColumngetChunkKeyColumn(Table table,@NullableString chunkKeyColumn){//获取表主键,如果没有主键直接报错List<Column> primaryKeys = table.primaryKeyColumns();if(primaryKeys.isEmpty()){thrownewValidationException(String.format("Incremental snapshot for tables requires primary key,"+" but table %s doesn't have primary key.",
table.id()));}//如果有配置chunkKeyColumn,则进行此列是否为主键列,是则返回改列,不是则报错if(chunkKeyColumn !=null){Optional<Column> targetPkColumn =
primaryKeys.stream().filter(col -> chunkKeyColumn.equals(col.name())).findFirst();if(targetPkColumn.isPresent()){return targetPkColumn.get();}thrownewValidationException(String.format("Chunk key column '%s' doesn't exist in the primary key [%s] of the table %s.",
chunkKeyColumn,
primaryKeys.stream().map(Column::name).collect(Collectors.joining(",")),
table.id()));}//对于联合主键表,默认使用第一个主键列作为chunkKeyColumn// use the first column of primary key columns as the chunk key column by defaultreturn primaryKeys.get(0);}
privateList<ChunkRange>splitTableIntoChunks(JdbcConnection jdbc,TableId tableId,Column splitColumn)throwsSQLException{//获取splitColumnName,即为chunkKeyColumn列名finalString splitColumnName = splitColumn.name();//通过"SELECT MIN(splitColumnName), MAX(splitColumnName) FROM tableId" 语句查询splitColumnName列最大值,最小值finalObject[] minMaxOfSplitColumn =queryMinMax(jdbc, tableId, splitColumnName);finalObject min = minMaxOfSplitColumn[0];finalObject max = minMaxOfSplitColumn[1];if(min ==null|| max ==null|| min.equals(max)){//如果是空表,或者只有一行数据,则把整张表作为一个chunk返回// empty table, or only one row, return full table scan as a chunkreturnCollections.singletonList(ChunkRange.all());}//获取配置的scan.incremental.snapshot.chunk.size,即split chunk size,默认8096finalint chunkSize = sourceConfig.getSplitSize();//获取配置的chunk-key.even-distribution.factor.upper-bound,默认1000.0dfinaldouble distributionFactorUpper = sourceConfig.getDistributionFactorUpper();//获取配置的chunk-key.even-distribution.factor.lower-bound,默认0.05dfinaldouble distributionFactorLower = sourceConfig.getDistributionFactorLower();//判断splitColumn是否为可分布均匀列,判断标准为:列类型为BIGINT,INTEGER,DECIMAL则认为是可分布均匀列,其他类型认为是不可均匀列if(isEvenlySplitColumn(splitColumn)){//使用SHOW TABLE STATUS LIKE 'tableId',快速查询表总行数long approximateRowCnt =queryApproximateRowCnt(jdbc, tableId);//计算表分布因子,approximateRowCnt=0,则distributionFactor = Double.MAX_VALUE,默认为distributionFactor = (max - min + 1) / approximateRowCntdouble distributionFactor =calculateDistributionFactor(tableId, min, max, approximateRowCnt);//计算distributionFactor是否在distributionFactorLower和distributionFactorUpper之间,在则认为表分布均匀,不在则认为表分布不均匀boolean dataIsEvenlyDistributed =doubleCompare(distributionFactor, distributionFactorLower)>=0&&doubleCompare(distributionFactor, distributionFactorUpper)<=0;if(dataIsEvenlyDistributed){// the minimum dynamic chunk size is at least 1//如果是分布均匀的,则按分布因子获取dynamicChunkSizefinalint dynamicChunkSize =Math.max((int)(distributionFactor * chunkSize),1);//对分布均匀的表进行拆分returnsplitEvenlySizedChunks(
tableId, min, max, approximateRowCnt, chunkSize, dynamicChunkSize);}else{//对分布不均匀的表进行拆分returnsplitUnevenlySizedChunks(
jdbc, tableId, splitColumnName, min, max, chunkSize);}}else{//对splitColumn为不可分布均匀列的表进行拆分returnsplitUnevenlySizedChunks(jdbc, tableId, splitColumnName, min, max, chunkSize);}}
对分布均匀的表进行拆分 splitEvenlySizedChunks方法:
publicList<ChunkRange>splitEvenlySizedChunks(TableId tableId,Object min,Object max,long approximateRowCnt,int chunkSize,int dynamicChunkSize){LOG.info("Use evenly-sized chunk optimization for table {}, the approximate row count is {}, the chunk size is {}, the dynamic chunk size is {}",
tableId,
approximateRowCnt,
chunkSize,
dynamicChunkSize);//如果配置或默认的chunkSize >= 表数据条数approximateRowCnt,则只返回一个chunk,不进行splitif(approximateRowCnt <= chunkSize){// there is no more than one chunk, return full table as a chunkreturnCollections.singletonList(ChunkRange.all());}finalList<ChunkRange> splits =newArrayList<>();Object chunkStart =null;//第一个chunk的End值为min+dynamicChunkSizeObject chunkEnd =ObjectUtils.plus(min, dynamicChunkSize);//按dynamicChunkSize为step大小,对min到max进行拆分while(ObjectUtils.compare(chunkEnd, max)<=0){
splits.add(ChunkRange.of(chunkStart, chunkEnd));
chunkStart = chunkEnd;try{
chunkEnd =ObjectUtils.plus(chunkEnd, dynamicChunkSize);}catch(ArithmeticException e){// Stop chunk split to avoid dead loop when number overflows.break;}}//添加(max-min)/dynamicChunkSize 的余数,即上面while循环后剩余的数据// add the ending split
splits.add(ChunkRange.of(chunkStart,null));return splits;}
对分布不均匀的表进行拆分 splitUnevenlySizedChunks方法:
privateList<ChunkRange>splitUnevenlySizedChunks(JdbcConnection jdbc,TableId tableId,String splitColumnName,Object min,Object max,int chunkSize)throwsSQLException{LOG.info("Use unevenly-sized chunks for table {}, the chunk size is {}", tableId, chunkSize);finalList<ChunkRange> splits =newArrayList<>();Object chunkStart =null;//主要使用"SELECT MAX(%s) FROM (SELECT %s FROM %s WHERE %s >= ? ORDER BY %s ASC LIMIT %s) AS T"来查询每个chunk的End位置,并对查询到的End进行了判断,详细可看源码nextChunkEnd方法Object chunkEnd =nextChunkEnd(jdbc, min, tableId, splitColumnName, max, chunkSize);int count =0;//循环添加chunkwhile(chunkEnd !=null&&ObjectUtils.compare(chunkEnd, max)<=0){// we start from [null, min + chunk_size) and avoid [null, min)
splits.add(ChunkRange.of(chunkStart, chunkEnd));// may sleep a while to avoid DDOS on MySQL servermaySleep(count++, tableId);
chunkStart = chunkEnd;
chunkEnd =nextChunkEnd(jdbc, chunkEnd, tableId, splitColumnName, max, chunkSize);}//添加未被循环到的数据// add the ending split
splits.add(ChunkRange.of(chunkStart,null));return splits;}