更多优秀文章，请扫码关注个人微信公众号或搜索“程序猿小杨”添加。
一、错误展现：
Caused by: io.debezium.connector.oracle.logminer.parser.DmlParserException: Failed to parse insert DML: 'insert into "HIS_DATA".
at io.debezium.connector.oracle.logminer.parser.LogMinerDmlParser.parseInsert(LogMinerDmlParser.java:109)
at io.debezium.connector.oracle.logminer.parser.LogMinerDmlParser.parse(LogMinerDmlParser.java:73)
at io.debezium.connector.oracle.logminer.processor.AbstractLogMinerEventProcessor.parseDmlStatement(AbstractLogMinerEventProcessor.java:1078)
... 16 common frames omitted
Caused by: java.lang.ArrayIndexOutOfBoundsException: 74
at io.debezium.connector.oracle.logminer.parser.LogMinerDmlParser.parseColumnListClause(LogMinerDmlParser.java:239)
at io.debezium.connector.oracle.logminer.parser.LogMinerDmlParser.parseInsert(LogMinerDmlParser.java:100)
... 18 common frames omitted二、问题原因：
通过分析源代码，发现debezium-connector-oracle-1.9.7.Final中的方法存在问题，造成数组越界，一般方式场景在：oracle某张表表结构发生新增字段，然后flink cdc的任务加载的还是旧的表字段信息，当被监控表业务场景写入数据的时候，由于sql里面已经带了新增字段，但是table里面的字段还是旧的，造成解析错误，源代码如下：
private int parseColumnListClause(String sql, int start, String[] columnNames) {
int index = start;
boolean inQuote = false;
for(int var6 = 0; index < sql.length(); ++index) {
char c = sql.charAt(index);
if (c == '(' && !inQuote) {
start = index + 1;
} else {
if (c == ')' && !inQuote) {
++index;
break;
}
if (c == '"') {
if (inQuote) {
inQuote = false;
columnNames[var6++] = sql.substring(start + 1, index);
start = index + 2;
} else {
inQuote = true;
}
}
}
}
return index;
}三、解决方案：
解决思路：重写io.debezium.connector.oracle.logminer.parser下的LogMinerDmlParser类中的方法即可，方案有两种：
方案1：如果新增字段的内容不需要，比如：只需获取主键信息及对应的值，那就忽略掉新增的字段，新增字段内容不包含在解析后的数据中。那只需要调整一下代码即可：
在代码中新建一个包路径为：io.debezium.connector.oracle.logminer.parser，
复制源码路径下的LogMinerDmlParser类中内容：
//
// Source code recreated from a .class file by IntelliJ IDEA
// (powered by Fernflower decompiler)
//
package io.debezium.connector.oracle.logminer.parser;
import io.debezium.DebeziumException;
import io.debezium.connector.oracle.logminer.LogMinerHelper;
import io.debezium.relational.Column;
import io.debezium.relational.Table;
public class LogMinerDmlParser implements DmlParser {
private static final String NULL_SENTINEL = "${DBZ_NULL}";
private static final String NULL = "NULL";
private static final String INSERT_INTO = "insert into ";
private static final String UPDATE = "update ";
private static final String DELETE_FROM = "delete from ";
private static final String AND = "and ";
private static final String OR = "or ";
private static final String SET = " set ";
private static final String WHERE = " where ";
private static final String VALUES = " values ";
private static final String IS_NULL = "IS NULL";
private static final String UNSUPPORTED = "Unsupported";
private static final String UNSUPPORTED_TYPE = "Unsupported Type";
private static final int INSERT_INTO_LENGTH = "insert into ".length();
private static final int UPDATE_LENGTH = "update ".length();
private static final int DELETE_FROM_LENGTH = "delete from ".length();
private static final int VALUES_LENGTH = " values ".length();
private static final int SET_LENGTH = " set ".length();
private static final int WHERE_LENGTH = " where ".length();
public LogMinerDmlParser() {
}
public LogMinerDmlEntry parse(String sql, Table table) {
if (table == null) {
throw new DmlParserException("DML parser requires a non-null table");
} else {
if (sql != null && sql.length() > 0) {
switch(sql.charAt(0)) {
case 'd':
return this.parseDelete(sql, table);
case 'i':
return this.parseInsert(sql, table);
case 'u':
return this.parseUpdate(sql, table);
}
}
throw new DmlParserException("Unknown supported SQL '" + sql + "'");
}
}
private LogMinerDmlEntry parseInsert(String sql, Table table) {
try {
int index = INSERT_INTO_LENGTH;
index = this.parseTableName(sql, index);
String[] columnNames = new String[table.columns().size()];
index = this.parseColumnListClause(sql, index, columnNames);
Object[] newValues = new Object[table.columns().size()];
this.parseColumnValuesClause(sql, index, columnNames, newValues, table);
return LogMinerDmlEntryImpl.forInsert(newValues);
} catch (Exception var6) {
throw new DmlParserException("Failed to parse insert DML: '" + sql + "'", var6);
}
}
private LogMinerDmlEntry parseUpdate(String sql, Table table) {
try {
int index = UPDATE_LENGTH;
index = this.parseTableName(sql, index);
Object[] newValues = new Object[table.columns().size()];
index = this.parseSetClause(sql, index, newValues, table);
Object[] oldValues = new Object[table.columns().size()];
this.parseWhereClause(sql, index, oldValues, table);
for(int i = 0; i < oldValues.length; ++i) {
oldValues[i] = ParserUtils.getColumnUnavailableValue(oldValues[i], (Column)table.columns().get(i));
if (newValues[i] == "${DBZ_NULL}") {
newValues[i] = null;
} else if (newValues[i] == null) {
newValues[i] = oldValues[i];
}
}
return LogMinerDmlEntryImpl.forUpdate(newValues, oldValues);
} catch (Exception var7) {
throw new DmlParserException("Failed to parse update DML: '" + sql + "'", var7);
}
}
private LogMinerDmlEntry parseDelete(String sql, Table table) {
try {
int index = DELETE_FROM_LENGTH;
index = this.parseTableName(sql, index);
Object[] oldValues = new Object[table.columns().size()];
this.parseWhereClause(sql, index, oldValues, table);
ParserUtils.setColumnUnavailableValues(oldValues, table);
return LogMinerDmlEntryImpl.forDelete(oldValues);
} catch (Exception var5) {
throw new DmlParserException("Failed to parse delete DML: '" + sql + "'", var5);
}
}
private int parseTableName(String sql, int index) {
for(boolean inQuote = false; index < sql.length(); ++index) {
char c = sql.charAt(index);
if (c == '"') {
if (inQuote) {
inQuote = false;
} else {
inQuote = true;
}
} else if ((c == ' ' || c == '(') && !inQuote) {
break;
}
}
return index;
}
private int parseColumnListClause(String sql, int start, String[] columnNames) {
int index = start;
boolean inQuote = false;
for(int var6 = 0; index < sql.length(); ++index) {
char c = sql.charAt(index);
if (c == '(' && !inQuote) {
start = index + 1;
} else {
if (c == ')' && !inQuote) {
++index;
break;
}
if (c == '"') {
if (inQuote) {
inQuote = false;
columnNames[var6++] = sql.substring(start + 1, index);
start = index + 2;
} else {
inQuote = true;
}
}
}
}
return index;
}
private int parseColumnValuesClause(String sql, int start, String[] columnNames, Object[] values, Table table) {
int nested = 0;
boolean inQuote = false;
boolean inValues = false;
if (sql.indexOf(" values ", start) != start) {
throw new DebeziumException("Failed to parse DML: " + sql);
} else {
int index = start + VALUES_LENGTH;
int columnIndex = 0;
for(StringBuilder collectedValue = null; index < sql.length(); ++index) {
char c = sql.charAt(index);
if (inQuote) {
if (c != '\'') {
collectedValue.append(c);
} else if (sql.charAt(index + 1) == '\'') {
collectedValue.append('\'');
++index;
continue;
}
}
if (c == '(' && !inQuote && !inValues) {
inValues = true;
start = index + 1;
} else if (c == '(' && !inQuote) {
++nested;
} else if (c == '\'') {
if (inQuote) {
inQuote = false;
} else {
inQuote = true;
collectedValue = new StringBuilder();
}
} else if (!inQuote && (c == ',' || c == ')')) {
if (c == ')' && nested != 0) {
--nested;
} else if (c != ',' || nested == 0) {
if (sql.charAt(start) == '\'' && sql.charAt(index - 1) == '\'') {
int position = LogMinerHelper.getColumnIndexByName(columnNames[columnIndex], table);
values[position] = collectedValue.toString();
collectedValue = null;
} else {
String s = sql.substring(start, index);
if (!s.equals("Unsupported Type") && !s.equals("NULL")) {
int position = LogMinerHelper.getColumnIndexByName(columnNames[columnIndex], table);
values[position] = s;
}
}
++columnIndex;
start = index + 1;
}
}
}
return index;
}
}
private int parseSetClause(String sql, int start, Object[] newValues, Table table) {
boolean inDoubleQuote = false;
boolean inSingleQuote = false;
boolean inColumnName = true;
boolean inColumnValue = false;
boolean inSpecial = false;
int nested = 0;
int set = sql.indexOf(" set ", start);
if (set == -1) {
throw new DebeziumException("Failed to parse DML: " + sql);
} else {
if (set != start) {
start = set;
}
start += SET_LENGTH;
int index = start;
String currentColumnName = null;
for(StringBuilder collectedValue = null; index < sql.length(); ++index) {
char c = sql.charAt(index);
char lookAhead = index + 1 < sql.length() ? sql.charAt(index + 1) : 0;
if (inSingleQuote) {
if (c != '\'') {
collectedValue.append(c);
} else if (lookAhead == '\'') {
collectedValue.append('\'');
++index;
continue;
}
}
if (c == '"' && inColumnName) {
if (inDoubleQuote) {
inDoubleQuote = false;
currentColumnName = sql.substring(start + 1, index);
start = index + 1;
inColumnName = false;
} else {
inDoubleQuote = true;
start = index;
}
} else if (c == '=' && !inColumnName && !inColumnValue) {
inColumnValue = true;
++index;
start = index + 1;
} else if (nested != 0 || c != ' ' || lookAhead != '|') {
int position;
if (nested == 0 && c == '|' && lookAhead == '|' && !inSingleQuote) {
for(position = index + 2; position < sql.length(); ++position) {
if (sql.charAt(position) != ' ') {
index = position - 1;
break;
}
}
} else if (c == '\'' && inColumnValue) {
if (inSingleQuote && lookAhead == '\'') {
++index;
} else if (inSingleQuote) {
inSingleQuote = false;
if (nested == 0) {
position = LogMinerHelper.getColumnIndexByName(currentColumnName, table);
newValues[position] = collectedValue.toString();
collectedValue = null;
start = index + 1;
inColumnValue = false;
inColumnName = false;
}
} else {
if (!inSpecial) {
start = index;
}
inSingleQuote = true;
collectedValue = new StringBuilder();
}
} else if (c == ',' && !inColumnValue && !inColumnName) {
inColumnName = true;
++index;
start = index;
} else if (inColumnValue && !inSingleQuote) {
if (!inSpecial) {
start = index;
inSpecial = true;
}
if (c == '(') {
++nested;
} else if (c == ')' && nested > 0) {
--nested;
} else if ((c == ',' || c == ' ' || c == ';') && nested == 0) {
String value = sql.substring(start, index);
int position;
if (!value.equals("NULL") && !value.equals("Unsupported Type")) {
if (!value.equals("Unsupported")) {
position = LogMinerHelper.getColumnIndexByName(currentColumnName, table);
newValues[position] = value;
start = index + 1;
inColumnValue = false;
inSpecial = false;
inColumnName = true;
}
} else {
if (value.equals("NULL")) {
position = LogMinerHelper.getColumnIndexByName(currentColumnName, table);
newValues[position] = "${DBZ_NULL}";
}
start = index + 1;
inColumnValue = false;
inSpecial = false;
inColumnName = true;
}
}
} else if (!inDoubleQuote && !inSingleQuote && c == 'w' && lookAhead == 'h' && sql.indexOf(" where ", index - 1) == index - 1) {
--index;
break;
}
}
}
return index;
}
}
private int parseWhereClause(String sql, int start, Object[] values, Table table) {
int nested = 0;
boolean inColumnName = true;
boolean inColumnValue = false;
boolean inDoubleQuote = false;
boolean inSingleQuote = false;
boolean inSpecial = false;
if (start >= sql.length()) {
return start;
} else {
int where = sql.indexOf(" where ", start);
if (where == -1) {
throw new DebeziumException("Failed to parse DML: " + sql);
} else {
if (where != start) {
start = where;
}
start += WHERE_LENGTH;
int index = start;
String currentColumnName = null;
for(StringBuilder collectedValue = null; index < sql.length(); ++index) {
char c = sql.charAt(index);
char lookAhead = index + 1 < sql.length() ? sql.charAt(index + 1) : 0;
if (inSingleQuote) {
if (c != '\'') {
collectedValue.append(c);
} else if (lookAhead == '\'') {
collectedValue.append('\'');
++index;
continue;
}
}
if (c == '"' && inColumnName) {
if (inDoubleQuote) {
inDoubleQuote = false;
currentColumnName = sql.substring(start + 1, index);
start = index + 1;
inColumnName = false;
} else {
inDoubleQuote = true;
start = index;
}
} else if (c == '=' && !inColumnName && !inColumnValue) {
inColumnValue = true;
++index;
start = index + 1;
} else if (c == 'I' && !inColumnName && !inColumnValue) {
if (sql.indexOf("IS NULL", index) == index) {
index += 6;
start = index;
}
} else {
int i;
if (c == '\'' && inColumnValue) {
if (inSingleQuote && lookAhead == '\'') {
++index;
} else if (inSingleQuote) {
inSingleQuote = false;
if (nested == 0) {
i = LogMinerHelper.getColumnIndexByName(currentColumnName, table);
values[i] = collectedValue.toString();
collectedValue = null;
start = index + 1;
inColumnValue = false;
inColumnName = false;
}
} else {
if (!inSpecial) {
start = index;
}
inSingleQuote = true;
collectedValue = new StringBuilder();
}
} else if (inColumnValue && !inSingleQuote) {
if (!inSpecial) {
start = index;
inSpecial = true;
}
if (c == '(') {
++nested;
} else if (c == ')' && nested > 0) {
--nested;
} else if (nested != 0 || c != ' ' || lookAhead != '|') {
if (nested == 0 && c == '|' && lookAhead == '|') {
for(i = index + 2; i < sql.length(); ++i) {
if (sql.charAt(i) != ' ') {
index = i - 1;
break;
}
}
} else if ((c == ';' || c == ' ') && nested == 0) {
String value = sql.substring(start, index);
if (!value.equals("NULL") && !value.equals("Unsupported Type")) {
if (!value.equals("Unsupported")) {
int position = LogMinerHelper.getColumnIndexByName(currentColumnName, table);
values[position] = value;
start = index + 1;
inColumnValue = false;
inSpecial = false;
inColumnName = true;
}
} else {
start = index + 1;
inColumnValue = false;
inSpecial = false;
inColumnName = true;
}
}
}
} else if (!inColumnValue && !inColumnName) {
if (c == 'a' && lookAhead == 'n' && sql.indexOf("and ", index) == index) {
index += 3;
start = index;
inColumnName = true;
} else if (c == 'o' && lookAhead == 'r' && sql.indexOf("or ", index) == index) {
index += 2;
start = index;
inColumnName = true;
}
}
}
}
return index;
}
}
}
}
调整其中的内容为：
if (inQuote) {
inQuote = false;
//调整方法即可
int columnNamesNum=columnIndex++;
// log.info("columnNamesNum的长度为：{}",columnNamesNum);
if(columnNamesNum < columnNames.length){
//log.info("columnNames[columnNamesNum]的值为："+sql.substring(start + 1, index));
columnNames[columnNamesNum] = sql.substring(start + 1, index);
start = index + 2;
}
} else {
inQuote = true;
}方案2：如果需要新增字段的值，那就利用sql中的字段去解析获取对应的值，不要使用table中的列，这块代码需要自己发挥。
扫一扫
1530
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
