hive从2.10升级到3.10后报错
query = Cannot find local variable 'query'
ne = {NucleusException@19314} "org.datanucleus.exceptions.NucleusException: Cannot add `SERDES`.`SERDE_ID` as referenced FK column for `CDS`"
this = {JDOQLQuery@19250} "SELECT FROM org.apache.hadoop.hive.metastore.model.MTableColumnStatistics WHERE dbName == ''"
parameters = {HashMap@19261} size = 0
candidateCmd = {ClassMetaData@19262} "<class name="MTableColumnStatistics"\n identity-type="DATASTORE"\n persistence-modifier="persistence-capable"\n table="TAB_COL_STATS"\n detachable="true"\n>\n<datastore-identity strategy="native">\n<column name="CS_ID"/>\n</datastore-identity>\n<inheritance strategy="new-table">\n</inheritance>\n<field name="avgColLen"\n persistence-modifier="PERSISTENT"\n null-value="NONE"\n default-fetch-group="true"\n embedded="true"\n unique="false">\n<column name="AVG_COL_LEN" jdbc-type="DOUBLE" allows-null="true"/>\n</field>\n<field name="colName"\n persistence-modifier="PERSISTENT"\n null-value="NONE"\n default-fetch-group="true"\n embedded="true"\n unique="false">\n<column name="COLUMN_NAME" jdbc-type="VARCHAR" allows-null="false" length="128"/>\n</field>\n<field name="colType"\n persistence-modifier="PERSISTENT"\n null-value="NONE"\n default-fetch-group="true"\n embedded="true"\n unique="false">\n<column name="COL"
startTime = 1562246434198
stmt = null
ne = {NucleusException@19314} "org.datanucleus.exceptions.NucleusException: Cannot add `SERDES`.`SERDE_ID` as referenced FK column for `CDS`"
result = null
candidateClass = {Class@12512} "class org.apache.hadoop.hive.metastore.model.MTableColumnStatistics"
datastoreCompilation = {RDBMSQueryCompilation@19264}
statementReturnsEmpty = false
ec = {ExecutionContextThreadedImpl@19216}
subclasses = true
这个报错主要是
Cannot add `SERDES`.`SERDE_ID` as referenced FK column for `CDS`"
一看这个就是HIVE元数据报错,但是为什么会出现外键问题我一直思考不出来,所以只能Debug源代码。这个现象开始出现在SparkSession无法查询到表,但是也没有报错信息,这个可是太恶心了,因为找不到堆栈问题,经过测试发现建表的时候会有堆栈信息,然后根据堆栈找到如下代码。
org.datanucleus.store.rdbms.table.TableImpl;
private Map getExistingForeignKeys(Connection conn)
throws SQLException
{
HashMap foreignKeysByName = new HashMap();
if (tableExistsInDatastore(conn))
{
StoreSchemaHandler handler = storeMgr.getSchemaHandler();
IdentifierFactory idFactory = storeMgr.getIdentifierFactory();
RDBMSTableFKInfo tableFkInfo = (RDBMSTableFKInfo)handler.getSchemaData(conn, "foreign-keys",
new Object[] {this});
Iterator fksIter = tableFkInfo.getChildren().iterator();
while (fksIter.hasNext())
{
ForeignKeyInfo fkInfo = (ForeignKeyInfo)fksIter.next();
DatastoreIdentifier fkIdentifier;
String fkName = (String)fkInfo.getProperty("fk_name");
if (fkName == null)
{
fkIdentifier = idFactory.newForeignKeyIdentifier(this, foreignKeysByName.size());
}
else
{
fkIdentifier = idFactory.newIdentifier(IdentifierType.FOREIGN_KEY, fkName);
}
short deferrability = ((Short)fkInfo.getProperty("deferrability")).shortValue();
boolean initiallyDeferred = deferrability == DatabaseMetaData.importedKeyInitiallyDeferred;
ForeignKey fk = (ForeignKey) foreignKeysByName.get(fkIdentifier);
if (fk == null)
{
fk = new ForeignKey(initiallyDeferred);
fk.setName(fkIdentifier.getIdentifierName());
foreignKeysByName.put(fkIdentifier, fk);
}
String pkTableName = (String)fkInfo.getProperty("pk_table_name");
DatastoreClass refTable = storeMgr.getDatastoreClass(
idFactory.newTableIdentifier(pkTableName));
if (refTable != null)
{
String fkColumnName = (String)fkInfo.getProperty("fk_column_name");
String pkColumnName = (String)fkInfo.getProperty("pk_column_name");
DatastoreIdentifier colName = idFactory.newIdentifier(IdentifierType.COLUMN, fkColumnName);
DatastoreIdentifier refColName = idFactory.newIdentifier(IdentifierType.COLUMN, pkColumnName);
Column col = columnsByName.get(colName);
Column refCol = refTable.getColumn(refColName);
if (col != null && refCol != null)
{
fk.addColumn(col, refCol);
}
else
{
//TODO throw exception?
}
}
}
}
return foreignKeysByName;
}
和
public void setColumn(int seq, Column col, Column refCol)
{
if (table == null)
{
table = col.getTable();
refTable = (DatastoreClass) refCol.getTable();
dba = table.getStoreManager().getDatastoreAdapter();
}
else
{
if (!table.equals(col.getTable()))
{
throw new NucleusException("Cannot add " + col + " as FK column for " + table).setFatal();
}
if (!refTable.equals(refCol.getTable()))
{
throw new NucleusException("Cannot add " + refCol + " as referenced FK column for " + refTable).setFatal();
}
}
setMinSize(columns, seq + 1);
setMinSize(refColumns, seq + 1);
columns.set(seq, col);
refColumns.set(seq, refCol);
}
其实这个也没事,但是这个东西恶心在他扫描的是整个库,不是连接库,所以我这个库里有以前版本的hive元数据表,他把他那里的外键也算进去了,所以在检验的时候报错,删除以前的库就正常了,我只能说太恶心了