hive2.3.4
presto0.215
使用hive2.3.4的beeline连接presto报错
$ beeline -d com.facebook.presto.jdbc.PrestoDriver -u "jdbc:presto://localhost:8080/hive" Error: Unrecognized connection property 'url' (state=,code=0)
增加--verbose查看详细报错
$ beeline -d com.facebook.presto.jdbc.PrestoDriver -u "jdbc:presto://localhost:8080/hive" --verbose Error: Unrecognized connection property 'url' (state=,code=0) java.sql.SQLException: Unrecognized connection property 'url' at com.facebook.presto.jdbc.PrestoDriverUri.validateConnectionProperties(PrestoDriverUri.java:316) at com.facebook.presto.jdbc.PrestoDriverUri.<init>(PrestoDriverUri.java:95) at com.facebook.presto.jdbc.PrestoDriverUri.<init>(PrestoDriverUri.java:85) at com.facebook.presto.jdbc.PrestoDriver.connect(PrestoDriver.java:87) at java.sql.DriverManager.getConnection(DriverManager.java:664) at java.sql.DriverManager.getConnection(DriverManager.java:208) at org.apache.hive.beeline.DatabaseConnection.connect(DatabaseConnection.java:145) at org.apache.hive.beeline.DatabaseConnection.getConnection(DatabaseConnection.java:209) at org.apache.hive.beeline.Commands.connect(Commands.java:1641) at org.apache.hive.beeline.Commands.connect(Commands.java:1536) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hive.beeline.ReflectiveCommandHandler.execute(ReflectiveCommandHandler.java:56) at org.apache.hive.beeline.BeeLine.execCommandWithPrefix(BeeLine.java:1273) at org.apache.hive.beeline.BeeLine.dispatch(BeeLine.java:1312) at org.apache.hive.beeline.BeeLine.connectUsingArgs(BeeLine.java:867) at org.apache.hive.beeline.BeeLine.initArgs(BeeLine.java:776) at org.apache.hive.beeline.BeeLine.begin(BeeLine.java:1010) at org.apache.hive.beeline.BeeLine.mainWithInputRedirection(BeeLine.java:519) at org.apache.hive.beeline.BeeLine.main(BeeLine.java:501) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.util.RunJar.run(RunJar.java:221) at org.apache.hadoop.util.RunJar.main(RunJar.java:136)
这里是connect过程报错,connect参数为
Usage: connect <url> <username> <password> [driver]
跟进hive和presto代码
org.apache.hive.beeline.BeeLine
boolean dispatch(String line) { ... if (isBeeLine) { if (line.startsWith(COMMAND_PREFIX)) { // handle SQLLine command in beeline which starts with ! and does not end with ; return execCommandWithPrefix(line); } else { return commands.sql(line, getOpts().getEntireLineAsCommand()); } } else { return commands.sql(line, getOpts().getEntireLineAsCommand()); }
beeline中的命令和sql都会经过dispatch,命令执行的是execCommandWithPrefix,然后会反射调用Commands
org.apache.hive.beeline.Commands
public boolean connect(String line) throws Exception { String example = "Usage: connect <url> <username> <password> [driver]" + BeeLine.getSeparator(); String[] parts = beeLine.split(line); if (parts == null) { return false; } if (parts.length < 2) { return beeLine.error(example); } String url = parts.length < 2 ? null : parts[1]; String user = parts.length < 3 ? null : parts[2]; String pass = parts.length < 4 ? null : parts[3]; String driver = parts.length < 5 ? null : parts[4]; Properties props = new Properties(); if (url != null) { String saveUrl = getUrlToUse(url); props.setProperty(JdbcConnectionParams.PROPERTY_URL, saveUrl); } String value = null; if (driver != null) { props.setProperty(JdbcConnectionParams.PROPERTY_DRIVER, driver); } else { value = Utils.parsePropertyFromUrl(url, JdbcConnectionParams.PROPERTY_DRIVER); if (value != null) { props.setProperty(JdbcConnectionParams.PROPERTY_DRIVER, value); } } if (user != null) { props.setProperty(JdbcConnectionParams.AUTH_USER, user); } else { value = Utils.parsePropertyFromUrl(url, JdbcConnectionParams.AUTH_USER); if (value != null) { props.setProperty(JdbcConnectionParams.AUTH_USER, value); } } if (pass != null) { props.setProperty(JdbcConnectionParams.AUTH_PASSWD, pass); } else { value = Utils.parsePropertyFromUrl(url, JdbcConnectionParams.AUTH_PASSWD); if (value != null) { props.setProperty(JdbcConnectionParams.AUTH_PASSWD, value); } } value = Utils.parsePropertyFromUrl(url, JdbcConnectionParams.AUTH_TYPE); if (value != null) { props.setProperty(JdbcConnectionParams.AUTH_TYPE, value); } return connect(props); } public boolean connect(Properties props) throws IOException { String url = getProperty(props, new String[] { JdbcConnectionParams.PROPERTY_URL, "javax.jdo.option.ConnectionURL", "ConnectionURL", }); String driver = getProperty(props, new String[] { JdbcConnectionParams.PROPERTY_DRIVER, "javax.jdo.option.ConnectionDriverName", "ConnectionDriverName", }); String username = getProperty(props, new String[] { JdbcConnectionParams.AUTH_USER, "javax.jdo.option.ConnectionUserName", "ConnectionUserName", }); String password = getProperty(props, new String[] { JdbcConnectionParams.AUTH_PASSWD, "javax.jdo.option.ConnectionPassword", "ConnectionPassword", }); if (url == null || url.length() == 0) { return beeLine.error("Property \"url\" is required"); } if (driver == null || driver.length() == 0) { if (!beeLine.scanForDriver(url)) { return beeLine.error(beeLine.loc("no-driver", url)); } } String auth = getProperty(props, new String[] {JdbcConnectionParams.AUTH_TYPE}); if (auth == null) { auth = beeLine.getOpts().getAuthType(); if (auth != null) { props.setProperty(JdbcConnectionParams.AUTH_TYPE, auth); } } beeLine.info("Connecting to " + url); if (Utils.parsePropertyFromUrl(url, JdbcConnectionParams.AUTH_PRINCIPAL) == null) { String urlForPrompt = url.substring(0, url.contains(";") ? url.indexOf(';') : url.length()); if (username == null) { username = beeLine.getConsoleReader().readLine("Enter username for " + urlForPrompt + ": "); } props.setProperty(JdbcConnectionParams.AUTH_USER, username); if (password == null) { password = beeLine.getConsoleReader().readLine("Enter password for " + urlForPrompt + ": ", new Character('*')); } props.setProperty(JdbcConnectionParams.AUTH_PASSWD, password); } try { beeLine.getDatabaseConnections().setConnection( new DatabaseConnection(beeLine, driver, url, props)); beeLine.getDatabaseConnection().getConnection(); if (!beeLine.isBeeLine()) { beeLine.updateOptsForCli(); } beeLine.runInit(); beeLine.setCompletions(); beeLine.getOpts().setLastConnectedUrl(url); return true; } catch (SQLException sqle) { beeLine.getDatabaseConnections().remove(); return beeLine.error(sqle); } catch (IOException ioe) { return beeLine.error(ioe); } }
注意props字段的赋值过程
org.apache.hive.beeline.DatabaseConnection
public DatabaseConnection(BeeLine beeLine, String driver, String url, Properties info) throws SQLException { this.beeLine = beeLine; this.driver = driver; this.url = url; this.info = info; } boolean connect() throws SQLException { ... Map<String, String> hiveVars = beeLine.getOpts().getHiveVariables(); if (hiveVars != null){ for (Map.Entry<String, String> var : hiveVars.entrySet()) { info.put(HIVE_VAR_PREFIX + var.getKey(), var.getValue()); } } Map<String, String> hiveConfVars = beeLine.getOpts().getHiveConfVariables(); if (hiveConfVars != null){ for (Map.Entry<String, String> var : hiveConfVars.entrySet()) { info.put(HIVE_CONF_PREFIX + var.getKey(), var.getValue()); } } if (isDriverRegistered) { // if the driver registered in the driver manager, get the connection via the driver manager setConnection(DriverManager.getConnection(getUrl(), info)); } else {
Commands中props赋值给DatabaseConnection的info
java.sql.DriverManager
public static Connection getConnection(String url, java.util.Properties info) throws SQLException { return (getConnection(url, info, Reflection.getCallerClass())); } private static Connection getConnection( String url, java.util.Properties info, Class<?> caller) throws SQLException { ... Connection con = aDriver.driver.connect(url, info);
DatabaseConnection的info传递给DriverManager然后传递给Driver,下面看具体的PrestoDriver
com.facebook.presto.jdbc.PrestoDriver
public Connection connect(String url, Properties info) throws SQLException { if (!acceptsURL(url)) { return null; } PrestoDriverUri uri = new PrestoDriverUri(url, info); OkHttpClient.Builder builder = httpClient.newBuilder(); uri.setupClient(builder); QueryExecutor executor = new QueryExecutor(builder.build()); return new PrestoConnection(uri, executor); }
info传递给PrestoDriverUri进行初始化
com.facebook.presto.jdbc.PrestoDriverUri
public PrestoDriverUri(String url, Properties driverProperties) throws SQLException { this(parseDriverUrl(url), driverProperties); } private PrestoDriverUri(URI uri, Properties driverProperties) throws SQLException { this.uri = requireNonNull(uri, "uri is null"); address = HostAndPort.fromParts(uri.getHost(), uri.getPort()); properties = mergeConnectionProperties(uri, driverProperties); validateConnectionProperties(properties); // enable SSL by default for standard port useSecureConnection = SSL.getValue(properties).orElse(uri.getPort() == 443); initCatalogAndSchema(); } private static void validateConnectionProperties(Properties connectionProperties) throws SQLException { for (String propertyName : connectionProperties.stringPropertyNames()) { if (ConnectionProperties.forKey(propertyName) == null) { throw new SQLException(format("Unrecognized connection property '%s'", propertyName)); } } for (ConnectionProperty<?> property : ConnectionProperties.allProperties()) { property.validate(connectionProperties); } }
所以报错“Unrecognized connection property 'url' ”意味着ConnectionProperties.forKey("url") == null
com.facebook.presto.jdbc.ConnectionProperties
ConnectionProperties中确实没有url,所以presto与hive的beeline不兼容;
尝试修改PrestoDriverUri.java,去掉校验:
com.facebook.presto.jdbc.PrestoDriverUri
private static void validateConnectionProperties(Properties connectionProperties) throws SQLException { // for (String propertyName : connectionProperties.stringPropertyNames()) { // if (ConnectionProperties.forKey(propertyName) == null) { // throw new SQLException(format("Unrecognized connection property '%s'", propertyName)); // } // } for (ConnectionProperty<?> property : ConnectionProperties.allProperties()) { property.validate(connectionProperties); } }
将以上几行注释
简单修改jar包中的class的方法,参考:https://www.cnblogs.com/barneywill/p/10552322.html
编译的时候发现一直报错,
presto-jdbc/com/facebook/presto/jdbc/PrestoDriverUri.java:145: error: method setupSocksProxy in class OkHttpUtil cannot be applied to given types; setupSocksProxy(builder, SOCKS_PROXY.getValue(properties)); ^ required: Builder,Optional<com.google.common.net.HostAndPort> found: Builder,Optional<com.facebook.presto.jdbc.internal.guava.net.HostAndPort> reason: argument mismatch; Optional<com.facebook.presto.jdbc.internal.guava.net.HostAndPort> cannot be converted to Optional<com.google.common.net.HostAndPort> presto-jdbc/com/facebook/presto/jdbc/PrestoDriverUri.java:146: error: method setupHttpProxy in class OkHttpUtil cannot be applied to given types; setupHttpProxy(builder, HTTP_PROXY.getValue(properties)); ^ required: Builder,Optional<com.google.common.net.HostAndPort> found: Builder,Optional<com.facebook.presto.jdbc.internal.guava.net.HostAndPort> reason: argument mismatch; Optional<com.facebook.presto.jdbc.internal.guava.net.HostAndPort> cannot be converted to Optional<com.google.common.net.HostAndPort> 2 errors
然后发现原来是presto-jdbc打包的时候设置了relocation,会将一些包做shade并且改package,详见:
https://github.com/prestodb/presto/blob/master/presto-jdbc/pom.xml
<properties> <air.main.basedir>${project.parent.basedir}</air.main.basedir> <shadeBase>com.facebook.presto.jdbc.internal</shadeBase> </properties> ... <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-shade-plugin</artifactId> <executions> <execution> <phase>package</phase> <goals> <goal>shade</goal> </goals> <configuration> <createSourcesJar>true</createSourcesJar> <shadeSourcesContent>true</shadeSourcesContent> <dependencyReducedPomLocation>${project.build.directory}/pom.xml</dependencyReducedPomLocation> <relocations> <relocation> <pattern>com.facebook.presto.client</pattern> <shadedPattern>${shadeBase}.client</shadedPattern> </relocation> <relocation> <pattern>com.facebook.presto.spi</pattern> <shadedPattern>${shadeBase}.spi</shadedPattern> </relocation> <relocation> <pattern>com.fasterxml.jackson</pattern> <shadedPattern>${shadeBase}.jackson</shadedPattern> </relocation> <relocation> <pattern>com.google.common</pattern> <shadedPattern>${shadeBase}.guava</shadedPattern> </relocation> <relocation> <pattern>com.google.thirdparty</pattern> <shadedPattern>${shadeBase}.guava</shadedPattern> </relocation> <relocation> <pattern>io.airlift</pattern> <shadedPattern>${shadeBase}.airlift</shadedPattern> </relocation> <relocation> <pattern>javax.annotation</pattern> <shadedPattern>${shadeBase}.javax.annotation</shadedPattern> </relocation> <relocation> <pattern>javax.inject</pattern> <shadedPattern>${shadeBase}.inject</shadedPattern> </relocation> <relocation> <pattern>org.openjdk.jol</pattern> <shadedPattern>${shadeBase}.jol</shadedPattern> </relocation> <relocation> <pattern>org.joda.time</pattern> <shadedPattern>${shadeBase}.joda.time</shadedPattern> </relocation> <relocation> <pattern>okhttp3</pattern> <shadedPattern>${shadeBase}.okhttp3</shadedPattern> </relocation> <relocation> <pattern>okio</pattern> <shadedPattern>${shadeBase}.okio</shadedPattern> </relocation> </relocations>
所以需要手工调整源代码中的package
okhttp3 -> com.facebook.presto.jdbc.internal.okhttp3
com.google.common -> com.facebook.presto.jdbc.internal.guava
com.facebook.presto.client -> com.facebook.presto.jdbc.internal.client
源代码地址:
https://github.com/prestodb/presto/blob/master/presto-jdbc/src/main/java/com/facebook/presto/jdbc/PrestoDriverUri.java
然后再编译就ok了,编译命令:
# javac -cp presto-jdbc-0.215.jar -d presto-jdbc presto-jdbc/com/facebook/presto/jdbc/PrestoDriverUri.java
然后重新打包,再使用beeline连接就ok了