Hive beeline 源代码解析

入口

Beeline 的入口是 org.apache.hive.beeline.Beeline 的 main 方法。在进行相关的初始化之后，进入execute，在 execute 方法里，每读取一个命令，进入 dispatch 执行。

private int execute(ConsoleReader reader, boolean exitOnError) {
    while (!exit) {
      try {
        // Execute one instruction; terminate on executing a script if there is an error
        // in silent mode, prevent the query and prompt being echoed back to terminal
        String line = (getOpts().isSilent() && getOpts().getScriptFile() != null) ? reader
            .readLine(null, mask) : reader.readLine(getPrompt());

        // trim line
        if (line != null) {
          line = line.trim();
        }

        if (!dispatch(line)) {
          lastExecutionResult = ERRNO_OTHER;
          if (exitOnError) {
            break;
          }
        } else if (line != null) {
          lastExecutionResult = ERRNO_OK;
        }

      } catch (Throwable t) {
        handleException(t);
        return ERRNO_OTHER;
      }
    }
    return lastExecutionResult;
  }

dispatch

dispatch 的主要逻辑如下，如果参数 String line 以 ‘!’ 开头，则执行 execCommandWithPrefix(line)。否则执行 commands.sql.

boolean dispatch(String line) {
    if (line == null) {
      // exit
      exit = true;
      return true;
    }

    if (line.trim().length() == 0) {
      return true;
    }

    if (isComment(line)) {
      return true;
    }

    line = line.trim();

    // save it to the current script, if any
    if (scriptOutputFile != null) {
      scriptOutputFile.addLine(line);
    }

    if (isHelpRequest(line)) {
      line = "!help";
    }

    if (isBeeLine) {
      if (line.startsWith(COMMAND_PREFIX)) {
        // handle SQLLine command in beeline which starts with ! and does not end with ;
        return execCommandWithPrefix(line);
      } else {
        return commands.sql(line, getOpts().getEntireLineAsCommand());
      }
    } else {
      return commands.sql(line, getOpts().getEntireLineAsCommand());
    }
  }

Command 的执行方式

所有 command 执行 execCommandWithPrefix。

execCommandWithPrefix

execCommandWithPrefix 先找到对应的 CommandHandler handler，再调用 handler.execute(line);

public boolean execCommandWithPrefix(String line) {
    Map<String, CommandHandler> cmdMap = new TreeMap<String, CommandHandler>();
    line = line.substring(1);
    for (int i = 0; i < commandHandlers.length; i++) {
      String match = commandHandlers[i].matches(line);
      if (match != null) {
        cmdMap.put(match, commandHandlers[i]);
      }
    }

    if (cmdMap.size() == 0) {
      return error(loc("unknown-command", line));
    }
    if (cmdMap.size() > 1) {
      // any exact match?
      CommandHandler handler = cmdMap.get(line);
      if (handler == null) {
        return error(loc("multiple-matches", cmdMap.keySet().toString()));
      }
      return handler.execute(line);
    }
    return cmdMap.values().iterator().next().execute(line);
  }

commandHandlers 列表如下：

final CommandHandler[] commandHandlers = new CommandHandler[] {
      new ReflectiveCommandHandler(this, new String[] {"quit", "done", "exit"},
          null),
      new ReflectiveCommandHandler(this, new String[] {"connect", "open"},
          new Completer[] {new StringsCompleter(getConnectionURLExamples())}),
      new ReflectiveCommandHandler(this, new String[] {"describe"},
          new Completer[] {new TableNameCompletor(this)}),
      new ReflectiveCommandHandler(this, new String[] {"indexes"},
          new Completer[] {new TableNameCompletor(this)}),
      new ReflectiveCommandHandler(this, new String[] {"primarykeys"},
          new Completer[] {new TableNameCompletor(this)}),
      new ReflectiveCommandHandler(this, new String[] {"exportedkeys"},
          new Completer[] {new TableNameCompletor(this)}),
      new ReflectiveCommandHandler(this, new String[] {"manual"},
          null),
      new ReflectiveCommandHandler(this, new String[] {"importedkeys"},
          new Completer[] {new TableNameCompletor(this)}),
      new ReflectiveCommandHandler(this, new String[] {"procedures"},
          null),
      new ReflectiveCommandHandler(this, new String[] {"tables"},
          null),
      new ReflectiveCommandHandler(this, new String[] {"typeinfo"},
          null),
      new ReflectiveCommandHandler(this, new String[] {"columns"},
          new Completer[] {new TableNameCompletor(this)}),
      new ReflectiveCommandHandler(this, new String[] {"reconnect"},
          null),
      new ReflectiveCommandHandler(this, new String[] {"dropall"},
          new Completer[] {new TableNameCompletor(this)}),
      new ReflectiveCommandHandler(this, new String[] {"history"},
          null),
      new ReflectiveCommandHandler(this, new String[] {"metadata"},
          new Completer[] {
              new StringsCompleter(getMetadataMethodNames())}),
      new ReflectiveCommandHandler(this, new String[] {"nativesql"},
          null),
      new ReflectiveCommandHandler(this, new String[] {"dbinfo"},
          null),
      new ReflectiveCommandHandler(this, new String[] {"rehash"},
          null),
      new ReflectiveCommandHandler(this, new String[] {"verbose"},
          null),
      new ReflectiveCommandHandler(this, new String[] {"run"},
          new Completer[] {new FileNameCompleter()}),
      new ReflectiveCommandHandler(this, new String[] {"batch"},
          null),
      new ReflectiveCommandHandler(this, new String[] {"list"},
          null),
      new ReflectiveCommandHandler(this, new String[] {"all"},
          null),
      new ReflectiveCommandHandler(this, new String[] {"go", "#"},
          null),
      new ReflectiveCommandHandler(this, new String[] {"script"},
          new Completer[] {new FileNameCompleter()}),
      new ReflectiveCommandHandler(this, new String[] {"record"},
          new Completer[] {new FileNameCompleter()}),
      new ReflectiveCommandHandler(this, new String[] {"brief"},
          null),
      new ReflectiveCommandHandler(this, new String[] {"close"},
          null),
      new ReflectiveCommandHandler(this, new String[] {"closeall"},
          null),
      new ReflectiveCommandHandler(this, new String[] {"isolation"},
          new Completer[] {new StringsCompleter(getIsolationLevels())}),
      new ReflectiveCommandHandler(this, new String[] {"outputformat"},
          new Completer[] {new StringsCompleter(
              formats.keySet().toArray(new String[0]))}),
      new ReflectiveCommandHandler(this, new String[] {"autocommit"},
          null),
      new ReflectiveCommandHandler(this, new String[] {"commit"},
          null),
      new ReflectiveCommandHandler(this, new String[] {"properties"},
          new Completer[] {new FileNameCompleter()}),
      new ReflectiveCommandHandler(this, new String[] {"rollback"},
          null),
      new ReflectiveCommandHandler(this, new String[] {"help", "?"},
          null),
      new ReflectiveCommandHandler(this, new String[] {"set"},
          getOpts().optionCompleters()),
      new ReflectiveCommandHandler(this, new String[] {"save"},
          null),
      new ReflectiveCommandHandler(this, new String[] {"scan"},
          null),
      new ReflectiveCommandHandler(this, new String[] {"sql"},
          null),
      new ReflectiveCommandHandler(this, new String[] {"sh"},
          null),
      new ReflectiveCommandHandler(this, new String[] {"call"},
          null),
      new ReflectiveCommandHandler(this, new String[] {"nullemptystring"},
          new Completer[] {new BooleanCompleter()}),
      new ReflectiveCommandHandler(this, new String[]{"addlocaldriverjar"},
          null),
      new ReflectiveCommandHandler(this, new String[]{"addlocaldrivername"},
          null),
      new ReflectiveCommandHandler(this, new String[]{"delimiter"},
          null)
  };

ReflectiveCommandHandler.execute

ReflectiveCommandHandler.execute 执行 Commands 的相应方法。

public boolean execute(String line) {
    lastException = null;
    ClientHook hook = ClientCommandHookFactory.get().getHook(beeLine, line);

    try {
      Object ob = beeLine.getCommands().getClass().getMethod(getName(),
          new Class[] {String.class})
          .invoke(beeLine.getCommands(), new Object[] {line});

      boolean result = (ob != null && ob instanceof Boolean && ((Boolean) ob).booleanValue());

      if (hook != null && result) {
        hook.postHook(beeLine);
      }

      return result;
    } catch (Throwable e) {
      lastException = e;
      return beeLine.error(e);
    }
  }

通过反射调用的是 Commands.connect(String)

public boolean connect(String line) throws Exception {
    String example = "Usage: connect <url> <username> <password> [driver]"
        + BeeLine.getSeparator();

    String[] parts = beeLine.split(line);
    if (parts == null) {
      return false;
    }

    if (parts.length < 2) {
      return beeLine.error(example);
    }

    String url = parts.length < 2 ? null : parts[1];
    String user = parts.length < 3 ? null : parts[2];
    String pass = parts.length < 4 ? null : parts[3];
    String driver = parts.length < 5 ? null : parts[4];

    Properties props = new Properties();
    if (url != null) {
      String saveUrl = getUrlToUse(url);
      props.setProperty(JdbcConnectionParams.PROPERTY_URL, saveUrl);
    }

    String value = null;
    if (driver != null) {
      props.setProperty(JdbcConnectionParams.PROPERTY_DRIVER, driver);
    } else {
      value = Utils.parsePropertyFromUrl(url, JdbcConnectionParams.PROPERTY_DRIVER);
      if (value != null) {
        props.setProperty(JdbcConnectionParams.PROPERTY_DRIVER, value);
      }
    }

    if (user != null) {
      props.setProperty(JdbcConnectionParams.AUTH_USER, user);
    } else {
      value = Utils.parsePropertyFromUrl(url, JdbcConnectionParams.AUTH_USER);
      if (value != null) {
        props.setProperty(JdbcConnectionParams.AUTH_USER, value);
      }
    }

    if (pass != null) {
      props.setProperty(JdbcConnectionParams.AUTH_PASSWD, pass);
    } else {
      value = Utils.parsePropertyFromUrl(url, JdbcConnectionParams.AUTH_PASSWD);
      if (value != null) {
        props.setProperty(JdbcConnectionParams.AUTH_PASSWD, value);
      }
    }

    value = Utils.parsePropertyFromUrl(url, JdbcConnectionParams.AUTH_TYPE);
    if (value != null) {
      props.setProperty(JdbcConnectionParams.AUTH_TYPE, value);
    }
    return connect(props);
  }

connect(String) 调用 connect(Properties)

public boolean connect(Properties props) throws IOException {
    String url = getProperty(props, new String[] {
        JdbcConnectionParams.PROPERTY_URL,
        "javax.jdo.option.ConnectionURL",
        "ConnectionURL",
    });
    String driver = getProperty(props, new String[] {
        JdbcConnectionParams.PROPERTY_DRIVER,
        "javax.jdo.option.ConnectionDriverName",
        "ConnectionDriverName",
    });
    String username = getProperty(props, new String[] {
        JdbcConnectionParams.AUTH_USER,
        "javax.jdo.option.ConnectionUserName",
        "ConnectionUserName",
    });
    String password = getProperty(props, new String[] {
        JdbcConnectionParams.AUTH_PASSWD,
        "javax.jdo.option.ConnectionPassword",
        "ConnectionPassword",
    });

    if (url == null || url.length() == 0) {
      return beeLine.error("Property \"url\" is required");
    }
    if (driver == null || driver.length() == 0) {
      if (!beeLine.scanForDriver(url)) {
        return beeLine.error(beeLine.loc("no-driver", url));
      }
    }

    String auth = getProperty(props, new String[] {JdbcConnectionParams.AUTH_TYPE});
    if (auth == null) {
      auth = beeLine.getOpts().getAuthType();
      if (auth != null) {
        props.setProperty(JdbcConnectionParams.AUTH_TYPE, auth);
      }
    }

    beeLine.info("Connecting to " + url);
    if (Utils.parsePropertyFromUrl(url, JdbcConnectionParams.AUTH_PRINCIPAL) == null) {
      String urlForPrompt = url.substring(0, url.contains(";") ? url.indexOf(';') : url.length());
      if (username == null) {
        username = beeLine.getConsoleReader().readLine("Enter username for " + urlForPrompt + ": ");
      }
      props.setProperty(JdbcConnectionParams.AUTH_USER, username);
      if (password == null) {
        password = beeLine.getConsoleReader().readLine("Enter password for " + urlForPrompt + ": ",
          new Character('*'));
      }
      props.setProperty(JdbcConnectionParams.AUTH_PASSWD, password);
    }

    try {
      beeLine.getDatabaseConnections().setConnection(
          new DatabaseConnection(beeLine, driver, url, props));
      beeLine.getDatabaseConnection().getConnection();

      if (!beeLine.isBeeLine()) {
        beeLine.updateOptsForCli();
      }
      beeLine.runInit();

      beeLine.setCompletions();
      beeLine.getOpts().setLastConnectedUrl(url);
      return true;
    } catch (SQLException sqle) {
      beeLine.getDatabaseConnections().remove();
      return beeLine.error(sqle);
    } catch (IOException ioe) {
      return beeLine.error(ioe);
    }
  }

connect(Properties) 的 beeLine.getDatabaseConnection().getConnection();

public Connection getConnection() throws SQLException {
    if (connection != null) {
      return connection;
    }
    connect();
    return connection;
  }

DatabaseConnection#connect

看到是通过 JDBC 的 DriverManager.getConnection(getUrl(), info) 创建的 JDBC Connection。

boolean connect() throws SQLException {
    try {
      if (driver != null && driver.length() != 0) {
        Class.forName(driver);
      }
    } catch (ClassNotFoundException cnfe) {
      return beeLine.error(cnfe);
    }

    boolean isDriverRegistered = false;
    try {
      isDriverRegistered = DriverManager.getDriver(getUrl()) != null;
    } catch (Exception e) {
    }

    try {
      close();
    } catch (Exception e) {
      return beeLine.error(e);
    }

    Map<String, String> hiveVars = beeLine.getOpts().getHiveVariables();
    if (hiveVars != null){
      for (Map.Entry<String, String> var : hiveVars.entrySet()) {
        info.put(HIVE_VAR_PREFIX + var.getKey(), var.getValue());
      }
    }

    Map<String, String> hiveConfVars = beeLine.getOpts().getHiveConfVariables();
    if (hiveConfVars != null){
      for (Map.Entry<String, String> var : hiveConfVars.entrySet()) {
        info.put(HIVE_CONF_PREFIX + var.getKey(), var.getValue());
      }
    }

    if (isDriverRegistered) {
      // if the driver registered in the driver manager, get the connection via the driver manager
      setConnection(DriverManager.getConnection(getUrl(), info));
    } else {
      beeLine.debug("Use the driver from local added jar file.");
      setConnection(getConnectionFromLocalDriver(getUrl(), info));
    }
    setDatabaseMetaData(getConnection().getMetaData());

    try {
      beeLine.info(beeLine.loc("connected", new Object[] {
          getDatabaseMetaData().getDatabaseProductName(),
          getDatabaseMetaData().getDatabaseProductVersion()}));
    } catch (Exception e) {
      beeLine.handleException(e);
    }

    try {
      beeLine.info(beeLine.loc("driver", new Object[] {
          getDatabaseMetaData().getDriverName(),
          getDatabaseMetaData().getDriverVersion()}));
    } catch (Exception e) {
      beeLine.handleException(e);
    }

    try {
      getConnection().setAutoCommit(beeLine.getOpts().getAutoCommit());
      // TODO: Setting autocommit should not generate an exception as long as it is set to false
      // beeLine.autocommitStatus(getConnection());
    } catch (Exception e) {
      beeLine.handleException(e);
    }

    try {
      beeLine.getCommands().isolation("isolation: " + beeLine.getOpts().getIsolation());
    } catch (Exception e) {
      beeLine.handleException(e);
    }

    return true;
  }

SQL 方式

SQL 方式执行的是 Commands#sql

public boolean sql(String line, boolean entireLineAsCommand) {
    return execute(line, false, entireLineAsCommand);
  }

Commands#execute

SQL 和 call 都执行 execute, 不同的是参数 call 是否为 true。

private boolean execute(String line, boolean call, boolean entireLineAsCommand) {
    if (line == null || line.length() == 0) {
      return false; // ???
    }

    // ### FIXME: doing the multi-line handling down here means
    // higher-level logic never sees the extra lines. So,
    // for example, if a script is being saved, it won't include
    // the continuation lines! This is logged as sf.net
    // bug 879518.

    // use multiple lines for statements not terminated by the delimiter
    try {
      line = handleMultiLineCmd(line);
    } catch (Exception e) {
      beeLine.handleException(e);
    }

    line = line.trim();
    List<String> cmdList = getCmdList(line, entireLineAsCommand);
    for (int i = 0; i < cmdList.size(); i++) {
      String sql = cmdList.get(i).trim();
      if (sql.length() != 0) {
        if (!executeInternal(sql, call)) {
          return false;
        }
      }
    }
    return true;
  }

executeInternal

executeInternal 方法执行一个 SQL 语句, 再判断这个语句是否是命令，如果是，以执行命令的执行方式，否则以 jdbc 的方式执行 。

private boolean executeInternal(String sql, boolean call) {
    if (!beeLine.isBeeLine()) {
      sql = cliToBeelineCmd(sql);
    }

    if (sql == null || sql.length() == 0) {
      return true;
    }

    if (beeLine.isComment(sql)) {
      //skip this and rest cmds in the line
      return true;
    }

    // is source CMD
    if (isSourceCMD(sql)) {
      return sourceFile(sql);
    }

    if (sql.startsWith(BeeLine.COMMAND_PREFIX)) {
      return beeLine.execCommandWithPrefix(sql);
    }

    String prefix = call ? "call" : "sql";

    if (sql.startsWith(prefix)) {
      sql = sql.substring(prefix.length());
    }

    // batch statements?
    if (beeLine.getBatch() != null) {
      beeLine.getBatch().add(sql);
      return true;
    }

    if (!(beeLine.assertConnection())) {
      return false;
    }

    ClientHook hook = ClientCommandHookFactory.get().getHook(beeLine, sql);

    try {
      Statement stmnt = null;
      boolean hasResults;
      Thread logThread = null;

      try {
        long start = System.currentTimeMillis();

        if (call) {
          stmnt = beeLine.getDatabaseConnection().getConnection().prepareCall(sql);
          hasResults = ((CallableStatement) stmnt).execute();
        } else {
          stmnt = beeLine.createStatement();
          // In test mode we want the operation logs regardless of the settings
          if (!beeLine.isTestMode() && beeLine.getOpts().isSilent()) {
            hasResults = stmnt.execute(sql);
          } else {
            InPlaceUpdateStream.EventNotifier eventNotifier =
                new InPlaceUpdateStream.EventNotifier();
            logThread = new Thread(createLogRunnable(stmnt, eventNotifier));
            logThread.setDaemon(true);
            logThread.start();
            if (stmnt instanceof HiveStatement) {
              HiveStatement hiveStatement = (HiveStatement) stmnt;
              hiveStatement.setInPlaceUpdateStream(
                  new BeelineInPlaceUpdateStream(
                      beeLine.getErrorStream(),
                      eventNotifier
                  ));
            }
            hasResults = stmnt.execute(sql);
            logThread.interrupt();
            logThread.join(DEFAULT_QUERY_PROGRESS_THREAD_TIMEOUT);
          }
        }

        beeLine.showWarnings();

        if (hasResults) {
          OutputFile outputFile = beeLine.getRecordOutputFile();
          if (beeLine.isTestMode() && outputFile != null && outputFile.isActiveConverter()) {
            outputFile.fetchStarted();
            if (!sql.trim().toLowerCase().startsWith("explain")) {
              outputFile.foundQuery(true);
            } else {
              outputFile.foundQuery(false);
            }
          }
          do {
            ResultSet rs = stmnt.getResultSet();
            try {
              int count = beeLine.print(rs);
              long end = System.currentTimeMillis();

              beeLine.info(
                  beeLine.loc("rows-selected", count) + " " + beeLine.locElapsedTime(end - start));
            } finally {
              if (logThread != null) {
                logThread.join(DEFAULT_QUERY_PROGRESS_THREAD_TIMEOUT);
                showRemainingLogsIfAny(stmnt);
                logThread = null;
              }
              rs.close();
            }
          } while (BeeLine.getMoreResults(stmnt));
          if (beeLine.isTestMode() && outputFile != null && outputFile.isActiveConverter()) {
            outputFile.fetchFinished();
          }
        } else {
          int count = stmnt.getUpdateCount();
          long end = System.currentTimeMillis();
          beeLine.info(
              beeLine.loc("rows-affected", count) + " " + beeLine.locElapsedTime(end - start));
        }
      } finally {
        if (logThread != null) {
          if (!logThread.isInterrupted()) {
            logThread.interrupt();
          }
          logThread.join(DEFAULT_QUERY_PROGRESS_THREAD_TIMEOUT);
          showRemainingLogsIfAny(stmnt);
        }
        if (stmnt != null) {
          stmnt.close();
        }
      }
    } catch (Exception e) {
      return beeLine.error(e);
    }
    beeLine.showWarnings();
    if (hook != null) {
      hook.postHook(beeLine);
    }
    return true;
  }