[root@leader bin]# echo $HADOOP_HOME
/opt/hadoop-3.2.2
[root@leader bin]# echo $HBASE_HOME
/opt/hbase-2.3.6
[root@leader bin]# echo $HIVE_HOME
/opt/hive-3.1.2
---
[root@leader conf]# vim sqoop-env.sh#export HADOOP_COMMON_HOME=exportHADOOP_COMMON_HOME=/opt/hadoop-3.2.2
#Set path to where hadoop-*-core.jar is available#export HADOOP_MAPRED_HOME=exportHADOOP_MAPRED_HOME=/opt/hadoop-3.2.2
#set the path to where bin/hbase is available#export HBASE_HOME=exportHBASE_HOME=/opt/hbase-2.3.6
#Set the path to where bin/hive is available#export HIVE_HOME=exportHIVE_HOME=/opt/hive-3.1.2
#Set the path for where zookeper config dir is#export ZOOCFGDIR=exportZOOCFGDIR=/opt/apache-zookeeper-3.6.3-bin
[root@leader sqoop-1.4.7.bin__hadoop-2.6.0]# /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/sqoop help
Warning: /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/../../hcatalog does not exist! HCatalog jobs will fail.
Please set$HCAT_HOME to the root of your HCatalog installation.
Warning: /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/../../accumulo does not exist! Accumulo imports will fail.
Please set$ACCUMULO_HOME to the root of your Accumulo installation.
2022-03-26 01:43:32,860 INFO sqoop.Sqoop: Running Sqoop version: 1.4.7
usage: sqoop COMMAND [ARGS]
Available commands:
codegen Generate code to interact with database records
create-hive-table Import a table definition into Hive
eval Evaluate a SQL statement and display the results
export Export an HDFS directory to a database table
help List available commands
import Import a table from a database to HDFS
import-all-tables Import tables from a database to HDFS
import-mainframe Import datasets from a mainframe server to HDFS
job Work with saved jobs
list-databases List available databases on a server
list-tables List available tables in a database
merge Merge results of incremental imports
metastore Run a standalone Sqoop metastore
version Display version information
See 'sqoop help COMMAND'for information on a specific command.
尝试连接mysql(官方坑!)
# 可以使用help命令来查看参数! [root@leader sqoop-1.4.7.bin__hadoop-2.6.0]# /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/sqoop help 查看参数[root@leader sqoop-1.4.7.bin__hadoop-2.6.0]# /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/sqoop list-databases help 查看参数# 坑: [root@leader sqoop-1.4.7.bin__hadoop-2.6.0]# /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/sqoop list-databases --connect jdbc:mysql://leader:3306/ --username root --password
Warning: /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/../../hcatalog does not exist! HCatalog jobs will fail.
Please set$HCAT_HOME to the root of your HCatalog installation.
Warning: /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/../../accumulo does not exist! Accumulo imports will fail.
Please set$ACCUMULO_HOME to the root of your Accumulo installation.
2022-03-26 01:48:47,096 INFO sqoop.Sqoop: Running Sqoop version: 1.4.7
2022-03-26 01:48:47,189 WARN tool.BaseSqoopTool: Setting your password on the command-line is insecure. Consider using -P instead.
2022-03-26 01:48:47,289 INFO manager.MySQLManager: Preparing to use a MySQL streaming resultset.
Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/commons/lang/StringUtils
at org.apache.sqoop.manager.MySQLManager.initOptionDefaults(MySQLManager.java:73)
at org.apache.sqoop.manager.SqlManager.<init>(SqlManager.java:89)
at com.cloudera.sqoop.manager.SqlManager.<init>(SqlManager.java:33)
at org.apache.sqoop.manager.GenericJdbcManager.<init>(GenericJdbcManager.java:51)
at com.cloudera.sqoop.manager.GenericJdbcManager.<init>(GenericJdbcManager.java:30)
at org.apache.sqoop.manager.CatalogQueryManager.<init>(CatalogQueryManager.java:46)
at com.cloudera.sqoop.manager.CatalogQueryManager.<init>(CatalogQueryManager.java:31)
at org.apache.sqoop.manager.InformationSchemaManager.<init>(InformationSchemaManager.java:38)
at com.cloudera.sqoop.manager.InformationSchemaManager.<init>(InformationSchemaManager.java:31)
at org.apache.sqoop.manager.MySQLManager.<init>(MySQLManager.java:65)
at org.apache.sqoop.manager.DefaultManagerFactory.accept(DefaultManagerFactory.java:67)
at org.apache.sqoop.ConnFactory.getManager(ConnFactory.java:184)
at org.apache.sqoop.tool.BaseSqoopTool.init(BaseSqoopTool.java:272)
at org.apache.sqoop.tool.ListDatabasesTool.run(ListDatabasesTool.java:44)
at org.apache.sqoop.Sqoop.run(Sqoop.java:147)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76)
at org.apache.sqoop.Sqoop.runSqoop(Sqoop.java:183)
at org.apache.sqoop.Sqoop.runTool(Sqoop.java:234)
at org.apache.sqoop.Sqoop.runTool(Sqoop.java:243)
at org.apache.sqoop.Sqoop.main(Sqoop.java:252)
Caused by: java.lang.ClassNotFoundException: org.apache.commons.lang.StringUtils
at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
at java.lang.ClassLoader.loadClass(ClassLoader.java:357)... 20more# 报错:缺少org.apache.commons.lang.StringUtils类,因此需要导入对应的jar包# 解决问题: 将commons-lang3-3.4.jar替换为commons-lang-2.6.jar ,commons-lang-2.6.jar 直接到https://mvnrepository.com/artifact/commons-lang/commons-lang/2.6处下载![root@leader sqoop-1.4.7.bin__hadoop-2.6.0]# cd lib[root@leader lib]# ls
ant-contrib-1.0b3.jar kite-data-hive-1.1.0.jar
ant-eclipse-1.0-jvm1.2.jar kite-data-mapreduce-1.1.0.jar
avro-1.8.1.jar kite-hadoop-compatibility-1.1.0.jar
avro-mapred-1.8.1-hadoop2.jar mysql-connector-java-5.1.48.jar
commons-codec-1.4.jar opencsv-2.3.jar
commons-compress-1.8.1.jar paranamer-2.7.jar
commons-io-1.4.jar parquet-avro-1.6.0.jar
commons-jexl-2.1.1.jar parquet-column-1.6.0.jar
commons-lang3-3.4.jar parquet-common-1.6.0.jar
commons-logging-1.1.1.jar parquet-encoding-1.6.0.jar
hsqldb-1.8.0.10.jar parquet-format-2.2.0-rc1.jar
jackson-annotations-2.3.1.jar parquet-generator-1.6.0.jar
jackson-core-2.3.1.jar parquet-hadoop-1.6.0.jar
jackson-core-asl-1.9.13.jar parquet-jackson-1.6.0.jar
jackson-databind-2.3.1.jar slf4j-api-1.6.1.jar
jackson-mapper-asl-1.9.13.jar snappy-java-1.1.1.6.jar
kite-data-core-1.1.0.jar xz-1.5.jar
[root@leader lib]# mv commons-lang3-3.4.jar commons-lang3-3.4.jar.bak[root@leader lib]# pwd
/opt/sqoop-1.4.7.bin__hadoop-2.6.0/lib
[root@leader lib]# ls
ant-contrib-1.0b3.jar kite-data-hive-1.1.0.jar
ant-eclipse-1.0-jvm1.2.jar kite-data-mapreduce-1.1.0.jar
avro-1.8.1.jar kite-hadoop-compatibility-1.1.0.jar
avro-mapred-1.8.1-hadoop2.jar mysql-connector-java-5.1.48.jar
commons-codec-1.4.jar opencsv-2.3.jar
commons-compress-1.8.1.jar paranamer-2.7.jar
commons-io-1.4.jar parquet-avro-1.6.0.jar
commons-jexl-2.1.1.jar parquet-column-1.6.0.jar
commons-lang-2.6.jar parquet-common-1.6.0.jar
commons-lang3-3.4.jar.bak parquet-encoding-1.6.0.jar
commons-logging-1.1.1.jar parquet-format-2.2.0-rc1.jar
hsqldb-1.8.0.10.jar parquet-generator-1.6.0.jar
jackson-annotations-2.3.1.jar parquet-hadoop-1.6.0.jar
jackson-core-2.3.1.jar parquet-jackson-1.6.0.jar
jackson-core-asl-1.9.13.jar slf4j-api-1.6.1.jar
jackson-databind-2.3.1.jar snappy-java-1.1.1.6.jar
jackson-mapper-asl-1.9.13.jar xz-1.5.jar
kite-data-core-1.1.0.jar
# 再次执行命令[root@leader lib]# /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/sqoop list-databases --connect jdbc:mysql://leader:3306/ --username root -P # 不建议直接输出明文密码!生产会被扣分!!!后续的--password都尽量用-P来实现
Warning: /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/../../hcatalog does not exist! HCatalog jobs will fail.
Please set$HCAT_HOME to the root of your HCatalog installation.
Warning: /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/../../accumulo does not exist! Accumulo imports will fail.
Please set$ACCUMULO_HOME to the root of your Accumulo installation.
2022-03-26 02:02:40,408 INFO sqoop.Sqoop: Running Sqoop version: 1.4.7
Enter password:
2022-03-26 02:02:43,985 INFO manager.MySQLManager: Preparing to use a MySQL streaming resultset.
information_schema
hive
mysql
performance_schema
---
# 其他节点的连接:(我其他节点的设置的问题! 本人的坑) # error:slave1节点的mysql不支持远程连接!!! [root@leader lib]# /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/sqoop list-databases --connect jdbc:mysql://slave1:3306/ --username root -P
Warning: /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/../../hcatalog does not exist! HCatalog jobs will fail.
Please set$HCAT_HOME to the root of your HCatalog installation.
Warning: /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/../../accumulo does not exist! Accumulo imports will fail.
Please set$ACCUMULO_HOME to the root of your Accumulo installation.
2022-03-26 02:05:43,843 INFO sqoop.Sqoop: Running Sqoop version: 1.4.7
Enter password:
2022-03-26 02:05:47,823 INFO manager.MySQLManager: Preparing to use a MySQL streaming resultset.
2022-03-26 02:05:48,072 ERROR manager.CatalogQueryManager: Failed to list databases
java.sql.SQLException: null, message from server: "Host 'leader' is not allowed to connect to this MySQL server"
at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:965)
at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:898)
at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:887)
at com.mysql.jdbc.MysqlIO.doHandshake(MysqlIO.java:1031)
at com.mysql.jdbc.ConnectionImpl.coreConnect(ConnectionImpl.java:2189)
at com.mysql.jdbc.ConnectionImpl.connectOneTryOnly(ConnectionImpl.java:2220)
at com.mysql.jdbc.ConnectionImpl.createNewIO(ConnectionImpl.java:2015)
at com.mysql.jdbc.ConnectionImpl.<init>(ConnectionImpl.java:768)
at com.mysql.jdbc.JDBC4Connection.<init>(JDBC4Connection.java:47)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at com.mysql.jdbc.Util.handleNewInstance(Util.java:425)
at com.mysql.jdbc.ConnectionImpl.getInstance(ConnectionImpl.java:385)
at com.mysql.jdbc.NonRegisteringDriver.connect(NonRegisteringDriver.java:323)
at java.sql.DriverManager.getConnection(DriverManager.java:664)
at java.sql.DriverManager.getConnection(DriverManager.java:247)
at org.apache.sqoop.manager.SqlManager.makeConnection(SqlManager.java:904)
at org.apache.sqoop.manager.GenericJdbcManager.getConnection(GenericJdbcManager.java:59)
at org.apache.sqoop.manager.CatalogQueryManager.listDatabases(CatalogQueryManager.java:57)
at org.apache.sqoop.tool.ListDatabasesTool.run(ListDatabasesTool.java:49)
at org.apache.sqoop.Sqoop.run(Sqoop.java:147)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76)
at org.apache.sqoop.Sqoop.runSqoop(Sqoop.java:183)
at org.apache.sqoop.Sqoop.runTool(Sqoop.java:234)
at org.apache.sqoop.Sqoop.runTool(Sqoop.java:243)
at org.apache.sqoop.Sqoop.main(Sqoop.java:252)2022-03-26 02:05:48,075 ERROR sqoop.Sqoop: Got exception running Sqoop: java.lang.RuntimeException: java.sql.SQLException: null, message from server: "Host 'leader' is not allowed to connect to this MySQL server"
java.lang.RuntimeException: java.sql.SQLException: null, message from server: "Host 'leader' is not allowed to connect to this MySQL server"
at org.apache.sqoop.manager.CatalogQueryManager.listDatabases(CatalogQueryManager.java:73)
at org.apache.sqoop.tool.ListDatabasesTool.run(ListDatabasesTool.java:49)
at org.apache.sqoop.Sqoop.run(Sqoop.java:147)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76)
at org.apache.sqoop.Sqoop.runSqoop(Sqoop.java:183)
at org.apache.sqoop.Sqoop.runTool(Sqoop.java:234)
at org.apache.sqoop.Sqoop.runTool(Sqoop.java:243)
at org.apache.sqoop.Sqoop.main(Sqoop.java:252)
Caused by: java.sql.SQLException: null, message from server: "Host 'leader' is not allowed to connect to this MySQL server"
at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:965)
at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:898)
at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:887)
at com.mysql.jdbc.MysqlIO.doHandshake(MysqlIO.java:1031)
at com.mysql.jdbc.ConnectionImpl.coreConnect(ConnectionImpl.java:2189)
at com.mysql.jdbc.ConnectionImpl.connectOneTryOnly(ConnectionImpl.java:2220)
at com.mysql.jdbc.ConnectionImpl.createNewIO(ConnectionImpl.java:2015)
at com.mysql.jdbc.ConnectionImpl.<init>(ConnectionImpl.java:768)
at com.mysql.jdbc.JDBC4Connection.<init>(JDBC4Connection.java:47)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at com.mysql.jdbc.Util.handleNewInstance(Util.java:425)
at com.mysql.jdbc.ConnectionImpl.getInstance(ConnectionImpl.java:385)
at com.mysql.jdbc.NonRegisteringDriver.connect(NonRegisteringDriver.java:323)
at java.sql.DriverManager.getConnection(DriverManager.java:664)
at java.sql.DriverManager.getConnection(DriverManager.java:247)
at org.apache.sqoop.manager.SqlManager.makeConnection(SqlManager.java:904)
at org.apache.sqoop.manager.GenericJdbcManager.getConnection(GenericJdbcManager.java:59)
at org.apache.sqoop.manager.CatalogQueryManager.listDatabases(CatalogQueryManager.java:57)... 7more# 解决问题: # 进入mysql中进行处理:
mysql> use mysql;
Reading table information for completion of table and column names
You can turn off this feature to get a quicker startup with -A
Database changed
mysql> update user sethost='%' where user ='root';
mysql>select user,host from user where user ='root';
+------+-----------+
| user |host|
+------+-----------+
| root | % || root |127.0.0.1 || root | ::1 || root | slave1 |
+------+-----------+
4 rows inset(0.00 sec)# 重启mysql服务: [root@slave1 opt]# systemctl restart mysqld# 查看mysql服务状态[root@slave1 opt]# systemctl status mysqld
● mysqld.service - MySQL Community Server
Loaded: loaded (/usr/lib/systemd/system/mysqld.service; enabled; vendor preset: disabled)
Active: active (running) since 六 2022-03-26 02:14:12 CST; 9s ago
Process: 76495ExecStartPost=/usr/bin/mysql-systemd-start post (code=exited, status=0/SUCCESS)
Process: 76477ExecStartPre=/usr/bin/mysql-systemd-start pre (code=exited, status=0/SUCCESS)
Main PID: 76493(mysqld_safe)
CGroup: /system.slice/mysqld.service
├─76493 /bin/sh /usr/bin/mysqld_safe --basedir=/usr
└─76683 /usr/sbin/mysqld --basedir=/usr --datadir=/var/lib/mysql --plugin-dir=/...
3月 26 02:14:10 slave1 systemd[1]: Starting MySQL Community Server...
3月 26 02:14:11 slave1 mysqld_safe[76493]: 220326 02:14:11 mysqld_safe Logging to '/var...'.3月 26 02:14:11 slave1 mysqld_safe[76493]: 220326 02:14:11 mysqld_safe Starting mysqld ...ql
3月 26 02:14:12 slave1 systemd[1]: Started MySQL Community Server.
Hint: Some lines were ellipsized, use -l to show in full.
# 再次执行命令: [root@leader lib]# /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/sqoop list-databases --connect jdbc:mysql://slave1:3306/ --username root -P
Warning: /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/../../hcatalog does not exist! HCatalog jobs will fail.
Please set$HCAT_HOME to the root of your HCatalog installation.
Warning: /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/../../accumulo does not exist! Accumulo imports will fail.
Please set$ACCUMULO_HOME to the root of your Accumulo installation.
2022-03-26 02:14:39,874 INFO sqoop.Sqoop: Running Sqoop version: 1.4.7
Enter password:
2022-03-26 02:14:43,167 INFO manager.MySQLManager: Preparing to use a MySQL streaming resultset.
information_schema
javaAndBigdata **比leader节点的mysql多了javaAndBigdata数据库!**
mysql
performance_schema
[root@leader lib]#