Flume 将数据导入Hive

Flume 将数据导入Hive

1:创建.conf文件

#cd   /usr/tmp

#mkdir  flumesrc

#vim  netcat_to_hive_user.conf

a1.sources=r1

a1.sinks=s1
a1.channels=c1

a1.sources.r1.type=netcat
a1.sources.r1.bind=master
a1.sources.r1.port=44444

a1.sinks.s1.type=hive
a1.sinks.s1.channel = c1
a1.sinks.s1.hive.metastore = thrift://master:9083
a1.sinks.s1.hive.database = bd18
a1.sinks.s1.hive.table = flume_user
a1.sinks.s1.useLocalTimeStamp = false
a1.sinks.s1.serializer = DELIMITED
a1.sinks.s1.serializer.delimiter = "\t"
a1.sinks.s1.serializer.serdeSeparator = '\t'
a1.sinks.s1.serializer.fieldnames=user_id,user_name,age

a1.channels.c1.type=memory
a1.channels.c1.capacity=1000

2:运行.conf文件

#flume-ng agent -c conf -f  netcat_to_hive_user.conf  --name a1 -Dflume.root.logger=INFO,console

3:将hive的四个jar包复制到flume安装目录的lib文件夹下

4:启动hive的metastore

#hive  --service metastore

5:进入hive,创建数据库和表

hive> create database bd18;

hive> create table flume_user(
    > user_id int,
    > user_name string,
    > age int
    > )
    > clustered by(user_id) into 2 buckets
    > stored as orc
    > tblproperties("transactional"='true');

hive> select * from flume_user;
FAILED: SemanticException [Error 10265]: This command is not allowed on an ACID table bd18.flume_user with a non-ACID transaction manager. Failed command: select * from flume_user

hive> set hive.support.concurrency=true；

hive> set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
hive> select * from flume_user;
OK
Time taken: 5.809 seconds

6:操作flume,向hive输入数据

[root@master ~]# telnet master 44444
Trying 192.168.2.100...
Connected to master.
Escape character is '^]'.
1     jim      18
OK

注：中间用Tab键隔开

7:查询Hive表中的内容