hadoop lzo安装

准备工作：
下载lzo-2.06、lzop-1.03、hadoop-lzo-0.4.21
上传到linux上
安装lzo开发环境等
yum -y install  lzo-devel  zlib-devel  gcc autoconf automake libtool

LZO安装：
tar -zxvf lzo-2.06.tar.gz 
cd到lzo-2.06目录下
./configure -enable-shared -prefix=/root/opt/lzo/
make && make test && make install
cp /root/opt/lzo/lib/* /usr/lib/
cp /root/opt/lzo/lib/* /usr/lib64/
配置环境变量
LZO_HOME=/root/opt/lzo/lzo-2.04
C_INCLUDE_PATH=/root/opt/lzo/include
LD_LIBRARY_PATH=/root/opt/lzo/lib

LZOP安装：
tar -zxvf lzop-1.03.tar.gz 
cd到lzop-1.03目录下
./configure -enable-shared -prefix=/root/opt/lzop/
make && make test && make install
把lzop复制到/usr/bin/
cp /root/opt/lzop/bin/lzop /usr/bin/lzop

hadoop-lzo编译：
tar -zxvf hadoop-lzo-0.4.21.tar.gz
cd到hadoop-lzo-master
修改pom.xml中的hadoop版本为你所使用的版本
<properties>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    <hadoop.current.version>2.2.0</hadoop.current.version>
    <hadoop.old.version>1.0.4</hadoop.old.version>
</properties>

加入jdk tool 1.6的引用

<dependency>
            <groupId>jdk.tools</groupId>
            <artifactId>jdk.tools</artifactId>
            <version>1.6</version>
            <scope>system</scope>
             <systemPath>/usr/java/default/lib/tools.jar</systemPath>
</dependency>
回到linux
export JAVA_HOME=/usr/java/default
export CFLAGS=-m64
export CXXFLAGS=-m64
export C_INCLUDE_PATH=/root/opt/lzo/include
export LIBRARY_PATH=/root/opt/lzo/lib
mvn clean package -Dmaven.test.skip=true
tar -cBf - -C target/native/Linux-amd64-64/lib . | tar -xBvf - -C /root/opt/hadoop-2.9.0/lib/native/
cp target/native/Linux-amd64-64/lib/* /root/opt/hadoop-2.9.0/lib/native/
cp target/hadoop-lzo-0.4.21-SNAPSHOT.jar /root/opt/hadoop-2.9.0/share/hadoop/common/
接下来就是将/root/hadoop/hadoop-2.9.0/share/hadoop/common/hadoop-lzo-0.4.21-SNAPSHOT.jar以及/home/hadoop/hadoop-2.2.0/lib/native/ 同步到其它所有的hadoop节点。
最好让运行hadoop的用户都有执行权限。
设置core-site.xml
<property>     
<name>io.compression.codecs</name>     
<value>org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec,org.apache.hadoop.io.compress.BZip2Codec</value>
</property>
<property>     
<name>io.compression.codec.lzo.class</name>     
<value>com.hadoop.compression.lzo.LzoCodec</value>
</property>
设置mapred-site.xml
<property>
<name>mapred.compress.map.output</name>     
<value>true</value>   
</property>   
<property>     
<name>mapred.map.output.compression.codec</name>      
<value>com.hadoop.compression.lzo.LzoCodec</value>   
</property>
<property>
<name>mapred.child.env</name>
<value>LD_LIBRARY_PATH=/root/opt/lzo/lib</value>
</property>
设置hadoop-env.sh
export LD_LIBRARY_PATH=/root/opt/lzo/lib
不设置也好像可以

重启Hadoop
进入hive
create table lzo_test(myid string,myname string) 
row format delimited 
fields terminated by ',' 
stored as inputformat "com.hadoop.mapred.DeprecatedLzoTextInputFormat" 
outputformat "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat";
导入数据
LOAD DATA Local INPATH '/root/allfiles.txt.lzo' INTO TABLE lzo_test
索引LZO
$HADOOP_HOME/bin/hadoop jar /root/opt/hadoop-2.9.0/share/hadoop/common/hadoop-lzo-0.4.21-SNAPSHOT.jar com.hadoop.compression.lzo.DistributedLzoIndexer /user/hive/warehouse/lzo_test/allfiles.txt.lzo
hiveQL使用
select * from lzo_test;

内容从网络学习整理