hadoop lzo配置

lzo相关环境安装编译部署

lzo配置

[root@hadoop001 tar]# wget http://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz
[root@hadoop001 tar]# tar -zxvf lzo-2.10.tar.gz -C /home/hadoop/app/
[root@hadoop001 app]# chown -R hadoop:hadoop lzo-2.10
[root@hadoop001 app]# cd lzo-2.10/
[root@hadoop001 lzo-2.10]# ./configure
[root@hadoop001 lzo-2.10]# make install

make[1]: Entering directory `/home/hadoop/app/lzo-2.10'
 /usr/bin/mkdir -p '/usr/local/lib'
 /bin/sh ./libtool   --mode=install /usr/bin/install -c   src/liblzo2.la '/usr/local/lib'
libtool: install: /usr/bin/install -c src/.libs/liblzo2.lai /usr/local/lib/liblzo2.la
libtool: install: /usr/bin/install -c src/.libs/liblzo2.a /usr/local/lib/liblzo2.a
libtool: install: chmod 644 /usr/local/lib/liblzo2.a
libtool: install: ranlib /usr/local/lib/liblzo2.a
libtool: finish: PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/sbin" ldconfig -n /usr/local/lib
----------------------------------------------------------------------
Libraries have been installed in:
   /usr/local/lib

If you ever happen to want to link against installed libraries
in a given directory, LIBDIR, you must either use libtool, and
specify the full pathname of the library, or use the '-LLIBDIR'
flag during linking and do at least one of the following:
   - add LIBDIR to the 'LD_LIBRARY_PATH' environment variable
     during execution
   - add LIBDIR to the 'LD_RUN_PATH' environment variable
     during linking
   - use the '-Wl,-rpath -Wl,LIBDIR' linker flag
   - have your system administrator add LIBDIR to '/etc/ld.so.conf'

See any operating system documentation about shared libraries for
more information, such as the ld(1) and ld.so(8) manual pages.
----------------------------------------------------------------------
 /usr/bin/mkdir -p '/usr/local/share/doc/lzo'
 /usr/bin/install -c -m 644 AUTHORS COPYING NEWS THANKS doc/LZO.FAQ doc/LZO.TXT doc/LZOAPI.TXT '/usr/local/share/doc/lzo'
 /usr/bin/mkdir -p '/usr/local/lib/pkgconfig'
 /usr/bin/install -c -m 644 lzo2.pc '/usr/local/lib/pkgconfig'
 /usr/bin/mkdir -p '/usr/local/include/lzo'
 /usr/bin/install -c -m 644 include/lzo/lzo1.h include/lzo/lzo1a.h include/lzo/lzo1b.h include/lzo/lzo1c.h include/lzo/lzo1f.h include/lzo/lzo1x.h include/lzo/lzo1y.h include/lzo/lzo1z.h include/lzo/lzo2a.h include/lzo/lzo_asm.h include/lzo/lzoconf.h include/lzo/lzodefs.h include/lzo/lzoutil.h '/usr/local/include/lzo'
make[1]: Leaving directory `/home/hadoop/app/lzo-2.10'

配置lzop

[root@hadoop001 tar]# wget http://www.lzop.org/download/lzop-1.04.tar.gz

[root@hadoop001 tar]# tar -zxvf lzop-1.04.tar.gz -C /home/hadoop/app/
[root@hadoop001 app]# cd lzop-1.04/
[root@hadoop001 lzop-1.04]# ./configure
 。。。
Type `make' to build lzop. Type `make install' to install lzop.
After installing lzop, please read the accompanied documentation.

[root@hadoop001 lzop-1.04]# make  && make install

编译hadoop-lzo

[root@hadoop001 tar]# wget https://github.com/twitter/hadoop-lzo/archive/master.zip
[root@hadoop001 tar]# unzip -d /home/hadoop/app/ master.zip 
修改hadoop version
  <properties>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    <hadoop.current.version>2.6.0-cdh5.7.0</hadoop.current.version>    
    <hadoop.old.version>1.0.4</hadoop.old.version>
  </properties>
  添加仓库
      <repository>
      <id>cloudera</id>
      <url>https://repository.cloudera.com/artifactory/cloudera-repos</url>
    </repository>
    readme文件需要设置
[hadoop@hadoop001 hadoop-lzo-master]$C_INCLUDE_PATH=/usr/local/include 
[hadoop@hadoop001 hadoop-lzo-master]$LIBRARY_PATH=/usr/local/lib
[hadoop@hadoop001 hadoop-lzo-master]$  mvn clean package -Dmaven.test.skip=true
[INFO] ------------------------------------------------------------------------
[INFO] BUILD SUCCESS
[INFO] ------------------------------------------------------------------------
[INFO] Total time: 02:41 min
[INFO] Finished at: 2019-04-14T12:03:05+00:00
[INFO] Final Memory: 37M/1252M
[INFO] ------------------------------------------------------------------------
拷贝相关文件放到本地库
[hadoop@hadoop001 hadoop-lzo-master]$ cd target/native/Linux-amd64-64/
[hadoop@hadoop001 Linux-amd64-64]$ tar -cBf - -C lib . | tar -xBvf - -C ~
[hadoop@hadoop001 ~]$ cp ~/libgplcompression* $HADOOP_HOME/lib/native/
把编译好的jar  加入 hadoop包下
[hadoop@hadoop001 target]$ cp hadoop-lzo-0.4.21-SNAPSHOT.jar $HADOOP_HOME/share/hadoop/common/

配置core-site.xml

    <property>
        <name>io.compression.codecs</name>
        <value>org.apache.hadoop.io.compress.GzipCodec,
                org.apache.hadoop.io.compress.DefaultCodec,
                org.apache.hadoop.io.compress.BZip2Codec,
                org.apache.hadoop.io.compress.SnappyCodec,
                com.hadoop.compression.lzo.LzoCodec,
                com.hadoop.compression.lzo.LzopCodec
        </value>
     </property>
        <!--支持LZO使用类-->
     <property>
         <name>io.compression.codec.lzo.class</name>
         <value>com.hadoop.compression.lzo.LzopCodec</value>
     </property>

配置mapred-site.xml

<!--启用map中间压缩类-->
<property>
   <name>mapred.map.output.compression.codec</name>
   <value>com.hadoop.compression.lzo.LzopCodec</value>
</property>
<!--启用mapreduce文件压缩-->
<property>
    <name>mapreduce.output.fileoutputformat.compress</name>
    <value>true</value>
</property> 
<!--启用mapreduce压缩类-->
<property>
   <name>mapreduce.output.fileoutputformat.compress.codec</name>
   <value>com.hadoop.compression.lzo.LzopCodec</value>
</property>
<!--配置Jar包-->
<property>
    <name>mapred.child.env</name>
    <value>LD_LIBRARY_PATH=/usr/local/lib</value>
</property>

下篇

lzo使用测试