1 给lzo文件建立索引
1
2
3
4
|
$HADOOP_HOMOE
/bin/hadoop
jar \
$HADOOP_HOMOE
/share/hadoop/mapreduce/lib/hadoop-lzo-0
.4.20-SNAPSHOT.jar \
com.hadoop.compression.lzo.LzoIndexer \
/hdfs/path/to/file
.lzo
|
1
2
3
4
|
$HADOOP_HOMOE
/bin/hadoop
jar \
$HADOOP_HOMOE
/share/hadoop/mapreduce/lib/hadoop-lzo-0
.4.20-SNAPSHOT.jar \
com.hadoop.compression.lzo.DistributedLzoIndexer \
/hdfs/path/to/file
.lzo
|
2 在mapreduce程序中使用lzo压缩
1
|
job.setInputFormatClass(LzoTextInputFormat.
class
);
|
1
2
3
4
5
|
<
dependency
>
<
groupId
>com.hadoop.gplcompression</
groupId
>
<
artifactId
>hadoop-lzo</
artifactId
>
<
version
>0.4.20</
version
>
</
dependency
>
|
3 在streaming程序中使用lzo压缩
1
2
3
4
5
6
7
8
|
$HADOOP_HOME/bin/hadoop jar \
$HADOOP_HOME/share/hadoop/tools/lib/hadoop-streaming-
2.2
.
0
.jar \
-D stream.map.input.ignoreKey=
true
\
-inputformat com.hadoop.mapred.DeprecatedLzoTextInputFormat \
-input /path/to/input \
-output /path/to/output \
-mapper cat \
-reducer wc
|
4 在hive中使用lzo压缩
1
2
3
4
5
|
create
table
lzo_test(
i
int
,
s string)
STORED
AS
INPUTFORMAT
'com.hadoop.mapred.DeprecatedLzoTextInputFormat'
OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
;
|