- 脚本数据
生成随机数到文件
for k in $(seq 1 10000)
do
echo $RANDOM>>test_data.txt;
done
创建目录
hdfs dfs -mkdir /data
上传脚本文件
hdfs dfs -put test_data.txt /data
package com.doop.gw.hadoop.reducer.order;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.partition.InputSampler;
import org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.junit.Test;
public class TotalSort extends Configured implements Tool {
static final String INPUT_PATH = "hdfs://192.168.33.104:8020/data/test_data.txt";
static final String OUT_PATH = "hdfs://192.168.33.104:8020/total_sort_output";
static final String OUT_PATH_1 = "hdfs://192.168.33.104:8020/total_sort_partitions";
//实现一个Kye比较器,用于比较两个key的大小,将key由字符串转化为Integer,然后进行比较。
public static class KeyComparator extends WritableComparator {
protected KeyComparator() {
super(Text.class, true);
}
@Override
public int compare(WritableComparable writableComparable1, WritableComparable writableComparable2) {
int num1 = Integer.parseInt(writableComparable1.toString());
int num2 = Integer.parseInt(writableComparable2.toString());
return num1 - num2;
}
}
@Override
public int run(String[] args) throws Exception {
Configuration conf = new Configuration();
FileSystem fileSystem = FileSystem.get(new URI(INPUT_PATH), conf);
Path outPath = new Path(OUT_PATH);
if (fileSystem.exists(outPath)) {
fileSystem.delete(outPath, true);
}
conf.set("mapreduce.totalorderpartitioner.naturalorder", "false");
Job job = Job.getInstance(conf, "Total Sort app");
job.setJarByClass(TotalSort.class);
//设置读取文件的路径,都是从HDFS中读取。读取文件路径从脚本文件中传进来
FileInputFormat.addInputPath(job,new Path(INPUT_PATH));
//设置mapreduce程序的输出路径,MapReduce的结果都是输入到文件中
FileOutputFormat.setOutputPath(job,new Path(OUT_PATH));
//设置比较器,用于比较数据的大小,然后按顺序排序,该例子主要用于比较两个key的大小
job.setSortComparatorClass(KeyComparator.class);
job.setNumReduceTasks(3);//设置reduce数量
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
job.setOutputKeyClass(IntWritable.class);
job.setOutputValueClass(NullWritable.class);
job.setInputFormatClass(KeyValueTextInputFormat.class);
//设置保存partitions文件的路径
TotalOrderPartitioner.setPartitionFile(job.getConfiguration(), new Path(OUT_PATH_1));
//key值采样,0.01是采样率,
InputSampler.Sampler<Text, Text> sampler = new InputSampler.RandomSampler<>(0.01, 1000, 100);
//将采样数据写入到分区文件中
InputSampler.writePartitionFile(job, sampler);
job.setMapperClass(TotalSortMap.class);
job.setReducerClass(TotalSortReduce.class);
//设置分区类。
job.setPartitionerClass(TotalOrderPartitioner.class);
return job.waitForCompletion(true) ? 0 : 1;
}
public static void main(String[] args)throws Exception{
int exitCode = ToolRunner.run(new TotalSort(), args);
System.exit(exitCode);
}
/**
* 读取文件
*/
@Test
public void testRead() {
Configuration conf = new Configuration();
try {
// 根据配置文件创建HDFS对象
conf.set("fs.defaultFS", "hdfs://192.168.33.104:8020"); //远程需要设定
FileSystem fs = FileSystem.get(conf);
Path path = new Path("hdfs://192.168.33.104:8020/total_sort_output/part-r-00000");
FSDataInputStream fis = fs.open(path);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
IOUtils.copyBytes(fis, baos, 1024);
fis.close();
System.out.println(new String(baos.toByteArray()));
} catch (IOException e) {
e.printStackTrace();
}
}
}
package com.doop.gw.hadoop.reducer.order;
import java.io.IOException;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
public class TotalSortMap extends Mapper<Text, Text, Text, IntWritable> {
@Override
protected void map(Text key, Text value, Context context) throws IOException, InterruptedException {
context.write(key, new IntWritable(Integer.parseInt(key.toString())));
}
}
package com.doop.gw.hadoop.reducer.order;
import java.io.IOException;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
public class TotalSortReduce extends Reducer<Text, IntWritable, IntWritable, NullWritable> {
@Override
protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
for (IntWritable value : values) {
context.write(value, NullWritable.get());
}
}
}
-
执行过程中遇到没有权限
hadoop user=xiaocai, access=WRITE, inode="/":root:supergroup:drw -
解决
修改hdfs-site.xml 增加
<property>
<name>dfs.permissions</name>
<value>false</value>
</property>