MapFileDemo

1. write方法

 @Test
    public void write() throws IOException {
        Configuration cfg = new Configuration();
        cfg.set("mapreduce.framework.name", "local");
        cfg.set("fs.defaultFs", "file:///");//使用本地文件系统
        Path mapFile = new Path("E:/test/a.map");

        //Writer内部类用于文件的写操作,假设Key和Value都为Text类型
        MapFile.Writer.Option optionKey = MapFile.Writer.keyClass(Text.class);
        SequenceFile.Writer.Option optionValue = MapFile.Writer.valueClass(IntWritable.class);

        MapFile.Writer writer = new MapFile.Writer(cfg, mapFile, optionKey, optionValue);

        //通过writer向文档中写入记录
        writer.append(new Text("lisi"), new IntWritable(2222));
        writer.append(new Text("mazi"), new IntWritable(4444));
        writer.append(new Text("wanger"), new IntWritable(3333));
        writer.append(new Text("zhangsan"), new IntWritable(1111));
        IOUtils.closeStream(writer);//关闭write流
    }

2.read方法

 @Test
    public void read() throws IOException {
        Configuration cfg = new Configuration();
        cfg.set("mapreduce.framework.name", "local");
        cfg.set("fs.defaultFs", "file:///");//使用本地文件系统
        Path mapFile = new Path("E:/test/a.map");

        MapFile.Reader reader = new MapFile.Reader(mapFile, cfg);

        //通过reader从文档中读取记录
        Text key = new Text();
        IntWritable value = new IntWritable();
        while (reader.next(key, value)) {
            System.out.println(key + " : " + value);
        }
        IOUtils.closeStream(reader);//关闭read流
    }

3.sequenceFile2MapFile方法

 @Test// 从SequenceFile中读取key、value写入到MapFile
    public void sequenceFile2MapFile() throws Exception {
        // 从SequenceFile中读取key、value写入到MapFile
        Configuration cfg = new Configuration();
        cfg.set("mapreduce.framework.name", "local");
        cfg.set("fs.defaultFs", "file:///");//使用本地文件系统
        FileSystem fs = FileSystem.get(cfg);

        //E:/test/sequenceFile2MapFile文件夹下放置的为重命名为data的sequencefile
        Path map = new Path("E:/test/sequenceFile2MapFile");
        // MapFile.DATA_FILE_NAME 为seq文件移动到文件夹下面的文件名称
        Path mapData = new Path(map, MapFile.DATA_FILE_NAME);  //文件名必须是data
        // 从data sequenceFile中读取key,value的类型
        SequenceFile.Reader.Option optionFile = SequenceFile.Reader.file(mapData);
        SequenceFile.Reader reader = new SequenceFile.Reader(cfg, optionFile);
        Class keyClass = reader.getKeyClass();
        Class valueClass = reader.getValueClass();
        reader.close();
        long entries = MapFile.fix(fs, map, keyClass, valueClass, false, cfg);
        System.out.printf("Created MapFile %s with %d entries\n", map, entries);
    }

4.测试类

@Test
    public void test() throws IOException {
        Configuration cfg = new Configuration();
        cfg.set("mapreduce.framework.name", "local");
        cfg.set("fs.defaultFs", "file:///");//使用本地文件系统
        Path mapFile = new Path("E:/test/sequenceFile2MapFile");

        MapFile.Reader reader = new MapFile.Reader(mapFile, cfg);

        //通过reader从文档中读取记录
        IntWritable key = new IntWritable();
        Text value = new Text();
        while (reader.next(key, value)) {
            System.out.println(key + " : " + value);
        }
        IOUtils.closeStream(reader);//关闭read流
    }