MapReduce任务输出到redis中

AnillegalName

于 2024-01-17 16:27:28 发布

阅读量401

点赞数 6

文章标签： mapreduce redis java

本文链接：https://blog.csdn.net/AnillegalName/article/details/135653967

版权

文章介绍了如何在Java中使用Redis连接池管理Jedis连接，并重写了FileOutputFormat，创建了一个专门用于将MapReduce任务处理结果写入Redis的RedisRecordWriter，实现了数据库记录的高效输出。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

主要包括redis连接池，重写FileOutputFormat函数。

redis连接池

/**
* redis 连接池
*/
public class RedisHelper {

    private static JedisPool jedisPool;

    static {
        init();
    }

    public synchronized static Jedis getJedis() {
        if (jedisPool != null) {
            Jedis resource = jedisPool.getResource();
            return resource;
        } else {
            return null;
        }
    }

    /**
     * 释放jedis资源
     *
     * @param jedis
     */
    public static void close(final Jedis jedis) {
        if (jedis != null) {
            jedis.close();
        }
    }

    public static void destroy() {
        if (jedisPool != null) {
            jedisPool.destroy();
        }
    }

public static void putValue(Jedis jedis, String key, String value) {
        jedis.set(key, value);
    }

    private static void init() {
        Properties prop = new Properties();
        try (InputStream in = RedisHelper.class.getResourceAsStream("/database_config.properties")) {
            prop.load(in);
            String redisHost = prop.getProperty("redisHost");
            int redisPort = Integer.valueOf(prop.getProperty("redisPort"));
            String redisPassword = prop.getProperty("redisPassword");
            JedisPoolConfig config = new JedisPoolConfig();
            config.setMaxTotal(Integer.valueOf(prop.getProperty("redisMaxTotal")));
            config.setMaxIdle(Integer.valueOf(prop.getProperty("redisMaxIdle")));
            config.setMinIdle(Integer.valueOf(prop.getProperty("redisMinIdle")));
            config.setMaxWaitMillis(Long.valueOf(prop.getProperty("redisMaxWaitMillis")));
            config.setTestOnBorrow(Boolean.parseBoolean(prop.getProperty("redisTestOnBorrow")));

            jedisPool = new JedisPool(config, redisHost, redisPort, 2000, redisPassword);
        } catch (IOException e) {
            log.error("Failed to initialize redis parameters. ", e);
            exit(1);
        }
    }

    private RedisHelper() {
    }
}

重写FileOutputFormat函数

public class ResultOutputFormat<K, V> extends FileOutputFormat<K, V> {

    /**
     * 定制一个RecordWriter类，每一条reduce处理后的记录，将该记录输出到数据库中
     */
    protected static class RedisRecordWriter<K, V> extends RecordWriter<K, V> {

        private Jedis jedis;

        public RedisRecordWriter(Jedis jedis) {
            this.jedis = jedis;
        }

        @Override
        public void write(K key, V value) throws IOException, InterruptedException {
            if (key == null || value == null) {
                return;
            }
            RedisHelper.putValue(jedis, key.toString(), value.toString());
        }

        @Override
        public void close(TaskAttemptContext context) throws IOException, InterruptedException {
            RedisHelper.close(jedis); //关闭链接
        }
    }

    @Override
    public RecordWriter<K, V> getRecordWriter(TaskAttemptContext job) throws IOException, InterruptedException {
        Jedis jedis = RedisHelper.getJedis();
        return new RedisRecordWriter<K, V>(jedis);
    }
}