Java连接kafka，并将数据写入hdfs

 public static void main(String[] args) throws IOException {
        Properties props = new Properties();
        //set brokers
        Gson gson = new Gson();
        props.put("bootstrap.servers", "139.198.XXX.XXX:9092,139.198.XXX.XXX:9093,139.198.XXX.XXX:9094");
        props.put("group.id", "bitautoHDFS");
        props.put("enable.auto.commit", "true");
        props.put("auto.commit.interval.ms", "1000");
        props.put("session.timeout.ms", "30000");
        props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props);
        consumer.subscribe(Arrays.asList(topicName));
        BitautoEntity bitautoEntity = null;
        while (true) {
            ConsumerRecords<String, String> records = consumer.poll(100);
            for (ConsumerRecord<String, String> record : records) {
                try {
//                    写为entity
                    bitautoEntity = gson.fromJson(record.value(), BitautoEntity.class);
//                    将Entity转换为json
                    String result = gson.toJson(bitautoEntity);
//                    写入大数据平台
                    WriteContent.write2hdfs(result, hdfs_path);
                    System.out.println("推送数据成功！");
                } catch (Exception ex) {
                    System.out.print(ex.getMessage());
                }
            }
        }
    }

 

转载于:https://my.oschina.net/u/3083459/blog/888511