public class MainBak2_ {
private static Logger logger = LoggerFactory.getLogger(MainBak2_.class);
private static KafkaCluster kafkaCluster = null;
public static void main(String[] args) {
String zookeeper = ConfigMgr.getZookeeper();
final String groupId = ConfigMgr.getKakfaGroupId();
String topicStr = ConfigMgr.getKafkaTopic();
String sparkAppName = ConfigMgr.getSparkAppName();
String master = ConfigMgr.getSparkMaster();
int duration = ConfigMgr.getSparkDuration();
String metadataBrokerList = ConfigMgr.getProByKey("metadata.broker.list");
String autoOffsetReset = ConfigMgr.getProByKey("auto.offset.reset");
logger.info("zk===========================" + zookeeper);
logger.info("groupId======================" + groupId);
logger.info("topic========================" + topicStr);
logger.info("sparkAppName=================" + sparkAppName);
logger.info("master=======================" + master);
logger.info("duration=====================" + duration);
logger.info("metadataBrokerList===========" + metadataBrokerList);
SparkConf sparkConf = new SparkConf().setAppName(sparkAppName).setMaster(master);
JavaStreamingContext jssc = new JavaStreamingContext(sparkConf, Durations.seconds(duration));
Set<String> topicSet = new HashSet<String>();
for (String topic : topicStr.split(",")) {
topicSet.add(topic);
}
Map<String, String> kafkaParams = new HashMap<String, String>();
kafkaParams.put("group.id", groupId);
kafkaParams.put("metadata.broker.list", metadataBrokerList);
kafkaParams.put("auto.offset.reset", autoOffsetReset);
scala.collection.mutable.Map<String, String> tmpScalaKafkaMap = JavaConversions.mapAsScalaMap(kafkaParams);
scala.collection.immutable.Map<String, String> scalaKafkaParams = tmpScalaKafkaMap
.toMap(new Predef.$less$colon$less<Tuple2<String, String>, Tuple2<String, String>>() {
private static final long serialVersionUID = 1L;
public Tuple2<String, String> apply(Tuple2<String, String> v1) {
return v1;
}
});
kafkaCluster = new KafkaCluster(scalaKafkaParams);
scala.collection.mutable.Set<String> tmpMutableTopicsSet = JavaConversions.asScalaSet(topicSet);
scala.collection.immutable.Set<String> tmpImmutableTopicsSet = tmpMutableTopicsSet.toSet();
scala.collection.immutable.Set<TopicAndPartition> tpScalaSet = kafkaCluster.getPartitions(tmpImmutableTopicsSet)
.right().get();
Map<TopicAndPartition, Long> offsetMap = new HashMap<TopicAndPartition, Long>();
if (kafkaCluster.getConsumerOffsets(groupId, tpScalaSet).isLeft()) {
System.out.println(" left : " + kafkaCluster.getConsumerOffsets(groupId, tpScalaSet).left().get());
Set<TopicAndPartition> tmpSet = JavaConversions.setAsJavaSet(tpScalaSet);
for (TopicAndPartition topicAndPartition : tmpSet) {
offsetMap.put(topicAndPartition, 0L);
}
} else {
scala.collection.immutable.Map<TopicAndPartition, Object> scalaOffsetMap = kafkaCluster
.getConsumerOffsets(groupId, tpScalaSet).right().get();
Map<TopicAndPartition, Object> consumerOffsets = JavaConversions.mapAsJavaMap(scalaOffsetMap);
Set<TopicAndPartition> tmpSet = JavaConversions.setAsJavaSet(tpScalaSet);
for (TopicAndPartition topicAndPartition : tmpSet) {
Long offset = (Long) consumerOffsets.get(topicAndPartition);
offsetMap.put(topicAndPartition, offset);
System.out.println("first getOffset:" + topicAndPartition.partition() + ":" + offset);
}
}
JavaInputDStream<String> jid = KafkaUtils.createDirectStream(jssc, String.class, String.class,
StringDecoder.class, StringDecoder.class, String.class, kafkaParams, offsetMap,
new Function<kafka.message.MessageAndMetadata<String, String>, String>() {
private static final long serialVersionUID = -6590667828252772663L;
@Override
public String call(MessageAndMetadata<String, String> arg0) throws Exception {
return arg0.message();
}
});
final VoidFunction<String> func0 = new VoidFunction<String>() {
private static final long serialVersionUID = -2520206838533422786L;
@Override
public void call(String arg0) throws Exception {
System.out.println(arg0);
}
};
VoidFunction<JavaRDD<String>> func = new VoidFunction<JavaRDD<String>>() {
private static final long serialVersionUID = 7679681553001908774L;
@Override
public void call(JavaRDD<String> arg0) throws Exception {
OffsetRange[] offsets = ((HasOffsetRanges) arg0.rdd()).offsetRanges();
for (OffsetRange o : offsets) {
TopicAndPartition tp = new TopicAndPartition(o.topic(), o.partition());
Map<TopicAndPartition, Object> tpMap = new HashMap<TopicAndPartition, Object>();
tpMap.put(tp, o.untilOffset());
System.out.println("write getOffset:" + tp.partition() + ":" + o.untilOffset());
scala.collection.mutable.Map<TopicAndPartition, Object> tpScalaMap = JavaConversions
.mapAsScalaMap(tpMap);
scala.collection.immutable.Map<TopicAndPartition, Object> scalaOffsetMap = tpScalaMap.toMap(
new Predef.$less$colon$less<Tuple2<TopicAndPartition, Object>, Tuple2<TopicAndPartition, Object>>() {
private static final long serialVersionUID = -3520206838533422786L;
public Tuple2<TopicAndPartition, Object> apply(Tuple2<TopicAndPartition, Object> v1) {
return v1;
}
});
kafkaCluster.setConsumerOffsets(groupId, scalaOffsetMap);
}
arg0.foreach(func0);
}
};
jid.foreachRDD(func);
jssc.start();
jssc.awaitTermination();
}
}
private static Logger logger = LoggerFactory.getLogger(MainBak2_.class);
private static KafkaCluster kafkaCluster = null;
public static void main(String[] args) {
String zookeeper = ConfigMgr.getZookeeper();
final String groupId = ConfigMgr.getKakfaGroupId();
String topicStr = ConfigMgr.getKafkaTopic();
String sparkAppName = ConfigMgr.getSparkAppName();
String master = ConfigMgr.getSparkMaster();
int duration = ConfigMgr.getSparkDuration();
String metadataBrokerList = ConfigMgr.getProByKey("metadata.broker.list");
String autoOffsetReset = ConfigMgr.getProByKey("auto.offset.reset");
logger.info("zk===========================" + zookeeper);
logger.info("groupId======================" + groupId);
logger.info("topic========================" + topicStr);
logger.info("sparkAppName=================" + sparkAppName);
logger.info("master=======================" + master);
logger.info("duration=====================" + duration);
logger.info("metadataBrokerList===========" + metadataBrokerList);
SparkConf sparkConf = new SparkConf().setAppName(sparkAppName).setMaster(master);
JavaStreamingContext jssc = new JavaStreamingContext(sparkConf, Durations.seconds(duration));
Set<String> topicSet = new HashSet<String>();
for (String topic : topicStr.split(",")) {
topicSet.add(topic);
}
Map<String, String> kafkaParams = new HashMap<String, String>();
kafkaParams.put("group.id", groupId);
kafkaParams.put("metadata.broker.list", metadataBrokerList);
kafkaParams.put("auto.offset.reset", autoOffsetReset);
scala.collection.mutable.Map<String, String> tmpScalaKafkaMap = JavaConversions.mapAsScalaMap(kafkaParams);
scala.collection.immutable.Map<String, String> scalaKafkaParams = tmpScalaKafkaMap
.toMap(new Predef.$less$colon$less<Tuple2<String, String>, Tuple2<String, String>>() {
private static final long serialVersionUID = 1L;
public Tuple2<String, String> apply(Tuple2<String, String> v1) {
return v1;
}
});
kafkaCluster = new KafkaCluster(scalaKafkaParams);
scala.collection.mutable.Set<String> tmpMutableTopicsSet = JavaConversions.asScalaSet(topicSet);
scala.collection.immutable.Set<String> tmpImmutableTopicsSet = tmpMutableTopicsSet.toSet();
scala.collection.immutable.Set<TopicAndPartition> tpScalaSet = kafkaCluster.getPartitions(tmpImmutableTopicsSet)
.right().get();
Map<TopicAndPartition, Long> offsetMap = new HashMap<TopicAndPartition, Long>();
if (kafkaCluster.getConsumerOffsets(groupId, tpScalaSet).isLeft()) {
System.out.println(" left : " + kafkaCluster.getConsumerOffsets(groupId, tpScalaSet).left().get());
Set<TopicAndPartition> tmpSet = JavaConversions.setAsJavaSet(tpScalaSet);
for (TopicAndPartition topicAndPartition : tmpSet) {
offsetMap.put(topicAndPartition, 0L);
}
} else {
scala.collection.immutable.Map<TopicAndPartition, Object> scalaOffsetMap = kafkaCluster
.getConsumerOffsets(groupId, tpScalaSet).right().get();
Map<TopicAndPartition, Object> consumerOffsets = JavaConversions.mapAsJavaMap(scalaOffsetMap);
Set<TopicAndPartition> tmpSet = JavaConversions.setAsJavaSet(tpScalaSet);
for (TopicAndPartition topicAndPartition : tmpSet) {
Long offset = (Long) consumerOffsets.get(topicAndPartition);
offsetMap.put(topicAndPartition, offset);
System.out.println("first getOffset:" + topicAndPartition.partition() + ":" + offset);
}
}
JavaInputDStream<String> jid = KafkaUtils.createDirectStream(jssc, String.class, String.class,
StringDecoder.class, StringDecoder.class, String.class, kafkaParams, offsetMap,
new Function<kafka.message.MessageAndMetadata<String, String>, String>() {
private static final long serialVersionUID = -6590667828252772663L;
@Override
public String call(MessageAndMetadata<String, String> arg0) throws Exception {
return arg0.message();
}
});
final VoidFunction<String> func0 = new VoidFunction<String>() {
private static final long serialVersionUID = -2520206838533422786L;
@Override
public void call(String arg0) throws Exception {
System.out.println(arg0);
}
};
VoidFunction<JavaRDD<String>> func = new VoidFunction<JavaRDD<String>>() {
private static final long serialVersionUID = 7679681553001908774L;
@Override
public void call(JavaRDD<String> arg0) throws Exception {
OffsetRange[] offsets = ((HasOffsetRanges) arg0.rdd()).offsetRanges();
for (OffsetRange o : offsets) {
TopicAndPartition tp = new TopicAndPartition(o.topic(), o.partition());
Map<TopicAndPartition, Object> tpMap = new HashMap<TopicAndPartition, Object>();
tpMap.put(tp, o.untilOffset());
System.out.println("write getOffset:" + tp.partition() + ":" + o.untilOffset());
scala.collection.mutable.Map<TopicAndPartition, Object> tpScalaMap = JavaConversions
.mapAsScalaMap(tpMap);
scala.collection.immutable.Map<TopicAndPartition, Object> scalaOffsetMap = tpScalaMap.toMap(
new Predef.$less$colon$less<Tuple2<TopicAndPartition, Object>, Tuple2<TopicAndPartition, Object>>() {
private static final long serialVersionUID = -3520206838533422786L;
public Tuple2<TopicAndPartition, Object> apply(Tuple2<TopicAndPartition, Object> v1) {
return v1;
}
});
kafkaCluster.setConsumerOffsets(groupId, scalaOffsetMap);
}
arg0.foreach(func0);
}
};
jid.foreachRDD(func);
jssc.start();
jssc.awaitTermination();
}
}