package com.tvm.writefile;
/**
* Created by zhangx on 2017/1/25.
*/
import com.alibaba.fastjson.JSONObject;
import kafka.consumer.ConsumerConfig;
import kafka.consumer.ConsumerIterator;
import kafka.consumer.KafkaStream;
import kafka.javaapi.consumer.ConsumerConnector;
import kafka.message.MessageAndMetadata;
import kafka.serializer.StringDecoder;
import kafka.utils.VerifiableProperties;
import lombok.extern.log4j.Log4j2;
import java.io.*;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@Log4j2
public class KafkaConsumer implements Runnable{
static boolean stop = false;
public static boolean isStop() {
return stop;
}
public static void setStop(boolean stop) {
KafkaConsumer.stop = stop;
}
private static Thread mainThread = Thread.currentThread();
public static Properties properties() throws IOException {
Properties properties_kafkainfo=new Properties();
InputStream in = new FileInputStream("/opt/logs/java_jar/kafkainfo.properties");
properties_kafkainfo.load(in);
return properties_kafkainfo;
}
public static Properties properties_topic() throws IOException {
Properties properties_kafkatopic=new Properties();
InputStream in = new FileInputStream("/opt/logs/java_jar/topic.properties");
properties_kafkatopic.load(in);
return properties_kafkatopic;
}
public final ConsumerConnector consumer;
public KafkaConsumer() throws IOException {
Properties props = new Properties();
//props.put("zookeeper.connect", "10.20.30.91:2181,10.20.30.92:2181,10.20.30.93:2181");
props.put("zookeeper.connect", properties().getProperty("zookeeper_connect"));
props.put("group.id", properties().getProperty("group"));
props.put("zookeeper.session.timeout.ms", properties().getProperty("session_timeout_ms"));
props.put("zookeeper.sync.time.ms", properties().getProperty("zookeeper_sync_time_ms"));
props.put("auto.commit.interval.ms", properties().getProperty("auto_commit_interval_ms"));
props.put("auto.commit.enable",properties().getProperty("auto_commit_enable"));
props.put("auto.offset.reset", properties().getProperty("auto_offset_reset")); //largest smallest
props.put("serializer.class", properties().getProperty("serializer_class"));
ConsumerConfig config = new ConsumerConfig(props);
consumer = kafka.consumer.Consumer.createJavaConsumerConnector(config);
}
public void run() {
final int numThreads = 6;
final Iterator<String> topic;
try {
topic = properties_topic().stringPropertyNames().iterator();
Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
List<String> topicList = new ArrayList<String>();
while(topic.hasNext()){
final String key_topic = topic.next();
topicList.add(key_topic);
topicCountMap.put(key_topic, new Integer(1));
}
StringDecoder keyDecoder = new StringDecoder(new VerifiableProperties());
StringDecoder valueDecoder = new StringDecoder(new VerifiableProperties());
final Map<String, List<KafkaStream<String, String>>> consumerMap =
consumer.createMessageStreams(topicCountMap,keyDecoder,valueDecoder);
for(int i=0;i<topicList.size();i++) {
final String key_topic1 = topicList.get(i);
for (int j = 0; j < 6; j++) {
new Thread(
new Runnable() {
@Override
public void run() {
List<KafkaStream<String, String>> stream = consumerMap.get(key_topic1);
ExecutorService executor = Executors.newFixedThreadPool(numThreads);
for (KafkaStream streams : stream) {
executor.submit(new KafkaConsumerThread(streams, key_topic1));
}
}
}).start();
}
}
} catch (IOException e) {
e.printStackTrace();
}
}
public static void main(String[] args) throws IOException {
Thread t = new Thread(new KafkaConsumer());
t.start();
Runtime.getRuntime().addShutdownHook(new Thread(){
@Override
public void run() {
KafkaConsumer.setStop(true);
try {
sleep(10000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
});
}
}
package com.tvm.writefile;
import com.alibaba.fastjson.JSONObject;
import kafka.consumer.ConsumerIterator;
import kafka.consumer.KafkaStream;
import kafka.message.MessageAndMetadata;
import java.io.*;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
/**
* Created by zhangx on 2017/3/10.
*/
public class KafkaConsumerThread implements Runnable{
private KafkaStream<String,String> stream;
private String topicNum;
public KafkaConsumerThread(KafkaStream<String,String> stream, String topicNum) {
this.stream = stream;
this.topicNum = topicNum;
}
public void run() {
ConsumerIterator<String,String> it = stream.iterator();
while (it.hasNext()){
if(KafkaConsumer.isStop()){
flushData();
break;
}
MessageAndMetadata<String,String> m = it.next();
JSONObject jsStr = JSONObject.parseObject(m.message());
String dateTime = new SimpleDateFormat("yyyyMMddHH").format(jsStr.get("createTime"));
String name = dateTime + ".log";
String path_send = "/opt/logs/kafkafile/";
String path_send_topic = path_send + topicNum;
String file_send = path_send_topic + name;//发放福利路径
FileWrapper writer = getFileByName(file_send);
writer.write(jsStr.toString());
}
}
static Map<String,FileWrapper> fileMap=new ConcurrentHashMap<>();
FileWrapper getFileByName(String name){
FileWrapper bufferedWriter = fileMap.get(name);
if(bufferedWriter!=null){
return bufferedWriter;
}
synchronized (fileMap){
bufferedWriter=fileMap.get(name);
if(bufferedWriter!=null){
return bufferedWriter;
}
File file = new File(name);
System.out.println("file.name==========="+file.getName());
FileWrapper result = new FileWrapper(name);
if(result==null){
new Throwable("result为空!");
}
fileMap.put(name,result);
return result;
}
}
private static class FileWrapper {
private final String name;
private AtomicInteger calc = new AtomicInteger();
private final BufferedWriter br;
private long starttime=new Date().getTime();
private FileOutputStream fos;
private OutputStreamWriter osw;
public FileWrapper(String name) {
this.name=name;
try {
fos=new FileOutputStream(name,true);
osw=new OutputStreamWriter(fos,"utf-8");
this.br = new BufferedWriter(osw);
} catch (FileNotFoundException e) {
e.printStackTrace();
throw new RuntimeException("文件不可用"+name,e);
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
throw new RuntimeException("文件不可用"+name,e);
}
}
public void write (String txt){
try {
//int i = calc.incrementAndGet();
String txt2=txt+"\n";
br.write(txt2);
//if(i%10000==0){
br.flush();
//calc.set(0);
//}
} catch (IOException e) {
e.printStackTrace();
}
}
public void close(){
try {
br.close();
} catch (IOException e) {
}
try {
osw.close();
} catch (IOException e) {
}
try {
fos.close();
fos=null;
} catch (IOException e) {
}
}
}
static void flushData(){
long nowtime = new Date().getTime();
synchronized (fileMap){
for (FileWrapper fileWrapper : fileMap.values()) {
//fileWrapper.br.flush();
if(nowtime-fileWrapper.starttime>2*60*60*1000){
fileMap.remove(fileWrapper.name);
fileWrapper.close();
}
}
}
}
static {
Timer timer = new Timer("flush-task", false);
timer.schedule(new TimerTask() {
@Override
public void run() {
flushData();
}
},60000,60000);
}
}
package com.tvm.util;
import java.text.SimpleDateFormat;
/**
* Created by zhangx on 2017/3/10.
*/
public class DateUtil {
//时间转换为yyyyMMddHH格式
public static String getTFormatString(Object date1) {
SimpleDateFormat format = new SimpleDateFormat("yyyyMMddHH");
String str = format.format(date1);
return str;
}
}
group 'com.kafka'
version '1.0-SNAPSHOT'
apply plugin: 'java'
apply plugin: 'idea'
[compileJava, compileTestJava, javadoc]*.options*.encoding = 'UTF-8'
sourceCompatibility = 1.7
targetCompatibility=1.7
repositories {
mavenLocal()
mavenCentral()
jcenter()
maven {
url = 'http://115.159.154.56:8081/nexus/content/groups/public'
}
}
dependencies {
testCompile group: 'junit', name: 'junit', version: '4.11'
compile "org.apache.kafka:kafka_2.10:0.8.2.2"
compile group: 'com.alibaba', name: 'fastjson', version: '1.2.21'
compile "org.projectlombok:lombok:1.16.6"
compile "org.apache.logging.log4j:log4j-api:2.3"
compile "org.apache.logging.log4j:log4j-core:2.3"
fileTree(dir: 'lib', include: '*.jar')
}
jar {
String someString = ''
configurations.runtime.each {someString = someString + " lib//"+it.name}//遍历项目的所有依赖的jar包赋值给变量someString
manifest {
attributes 'Main-Class': 'com.tvm.writefile.KafkaConsumer'
attributes 'Class-Path': someString
}
}
//清除上次的编译过的文件
task clearPj(type:Delete){
delete 'build','target'
}
task copyJar(type:Copy){
from configurations.runtime
into ('build/libs/lib')
}
//把JAR复制到目标目录
task release(type: Copy,dependsOn: [build,copyJar]) {
// from 'conf'
// into ('build/libs/eachend/conf') // 目标位置
}
通过
gradle release
命令来进行打包
在linux系统中
nohup
java -jar KafkaConsumer_log-1.0-SNAPSHOT.jar
> /dev/null 2>&1 &
后台运行