背景
我们数据加工使用海豚调度DolpinScheduler做任务调度。流程是 提交一个shell 到海豚,shell里使用
spark-submit 提交 java 程序,然后使用海豚的api 去获取日志(海豚管理端也能看到,我们换皮定制化了),但是在集群模式下,获取到的日志是spark运行的日志(拉取jar,accept,running之类的),我们需要的是jar包里打印的log.info()这一类的,在jar包里打印了任务进行到流程的哪个阶段,这个日志在hadoop里能看,海豚调度里看不了,又不想大费周章去部署logtash搞elk,或者去配flume,然后我们就想着把日志输出到ES里,能看能查。之前的业务日志又全都是log.info打印的,又懒得去改造,调研后发现log4j2可以自定义Appender,那么咱们今天就搞定他。
在海豚里提交的shell脚本
下面这个脚本我们换皮搞定制化的提交到海豚的
/home/software/spark-3.1.2/bin/spark-submit --class cnki.bdms.servicespark.BdcServiceSparkApplication \
--conf spark.yarn.jars="hdfs://cluster1:8020/bdclib/*" \
--driver-java-options "-Dspark.yarn.dist.files=/home/software/hadoop-3.3.0/etc/hadoop/yarn-site.xml" \
--master yarn \
--deploy-mode cluster \
--driver-memory 1g \
--executor-memory 3g \
--executor-cores 3 \
--num-executors 3 \
--conf spark.yarn.maxAppAttempts=5 \
--conf spark.yarn.preserve.staging.files=true \
/data/jar/bdcServiceSpark-1.0.0.jar 3627
自定义EsAppender
先引入pom,我们提交的是springboot 的jar包
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-log4j2</artifactId>
</dependency>
EsAppender实现
package cnki.bdms.servicespark.utils;
import com.fasterxml.jackson.core.JsonProcessingException;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.nio.reactor.IOReactorConfig;
import org.apache.logging.log4j.core.Filter;
import org.apache.logging.log4j.core.Layout;
import org.apache.logging.log4j.core.LogEvent;
import org.apache.logging.log4j.core.appender.AbstractAppender;
import org.apache.logging.log4j.core.config.plugins.Plugin;
import org.apache.logging.log4j.core.config.plugins.PluginAttribute;
import org.apache.logging.log4j.core.config.plugins.PluginElement;
import org.apache.logging.log4j.core.config.plugins.PluginFactory;
import org.apache.logging.log4j.core.layout.PatternLayout;
import org.apache.logging.log4j.util.ReadOnlyStringMap;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.slf4j.MDC;
import org.springframework.util.CollectionUtils;
import java.io.IOException;
import java.io.Serializable;
import java.net.InetAddress;
import java.net.NetworkInterface;
import java.net.UnknownHostException;
import java.util.*;
@Plugin(name = "elasticSearchAppender", category = "Core", elementType = "appender", printObject = true)
public class EsAppender extends AbstractAppender {
private String address;
private Integer port;
private String user;
private String password;
private String threshold;
/**
* 日志缓冲池大小,如果缓冲池大小为1则日志会被立即同步到ES中</br>
* 否则需要等到缓冲池Size达到bufferSize了后才会将日志刷新至ES</br>
* bufferSize默认初始化为1
*/
private int bufferSize;
/**
* 日志缓冲数据
*/
private List<Map<String, Object>> buffers = new ArrayList<>();
/**
* 操作ES集群的客户端
*/
private RestHighLevelClient client;
/**
* 插入索引
*/
private String index;
protected EsAppender(String name, Filter filter, Layout<? extends Serializable> layout,
String address,
Integer port,
String index,
String user,
String password,
String threshold,
int bufferSize) {
super(name, filter, layout);
this.address = address;
this.port = port;
this.index = index;
this.user = user;
this.password = password;
this.threshold = threshold;
this.bufferSize = bufferSize;
}
private void parseLog(LogEvent event) throws JsonProcessingException, UnknownHostException {
ReadOnlyStringMap contextData = event.getContextData();
//System.out.println(LOG_LEVEL_PATTERN);
String applicationId = MDC.get("applicationId");
Map<String, Object> item = new HashMap<>();
item.put("className", event.getSource().getClassName());
item.put("fileName", event.getSource().getFileName());
item.put("lineNumber", event.getSource().getLineNumber());
item.put("methodName", event.getSource().getMethodName());
item.put("serverIp", getIp());
item.put("logName", event.getLoggerName());
item.put("logLevel", event.getLevel().toString());
item.put("logThread", event.getThreadName());
item.put("logMills", new Date(event.getTimeMillis()));
item.put("logMessage", JsonHelper.toJSONString(event.getMessage().getFormattedMessage()));
item.put("applicationId", applicationId);
//全部或者获取对应的日志级别
if ((threshold.equalsIgnoreCase(event.getLevel().toString()) || threshold.equalsIgnoreCase("all"))
&& StringUtil.isNotBlank(applicationId)) {
buffers.add(item);
}
}
/**
* 将数据写入到ES中
*/
private void flushBuffer() {
if (!CollectionUtils.isEmpty(buffers)) {
try {
bulkLoadMap(index, buffers);
buffers.clear();
} catch (IOException e) {
e.printStackTrace();
}
}
}
/**
* 批量写入
* @param esIndex 索引
* @param datalist 数据
* @throws IOException
*/
public void bulkLoadMap(String esIndex, List<Map<String, Object>> datalist) throws IOException {
//获取客户端
RestHighLevelClient client = getRestHighLevelClient();
BulkRequest bulkRequest = new BulkRequest();
for (Map<String, Object> data : datalist) {
Object id = data.get("sys_uuid");
//如果数据包含id字段,使用数据id作为文档id
if (id != null) {
bulkRequest.add(new IndexRequest(esIndex).id(id.toString()).source(data));
} else {//让es自动生成id
bulkRequest.add(new IndexRequest(esIndex).source(data));
}
}
client.bulk(bulkRequest, RequestOptions.DEFAULT);
}
/**
* 获取ES RestHighLevelClient客户端
*/
private RestHighLevelClient getRestHighLevelClient() {
if (client == null) {
RestClientBuilder builder = RestClient.builder(new HttpHost(address, port, "http"));
builder.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder
.setDefaultIOReactorConfig(IOReactorConfig.custom()
.setSoKeepAlive(true)
.build())).setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder
.setConnectTimeout(40000)
.setSocketTimeout(40000));
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(user, password));
builder.setHttpClientConfigCallback(f -> f.setDefaultCredentialsProvider(credentialsProvider));
client = new RestHighLevelClient(builder);
}
return client;
}
/**
* 获取ip
* @return
* @throws UnknownHostException
*/
private String getIp() throws UnknownHostException {
try {
InetAddress candidateAddress = null;
// 遍历所有的网络接口
for (Enumeration ifaces = NetworkInterface.getNetworkInterfaces(); ifaces.hasMoreElements(); ) {
NetworkInterface iface = (NetworkInterface) ifaces.nextElement();
// 在所有的接口下再遍历IP
for (Enumeration inetAddrs = iface.getInetAddresses(); inetAddrs.hasMoreElements(); ) {
InetAddress inetAddr = (InetAddress) inetAddrs.nextElement();
// 排除loopback类型地址
if (!inetAddr.isLoopbackAddress()) {
if (inetAddr.isSiteLocalAddress()) {
// 如果是site-local地址,就是它了
return inetAddr.getHostAddress();
} else if (candidateAddress == null) {
// site-local类型的地址未被发现,先记录候选地址
candidateAddress = inetAddr;
}
}
}
}
if (candidateAddress != null) {
return candidateAddress.getHostAddress();
}
// 如果没有发现 non-loopback地址.只能用最次选的方案
InetAddress jdkSuppliedAddress = InetAddress.getLocalHost();
if (jdkSuppliedAddress == null) {
throw new UnknownHostException("The JDK InetAddress.getLocalHost() method unexpectedly returned null.");
}
return jdkSuppliedAddress.getHostAddress();
} catch (Exception e) {
UnknownHostException unknownHostException = new UnknownHostException(
"Failed to determine LAN address: " + e);
unknownHostException.initCause(e);
throw unknownHostException;
}
}
/**
* 类结束之前调用方法
*/
@Override
public void stop() {
if (buffers.size() > 0) {
flushBuffer();
}
try {
client.close();
} catch (IOException e) {
e.printStackTrace();
}
super.stop();
}
@Override
public void append(LogEvent event) {
try {
parseLog(event);
if (buffers.size() >= (bufferSize == 0 ? 50 : bufferSize)) {
flushBuffer();
}
} catch (JsonProcessingException | UnknownHostException e) {
e.printStackTrace();
}
}
@PluginFactory
public static EsAppender createAppender(@PluginAttribute("name") String name,
@PluginElement("Filter") final Filter filter,
@PluginElement("Layout") Layout<? extends Serializable> layout,
@PluginAttribute("address") String address,
@PluginAttribute("port") Integer port,
@PluginAttribute("index") String index,
@PluginAttribute("user") String user,
@PluginAttribute("password") String password,
@PluginAttribute("threshold") String threshold,
@PluginAttribute("bufferSize") int bufferSize
) {
if (name == null) {
LOGGER.error("No name provided for MyCustomAppenderImpl");
return null;
}
if (layout == null) {
layout = PatternLayout.createDefaultLayout();
}
return new EsAppender(name, filter, layout, address, port, index, user, password, threshold, bufferSize);
}
}
配置文件log4j2.xml,这里也可以使用yml,看个人喜好,这里几个参数和上面的保持一致
<?xml version="1.0" encoding="UTF-8"?>
<Configuration packages="org.apache.logging.log4j.core,io.sentry.log4j2,cnki.bdms.servicespark.utils">
<appenders>
<Console name="Console" target="SYSTEM_OUT">
<PatternLayout pattern="%X{applicationId} %d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
</Console>
<elasticSearchAppender name="esAppender" address="你的es ip地址" port="9200" index="spark_log"
user="admin" password="admin" threshold="info" bufferSize="100">
</elasticSearchAppender>
</appenders>
<loggers>
<root level="INFO">
<appender-ref ref="Console"/>
<appender-ref ref="esAppender"/>
</root>
</loggers>
</Configuration>
看完上面你一定注意到我们使用MDC了,这个是为了记录spark 运行的appincationId
所以你在使用的时候可能是这样
try {
spark = SparkSession.builder()
.enableHiveSupport()
.master(master)
.appName(appName)
.config(conf)
.config("spark.sql.shuffle.partition", shufflePartition)
.config("hive.metastore.uris", metastoreUris)
.config("spark.yarn.jars", yarnJars)
.config("spark.sql.warehouse.dir", spark_warehouse_home)
.config("spark.sql.legacy.timeParserPolicy", timeParserPolicy)
String applicationId = spark.sparkContext().applicationId();
MDC.put("applicationId", applicationId);
//处理你的业务
...
//处理完之后
MDC.remove("applicationId");
//触发缓存池的信息写入到es
removeEsAppender();
}
catch (Exception ex) {
log.info("spark对象初始化成功:master=" + ex.getMessage());
throw new RuntimeException(ex);
}
removeEsAppender ,为啥要加这个,我们使用这个appender后,程序执行完了,在idea里一直是running状态,而且没法触发appender的stop,咱们可是在stop里写有业务的(将剩余的不足缓存池大小的日志也要写进es)
/**
* 停止并移除EsAppender(先触发缓存池的信息写入到es里,在停止后移除)
*/
private void removeEsAppender() {
LoggerContext context = LoggerContext.getContext(false);
org.apache.logging.log4j.core.config.Configuration config = context.getConfiguration();
Appender esAppender = config.getAppender("esAppender");
if (config.getAppenders().remove(esAppender.getName()) != null) {
esAppender.stop();
config.getRootLogger().removeAppender(esAppender.getName());
context.updateLoggers(config);
}
}
至此搞定,再说下我们怎么从海豚里拿applicationid的吧,海豚里每次执行日志里都有
从海豚的api里能拿到这个日志,然后咱们可以写个正则,拿到applicationId再去找同一个applicationId 的记录,完事了。
Pattern pattern = Pattern.compile("(?<=find app id: )([a-z0-9_]+)");
Matcher matcher = pattern.matcher(data);
if (matcher.find()) {
String applicationId = matcher.group();
System.out.println(applicationId );
}