ETL采集器-开发者模式

最新推荐文章于 2024-07-18 20:13:39 发布

chuangjiji0839

最新推荐文章于 2024-07-18 20:13:39 发布

阅读量155

点赞数

文章标签： java

原文链接：https://my.oschina.net/u/2470985/blog/511266

版权

ETL采集器-开发者模式

ETL采集器一共有4层分别为：采集层、任务拆分层、清洗层、存储层。除了

采集层开发者API

实现ICollectData接口,存储本地文件

public interface ICollectData { 
	//执行采集过程
	public IData<String, String>  execute(String id,CollectorJobRunState collectorJobRunState) throws Exception ;
	//采集同步
	public IData<String, String> synchronous(String id,CollectorJobRunState collectorJobRunState) throws Exception;
	//自动采集
	public IData<String, String> collectAuto(String id,CollectorJobRunState collectorJobRunState) throws Exception;
	//获取采集参数
	public Map<String, String> getParameter();
	//初始化采集参数
	public void initCollectParameter(String id);
	//设置参数
	public void setCollectParameter(String id,IData<String,String> data);
}

将实现类加入spring容器中

</bean>

清洗开发者API

实现CleanDataMethod清洗接口

public interface CleanDataMethod {
	

	/**
	 * 
	* @Title: getForMatData 
	* @Description: TODO(清洗数据方法) 
	* @param valueThis 原始日志
	* @param parameters 参数
	* @param elementXml xml属性
	* @param data 保存数据  
	* @param key 字段
	* @param id id
	* @return
	* @throws CollectorExceptionLog 
	* @return ForMatReturn    返回类型 
	* @throws
	 */
	public IData<String,Object> executeCleanData(String valueThis,String parameters,String elementXml, IData<String,Object> data,String key,String id)throws CollectorExceptionLog;
}

将实现类加入spring容器中

<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://www.springframework.org/schema/beans
        classpath:spring-beans.xsd">
        
	<bean id="mappingArray"
		class="com.venustech.collector.service.fromat.method.mapping.MappingArray" />
	<bean id="mappingID"
		class="com.venustech.collector.service.fromat.method.mapping.MappingID" />
	<bean id="mappingValueArray"
		class="com.venustech.collector.service.fromat.method.mapping.MappingValueArray" />
	<bean id="mappingValueID"
		class="com.venustech.collector.service.fromat.method.mapping.MappingValueID" />
	<bean id="dateUnix"
		class="com.venustech.collector.service.fromat.method.dateunix.DateUnix" />
	<bean id="dateUnixArray"
		class="com.venustech.collector.service.fromat.method.dateunix.DateUnixArray" />
	<bean id="dateUnixArrayTime"
		class="com.venustech.collector.service.fromat.method.dateunix.DateUnixArrayTime" />
	<bean id="dateUnixID"
		class="com.venustech.collector.service.fromat.method.dateunix.DateUnixID" />
	<bean id="dateUnixIDTime"
		class="com.venustech.collector.service.fromat.method.dateunix.DateUnixIDTime" />
	<bean id="iPIntArray"
		class="com.venustech.collector.service.fromat.method.iptoint.IPIntArray" />
	<bean id="iPIntID"
		class="com.venustech.collector.service.fromat.method.iptoint.IPIntID" />
	<bean id="regexpArray"
		class="com.venustech.collector.service.fromat.method.regexp.RegexpArray" />
	<bean id="regexpID"
		class="com.venustech.collector.service.fromat.method.regexp.RegexpID" />
	<bean id="regexpThis"
		class="com.venustech.collector.service.fromat.method.regexp.RegexpThis" />
	<bean id="splitArray"
		class="com.venustech.collector.service.fromat.method.split.SplitArray" />
	<bean id="splitID"
		class="com.venustech.collector.service.fromat.method.split.SplitID" />
	<bean id="splitThis"
		class="com.venustech.collector.service.fromat.method.split.SplitThis" />
	<bean id="textArray"
		class="com.venustech.collector.service.fromat.method.text.TextArray" />
	<bean id="textID"
		class="com.venustech.collector.service.fromat.method.text.TextID" />
		<bean id="info"
		class="com.venustech.collector.service.fromat.method.info.Info" />
		<bean id="subStringArray"
		class="com.venustech.collector.service.fromat.method.substring.SubStringArray" />	
		<bean id="filterRegexpArray"
		class="com.venustech.collector.service.fromat.method.regexp.FilterRegexpArray" />	
		<bean id="filterRegexpID"
		class="com.venustech.collector.service.fromat.method.regexp.FilterRegexpID" />	
		<bean id="filterRegexpThis"
		class="com.venustech.collector.service.fromat.method.regexp.FilterRegexpThis" />	
		<bean id="splitKeyValueID"
		class="com.venustech.collector.service.fromat.method.split.SplitKeyValueID" />	

	<bean id="subStringID"
		class="com.venustech.collector.service.fromat.method.substring.SubStringID" />	
		<bean id="completionData"
		class="com.venustech.collector.service.fromat.method.completion.CompletionData" />	
		<bean id="completionDataDefault"
		class="com.venustech.collector.service.fromat.method.completion.CompletionDataDefault" />	
		<bean id="subIndexStringArray"
		class="com.venustech.collector.service.fromat.method.substring.SubIndexStringArray" />	
		<bean id="subIndexStringId"
		class="com.venustech.collector.service.fromat.method.substring.SubIndexStringId" />	
		<bean id="urlToID" class="com.venustech.collector.service.fromat.method.urltoip.UrlToID" />	
		<bean id="urlToIpArray"
		class="com.venustech.collector.service.fromat.method.urltoip.UrlToIpArray" />	
		<bean id="mappingKey"
		class="com.venustech.collector.service.fromat.method.mapping.MappingKey" />	
		<bean id="mappingValueKey"
		class="com.venustech.collector.service.fromat.method.mapping.MappingValueKey" />	
	 
	  <bean id="toDateArray"
		class="com.venustech.collector.service.fromat.method.todate.ToDateArray" />	
	 
	  <bean id="toDateIDTime"
		class="com.venustech.collector.service.fromat.method.todate.ToDateIDTime" />	
	 
	  <bean id="mainformat" class="com.venustech.collector.service.fromat.MainForMat">
		<property name="method">
			<map>
				<entry key="mappingValueKey" value-ref="mappingValueKey" />
				<entry key="mappingKey" value-ref="mappingKey" />
				<entry key="mappingArray" value-ref="mappingArray" />
				<entry key="mappingID" value-ref="mappingID" />
				<entry key="mappingValueArray" value-ref="mappingValueArray" />
				<entry key="mappingValueID" value-ref="mappingValueID" />
				<entry key="dateUnix" value-ref="dateUnix" />
				<entry key="dateUnixArray" value-ref="dateUnixArray" />
				<entry key="dateUnixArrayTime" value-ref="dateUnixArrayTime" />
				<entry key="dateUnixID" value-ref="dateUnixID" />
				<entry key="dateUnixIDTime" value-ref="dateUnixIDTime" />
				<entry key="iPIntArray" value-ref="iPIntArray" />
				<entry key="iPIntID" value-ref="iPIntID" />
				<entry key="regexpArray" value-ref="regexpArray" />
				<entry key="regexpID" value-ref="regexpID" />
				<entry key="regexpThis" value-ref="regexpThis" />
				<entry key="splitArray" value-ref="splitArray" />
				<entry key="splitID" value-ref="splitID" />
				<entry key="splitThis" value-ref="splitThis" />
				<entry key="textArray" value-ref="textArray" />
				<entry key="textID" value-ref="textID" />
				<entry key="subStringID" value-ref="subStringID" />
				<entry key="subStringArray" value-ref="subStringArray" />
				<entry key="completionData" value-ref="completionData" />
				<entry key="subIndexStringArray" value-ref="subIndexStringArray" />
				<entry key="subIndexStringId" value-ref="subIndexStringId" />
				<entry key="completionDataDefault" value-ref="completionDataDefault" />
				<entry key="urlToID" value-ref="urlToID" />
				<entry key="urlToIpArray" value-ref="urlToIpArray" />
				<entry key="filterRegexpArray" value-ref="filterRegexpArray" />
				<entry key="filterRegexpID" value-ref="filterRegexpID" />
				<entry key="filterRegexpThis" value-ref="filterRegexpThis" />
				<entry key="splitKeyValueID" value-ref="splitKeyValueID" />
				<!--映射 -->
				<entry key="mappingArray" value-ref="mappingArray" />
				<entry key="mappingID" value-ref="mappingID" />
				<entry key="mappingValueArray" value-ref="mappingValueArray" />
				<entry key="mappingValueID" value-ref="mappingValueID" />
				<entry key="mappingKey" value-ref="mappingKey" />
				<entry key="mappingValueKey" value-ref="mappingValueKey" />
				<entry key="info" value-ref="info" />
				<!--補全 -->
				<entry key="completionData" value-ref="completionData" />
				<entry key="completionDataDefault" value-ref="completionDataDefault" />
				<!--data转换-->
				<entry key="toDateIDTime" value-ref="toDateIDTime" />
				<entry key="toDateArray" value-ref="toDateArray" />
			</map> 
		</property>
	</bean> 
	
</beans>

存储层开发者API

实现SaveCollector接口

public interface SaveCollector{
	public String save(String id, String[] tableName,
			IDataset<IData<String, Object>> list,FileAttribute fileAttribute) throws Exception;
	
}

将实现类加入spring容器中

转载于:https://my.oschina.net/u/2470985/blog/511266

chuangjiji0839

关注

0
点赞
踩
0

收藏

觉得还不错? 一键收藏
0
评论
ETL采集器-开发者模式

ETL采集器-开发者模式 ETL采集器一共有4层分别为：采集层、任务拆分层、清洗层、存储层。除了采集层开发者API 实现ICollectData接口,存储本地文件 public interface ICollectData { //执行采...
复制链接

扫一扫