使用反射机制编写数据与文件映射工具

背景

• 最近公司需要开发许多文件数据同步的程序, 将数据写入文件, 通过SFTP协议同步给其他平台, 基于面向对象编程思想(其实就是懒), 就想弄个一劳永逸的工具。

思路

1. 系统启动时, 创建一个Map结构的容器, 容器中存储文件生成规则与数据模型的映射属性配置。
2. 文件生成时, 根据根据配置Key查询配置的值(配置值包括文件编码, 文件是否存在文件头, 文件每一列的分隔符), 使用反射机制反射Entity拿到值, 根据配置的规则写入文件。
3. 读取文件时, 根据根据配置Key查询配置的值, 创建List集合, 读取到文件, 根据配置中的数据模型属性与文件属性映射关系将每一行数据都转换为模型对象, 存入List集合。

实现

• FileMapping.java

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.lang.reflect.Field;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import javax.xml.bind.JAXBContext;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.stereotype.Component;

import com.ctid.core.exception.ServiceException;
import com.ctid.mainlib.system.util.datehandle.DateHandle;
import com.ctid.mainlib.system.util.datehandle.PathHandler;
import com.ctid.mainlib.system.util.datehandle.StringHandle;
import com.ctid.mainlib.system.util.file.FileMapping.Context.FileMapper;

/**
 * @功能描述 文件映射工具类
 * @author 索睿君
 * @date 2019年7月5日14:55:35
 */
public class FileMapping {
	private static final Log Log = LogFactory.getLog(FileMapping.class);
	@Component public static class FileMapperContext{
		private final static String CLASSPATH = "com/config/fileMapper/fileMapper.xml";
		private final static Log Log = LogFactory.getLog(FileMapperContext.class);
		public final static Map<String, Context.FileMapper> MAPPER_CONTEXT = new HashMap<>();
		public final static Map<String,BufferedWriter> BufferedWriter_CONTEXT = new HashMap<>();
		static{
			try {
				File file = new File(Context.class.getClassLoader().getResource(CLASSPATH).getFile());
				JAXBContext jaxbContext = JAXBContext.newInstance(Context.class);
				Context context = (Context) jaxbContext.createUnmarshaller().unmarshal(file);
				context.fileMapperList.forEach(fileMapper -> {
					fileMapper.mapperList.forEach(mapper -> {
						fileMapper.mapper.put(mapper.entityAttribute, mapper.fileAttribute);
					});
					MAPPER_CONTEXT.put(fileMapper.id, fileMapper);
					//创建文件写入流
					if(fileMapper.fileTemplate != null && fileMapper.fileTemplate.trim().length() > 0){
						try {
							String collect = fileMapper.mapper.values().stream().collect(Collectors.joining(fileMapper.separator));
							fileMapper.filePath = initFile(fileMapper.fileTemplate);
							if(!new File(fileMapper.filePath).isFile()){
								PathHandler.initPath(fileMapper.filePath);
								BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(fileMapper.filePath,true));
								BufferedWriter_CONTEXT.put(fileMapper.id, bufferedWriter);
								if(fileMapper.isFileHead){
									bufferedWriter.write(collect + System.getProperty("line.separator"));		//生成文件头
									bufferedWriter.flush();
								}
							}else{
								PathHandler.initPath(fileMapper.filePath);
								BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(fileMapper.filePath,true));
								BufferedWriter_CONTEXT.put(fileMapper.id, bufferedWriter);
							}
						} catch (Exception e) {
							Log.error(e);
						}
					}
					Log.info("=============文件映射配置[" + fileMapper.id + "]加载完毕!~");
				});
			} catch (Exception e) {
				Log.error("文件映射配置加载异常, 嵌套异常: \n" + e.toString());
			}
		}
		public static String initFile(String fileTemplate) throws ServiceException {
			try {
				if(fileTemplate == null || fileTemplate.trim().length() < 1){
					return null;
				}
				return StringHandle.dateFormatPath(fileTemplate, new Date()) + ".MQ";
			} catch (Exception e) {
				throw new ServiceException("静态缓冲输出流加载异常, 嵌套异常: \n" + e.toString());
			}
		}
	}
	
	/**
	 * @功能描述 读取多个路径
	 * @param mapperKey
	 * @param filePaths
	 * @param clazz
	 * @return
	 * @throws ServiceException
	 */
	public static <T> List<T> readFiles(String mapperKey, String[] filePaths , Class<T> clazz) throws ServiceException {
		if(filePaths  == null || filePaths.length < 1){
			throw new ServiceException("文件读取数据异常, 异常信息: 无效路径!~");
		}
		ArrayList<T> arrayList = new ArrayList<>();
		for (String filePath : filePaths) {
			List<T> readFile = readFile(mapperKey,filePath, clazz);
			if(readFile != null && readFile.size() > 0){
				arrayList.addAll(readFile);
			}
		}
		return arrayList;
	}
	
	/**
	 * @功能描述 将一行数据写入文件
	 * @param mapperKey
	 * @param filePath
	 * @param datas
	 * @throws ServiceException
	 */
	public static void writeFile(String mapperKey, String filePath, String[] datas) throws ServiceException {
		//参数验证
		if (mapperKey == null || mapperKey.trim().length() < 1 || filePath == null || !PathHandler.initPath(filePath).isFile()) {
			throw new ServiceException("数据写入文件异常, 异常信息: 非法参数!~");
		}
		//获取分隔符, 字符编码
		FileMapper fileMapper = FileMapperContext.MAPPER_CONTEXT.get(mapperKey);
		String separator = mapperKey;
		Charset charEncod = Charset.forName("UTF-8");
		if(fileMapper != null){
			separator = fileMapper.separator;
			charEncod = Charset.forName(fileMapper.charEncod);
		}
		String fileLine = Stream.of(datas).collect(Collectors.joining(separator));
		try(BufferedWriter writer = Files.newBufferedWriter(Paths.get(filePath), charEncod)){
			writer.write(fileLine);
		} catch (IOException e) {
			throw new ServiceException("数据写入文件IO异常, 嵌套异常: " + e.toString());
		}
	}
	/**
	 * @param <T>
	 * @功能描述 文件读取数据
	 * @param mapperKey
	 * @param filePath
	 * @return
	 * @throws ServiceException
	 */
	public static < T> List<T> readFile(String mapperKey,String filePath,Class<T> javaClass) throws ServiceException{
		//参数验证
		if(mapperKey == null || mapperKey.trim().length() < 1 || !new File(filePath).isFile()){
			throw new ServiceException("文件读取数据异常, 异常信息: 非法参数!~");
		}
		long startTime = System.currentTimeMillis();
		Log.info("开始读取文件["+PathHandler.getFileName(filePath)+"]");
		try {
			//获取映射
			Context.FileMapper fileMapperObject = FileMapperContext.MAPPER_CONTEXT.get(mapperKey);
			if(fileMapperObject == null){
				throw new ServiceException("文件读取数据异常, 异常信息: 无效的文件映射键!~");
			}
			Charset charEncod = Charset.forName(fileMapperObject.charEncod);			//获取字符编码
			Map<String, String> fileMapperMap = fileMapperObject.mapper;						//获取文件对象映射
			String[]  variableArray = fileMapperMap.keySet().toArray(new String[]{});	//获取属性数组
			String separator = fileMapperObject.separator;															//获取文件列分隔符
			boolean isFileHead = fileMapperObject.isFileHead;													//文件是否存文件头
			//读取文件
			List<String> fileContext = Files.readAllLines(Paths.get(filePath),charEncod);	
			if(fileContext == null || fileContext.size() < 1) return null;
			List<T> dataList = new LinkedList<>();
			//开始解析
			for (int lineIndex = isFileHead ? 1 : 0; lineIndex < fileContext.size(); lineIndex++) {
				//获取行内容
				String fileLine =  fileContext.get(lineIndex);
				T newInstance = javaClass.newInstance();
				if(fileLine == null || fileLine.trim().length() < 1) continue;
				String[] fileColumn = fileLine.split("["+separator+"]");
				for (int columnIndex = 0; columnIndex < fileColumn.length; columnIndex++) {
					Field declaredField = javaClass.getDeclaredField(variableArray[columnIndex]);
					declaredField.setAccessible(true);
					declaredField.set(newInstance, baseClass(fileColumn[columnIndex], declaredField.getType()) );
					declaredField.setAccessible(false);
				}
				dataList.add(newInstance);
			}
			fileContext.clear();
			long endTime = System.currentTimeMillis();
			Log.info("文件["+PathHandler.getFileName(filePath)+"]读取完毕, 耗时: " + (endTime - startTime) + "ms");
			return dataList;
		} catch (IOException e) {
			throw new ServiceException("文件读取数据IO读取异常, 异常信息: " + e.toString());
		} catch (InstantiationException | IllegalAccessException e) {
			throw new ServiceException("文件读取数据反射对象异常, 异常信息: " + e.toString());
		}  catch (NoSuchFieldException | SecurityException e) {
			throw new ServiceException("文件读取数据反射属性异常, 异常信息: " + e.toString());
		} 
	}
	/**
	 * @功能描述 数据写入文件
	 * @param mapperKey
	 * @param filePath
	 * @param dataList
	 * @throws ServiceException
	 */
	public static <T> void writeFile(String mapperKey, String filePath, List<T> dataList) throws ServiceException {
		// 参数验证
		if (mapperKey == null || mapperKey.trim().length() < 1 || filePath == null || !PathHandler.initPath(filePath).isFile()) {
			throw new ServiceException("数据写入文件异常, 异常信息: 非法参数!~");
		}
		long startTime = System.currentTimeMillis();
		Log.info("开始写入文件["+PathHandler.getFileName(filePath)+"]");
		// 获取映射
		Context.FileMapper fileMapperObject = FileMapperContext.MAPPER_CONTEXT.get(mapperKey);
		if (fileMapperObject == null) {
			throw new ServiceException("数据写入文件异常, 异常信息: 无效的文件映射键!~");
		}
		Charset charEncod = Charset.forName(fileMapperObject.charEncod);				//字符编码
		boolean isFileHead = fileMapperObject.isFileHead;														//是否拥有文件头
		String separator = fileMapperObject.separator;																//列分隔符
		Map<String, String> mapper = fileMapperObject.mapper;											//属性文件映射
		String fileHeads = mapper.values().stream().collect(Collectors.joining(separator));	//文件头
		List<String> variableName = mapper.keySet().stream().collect(Collectors.toList());	//属性名
		try (BufferedWriter writer = Files.newBufferedWriter(Paths.get(filePath),charEncod)) {
			if(isFileHead) writer.write(fileHeads + System.getProperty("line.separator"));
			if(dataList == null || dataList.size() < 1) return;
			for (T data : dataList) {
				ArrayList<String> variableValueList = new ArrayList<>();
				Class<? extends Object> clazz = data.getClass();
				for (String variable : variableName) {
					Field declaredField = clazz.getDeclaredField(variable);
					declaredField.setAccessible(true);
					Object object = declaredField.get(data);
					if(object == null){
						variableValueList.add("");
					}else if(declaredField.getType().getName().equals(Date.class.getName())){
						String format = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss").format(object);
						variableValueList.add(format);
					}else{
						variableValueList.add(String.valueOf(object));
					}
				}
				String fileLine = variableValueList.stream().collect(Collectors.joining(separator));
				writer.write(fileLine + System.getProperty("line.separator"));
			}
			Long endTime = System.currentTimeMillis();
			Log.info("文件["+PathHandler.getFileName(filePath)+"]写入结束, 耗时: " + (endTime - startTime) + "ms");
		} catch (IOException e) {
			throw new ServiceException("数据写入文件IO异常, 嵌套异常: " + e.toString());
		} catch (NoSuchFieldException | SecurityException e) {
			throw new ServiceException("数据写入文件反射属性异常, 嵌套异常: " + e.toString());
		} catch (IllegalArgumentException | IllegalAccessException e) {
			throw new ServiceException("数据写入文件读取属性异常, 嵌套异常: " + e.toString());
			
		} 
	}
	
	/**
	 * @功能描述 将单条记录写入文件(续写)
	 * @param mapperKey
	 * @param filePath
	 * @param data
	 * @throws ServiceException
	 * @throws  
	 */
	public static <T> String writeFile(String mapperKey, T data) throws Exception {
		// 参数验证
		if (mapperKey == null || mapperKey.trim().length() < 1 || data == null) {
			throw new ServiceException("数据写入文件异常, 异常信息: 非法参数!~");
		}
		// 获取映射
		Context.FileMapper fileMapperObject = FileMapperContext.MAPPER_CONTEXT.get(mapperKey);
		BufferedWriter bufferedWriter = FileMapperContext.BufferedWriter_CONTEXT.get(mapperKey);
		if (fileMapperObject == null) {
			throw new ServiceException("数据写入文件异常, 异常信息: 无效的文件映射键!~");
		}
		String separator = fileMapperObject.separator;																			//列分隔符
		Map<String, String> mapper = fileMapperObject.mapper;										//属性文件映射
		List<String> variableName = mapper.keySet().stream().collect(Collectors.toList());	//属性名
		Class<? extends Object> clazz = data.getClass();
		try {
			List<String> fileLine = new LinkedList<>();
			for (String variable : variableName) {
				Field declaredField = clazz.getDeclaredField(variable);
				declaredField.setAccessible(true);
				Object variableValue = declaredField.get(data);
				if(variableValue == null){
					fileLine.add("");
				}else if(declaredField.getType().getName().equals(Date.class)){
					fileLine.add(new SimpleDateFormat("yyyy-MM-dd hh:mm:ss").format(variableValue));
				}else{
					fileLine.add(variableValue.toString());
				}
				declaredField.setAccessible(false);
			}
			String filePath = fileMapperObject.filePath;
			Date charExtractDate = StringHandle.charExtractDate(filePath);
			if(!DateHandle.dateEquals(charExtractDate,new Date())){
				bufferedWriter.close();
				String collect = mapper.values().stream().collect(Collectors.joining(separator));
				fileMapperObject.filePath = FileMapperContext.initFile(fileMapperObject.fileTemplate);
				PathHandler.initPath(fileMapperObject.filePath);
				bufferedWriter = new BufferedWriter(new FileWriter(filePath,true));
				if(fileMapperObject.isFileHead){
					bufferedWriter.write(collect + System.getProperty("line.separator"));		//生成文件头
				}
				FileMapperContext.BufferedWriter_CONTEXT.put(mapperKey, bufferedWriter);
			}
			bufferedWriter.write(fileLine.stream().collect(Collectors.joining(separator)) + System.getProperty("line.separator"));
			return  fileMapperObject.filePath;
		}catch (Exception e) {
			throw new ServiceException("数据文件续写执行异常, 嵌套异常: \n" + e.toString());
		}finally {
			bufferedWriter.flush();
		}
	}
	/**
	 * @功能描述 文件映射上下文
	 * @author 索睿君
	 * @date 2019年7月5日10:01:21
	 */
	@XmlAccessorType(XmlAccessType.FIELD)
	@XmlRootElement(name = "context")
	public static class Context {
		@XmlElement(name = "fileMapper")
		public List<FileMapper> fileMapperList;

		@XmlAccessorType(XmlAccessType.FIELD)
		@XmlRootElement(name = "fileMapper")
		public static class FileMapper {
			/** 唯一标识 */
			@XmlAttribute
			private String id;
			/** 是否拥有文件头 (true是false否) */
			@XmlAttribute
			private boolean isFileHead;
			/** 每列的分隔符 */
			@XmlAttribute
			private String separator;
			/** 字符编码 */
			@XmlAttribute
			private String charEncod = "UTF-8";
			/** 文件路径 */
			@XmlAttribute
			private String fileTemplate;
			private String filePath;
			/** Java属性名与文件头名称映射 */
			@XmlElement(name = "mapper")
			private List<Mapper> mapperList;
			private Map<String, String> mapper = new LinkedHashMap<>();
			public String getFileTemplate() {
				return fileTemplate;
			}
			@XmlAccessorType(XmlAccessType.FIELD)
			@XmlRootElement(name = "mapper")
			public static class Mapper {
				@XmlAttribute
				private String entityAttribute;
				@XmlAttribute
				private String fileAttribute;
			}
			@Override
			public String toString() {
				return "FileMapper [id=" + id + ", isFileHead=" + isFileHead + ", separator=" + separator
						+ ", charEncod=" + charEncod + ", fileTemplate=" + fileTemplate + ", filePath=" + filePath
						+ ", mapperList=" + mapperList + ", mapper=" + mapper + "]";
			}
		}
	}
	
	@SuppressWarnings("unchecked")
	public static <T> T baseClass(Object value,Class<?> classType) {
		try {
			if(String.class.equals(classType)){
				return (T) new String(value.toString());
			}
			if(Character.class.equals(classType) || "char".equals(classType.getName())){
				return (T) new Character(value.toString().charAt(0));
			}
			if(Integer.class.equals(classType) || "int".equals(classType.getName())){
				return (T) new Integer(value.toString());
			}
			if(Long.class.equals(classType) || "long".equals(classType.getName())){
				return (T) new Long(value.toString());
			}
			if(Short.class.equals(classType) || "short".equals(classType.getName())){
				return (T) new Short(value.toString());
			}
			if(Double.class.equals(classType) || "double".equals(classType.getName())){
				return (T) new Double(value.toString());
			}
			if(Float.class.equals(classType) || "float".equals(classType.getName())){
				return (T) new Float(value.toString());
			}
			if(Boolean.class.equals(classType) || "boolean".equals(classType.getName())){
				return (T) new Boolean(value.toString());
			}
			if(Date.class.equals(classType)){
				return (T) DateHandle.format(value.toString());
			}
			throw new ClassCastException(value+"转换为" + classType+"类型出现异常,未收录的类型!");
		} catch (Exception e) {
			throw new ClassCastException(value+"转换为" + classType+"类型出现异常," + e.toString());
		}
	}
}

• fileMapper.xml

<?xml version="1.0" encoding="UTF-8" standalone="yes" ?> 
<!DOCTYPE context [
   <!ELEMENT context (fileMapper | fileTemplate)*>
   <!ELEMENT fileMapper (mapper+)>
   <!ELEMENT mapper EMPTY>
   <!ATTLIST fileMapper id ID #REQUIRED>
   <!ATTLIST fileMapper isFileHead (true|false) "true">
   <!ATTLIST fileMapper separator CDATA #REQUIRED>
   <!ATTLIST fileMapper charEncod CDATA "UTF-8">
   <!ATTLIST fileMapper fileTemplate CDATA #IMPLIED >
   <!ATTLIST mapper entityAttribute CDATA #REQUIRED>
   <!ATTLIST mapper fileAttribute CDATA #REQUIRED>
]>

<context>
    <!--  
        一, fileMapper属性注释: 
            id = "配置的唯一值, 根据ID可从容器中获取该条配置"
            isFileHead = "是否存在文件头 true存在, false不存在"
            separator = "每一列数据的分隔符"
            charEncod = "文件编码"
        二, mapper属性注释
            entityAttribute = "Java类的属性名"
            fileAttribute = "文件的列名"
    -->
    <fileMapper id="userInfo_MIIT" isFileHead="false" separator="," charEncod="UTF-8" >
        <mapper entityAttribute="name" fileAttribute="fullName" />
        <mapper entityAttribute="age" fileAttribute="age" />
        <mapper entityAttribute="sex" fileAttribute="gender" />
        <mapper entityAttribute="phone" fileAttribute="phoneNumber" />
        <mapper entityAttribute="company" fileAttribute="organize" />
        <mapper entityAttribute="department" fileAttribute="department" />
        <mapper entityAttribute="location" fileAttribute="province" />
    </fileMapper>
    
</context>

使用

//生成文件
FileMapping.writeFile('配置ID','文件路径', new List<数据模型>list);
//读取文件
List<数据模型> list = FileMapping.readFile('配置ID','文件路径',数据模型.class);

总结

1. 生成500万数据耗时13113ms, 文件大小352MB
2. 读取500万数据耗时242876ms, 文件大小352MB
3. 有问题或优化点及时评论O(∩_∩)O~