Java 分割功能实现-CSDN博客

在项目针对比较大的文件需要文件的分割功能,特意写了一个具有分割功能代码,以供大家拍砖之用,目的进行沟通交流,如有好的建议和联系本人沟通交流谢谢!

package easyway.tbs.file.transport.core.split;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.io.Serializable;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

/**
 * 文件读写访问器
 * 
 * @author longgangbai
 *
 */
public class DataFileAccessor implements Serializable {
	 private final static Log logger = LogFactory.getLog(DataFileAccessor.class);
     
	/**
	 * 
	 */
	private static final long serialVersionUID = 1L;
	/**
	 * 只读模式
	 */
	public static final String RANDOMACESSFILE_READ_MODE="r";
	/**
	 * 只写模式
	 */
	public static final String RANDOMACESSFILE_WRITE_MODE="w";
	/**
	 * 读写模式
	 */
	public static final String RANDOMACESSFILE_WRITE_READ_MODE="rw";
	
	/**
	 * 随机读取的缓存文件的长度
	 */
	public static final int RANDOMACESSFILE_BUFFER_LENGTH=1024*1000;
				
				
	/**
	 * 文件读取的对象操作对象
	 */
	private RandomAccessFile fileAcessor;
	/**
	 * 在文件的路径
	 */
    private String filePath;
    
    /**
     * 
     * @throws IOException
     */
	public DataFileAccessor() throws IOException {
		this("", RANDOMACESSFILE_WRITE_READ_MODE);
	}

	/**
	 * 文件读取访问其
	 * @param sName 
	 * @param nPos
	  * @param nPos
	 * @throws IOException
	 */
	public DataFileAccessor(String filePath, String mode) throws IOException {
		this.filePath=filePath;
		//支持读写操作
		if(logger.isDebugEnabled()){
			logger.info("filePath="+filePath+"======mode===="+mode);
		}
		fileAcessor = new RandomAccessFile(filePath, mode);
	}

	/**
	 * 写文件的片段
	 * @param buffer 
	 *             临时存储的数据的数组
	 * @param nStart
	 * @param nLen
	 * @return
	 */
	public synchronized int write(byte[] buffer, int nStart, int nLen) {
		if(logger.isDebugEnabled()){
			logger.info("nStart="+nStart+"  nLen="+nLen);
		}
		int n = -1;
		try {
			//移动文件指针
			fileAcessor.seek(nStart);
			fileAcessor.write(buffer, nStart, nLen);
			n = nLen;
		} catch (IOException e) {
			logger.error("write file "+filePath +" nStart="+nStart+"   nLen="+nLen,e);
			
		}
		return n;
	}
	
	/**
	 * 写文件的片段
	 * @param buffer 
	 *             临时存储的数据的数组
	 * @param nStart
	 * @param nLen
	 * @return
	 */
	public synchronized int readFile(byte[] buffer, int offset, int nLen) {
		try {
			fileAcessor.seek(offset);
			long length = nLen + 1;// 文件段的长度
			byte[] b = new byte[RANDOMACESSFILE_BUFFER_LENGTH];
			long m = length / b.length;// 读和写入2048个字节的次数
			int leftsize = (int) (length - m * b.length);// 剩下的字节数
			long timestart = System.currentTimeMillis();
			for (long i = 1; i <= m; i++) {
				fileAcessor.read(b);// 将数据从from文件读入
			}
			fileAcessor.read(b, 0, leftsize);// 读入最后剩下的字节
			fileAcessor.close();// 关闭输入
			long timeend = System.currentTimeMillis();
			if(logger.isDebugEnabled()){
				logger.info("线程" + Thread.currentThread().getName()
						+ "读写完毕,共使用" + (timeend - timestart) + "毫秒");
			}
			
		} catch (IOException e) {
			e.printStackTrace();
		}
		
		return -1;
	}
	/**
	 * 
	 * @param file
	 * @param tempFile
	 * @param begin
	 * @param end
	 * @return
	 * @throws IOException
	 */
	  public  long writeTempFile(File file, File tempFile, long begin ,long end) throws IOException{
		  RandomAccessFile in=new RandomAccessFile(file,RANDOMACESSFILE_READ_MODE);
		  RandomAccessFile out=new RandomAccessFile(tempFile,RANDOMACESSFILE_WRITE_READ_MODE);
		  byte[] b=new byte[RANDOMACESSFILE_BUFFER_LENGTH];
		  int n=0;
		  in.seek(begin);
		  while(in.getFilePointer()<=end&&(n=in.read(b))!=-1){
			  out.write(b,0,n);
		  }
		  long endPointer=in.getFilePointer();
		  in.close();
		  out.close();
		  return endPointer;
	  }
	  
	
	public synchronized int read(byte[] buffer, int offset, int nLen) {
		int n = -1;
		FileInputStream is=null;
		try {
			 is=new FileInputStream(filePath);
			 //忽略前面的读取信息
			 is.skip(offset);
			 fileAcessor.seek(offset);
			 //定义读取输入流内容的缓存数据
			 byte[] buff=new byte[RANDOMACESSFILE_BUFFER_LENGTH];
			 //定义最多需要读取几次就可以完成本次的读取
			 long times=(nLen%RANDOMACESSFILE_BUFFER_LENGTH==0)?nLen/RANDOMACESSFILE_BUFFER_LENGTH:((nLen/RANDOMACESSFILE_BUFFER_LENGTH)+1);
			 int hasRead=0;
			 for (int i = 0; i < times; i++) {
				 hasRead=is.read(buff);
				 if(hasRead<0){
					 break;
				 }
				 fileAcessor.read(buffer);
			}
			n = nLen;
		} catch (IOException e) {
			e.printStackTrace();
		}finally{
			if(is!=null){
				try {
					is.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
				is=null;
			}
		}
		return n;
	}
	
	/**
	 * 关闭流对象
	 *
	 */
	public void close(){
	    if (fileAcessor != null) {
            try {
            	fileAcessor.close();
            } catch (Throwable ignore) {
            }
            fileAcessor = null;
        }
	}


	public String getFilePath() {
		return filePath;
	}

	public void setFilePath(String filePath) {
		this.filePath = filePath;
	}
	
	
}

package easyway.tbs.file.transport.core.split;

import java.io.File;
import java.io.IOException;
import java.util.concurrent.CountDownLatch;

import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
/**
 * 文件分割线程
 * @author longgangbai
 *
 */
public class FileSplitterFetch implements Runnable {
	 private final static Log logger = LogFactory.getLog(DataFileAccessor.class);
	  
    /**
     * 将读取到的字节输出到raf中
     */
    private DataFileAccessor raf;
    /**
     * 文件整个完成标识
     */
	private CountDownLatch masterLatch;
    /**
     * 当前文件读取执行完毕的标识
     */
	private CountDownLatch childrenLatch;
    /**
     * 文件分割的文件片段
     */
    private FileSplitterMessage message;
    
    

    /**
     * 构造器 
     * @param filepath 	服务器文件的路径
     * @param offset	输出流和读取起始点
     * @param length   输出流和读取结束点
     * @param buffer   数组的大小
     * @throws IOException
     */
    public FileSplitterFetch(FileSplitterMessage message,CountDownLatch masterLatch,CountDownLatch childrenLatch) throws IOException {
    	super();
    	this.message=message;
    	this.masterLatch=masterLatch;
    	this.childrenLatch=childrenLatch;
    	this.raf=new DataFileAccessor(message.getFilePath(),DataFileAccessor.RANDOMACESSFILE_READ_MODE);
    	System.out.println("raf=="+this.raf);
    }

    /**
     * 执行读取的方法
     */
    public void run() {

    	File tempFile = message.getTempFile();
    	try {
        	if(!tempFile.exists()){
        		tempFile.createNewFile();
        	}
			//masterLatch.await();
    		this.raf.writeTempFile(new File(message.getFilePath()), tempFile, message.getStartPos(), message.getEndPos());
        } catch (Exception ex) {
        	if(tempFile.exists()){
        		try {
					FileUtils.forceDelete(tempFile);
				} catch (IOException e) {
					
				}
        	}
        }finally {  
        	//使用finally块来关闭当前线程的输入流、输出流
        	this.raf.close();
        	//childrenLatch.countDown();
        }
        
    }


}

package easyway.tbs.file.transport.core.split;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * 文件分割管理器
 * 当单个线程执行完毕之后，开始发送文件
 * @author longgangbai
 *
 */
public class FileSplitterManager {
	private static String File_Split_Separator="_";
	 private static final transient Logger logger = LoggerFactory.getLogger(FileSplitterManager.class);

	/**
	 * 文件的总长度
	 */
	private long fileLength;

	/**
	 * 整个文件分割完成的标识
	 * 开始执行发送消息
	 */
	private CountDownLatch masterLatch = new CountDownLatch(1);
	/**
	 *文件分割线程的数量
	 */
	private int splitThreadNum;
    /**
     * 
     * 当每一个线程执行完毕时候，则报告执行执行线程完成，当所有的执行线程
     * 执行完毕，则开始发送文件
     */
	private CountDownLatch childrenLatch = null;
	
	/**
	 * 文件片段的集合
	 */
	private List<FileSplitterMessage> fileSplitterMessageList=null;
	/**
	 * 创建文件锁
	 */
	private FileTransportLock fileLock=null;
	//发送文件的路径
	private String filepath;
	//发送文件的限制大小
	private int filepageSize;
	
	
	private String fileChannel;
	
	private File file;
	
	/**
	 * 
	 * @param filepath
	 * @param filepageSize
	 */
	public FileSplitterManager(String filepath,int filepageSize,String fileChannel){
		this.filepath=filepath;
		this.filepageSize=filepageSize;
		this.fileChannel=fileChannel;
	}

	/**
	 * 文件分割管理器的方法
	 *
	 */
	public void splitFileMgr(){
		file=new File(filepath);
		 //获取文件的大小
    	long fileTotalSize=file.length();
    	//如果文件的大小
    	if(filepageSize>0&&(filepageSize<fileTotalSize)){
    		initSplitFileMgr(file);
    	}else{
    		//直接发送文件
				fileSplitterMessageList=new ArrayList<FileSplitterMessage>(1);
				
				FileSplitterMessage fileSplitMessage=new  FileSplitterMessage();
	    		fileSplitMessage.setFilePath(file.getAbsolutePath());
	    		fileSplitMessage.setFileLength(file.length());
	    		fileSplitMessage.setFileName(file.getName());
	    		fileSplitMessage.setSplitThreadNum(1);
	    		fileSplitMessage.setCurrentThreadNum(1);
	    		fileSplitMessage.setStartPos(0);
	    		fileSplitMessage.setEndPos(filepageSize);
	    		fileSplitMessage.setLen(fileTotalSize);
	    		
	    		String threadNum="1";
	    		String currentThread="1";
	    		//临时文件的命名规则：源文件名称+“_”+"总线程数"+“_”+"当前线程编号"
	    		String tempFileName=file.getName()+File_Split_Separator+threadNum+File_Split_Separator+currentThread+".tmp";
	    		//文件相对比较小，文件的总线程数为1，当前线程也为1
	    		File tempFile=getTempFileName(file.getParent(),tempFileName);
	    		
	    		fileSplitMessage.setTempFile(tempFile);
	    		fileSplitterMessageList.add(fileSplitMessage);
	    		
	    		//重新命名原来的文件，标识文件已经开始发送
	    		file.renameTo(new File(file.getAbsolutePath()+".bak"));
	    		
	    		if(logger.isDebugEnabled()){
					logger.debug("the direct send file "+file.getAbsolutePath());
				}
    	}
    	System.out.println("fileSplitterMessageList ="+fileSplitterMessageList.size());
	}
	/**
	 * 创建临时文件
	 * @param parentPath
	 * @param tempFileName
	 * @return
	 */
	public File getTempFileName(String parentPath,String tempFileName){
		System.out.println(parentPath+File.separator+tempFileName);
			File tempFile=new File(parentPath+File.separator+tempFileName);
	
		try {
			if(tempFile.exists()){
				FileUtils.forceDelete(tempFile);
				if(logger.isDebugEnabled()){
					logger.info("文件存在，已经删除！");
				}
			}
			tempFile.createNewFile();
		} catch (IOException e) {
			if(logger.isDebugEnabled()){
				logger.info("文件创建失败："+e);
			}
		}
		return tempFile;
	}

	/**
	 * 分割文件的方法
	 *
	 */
	public void initSplitFileMgr(File file) {
		if(file.exists()){
			//创建文件锁
			//fileLock=new FileTransportLock(file,false);
			try {
				//设置文件锁
				//fileLock.lock();
				String fileName=file.getName();
	            //获取文件的长度
	        	fileLength=file.length();
	        	//获取文件对应的线程数
	        	splitThreadNum=(int)getThreadNum(fileLength,filepageSize);
	        	//线程同步线程
	        	childrenLatch = new CountDownLatch(splitThreadNum);
	        	//
	        	fileSplitterMessageList=new ArrayList<FileSplitterMessage>(splitThreadNum);
	        	//文件分割的过程
	        	splitFile(filepath,filepageSize,fileName);
			} catch (Exception e) {
				e.printStackTrace();
			}finally{
				//释放文件锁
				//fileLock.unlock();
			}
		}
	}
	

	/**
	 * 总的线程数
	 * @param fileSize
	 * @param filepageSize
	 * @return
	 */
	public long getThreadNum(long fileSize,long filepageSize){
		long 	threadNum=(fileSize%filepageSize==0)?(fileSize%filepageSize):((fileSize/filepageSize)+1);
		return threadNum;
	}
	
	/**
	 * 分解文件
	 * @param filepath  文件路径
	 * @param filepageSize  文件的大小
	 */ 
	public void splitFile(String filepath,int filepageSize,String fileName){
				
		        //当前线程真正读取的长度
	        	int length=filepageSize;
	        	//创建一个线程池
	        	ExecutorService threadExePool = Executors.newFixedThreadPool(splitThreadNum);
	        	//循环创建线程
	        	for(int i=1;i<=splitThreadNum;i++){
	        		//起始点
	        		int startPos=(i-1)*filepageSize;
	        		
	        		int endPos=i*filepageSize;
	        		
	        		FileSplitterMessage  fileSplitterMessage=new FileSplitterMessage();
	        		
	        		String tempFileName=file.getName()+File_Split_Separator+splitThreadNum+File_Split_Separator+i+".tmp";
	        		
	        		File tempFile=getTempFileName(file.getParent(), tempFileName);
	        		
	        		//针对文件最后一个片度结束点和片段长度
	        		if((splitThreadNum==i)&&(fileLength%filepageSize!=0)){
	        			endPos=(int)fileLength;
	        			length=(int)fileLength-startPos;
	        		}
	        		//设置文件片段内容
        			fileSplitterMessage.setTempFile(tempFile);
        			//文件读取起始点
        			fileSplitterMessage.setStartPos(startPos);
        			//文件读取起始点
        			fileSplitterMessage.setEndPos(endPos);
        			//文件总长度
        			fileSplitterMessage.setFileLength(fileLength);
        			//文件路径
        			fileSplitterMessage.setFilePath(filepath);
        			//文件分割线程数
        			fileSplitterMessage.setSplitThreadNum(splitThreadNum);
        			//当前线程编号数
        			fileSplitterMessage.setCurrentThreadNum(i);
        			//读取的长度
        			fileSplitterMessage.setLen(length);
        			
        			System.out.println("fileSplitterMessage="+fileSplitterMessage);
        			//文件的名称
        			fileSplitterMessage.setFileName(file.getName());
        			fileSplitterMessageList.add(fileSplitterMessage);
	        }
	       try {
	    		if(logger.isDebugEnabled()){
					logger.debug("文件"+fileName+"开始分割....");
				}
	    		//线程池开始执行线程
	        	for (FileSplitterMessage fileMessage : fileSplitterMessageList) {
	        		threadExePool.execute(new FileSplitterFetch(fileMessage,masterLatch,childrenLatch));					
				}
				masterLatch.countDown();//宣布开始
				if(logger.isDebugEnabled()){
					logger.debug("文件"+fileName+"分割完毕....");
				}
				//等待CountdownLatch信号为0，表示所有子线程都结束。 
				childrenLatch.await();//等待结束
		 
				System.out.println("size ================="+fileSplitterMessageList.size());
			} catch (Exception e) {
				e.printStackTrace();
			} finally {
				System.out.println("整个文件"+fileName+"分割完毕");
				//开始执行发送文件
			}
			//注意线程已经要结束了，但是threadExePool线程如果不关闭是不会结束的
			threadExePool.shutdown();
	}


	public int getFilepageSize() {
		return filepageSize;
	}


	public void setFilepageSize(int filepageSize) {
		this.filepageSize = filepageSize;
	}


	public String getFilepath() {
		return filepath;
	}


	public void setFilepath(String filepath) {
		this.filepath = filepath;
	}


	public String getFileChannel() {
		return fileChannel;
	}

	public void setFileChannel(String fileChannel) {
		this.fileChannel = fileChannel;
	}

}

package easyway.tbs.file.transport.core.split;

import java.io.File;

import org.apache.commons.lang.builder.ToStringBuilder;
import org.apache.commons.lang.builder.ToStringStyle;

/**
 * 文件分割一段的信息
 * 
 * 
 * @author longgangbai
 *
 */
public class FileSplitterMessage {
	
		/**
		 * 文件路径
		 */
		private String filePath;
		/**
		 * 文件总长度
		 */
		private long fileLength;
		
		/**
		 * 文件名称
		 */
		private String fileName;
		/**
		 * 分割线程数量
		 */
		private int splitThreadNum;
		
		/**
		 * 当前分割线程编号
		 */
		private int currentThreadNum;
		/**
		 * 文件开始读取的起始点
		 */
		private int startPos;
		/**
		 * 文件开始读取的结束点
		 */
		private int endPos;
		/**
		 * 文件起止点之间的数据信息
		 */
		private byte[] data;
		/**
		 * 临时文件
		 */
		private File tempFile;
		/**
		 * 临时文件的长度
		 */
		private long len;
		
		
		
		public byte[] getData() {
			return data;
		}
		public void setData(byte[] data) {
			this.data = data;
		}
		public int getEndPos() {
			return endPos;
		}
		public void setEndPos(int endPos) {
			this.endPos = endPos;
		}
		public long getFileLength() {
			return fileLength;
		}
		public void setFileLength(long fileLength) {
			this.fileLength = fileLength;
		}
		public String getFilePath() {
			return filePath;
		}
		public void setFilePath(String filePath) {
			this.filePath = filePath;
		}
		public int getSplitThreadNum() {
			return splitThreadNum;
		}
		public void setSplitThreadNum(int splitThreadNum) {
			this.splitThreadNum = splitThreadNum;
		}
		public int getStartPos() {
			return startPos;
		}
		public void setStartPos(int startPos) {
			this.startPos = startPos;
		}
		public int getCurrentThreadNum() {
			return currentThreadNum;
		}
		public void setCurrentThreadNum(int currentThreadNum) {
			this.currentThreadNum = currentThreadNum;
		}
		

		public String getFileName() {
			return fileName;
		}
		public void setFileName(String fileName) {
			this.fileName = fileName;
		}
		public File getTempFile() {
			return tempFile;
		}
		public void setTempFile(File tempFile) {
			this.tempFile = tempFile;
		}
	
	    @Override
		public String toString(){
			return ToStringBuilder.reflectionToString(this, ToStringStyle.MULTI_LINE_STYLE);
		}
		public long getLen() {
			return len;
		}
		public void setLen(long len) {
			this.len = len;
		}
}

package easyway.tbs.file.transport.core.split;

import java.io.File;

import org.apache.commons.lang.builder.ToStringBuilder;
import org.apache.commons.lang.builder.ToStringStyle;

/**
 * 文件分割一段的信息
 * 
 * 
 * @author longgangbai
 *
 */
public class FileSplitterMessage {
	
		/**
		 * 文件路径
		 */
		private String filePath;
		/**
		 * 文件总长度
		 */
		private long fileLength;
		
		/**
		 * 文件名称
		 */
		private String fileName;
		/**
		 * 分割线程数量
		 */
		private int splitThreadNum;
		
		/**
		 * 当前分割线程编号
		 */
		private int currentThreadNum;
		/**
		 * 文件开始读取的起始点
		 */
		private int startPos;
		/**
		 * 文件开始读取的结束点
		 */
		private int endPos;
		/**
		 * 文件起止点之间的数据信息
		 */
		private byte[] data;
		/**
		 * 临时文件
		 */
		private File tempFile;
		/**
		 * 临时文件的长度
		 */
		private long len;
		
		
		
		public byte[] getData() {
			return data;
		}
		public void setData(byte[] data) {
			this.data = data;
		}
		public int getEndPos() {
			return endPos;
		}
		public void setEndPos(int endPos) {
			this.endPos = endPos;
		}
		public long getFileLength() {
			return fileLength;
		}
		public void setFileLength(long fileLength) {
			this.fileLength = fileLength;
		}
		public String getFilePath() {
			return filePath;
		}
		public void setFilePath(String filePath) {
			this.filePath = filePath;
		}
		public int getSplitThreadNum() {
			return splitThreadNum;
		}
		public void setSplitThreadNum(int splitThreadNum) {
			this.splitThreadNum = splitThreadNum;
		}
		public int getStartPos() {
			return startPos;
		}
		public void setStartPos(int startPos) {
			this.startPos = startPos;
		}
		public int getCurrentThreadNum() {
			return currentThreadNum;
		}
		public void setCurrentThreadNum(int currentThreadNum) {
			this.currentThreadNum = currentThreadNum;
		}
		

		public String getFileName() {
			return fileName;
		}
		public void setFileName(String fileName) {
			this.fileName = fileName;
		}
		public File getTempFile() {
			return tempFile;
		}
		public void setTempFile(File tempFile) {
			this.tempFile = tempFile;
		}
	
	    @Override
		public String toString(){
			return ToStringBuilder.reflectionToString(this, ToStringStyle.MULTI_LINE_STYLE);
		}
		public long getLen() {
			return len;
		}
		public void setLen(long len) {
			this.len = len;
		}
}

package easyway.tbs.file.transport.core.split;

/**
 * 分割文件测试类
 * @author longgangbai
 *
 */
public class FileSplitterManagerMain {
	
	public static void main(String[] args) {
		FileSplitterManager manager=new FileSplitterManager("E:/TestJoltProxy/RMAN.txt",10*1024*1024,"queue-svn");
		manager.splitFileMgr();
	}

}