package com.dc.test.ftp; import java.io.IOException; import java.net.Socket; import java.net.UnknownHostException; import org.apache.commons.net.DefaultSocketFactory; /** * */ public class VpcSocketFactory extends DefaultSocketFactory { private int connectTimeout = 0; public VpcSocketFactory() { super(); } public Socket createSocket(String host, int port) throws UnknownHostException, IOException { Socket socket = new java.net.Socket(); java.net.InetSocketAddress addr = new java.net.InetSocketAddress(host, port); socket.connect(addr, connectTimeout); return socket; } public int getConnectTimeout() { return connectTimeout; } public void setConnectTimeout(int timeout) { connectTimeout = timeout; } } /* * Created on 14-Jun-2005 * * An ftp tool for transfering files in a given source directory/list of files paths to a destination * directory. Provides the option to create archives in both source and destination * using standard archiving sub-directory named with the local timestamp of transfer. */ package com.dc.test.ftp; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.net.SocketException; import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Calendar; import java.util.Date; import java.util.GregorianCalendar; import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.commons.net.ftp.FTP; import org.apache.commons.net.ftp.FTPClient; import org.apache.commons.net.ftp.FTPConnectionClosedException; import org.apache.commons.net.ftp.FTPFile; import org.apache.commons.net.ftp.FTPReply; import org.apache.log4j.PropertyConfigurator; import com.dc.test.ftp.ex.ArchiveException; import com.dc.test.ftp.ex.InvalidFTPAccountException; import com.dc.test.ftp.ex.InvalidFTPServerException; import com.dc.test.ftp.ex.InvalidFileException; /** * TODO proper exceptions handling */ public class FtpAgent { private static final Log _log = LogFactory.getLog(FtpAgent.class); private final static char LOCAL_ARCHIVE = 1; private final static char REMOTE_ARCHIVE = 2; private final long WAIT = 10000; private FTPClient ftp; public String server; public String user; public String password; public static void main(String[] args) { FtpAgent ftpAgent = new FtpAgent(); PropertyConfigurator.configure("./log4j.properties"); String operation = null, server = null, username = null, password = null, source = null, dest = null; boolean archiveSource = false , archiveDest = false, overwrite = true; if (args.length == 14) { for(int i=0; i<args.length; i=i+2) { if(args[i].equals("-o") || args[i].equals("-O")) operation = args[i+1]; else if(args[i].equals("-v") || args[i].equals("-v")) server = args[i+1]; else if(args[i].equals("-u") || args[i].equals("-U")) username = args[i+1]; else if(args[i].equals("-p") || args[i].equals("-P")) password = args[i+1]; else if(args[i].equals("-s") || args[i].equals("-S")) source = args[i+1]; else if(args[i].equals("-d") || args[i].equals("-D")) dest = args[i+1]; else if(args[i].equals("-a") || args[i].equals("-A")) { if (args[i+1].equals("s") || args[i].equals("S")) { archiveSource = true; archiveDest = false; } else if (args[i+1].equals("d") || args[i].equals("D")) { archiveSource = false; archiveDest = true; } else if (args[i+1].equals("b") || args[i].equals("B")) { archiveSource = true; archiveDest = true; } } } if (operation.equals("g") || operation.equals("G")) { try { ftpAgent.getFiles(server, username, password, source, archiveSource, dest, archiveDest); System.out.println("Files Transferred Successfully"); System.exit(0); } catch (Exception e) { System.err.println(e); System.exit(1); } } else if (operation.equals("p") || operation.equals("P")) { try { ftpAgent.putFiles(server, username, password, source, archiveSource, true, dest, archiveDest, overwrite); System.out.println("Files Transferred Successfully"); System.exit(0); } catch (Exception e){ System.err.println(e); System.exit(1); } } } else if(args.length == 7) { if (args[6].equals("s") || args[6].equals("S")) { archiveSource = true; archiveDest = false; } else if (args[6].equals("d") || args[6].equals("D")) { archiveSource = false; archiveDest = true; } else if (args[6].equals("b") || args[6].equals("B")) { archiveSource = true; archiveDest = true; } if (args[0].equals("g") || args[0].equals("G") || args[0].equals("getFiles")) { try { ftpAgent.getFiles(args[1], args[2], args[3], args[4], archiveSource, args[5], archiveDest); System.out.println("Files Transferred Successfully"); System.exit(0); } catch (Exception e) { System.err.println(e); System.exit(1); } } else if (args[0].equals("p") || args[0].equals("P") || args[0].equals("putFiles")) { try { ftpAgent.putFiles(args[1], args[2], args[3], args[4], archiveSource, true, args[5], archiveDest, overwrite); System.out.println("Files Transferred Successfully"); System.exit(0); } catch (Exception e){ System.err.println(e); System.exit(1); } } } else { System.err.println("/nInvalid arguments."); } System.out.println("NOTE: Source files will be deleted after successful transfer!"); System.out.println("Use archiving function to retain a copy of files"); System.out.println("/nUsage : java -jar ftpAgent.jar ATTRIBUTES"); System.out.println("Example: java -jar ftpAgent.jar -o p -v 192.168.0.1 -u user -p myPassword -s ./source -d ./dest -a s"); System.out.println("/nAttribute = PREFIX ARGUMENT"); System.out.println("Prefixes /t/tArguments"); System.out.println("-o --operation/t/t g|p type of operation: g to get files from server or p to put files on server"); System.out.println("-v --ftp server/t/t server address"); System.out.println("-u --ftp user/t/t ftp account user name"); System.out.println("-p --password/t/t ftp account password"); System.out.println("-s --source/t/t source directory"); System.out.println("-d --destination/t destination directory"); System.out.println("-a --achiving options/t n|s|d|b n-no archiving, s-archive source, d-archive destination, b-both"); System.out.println("/nAlternative: java -jar ftpAgent.jar OPERATION SERVER USER PASSWORD SOURCE_DIRECTORY DESTINATION_DIRECTORY ARCHIVING"); System.out.println("Example: java -jar ftpAgent.jar p 192.168.0.1 user myPassword ./source ./dest s"); } /* * get files from sourceDir in the source ftp server using username and password * and store in destDir. If sourceArchive and or destArchive are given then a copy * of downloaded files are archived in the respective directories * */ public FtpAgent() { ftp = new FTPClient(); } public void getFiles(String sourceServer, String userName, String pw, String sourceDir, boolean archiveSource, String destDir, boolean archiveDest) throws InvalidFTPServerException, InvalidFTPAccountException, ArchiveException, IOException, Exception{ server = sourceServer; user = userName; password = pw; List recvFiles = null; File dir = null; String destArchDir = null; Calendar cal = new GregorianCalendar(); SimpleDateFormat dFormat = new SimpleDateFormat("yyyyMMdd"); String archiveDir = dFormat.format(cal.getTime()) + File.separator; if (!sourceDir.endsWith(File.separator) && !sourceDir.equals("")) sourceDir = sourceDir.concat(File.separator); if (!destDir.endsWith(File.separator) && !destDir.equals("")) destDir = destDir.concat(File.separator); dir = new File(destDir); if (!dir.exists()) { dir.mkdir(); } try { connectFTP(); recvFiles = performGetFromDir(sourceDir, destDir, archiveSource, archiveDir); if (_log.isDebugEnabled()) _log.debug("Received files : " + recvFiles); } catch (FTPConnectionClosedException e) { _log.error("Premature disconnection of server: " + sourceServer, e); throw new FTPConnectionClosedException("Premature disconnection of server: " + sourceServer + "/n" + e.getLocalizedMessage()); } catch (IOException e) { _log.error("IO Error on getting files from server:" + sourceServer, e); throw new IOException("IO Error on getting files from server:" + sourceServer + "/n" + e.getLocalizedMessage() ); } finally { if (ftp != null) disconnectFTP(); } // Copy downloaded files to destination archive if required if (archiveDest) { destArchDir = destDir + archiveDir; if (_log.isDebugEnabled()) _log.debug("remote archive: " + destArchDir); performLocalArchive(recvFiles, destArchDir); } } public void getFiles(String sourceServer, String userName, String pw, List pathList, boolean archiveSource, String destDir, boolean archiveDest) throws InvalidFTPServerException, InvalidFTPAccountException, ArchiveException, IOException, Exception{ server = sourceServer; user = userName; password = pw; List recvFiles = null; File dir = null; String destArchDir = null; Calendar cal = new GregorianCalendar(); SimpleDateFormat dFormat = new SimpleDateFormat("yyyyMMdd"); String archiveDir = dFormat.format(cal.getTime()) + File.separator; if (!destDir.endsWith(File.separator) && !destDir.equals("")) destDir = destDir.concat(File.separator); dir = new File(destDir); if (!dir.exists()) { dir.mkdir(); } try { connectFTP(); recvFiles = performGetFiles(pathList, destDir, archiveSource, archiveDir); } catch (FTPConnectionClosedException e) { _log.error("Premature disconnection of server: " + sourceServer, e); throw new FTPConnectionClosedException("Premature disconnection of server: " + sourceServer + "/n" + e); } catch (IOException e) { _log.error("IO Error on getting files from server:" + sourceServer, e); throw new IOException("IO Error on getting files from server:" + sourceServer + "/n" + e); } finally { if (ftp != null) disconnectFTP(); } // Copy downloaded files to destination archive if required if (archiveDest) { destArchDir = destDir + archiveDir; if (_log.isDebugEnabled()) _log.debug("remote archive: " + destArchDir); performLocalArchive(recvFiles, File.separator + destArchDir); } } /* * takes a directory and put files in the directory in destination directory */ public void putFiles(String destServer, String userName, String pw, String sourceDir, boolean archiveSource, boolean cleanup, String destDir, boolean archiveDest, boolean overwrite) throws InvalidFTPServerException, InvalidFTPAccountException, ArchiveException, IOException, SocketException, Exception { server = destServer; user = userName; password = pw; // list of file objects List fileList = null; fileList = getFileList(sourceDir); Calendar cal = new GregorianCalendar(); SimpleDateFormat dFormat = new SimpleDateFormat("yyyyMMdd"); String archiveDir = dFormat.format(cal.getTime()) + File.separator; if (!sourceDir.endsWith(File.separator) && !sourceDir.equals("")) sourceDir = sourceDir.concat(File.separator); if (!destDir.endsWith(File.separator) && !destDir.equals("")) destDir = destDir.concat(File.separator); try { connectFTP(); performPutFiles(fileList, destDir, overwrite); if (archiveDest) { String remoteArchDir = destDir + archiveDir; if (_log.isDebugEnabled()) _log.debug("remote archive: " + remoteArchDir); performPutFiles(fileList, remoteArchDir, overwrite); } } catch (FTPConnectionClosedException e) { _log.error("Premature disconnection of server: " + destServer, e); throw new FTPConnectionClosedException("Premature disconnection of server: " + destServer + "/n" + e.getLocalizedMessage()); } catch (IOException e) { _log.error("IO Error on putting files on server:" + destServer, e); throw new IOException("IO Error putting files on server:" + destServer + "/n" + e.getLocalizedMessage()); } finally { if (ftp != null) disconnectFTP(); } // Copy downloaded files to source archive if required if (archiveSource) { String sourceArchDir = sourceDir + archiveDir; performLocalArchive(fileList, sourceArchDir); } if (cleanup) { performLocalDelete(fileList); } } /* * Takes a list of file paths and puts them on remote ftp server */ public void putFiles(String destServer, String userName, String pw, List sourcePaths, boolean archiveSource, boolean cleanup, String destDir, boolean archiveDest, boolean overwrite) throws InvalidFTPServerException, InvalidFTPAccountException, InvalidFileException,ArchiveException,IOException, SocketException, Exception { server = destServer; user = userName; password = pw; List fileList = new ArrayList(); String sourceArchDir = null; Calendar cal = new GregorianCalendar(); SimpleDateFormat dFormat = new SimpleDateFormat("yyyyMMdd"); String archiveDir = dFormat.format(cal.getTime()) + File.separator; if (!destDir.endsWith(File.separator) && !destDir.equals("")) destDir = destDir.concat(File.separator); try { fileList = getFileList(sourcePaths); connectFTP(); if (_log.isDebugEnabled()) _log.debug("Sending files to remote dir: " + destDir); performPutFiles(fileList, destDir, overwrite); if (archiveDest) { String remoteArchDir = destDir + archiveDir; if (_log.isDebugEnabled()) _log.debug("remote archive: " + remoteArchDir); performPutFiles(fileList, remoteArchDir, overwrite); } } catch (FTPConnectionClosedException e) { _log.error("Premature disconnection of server: " + destServer, e); throw new FTPConnectionClosedException("Premature disconnection of server: " + destServer + "/n" + e.getLocalizedMessage()); } catch (IOException e) { _log.error("IO Error on putting files on server:" + destServer, e); throw new IOException("IO Error putting files on server:" + destServer + "/n" + e.getLocalizedMessage()); } catch (Exception e) { throw new Exception("FTP exception " + e); } finally { if (ftp != null) disconnectFTP(); } // Send files to destination archive if required if (archiveSource) { String aPath = ((File) fileList.get(0)).getPath(); sourceArchDir = (aPath.substring(0, aPath.lastIndexOf(File.pathSeparator) + 1) + archiveDir); performLocalArchive(fileList, sourceArchDir); } if (cleanup) { performLocalDelete(fileList); } } /* * takes a list of file paths and returns an ArrayList of File Objects */ private List getFileList (List pathList) throws InvalidFileException{ List fileList = new ArrayList(pathList.size()); File tempFile = null; for (int i = 0; i < pathList.size(); i++) { tempFile = new File((String) pathList.get(i)); if (tempFile.exists()) { fileList.add(tempFile); } else { _log.error("File do not exist: " + tempFile.getName()); throw new InvalidFileException("Invalid file: " + pathList.get(i)); } } return fileList; } private List getFileList (String directoryPath) { /* gets a list of files from the directory. */ File directory = new File(directoryPath); ArrayList fileList = new ArrayList(); if(directory.exists()) { File[] filesInDir = directory.listFiles(); if (filesInDir != null) { int length = filesInDir.length; for (int i = 0; i < length; ++i) { File file = filesInDir[i]; if (file.isFile()) { if (file.canRead()) { fileList.add(file); } } } } } return fileList; } public void connectFTP () throws InvalidFTPServerException, InvalidFTPAccountException, IOException, Exception { int timeout = 15000; //connection timeout interval in milliseconds, 15 seconds //Overriding default socketfactory to set the timeout interval for connecting to remote server VpcSocketFactory socketFactory = new VpcSocketFactory(); socketFactory.setConnectTimeout(timeout); ftp.setSocketFactory(socketFactory); // Connect and login to source FTP Server ftp.connect( server ); int reply = ftp.getReplyCode(); if(!FTPReply.isPositiveCompletion(reply)) { ftp.logout(); ftp.disconnect(); _log.warn("FTP connection refused by server: " + server); throw new InvalidFTPServerException("Unable to connect to server: " + server); } if(!ftp.login(user, password)) { ftp.logout(); ftp.disconnect(); _log.warn("Unable to login with user account: " + user); throw new InvalidFTPAccountException("Unable to login with user account:" + user); } ftp.setFileType(FTP.IMAGE_FILE_TYPE); } public void disconnectFTP () throws IOException { // Logout from the FTP Server and disconnect if (ftp.isConnected()) { ftp.logout(); ftp.disconnect(); } } private List performGetFromDir (String remoteDir, String localDir, boolean archiveRemote, String archiveDir) throws FTPConnectionClosedException, ArchiveException, IOException, InvalidFTPServerException, InvalidFTPAccountException, Exception{ List remoteFiles = null; FTPFile[] remoteFiles2 = null; FTPFile remoteFile = null; List recvFiles = new ArrayList(); File file = null; File newName = null; try { if (_log.isDebugEnabled()) _log.debug("getting files from dir:" + remoteDir + "with ftpClient " + ftp); // List the files in the directory if (!ftp.changeWorkingDirectory( remoteDir )){ throw new IOException("Invalid source directory: " + remoteDir); } remoteFiles = getFTPFileList(remoteDir); for( int i=0; i<remoteFiles.size(); i++ ) { remoteFile = (FTPFile) remoteFiles.get(i); if (remoteFile.isFile()) { Date fileDate = remoteFile.getTimestamp().getTime(); ftp.setFileType(FTP.IMAGE_FILE_TYPE); // Download a file from the source FTP Server file = new File( localDir + remoteFile.getName() + ".tmp" ); FileOutputStream fos = new FileOutputStream( file ); ftp.retrieveFile( remoteFile.getName(), fos ); fos.close(); if (_log.isDebugEnabled()) _log.debug("remoteFile size: " + remoteFile.getSize() + "/t downloaded file size: " + file.length()); // checking size of downloaded file with remote original remoteFiles2 = ftp.listFiles( remoteFile.getName()) ; if (remoteFiles2.length != 0 && remoteFiles2[0].isFile()) { remoteFile = remoteFiles2[0]; if (remoteFile.getSize() != file.length()) { //downloaded file is incomplete, discarding if (_log.isDebugEnabled()) _log.debug("Discarding downloaded file with size different from remote original : " + file.getName()); file.delete(); continue; } } else { if (_log.isDebugEnabled()) _log.debug("Discarding downloaded file due to missing remote original after download: " + file.getName()); file.delete(); continue; } newName = new File(localDir + remoteFile.getName()); if (file.renameTo(newName)) { if (_log.isDebugEnabled()) _log.debug("Renamed temporary download file from: " + file.getName() + " to : " + newName.getName()); file.setLastModified( fileDate.getTime() ); // maintain list of received files recvFiles.add(newName); } else { _log.error("Failed to rename temporary download file"); } } } if (archiveRemote) { if (_log.isDebugEnabled()) _log.debug("remote archiving: " + remoteDir + archiveDir); performRemoteArchiveWithFiles(remoteFiles, archiveDir); } else { performRemoteDelete(remoteFiles); } } catch (FTPConnectionClosedException e) { _log.error("Premature disconnection while getting files from directory: " + remoteDir, e); throw new FTPConnectionClosedException("Premature disconnection while getting files from directory: " + remoteDir + "/n" + e); } catch (IOException e) { _log.error("IO Error while getting file from directory: " + remoteDir, e); throw new IOException("IO Error while getting file from directory: " + remoteDir + "/n" + e); } return recvFiles; } private List performGetFiles (List pathList, String localDir, boolean archiveRemote, String archiveDir) throws FTPConnectionClosedException, ArchiveException, IOException, InvalidFTPServerException, InvalidFTPAccountException, Exception{ List remoteFiles = null; FTPFile[] remoteFiles2 = null; FTPFile remoteFile = null; List recvFiles = new ArrayList(); File file = null; File newName = null; String remoteFileName = (String) pathList.get(0); String remoteDir = remoteFileName.substring(0, remoteFileName.lastIndexOf('/')); try { remoteFiles = getFTPFileList(pathList); for( int i=0; i<remoteFiles.size(); i++ ) { remoteFile = (FTPFile) remoteFiles.get(i); remoteFileName = remoteFile.getName().substring(remoteFile.getName().lastIndexOf('/')); Date fileDate = remoteFile.getTimestamp().getTime(); // Download a file from the source FTP Server file = new File( localDir + remoteFileName + ".tmp"); FileOutputStream fos = new FileOutputStream( file ); ftp.retrieveFile( remoteFile.getName(), fos ); fos.close(); // check size of downloaded file with remote original remoteFiles2 = ftp.listFiles( remoteFile.getName()) ; if (remoteFiles2.length != 0 && remoteFiles2[0].isFile()) { remoteFile = remoteFiles2[0]; if (remoteFile.getSize() != file.length()) { //downloaded file is incomplete, discarding if (_log.isDebugEnabled()) _log.debug("Discarding downloaded file with different size from remote original : " + file.getName()); file.delete(); continue; } } else { if (_log.isDebugEnabled()) _log.debug("Discarding downloaded file due to missing remote original after download: " + file.getName()); file.delete(); continue; } newName = new File(localDir + remoteFileName); file.renameTo(newName); file.setLastModified( fileDate.getTime() ); // maintain list of received files recvFiles.add(newName); if (archiveRemote) { String sourceArchDir = remoteDir + archiveDir; if (_log.isDebugEnabled()) _log.debug("Performing remote archive using " + sourceArchDir); performRemoteArchive((String)pathList.get(i), sourceArchDir); } else { performRemoteDelete(remoteFile); } } } catch (FTPConnectionClosedException e) { _log.error("Premature disconnection while getting files with path list ", e); throw new FTPConnectionClosedException("Premature disconnection while getting files with path list due to/n " + e); } catch (IOException e) { _log.error("IO Error while getting files ", e); throw new IOException("IO Error while getting files due to/n" + e); } return recvFiles; } private void createDirectories(String remoteDir) { //Create required parent directories _log.debug("About to create dir:"+remoteDir); String dirs[] = remoteDir.split("/"); int index = 0; while (index < dirs.length) { try { ftp.makeDirectory(dirs[index]); ftp.changeWorkingDirectory(dirs[index]); } catch (IOException e) { _log.error("Failed to create directory: " + dirs[index] + "/ due to: " + e.getMessage()); } index++; } } private void performPutFiles (List listOfFiles, String remoteDir, boolean overwrite) throws FTPConnectionClosedException, IOException, SocketException, Exception { File file = null; String originalDir = ftp.printWorkingDirectory(); if (_log.isDebugEnabled()) _log.debug("performPutFiles(" + listOfFiles + ", " + remoteDir + " )"); try { // check and try to create remote directory if it does not exist if (!remoteDir.equals("") && remoteDir != null) { if (!ftp.changeWorkingDirectory(remoteDir)) { // destination directory do not exist, try to create directories on remote server createDirectories(remoteDir); } } if (_log.isDebugEnabled()) _log.debug("Creating temp directory (ftptmp/) in remote working directory:" + ftp.printWorkingDirectory()); // try to create the temporary ftp working directory if (!ftp.changeWorkingDirectory("ftptmp/")) { if (!ftp.makeDirectory("ftptmp/")) { _log.error("Failed to create temp directory: ftptmp/ in remote directory: " + remoteDir); throw new IOException("Unable to create directory on remote server: " + remoteDir + "ftptmp/"); } } ftp.changeWorkingDirectory(originalDir); if (_log.isDebugEnabled()) _log.debug("Current remote directory:" + ftp.printWorkingDirectory()); for( int i=0; i<listOfFiles.size(); i++ ) { file = (File) listOfFiles.get(i); if (!file.exists()) { throw new IOException("Missing file: " + file.getName() + ", ftp transfer aborted"); } // upload a file to destination FTP Server FileInputStream fis = new FileInputStream( file ); if (!ftp.storeFile( remoteDir + "ftptmp/" + file.getName(), fis )) { _log.error("FTP file transfer failed while transferring " + file.getName() + " to :" + remoteDir + "ftptmp/"); throw new IOException("FTP file transfer failed while transferring " + file.getName()); } else { _log.debug("File uploaded to FTP server:"+remoteDir + "ftptmp/" + file.getName() +" File No:"+ (i+1) +" of "+listOfFiles.size()); } fis.close(); } /* * Delay, not sure why but others do it. * Server must be slow to recognise the new copied file * Fix for export bug. */ try { Thread.sleep(1100); } catch (InterruptedException e) { // ignore } for( int i=0; i<listOfFiles.size(); i++ ) { file = (File) listOfFiles.get(i); String uniqueFilename = null; if (!overwrite) uniqueFilename = getUniqueFileName(remoteDir, file.getName()); if (uniqueFilename == null) uniqueFilename = file.getName(); String renameFile = remoteDir + uniqueFilename; // rename temp files on FTP Server if (!ftp.rename( remoteDir + "ftptmp/" + file.getName(), renameFile)) { _log.error("Failed to rename temp file: " + renameFile); throw new IOException("Failed to rename temp file: " + renameFile); } else { _log.debug("File renamed:"+remoteDir + "ftptmp/" + file.getName() +" to:" + renameFile + " File No:"+ (i+1) +" of "+listOfFiles.size()); } } ftp.removeDirectory(remoteDir + "ftptmp/"); } catch (FTPConnectionClosedException e) { _log.error("Premature disconnection while putting files with path list ", e); throw new FTPConnectionClosedException("Premature disconnection while putting files with path list due to:/n" + e); } catch (IOException e) { _log.error("IO Error while putting files ", e); throw new IOException("IO Error while putting files due to: /n" + e); } } private String getUniqueFileName(String remoteDir, String filename) { String[] ftpFileNames; try { ftpFileNames = ftp.listNames(remoteDir + filename); if (ftpFileNames != null && ftpFileNames.length > 0) { //File exist in remote server, do not overwrite it, change filename by assigning a unique index String [] renameFileNames = ftp.listNames(remoteDir); int k = 0, index = 1; String name = filename.substring(0, filename.indexOf(".")); //Find out the largest index for the same filename while (k < renameFileNames.length) { int beginIndex = renameFileNames[k].lastIndexOf('/') + 1; int endIndex = renameFileNames[k].lastIndexOf('('); if (beginIndex == -1) beginIndex = 0; if (endIndex == -1) { k++; continue; } String serverFilename = renameFileNames[k].substring(beginIndex, endIndex); if (name.equals(serverFilename)) { int curIndex = Integer.parseInt(String.valueOf(renameFileNames[k].charAt(endIndex + 1))) + 1; if (curIndex > index); index = curIndex; } k++; } return name + "(" + index + ")" + filename.substring(filename.indexOf(".")); } else return filename; } catch (IOException e) { _log.error("IO Error while getting unique filename ", e); return null; } } private void performLocalArchive (List files, String localArchDir) throws ArchiveException, IOException{ if (files.size() == 0) return; File dir = new File(localArchDir); if (!dir.exists()) { dir.mkdir(); } FileChannel in = null, out = null; File archiveFile = null; int i = 0; try { for ( ; i < files.size(); i++) { if (_log.isDebugEnabled()) _log.debug("LocalArchSource " + i + ":" + ((File) files.get(i)).getPath()); in = new FileInputStream(((File) files.get(i))).getChannel(); archiveFile = new File(localArchDir + ((File) files.get(i)).getName()); if (_log.isDebugEnabled()) _log.debug("LocalArchive " + i + ":" + archiveFile.getPath()); out = new FileOutputStream(archiveFile).getChannel(); long size = in.size(); MappedByteBuffer buf = in.map(FileChannel.MapMode.READ_ONLY, 0, size); out.write(buf); } } catch( Exception e ) { throw new ArchiveException("Local archive failed in dir:" + localArchDir + "/n" + e, LOCAL_ARCHIVE); } finally { try { if (in != null) in.close(); if (out != null) out.close(); } catch( IOException e) { _log.error("Failed to close channels for local archiving:", e); throw new IOException("Failed to close channels for local archiving:" + e); } } } private void performRemoteArchiveWithFiles (List files, String remoteArchDir) throws FTPConnectionClosedException, ArchiveException, IOException { if (files.size() == 0 ) return; String originalDir = ftp.printWorkingDirectory(); if (!ftp.changeWorkingDirectory(remoteArchDir)) { ftp.makeDirectory(remoteArchDir); } else { ftp.changeWorkingDirectory(originalDir); } for( int i=0; i<files.size(); i++ ) { if (files.get(i) != null && ((FTPFile) files.get(i)).isFile()) { if (_log.isDebugEnabled()) { _log.debug("fileName: " + ((FTPFile) files.get(i)).getName()); _log.debug("remoteArchDir: " + remoteArchDir); } if (!ftp.rename(((FTPFile) files.get(i)).getName(), remoteArchDir + ((FTPFile) files.get(i)).getName())) { _log.error("Remote archive failed in dir: " + remoteArchDir + "/n" + ftp.getReplyString()); throw new ArchiveException("Remote archive failed in dir: " + remoteArchDir + "/n" + ftp.getReplyString(), REMOTE_ARCHIVE); } } } } private void performRemoteArchive (String path, String remoteArchDir) throws FTPConnectionClosedException, ArchiveException, IOException { if (path == null || path.length() == 0 ) return; String originalDir = ftp.printWorkingDirectory(); if (!ftp.changeWorkingDirectory(remoteArchDir)) { ftp.makeDirectory(remoteArchDir); } else { ftp.changeWorkingDirectory(originalDir); } int k; String fileName = null; k = path.lastIndexOf(File.separatorChar); // fileName includes separator fileName = path.substring(k, path.length()); if (!ftp.rename(path, remoteArchDir + fileName)) { _log.error("Remote archive with pathList failed in dir:" + remoteArchDir); throw new ArchiveException("Remote archive with pathList failed in dir:" + remoteArchDir, REMOTE_ARCHIVE); } } private void performLocalDelete(List fileList) { for (int i=0; i<fileList.size(); i++) { if (!((File) fileList.get(i)).delete()) _log.info("Unable to delete file: " + ((File) fileList.get(i)).getAbsolutePath()); } fileList.clear(); } private void performRemoteDelete(List files) throws FTPConnectionClosedException, IOException { for( int i=0; i<files.size(); i++ ) { if (files.get(i) != null) ftp.deleteFile(((FTPFile) files.get(i)).getName()); } } private void performRemoteDelete(FTPFile file) throws FTPConnectionClosedException, IOException { ftp.deleteFile(file.getName()); } private synchronized List getFTPFileList(List pathList) throws IOException, InvalidFTPServerException, InvalidFTPAccountException, Exception{ List remoteList = new ArrayList(); FTPFile[] remoteFiles = null; FTPFile remoteFile = null; try { for( int i=0; i<pathList.size(); i++ ) { remoteFiles = ftp.listFiles( (String) pathList.get(i)) ; if (remoteFiles.length != 0 && remoteFiles[0].isFile()) { remoteList.add(remoteFiles[0]); } else { _log.error("File not found on remote server : " + pathList.get(i)); } } if (remoteList.size() == 0) return remoteList; if (_log.isDebugEnabled()) _log.debug("Disconnecting FTPClient: " + ftp); this.disconnectFTP(); if (_log.isDebugEnabled()) _log.debug("Activating wait"); // go away and wait for ten seconds this.wait(WAIT); if (_log.isDebugEnabled()) _log.debug("Back from wait"); this.connectFTP(); if (_log.isDebugEnabled()) _log.debug("Reconnected with FTPClient: " + ftp); for( int i=0; i<remoteList.size(); i++ ) { remoteFiles = ftp.listFiles( ((FTPFile) remoteList.get(i)).getName()) ; if (remoteFiles.length != 0 && remoteFiles[0].isFile()) { remoteFile = remoteFiles[0]; if(remoteFile.getSize() != ((FTPFile) remoteList.get(i)).getSize() ) { remoteList.set(i, null); if (_log.isDebugEnabled()) _log.debug("FTP file testing rejected : " + remoteFile.getName()); } } else { remoteList.set(i, null); _log.error("File went missing on remote server after wait test: " + pathList.get(i)); } } } catch (IOException e) { _log.error("Failed to get list of valid remoteFiles for download", e); throw new IOException("Failed to get list of valid remoteFiles for download due to /n" + e.getLocalizedMessage()); } catch (InterruptedException e) { _log.error("Ftp wait for file testing interrupted"); } return remoteList; } private synchronized List getFTPFileList(String remoteDir) throws InvalidFTPServerException, InvalidFTPAccountException, IOException, Exception{ List remoteList = new ArrayList(); FTPFile[] remoteFiles = null; FTPFile[] remoteFiles2 = null; FTPFile remoteFile = null; try { // getting first version of directroty listing ftp.changeWorkingDirectory(remoteDir); remoteFiles = ftp.listFiles(); if (_log.isDebugEnabled()) _log.debug("Retrieved remoteFiles: " + remoteFiles + " with length: " + remoteFiles.length); if (remoteFiles.length == 0) return remoteList; if (_log.isDebugEnabled()) _log.debug("Disconnecting FTPClient: " + ftp); this.disconnectFTP(); if (_log.isDebugEnabled()) _log.debug("Activating wait"); // go away and wait for ten seconds this.wait(WAIT); if (_log.isDebugEnabled()) _log.debug("Back from wait"); this.connectFTP(); if (_log.isDebugEnabled()) _log.debug("Reconnected with FTPClient: " + ftp); ftp.changeWorkingDirectory(remoteDir); // getting previously retrieved remote file using first listing // and check that their sizes are the same for( int i=0; i<remoteFiles.length; i++ ) { if (remoteFiles[i].isFile()) { if (_log.isDebugEnabled()) _log.debug("Trying to retrieve again file: " + remoteFiles[i].getName()); remoteFiles2 = ftp.listFiles( remoteFiles[i].getName()) ; if (remoteFiles2.length != 0 && remoteFiles2[0].isFile()) { remoteFile = remoteFiles2[0]; if(remoteFile.getSize() == (remoteFiles[i].getSize())) { remoteList.add(remoteFile); if (_log.isDebugEnabled()) _log.debug("FTP file testing accepted: " + remoteFile.getName()); } } else { if (_log.isDebugEnabled()) _log.debug("File went missing on remote server after wait test: " + ((FTPFile) remoteList.get(i)).getName()); } } } } catch (IOException e) { _log.error("Failed to get list of valid remoteFiles for download", e); throw new IOException("Failed to get list of valid remoteFiles for download due to /n" + e.getLocalizedMessage()); } catch (InterruptedException e) { _log.error("Ftp wait for file testing interrupted"); } return remoteList; } }