文件处理入solr第二步

开发思路：从ftp服务器上下载文件压缩包，然后把ftp上已经下载的文件删除，最后把下载好的文件解压到指定目录。

上代码：

A.pom.xml

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
	<modelVersion>4.0.0</modelVersion>

	<groupId>com.hisign</groupId>
	<artifactId>Test</artifactId>
	<version>0.0.1-SNAPSHOT</version>
	<packaging>jar</packaging>

	<name>Test</name>
	<url>http://maven.apache.org</url>

	<properties>
		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
	</properties>

	<dependencies>

		<dependency>
			<groupId>commons-net</groupId>
			<artifactId>commons-net</artifactId>
			<version>2.2</version>
			<scope>compile</scope>
		</dependency>
		<!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-nop -->
		<dependency>
			<groupId>org.slf4j</groupId>
			<artifactId>slf4j-nop</artifactId>
			<version>1.7.21</version>
		</dependency>

		<!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-api -->
		<dependency>
			<groupId>org.slf4j</groupId>
			<artifactId>slf4j-api</artifactId>
			<version>1.7.21</version>
		</dependency>

		<dependency>
			<groupId>junit</groupId>
			<artifactId>junit</artifactId>
			<version>3.8.1</version>
			<scope>test</scope>
		</dependency>

		<dependency>
			<groupId>org.quartz-scheduler</groupId>
			<artifactId>quartz</artifactId>
			<version>2.2.1</version>
		</dependency>

		<dependency>
			<groupId>org.quartz-scheduler</groupId>
			<artifactId>quartz-jobs</artifactId>
			<version>2.2.1</version>
		</dependency>

	</dependencies>
	<build>
		<plugins>

			<plugin>
				<groupId>org.apache.maven.plugins</groupId>
				<artifactId>maven-shade-plugin</artifactId>
				<version>2.4.1</version>
				<executions>
					<execution>
						<phase>package</phase>
						<goals>
							<goal>shade</goal>
						</goals>
						<configuration>
							<transformers>
								<transformer
									implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
									<mainClass>com.hisign.QuartzI</mainClass>
								</transformer>
							</transformers>
						</configuration>
					</execution>
				</executions>
			</plugin>

		</plugins>
	</build>
</project>

B.QuartzI.java

package com.hisign;

import static org.quartz.CronScheduleBuilder.cronSchedule;
import static org.quartz.JobBuilder.newJob;
import static org.quartz.TriggerBuilder.newTrigger;

import java.util.Date;
import java.util.Properties;

import org.quartz.CronTrigger;
import org.quartz.JobDetail;
import org.quartz.Scheduler;
import org.quartz.SchedulerException;
import org.quartz.SchedulerFactory;
import org.quartz.SchedulerMetaData;
import org.quartz.impl.StdSchedulerFactory;

public class QuartzI {
	
	public void run(){
		try {
			Properties prop = PropertiesUtil.getProp();
			String dayQuartz = prop.getProperty("day.quartz");
			SchedulerFactory sf = new StdSchedulerFactory();
			Scheduler scheduler = sf.getScheduler();
			
			JobDetail job = newJob(Main.class).withIdentity("day", "group")
		            .build();
			CronTrigger cronTrigger = newTrigger().withIdentity("cronTrigger","group")
					.withSchedule(cronSchedule(dayQuartz)).build();
			Date ft = scheduler.scheduleJob(job,cronTrigger);
			scheduler.start();
			SchedulerMetaData metaData = scheduler.getMetaData();
		} catch (SchedulerException e) {
			e.printStackTrace();
		}
	}
	
	public static void main(String[] args) {
		QuartzI quartzi = new QuartzI();
		quartzi.run();
	}
}

C.Main.java

package com.hisign;

import org.quartz.DisallowConcurrentExecution;
import org.quartz.Job;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;

@DisallowConcurrentExecution
public class Main implements Job {

	@Override
	public void execute(JobExecutionContext context) throws JobExecutionException {
		DispatchProcess dp = new DispatchProcess();
		do {
			dp.startThreadPool();
		} while (dp.execute());
	}

}

D.DispatchProcess.java

package com.hisign;

import java.util.Properties;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;

import org.apache.commons.net.ftp.FTPFile;

public class DispatchProcess {
	private ExecutorService exeSvr = null;

	public void startThreadPool() {
		int threadCount = Runtime.getRuntime().availableProcessors() * 4;
		exeSvr = Executors.newFixedThreadPool(threadCount);
	}

	public boolean execute() {
		Properties prop = PropertiesUtil.getProp();
		String url = prop.getProperty("url");
		int port = Integer.parseInt(prop.getProperty("port"));
		String username = prop.getProperty("username");
		String password = prop.getProperty("password");
		String ftppath = prop.getProperty("ftppath");

		FTPFile[] filess = DownFile.downLoadFile(url,port,username,password,ftppath);
		
		for(int i=0;i<filess.length;i++){
			try {
				if(filess[i].getName().contains(".tar.gz")){
					exeSvr.execute(new ThreadClass(filess[i].getName()));
				}
			} catch (Exception e) {
				e.printStackTrace();
			}
			
		}
		exeSvr.shutdown();
		try {
			exeSvr.awaitTermination(5, TimeUnit.MINUTES);
		} catch (InterruptedException e) {
			e.printStackTrace();
			return false;
		}
		return false;
	}
}

E.ThreadClass.java

package com.hisign;

import java.io.File;
import java.util.Properties;

public class ThreadClass implements Runnable {

	private String fileName;

	public ThreadClass(String fileName) {
		this.fileName = fileName;
	}

	Properties prop = PropertiesUtil.getProp();
	String url = prop.getProperty("url");
	int port = Integer.parseInt(prop.getProperty("port"));
	String username = prop.getProperty("username");
	String password = prop.getProperty("password");
	String ftppath = prop.getProperty("ftppath");
	String localftppath = prop.getProperty("localftppath");
	String localpath = prop.getProperty("localpath");

	@Override
	public void run() {
		boolean flag = DownFile.downFile(url,port,username,password,ftppath,
				fileName, localftppath);
		if(flag){//删除文件
			DownFile.deleteFileFtp(url, port, username, password, ftppath, fileName);
		}
		try {
			File file = new File(localftppath+"//" + fileName);
			Long len = file.length();
			Thread.sleep(300);
			while(len != file.length()){
				len = file.length();
				Thread.sleep(300);
			}
			UnTarGZ.unTarGz(file, localpath);
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
}

F.DownFile.java

package com.hisign;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.List;

import org.apache.commons.net.ftp.FTPClient;
import org.apache.commons.net.ftp.FTPFile;
import org.apache.commons.net.ftp.FTPReply;

public class DownFile {

	public static FTPFile[] downLoadFile(String url, int port, String username, String password, String remotePath){
		try {
			int reply;
			FTPClient ftp = new FTPClient();
			ftp.connect(url, port);
			// 如果采用默认端口，可以使用ftp.connect(url)的方式直接连接FTP服务器
			ftp.login(username, password);// 登录
			reply = ftp.getReplyCode();
			if (!FTPReply.isPositiveCompletion(reply)) {
				ftp.disconnect();
				return null;
			}
			ftp.changeWorkingDirectory(remotePath);// 转移到FTP服务器目录
			return ftp.listFiles();
		} catch (Exception e) {
			e.printStackTrace();
		}
		
		return null;
	}
	
	/**
	 * Description: 从FTP服务器下载文件 @Version. Jul ,
	 * 
	 * @param url
	 *            FTP服务器hostname
	 * @param port
	 *            FTP服务器端口
	 * @param username
	 *            FTP登录账号
	 * @param password
	 *            FTP登录密码
	 * @param remotePath
	 *            FTP服务器上的相对路径
	 * @param fileName
	 *            要下载的文件名
	 * @param localPath
	 *            下载后保存到本地的路径
	 * @return
	 */
	public static boolean downFile(String url, int port, String username, String password, String remotePath,
			String fileName, String localPath) {
		boolean success = false;
		FTPClient ftp = new FTPClient();
		try {
			int reply;
			ftp.connect(url, port);
			// 如果采用默认端口，可以使用ftp.connect(url)的方式直接连接FTP服务器
			ftp.login(username, password);// 登录
			reply = ftp.getReplyCode();
			if (!FTPReply.isPositiveCompletion(reply)) {
				ftp.disconnect();
				return success;
			}
			ftp.changeWorkingDirectory(remotePath);// 转移到FTP服务器目录
			FTPFile[] fs = ftp.listFiles();
			for (FTPFile ff : fs) {
				if (ff.getName().equals(fileName)) {
					File localFile = new File(localPath + "/" + ff.getName());
					OutputStream is = new FileOutputStream(localFile);
					ftp.retrieveFile(ff.getName(), is);
					is.close();
				}
			}
			ftp.logout();
			success = true;
		} catch (IOException e) {
			e.printStackTrace();
		} finally {
			if (ftp.isConnected()) {
				try {
					ftp.disconnect();
				} catch (IOException ioe) {
				}
			}
		}
		return success;
	}
	/**
	 * 删除文件
	 * @param ftpHost 
	 * @param port
	 * @param userName
	 * @param passWord
	 * @param path 文件路径
	 * @param fileName 文件名
	 */
	public static void deleteFileFtp(String ftpHost, int port, String userName, String passWord, String path,
			String fileName) {
		try {
			FTPClient ftpClient = new FTPClient();// ftpHost为FTP服务器的IP地址，port为FTP服务器的登陆端口,ftpHost为String型,port为int型。
			ftpClient.connect(ftpHost, port);
			ftpClient.login(userName, passWord);// userName、passWord分别为FTP服务器的登陆用户名和密码

			int reply = ftpClient.getReplyCode();
			if (!FTPReply.isPositiveCompletion(reply)) {
				ftpClient.disconnect();
				return;
			}
			ftpClient.changeWorkingDirectory(path);
			ftpClient.deleteFile(fileName);
			ftpClient.logout();
		} catch (Exception e) {
			System.out.println("删除文件失败！");
		}
	}
}

G.UnTarGZ.java

package com.hisign;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.zip.GZIPInputStream;

import org.apache.tools.tar.TarEntry;
import org.apache.tools.tar.TarInputStream;

public class UnTarGZ {

	/**
	 * 3 * 解压tar.gz 文件 4 * @param file 要解压的tar.gz文件对象 5 * @param outputDir
	 * 要解压到某个指定的目录下 6 * @throws IOException 7
	 */
	public static void unTarGz(File file, String outputDir) throws IOException {
		TarInputStream tarIn = null;
		try {
			tarIn = new TarInputStream(new GZIPInputStream(new BufferedInputStream(new FileInputStream(file))),
					1024 * 2);

			createDirectory(outputDir, null);// 创建输出目录

			TarEntry entry = null;
			while ((entry = tarIn.getNextEntry()) != null) {

				if (entry.isDirectory()) {// 是目录
					entry.getName();
					createDirectory(outputDir, entry.getName());// 创建空目录
				} else {// 是文件
					File tmpFile = new File(outputDir + "/" + entry.getName());
					createDirectory(tmpFile.getParent() + "/", null);// 创建输出目录
					OutputStream out = null;
					try {
						out = new FileOutputStream(tmpFile);
						int length = 0;

						byte[] b = new byte[2048];

						while ((length = tarIn.read(b)) != -1) {
							out.write(b, 0, length);
						}

					} catch (IOException ex) {
						throw ex;
					} finally {

						if (out != null)
							out.close();
					}
				}
			}
		} catch (IOException ex) {
			throw new IOException("解压归档文件出现异常", ex);
		} finally {
			try {
				if (tarIn != null) {
					tarIn.close();
				}
			} catch (IOException ex) {
				throw new IOException("关闭tarFile出现异常", ex);
			}
		}
	}

	/**
	 * 构建目录
	 * 
	 * @param outputDir
	 * @param subDir
	 */
	public static void createDirectory(String outputDir, String subDir) {
		File file = new File(outputDir);
		if (!(subDir == null || subDir.trim().equals(""))) {// 子目录不为空
			file = new File(outputDir + "/" + subDir);
		}
		if (!file.exists()) {
			if (!file.getParentFile().exists())
				file.getParentFile().mkdirs();
			file.mkdirs();
		}
	}
}

H.PropertiesUtil.java

package com.hisign;

import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;

public class PropertiesUtil {
	
	public static Properties getProp(){
		Properties prop = null;
		try {
			InputStream in = PropertiesUtil.class.getClassLoader().getResourceAsStream("file.properties");
			prop = new Properties();
			prop.load(in);
		} catch (IOException e) {
			e.printStackTrace();
		}
		return prop;
	}
}

I.file.properties

url=10.1
port=21
username=
password=
ftppath=/home/data/ftpdir
localftppath=H:\\middle_ftp\\
localpath=H:\\middle_local\\
day.quartz= 0 0/30 * * * ?

 

转载于:https://my.oschina.net/u/2954291/blog/917972