功能描述:定时从ftp上下载文件到本地,读取文件中的电话号码,添加到数据库中。
1.从ftp下载文件到D:/ftpDownLoad
2.读取文件写入数据库,然后转移写入后的文件到E:/localhostDowload
3..定时器(spring 的Quartz)定时作业
这里引入的jar包是commons-net-1.4.1.jar 及 jakarta-oro-2.0.8.jar;
因为涉及到大量数据文件的下载和io操作及入库 只是单一的这么写下载就够呛入库操作更是显得坑爹异常。所以还得另想办法。
在这儿目前想到的解决方案先记下来:
因为下载的文件可能很多虽然每个文件的数据量不是很多也就不到700kb也就是6.5万条记录差不多的样子。但是动辄几十上百甚至更多的文件,入库的时候自是不必说的,光下载也是很吃力的。所以只能考虑用多线程来解决一下。不过之前没怎么弄过线程这块儿的,这次就当是练练刀 呵呵。。。
首先如开头所述,这是个基于ssh的项目。下载及入库操作又都是以定时作业的方式来实现。那么就在原有的框架基础上加一个任务的,我且称之为quartz层。那么整个流程就变成 这样了:
quartz-->传入service 与文件路径-->启用多线程(这边写一个线程类)-->调用service-->调用Dao进行入库操作
首先因为用的是ssh定时任务我选择用sping 的定时任务
jobClass的代码如下
package com.soarsky.service.quartz;
import org.apache.commons.net.ftp.FTPClient;
import org.apache.commons.net.ftp.FTPClientConfig;
import org.apache.commons.net.ftp.FTPFile;
import org.apache.commons.net.ftp.FTPReply;
import org.apache.log4j.Logger;
import org.quartz.Job;
import org.quartz.JobDataMap;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;
import com.soarsky.common.util.DownloadFromFtpThread;
import com.soarsky.common.util.GetConfig;
import com.soarsky.service.IDownLoadFileFromFtpService;
/**
* 定时从ftp下载文件
* @author JohnYang
*
*/
public class FtpDownLoadQuartz implements Job {
private static Logger logger = Logger.getLogger(FtpDownLoadQuartz.class);
private IDownLoadFileFromFtpService downLoadFileFromFtpService;
private String fileName;
public String getFileName() {
return fileName;
}
public void setFileName(String fileName) {
this.fileName = fileName;
}
public IDownLoadFileFromFtpService getDownLoadFileFromFtpService() {
return downLoadFileFromFtpService;
}
public void setDownLoadFileFromFtpService(
IDownLoadFileFromFtpService downLoadFileFromFtpService) {
this.downLoadFileFromFtpService = downLoadFileFromFtpService;
}
public FtpDownLoadQuartz(
IDownLoadFileFromFtpService downLoadFileFromFtpService,
String fileName) {
this.downLoadFileFromFtpService = downLoadFileFromFtpService;
this.fileName = fileName;
}
public FtpDownLoadQuartz() {
}
@Override
public void execute(JobExecutionContext context)
throws JobExecutionException {
try {
// 获取map
JobDataMap dataMap = context.getJobDetail().getJobDataMap();
// 获取map中的service
downLoadFileFromFtpService = (IDownLoadFileFromFtpService) dataMap
.get("downLoadFileFromFtpService");
FTPClient ftp = new FTPClient();
// 获取配置信息
String url = GetConfig.getInstance().getValue("ftp.Ip");
Integer port = Integer.valueOf(GetConfig.getInstance().getValue(
"ftp.port"));
String username = GetConfig.getInstance().getValue("ftp.userName");
String password = GetConfig.getInstance().getValue("ftp.password");
String ftpDir = GetConfig.getInstance().getValue("ftp.ftpDir");
int reply;
ftp.connect(url, port);
ftp.setControlEncoding("UTF-8");
FTPClientConfig conf = new FTPClientConfig(FTPClientConfig.SYST_NT);
conf.setServerLanguageCode("zh");
// 如果采用默认端口,可以使用ftp.connect(url)的方式直接连接FTP服务器
ftp.login(username, password);// 登录
ftp.setFileType(FTPClient.BINARY_FILE_TYPE);
reply = ftp.getReplyCode();
if (!FTPReply.isPositiveCompletion(reply)) {
ftp.disconnect();
logger.info("登录失败!");
}
logger.info("登录成功!");
ftp.changeWorkingDirectory(ftpDir);// 转移到FTP服务器目录
logger.info("ftp.getRemoteAddress:" + ftp.getRemoteAddress());
//获得相应目录下要下载的文件列表
FTPFile[] fs = ftp.listFiles();
//实例化下载文件线程对象
DownloadFromFtpThread th = null;
for (int k = 0; k < fs.length; k++) {
//获得要下载文件名
fileName = fs[k].getName();
th = new DownloadFromFtpThread(downLoadFileFromFtpService,fileName);
new Thread(th).start();
}
} catch (Exception e) {
logger.error("quartz error......");
e.printStackTrace();
}
logger.info("quartz end......");
}
}
package com.soarsky.service.quartz;
import java.io.File;
import org.apache.log4j.Logger;
import org.quartz.Job;
import org.quartz.JobDataMap;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;
import com.soarsky.common.util.GetConfig;
import com.soarsky.common.util.ReadFromLoaclThread;
import com.soarsky.service.IDownLoadFileFromFtpService;
/**
* 转移下载下来的文件并且入库
* @author Johnyang
*
*/
public class GetDownLoadDateQuartz implements Job {
private static Logger log = Logger.getLogger(GetDownLoadDateQuartz.class);
private IDownLoadFileFromFtpService downLoadFileFromFtpService;
private String fileName;
public String getFileName() {
return fileName;
}
public void setFileName(String fileName) {
this.fileName = fileName;
}
public IDownLoadFileFromFtpService getDownLoadFileFromFtpService() {
return downLoadFileFromFtpService;
}
public void setDownLoadFileFromFtpService(
IDownLoadFileFromFtpService downLoadFileFromFtpService) {
this.downLoadFileFromFtpService = downLoadFileFromFtpService;
}
public GetDownLoadDateQuartz(IDownLoadFileFromFtpService downLoadFileFromFtpService,String fileName) {
this.downLoadFileFromFtpService = downLoadFileFromFtpService;
this.fileName = fileName;
}
public GetDownLoadDateQuartz() {
}
@Override
public void execute(JobExecutionContext context)
throws JobExecutionException {
try {
// 获取map
JobDataMap dataMap = context.getJobDetail().getJobDataMap();
// // 获取map中的service
downLoadFileFromFtpService = (IDownLoadFileFromFtpService) dataMap.get("downLoadFileFromFtpService");
// //读取下载文件并入库
String SourcePath = GetConfig.getInstance().getValue("ftp.sourceDir");
File file = new File(SourcePath);
String[] nameList = file.list();
if (nameList.length > 0) {
for (int i = 0; i < nameList.length; i++) {
fileName=nameList[i];
ReadFromLoaclThread th = new ReadFromLoaclThread(downLoadFileFromFtpService, fileName);
new Thread(th).start();
}
}
} catch (Exception e) {
log.error("quartz error......");
e.printStackTrace();
}
log.info("quartz end......");
}
}
spring bean.xm.中配置如下:
<!-- quartz配置 -->
<!-- 从ftp 下载文件 -->
<bean name="FtpDownTask" class="org.springframework.scheduling.quartz.JobDetailBean">
<property name="jobClass"
value="com.soarsky.service.quartz.FtpDownLoadQuartz" /> <!-- 指定ftp 下载文件job -->
<property name="jobDataAsMap">
<map>
<entry key="downLoadFileFromFtpService" value-ref="downLoadFileFromFtpService" />
</map>
</property>
</bean>
<bean id="downLoadTrigger" class="org.springframework.scheduling.quartz.CronTriggerBean">
<property name="jobDetail">
<ref bean="FtpDownTask" />
</property>
<property name="cronExpression">
<!--两分钟执行一次
<value>0 0/2 * * * ?</value>-->
<!--每天9点执行 -->
<value>0 0 9 * * ?</value>
</property>
</bean>
<!-- 读取本地的下载文件 并加入数据库 -->
<bean name="getDownLoadDateTask" class="org.springframework.scheduling.quartz.JobDetailBean">
<property name="jobClass"
value="com.soarsky.service.quartz.GetDownLoadDateQuartz" /><!-- 指定定时任务的job -->
<property name="jobDataAsMap">
<map>
<entry key="downLoadFileFromFtpService" value-ref="downLoadFileFromFtpService" />
</map>
</property>
</bean>
<bean id="getDownLoadDateTrigger" class="org.springframework.scheduling.quartz.CronTriggerBean">
<property name="jobDetail">
<ref bean="getDownLoadDateTask" />
</property>
<property name="cronExpression">
<value>0 0 10 * * ?</value>
</property>
</bean>
<!--启用定时任务 -->
<bean class="org.springframework.scheduling.quartz.SchedulerFactoryBean">
<property name="triggers">
<list>
<ref local="cronTrigger" />
<ref local="downLoadTrigger"/>
<ref local="getDownLoadDateTrigger"/>
</list>
</property>
<property name="startupDelay" value="60"></property>
</bean>
package com.soarsky.common.util;
import com.soarsky.service.IDownLoadFileFromFtpService;
/**
* 多线程下载Ftp上的文件
* @author JohnYang
*
*/
public class DownloadFromFtpThread implements Runnable {
private IDownLoadFileFromFtpService downLoadFileFromFtpService;
private String fileName;
public String getFileName() {
return fileName;
}
public void setFileName(String fileName) {
this.fileName = fileName;
}
public DownloadFromFtpThread(
IDownLoadFileFromFtpService downLoadFileFromFtpService,
String fileName) {
this.downLoadFileFromFtpService = downLoadFileFromFtpService;
this.fileName = fileName;
}
public DownloadFromFtpThread() {
//注:如果为线程类声明了其他的构造函数,那也要把它的无参构造函数显示写出不然会报异常 (这个有时间再做研究)
}
/**
*
*/
public synchronized void run() {
try {
downLoadFileFromFtpService.downFile(fileName);
} catch (Exception e1) {
e1.printStackTrace();
}
}
public IDownLoadFileFromFtpService getDownLoadFileFromFtpService() {
return downLoadFileFromFtpService;
}
public void setDownLoadFileFromFtpService(
IDownLoadFileFromFtpService downLoadFileFromFtpService) {
this.downLoadFileFromFtpService = downLoadFileFromFtpService;
}
}
最后serviceImpl实现就可以了
/**
* Description: 从FTP服务器下载文件
*
* @param url
* FTP服务器hostname
* @param port
* FTP服务器端口
* @param username
* FTP登录账号
* @param password
* FTP登录密码
* @param remotePath
* FTP服务器上的相对路径
* @param localPath
* 下载后保存到本地的路径
* @return
*/
public void downFile(String fileName) {
OutputStream output = null;
FTPClient ftp = new FTPClient();
//获取配置信息
String url = GetConfig.getInstance().getValue("ftp.Ip");
Integer port = Integer.valueOf(GetConfig.getInstance().getValue("ftp.port"));
String username = GetConfig.getInstance().getValue("ftp.userName");
String password = GetConfig.getInstance().getValue("ftp.password");
String ftpDir = GetConfig.getInstance().getValue("ftp.ftpDir");
String sourceDir = GetConfig.getInstance().getValue("ftp.sourceDir");
try {
int reply;
ftp.connect(url, port);
ftp.setControlEncoding("UTF-8");
FTPClientConfig conf = new FTPClientConfig(FTPClientConfig.SYST_NT);
conf.setServerLanguageCode("zh");
// // 如果采用默认端口,可以使用ftp.connect(url)的方式直接连接FTP服务器
ftp.login(username, password);// 登录
ftp.setFileType(FTPClient.BINARY_FILE_TYPE);
reply = ftp.getReplyCode();
if (!FTPReply.isPositiveCompletion(reply)) {
ftp.disconnect();
logger.info("登录失败!");
}
logger.info("登录成功!");
ftp.changeWorkingDirectory(ftpDir);// 转移到FTP服务器目录
output = new FileOutputStream(new File(sourceDir + "/" + fileName));
ftp.retrieveFile(new String(fileName.getBytes("UTF-8"),"ISO-8859-1"), output);
output.flush();
output.close();
} catch (IOException e) {
logger.error("下载文件失败!");
this.downFile(fileName);
e.printStackTrace();
} finally {
if (ftp.isConnected()) {
try {
ftp.logout();
ftp.disconnect();
} catch (IOException e) {
e.printStackTrace();
}
}
logger.info("操作结束!");
}
}
/**
* 遍历ftp上所有要下载的txt文件 并录入数据库
*/
public void saveUsersByDownload(String fileName) {
// 读取到的数据录入数据库
String SourcePath = GetConfig.getInstance().getValue("ftp.sourceDir");
String toPath = GetConfig.getInstance().getValue("ftp.toDir");
File newFile = new File(toPath);
//临时文件
File fs =new File(SourcePath + fileName);
BufferedReader br = null;
String str = null;
try {
br = new BufferedReader(new InputStreamReader(new FileInputStream(fs)));
// 添加群组信息到用户群组信息表
UserGroup oldUserGroup = userGroupDAO.queryByGroupName(fileName
.substring(0, fileName.length() - 4));
UserGroup userGroup = new UserGroup();
if (oldUserGroup == null) {
userGroup.setGroupName(fileName.substring(0,
fileName.length() - 4));
userGroup.setGroupType(1);
userGroup.setAddTime(DateUtil.getCursorDate());
userGroupDAO.saveUserGroup(userGroup);
} else {
userGroup = oldUserGroup;
}
while ((str = br.readLine()) != null) {
// 添加群组数据到用户信息表
if (str.matches("^[1][3-8]\\d{9}$")) {
Userinfo oldUserinfo = userinfoDAO.queryByPhone(str);
Userinfo userinfo = new Userinfo();
if (oldUserinfo == null) {
userinfo.setAddTime(DateUtil.getCursorDate());
userinfo.setPhone(str);
userinfoDAO.saveUserinfo(userinfo);
} else {
userinfo = oldUserinfo;
}
// 更新用户群组关系表
UserUsergroup userUsergroup = new UserUsergroup();
userUsergroup.setUserId(userinfo.getId());
userUsergroup.setUserGroupId(userGroup.getId());
userUsergroupDAO.saveUserUsergroup(userUsergroup);
}
}
} catch (Exception e) {
e.printStackTrace();
}
if (!newFile.exists()) {
newFile.mkdir();
}
// 将文件移到新文件里
removeFile(SourcePath + fileName, toPath + fileName);
// 删除源文件
// fs.delete();
}
至此 算是告一段落,运行起来速度是快了测试的时候数据量正常下载还好,入库的速度仍然不理想。因为是初次写着个所以有点儿乱。下载那块儿怎么看怎么别扭,没办法时间仓促, 有时间我会继续整理,并找找到一个好点儿的解决方案来。