前两章记录了Maven项目开发过程的配置,下面就是简单的开发过程,这里是一个定时任务的开发,初衷是能够定时根据保存的URL去网上搜索资源,但没那么多时间去搞,算是个半成品吧,算是记录一下开发的过程。
Web.xml
接上面的配置,在web.xml中配置一个listener:
<?xml version="1.0" encoding="UTF-8"?>
<web-app version="2.4"
xmlns="http://java.sun.com/xml/ns/j2ee"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://java.sun.com/xml/ns/j2ee http://java.sun.com/xml/ns/j2ee/web-app_2_4.xsd">
<!-- Servlet Filters ================================================ -->
<!--
- Declare a filter for multipart MIME handling
-->
<context-param>
<param-name>contextConfigLocation</param-name>
<param-value>classpath:/applicationContext.xml</param-value>
</context-param>
<listener>
<listener-class>org.springframework.web.context.ContextLoaderListener</listener-class>
</listener>
<listener>
<listener-class>org.springframework.web.context.request.RequestContextListener</listener-class>
</listener>
<listener>
<listener-class>
com.cff.study.TaskContextListener
</listener-class>
</listener>
<filter>
<filter-name>encodingFilter</filter-name>
<filter-class>org.springframework.web.filter.CharacterEncodingFilter</filter-class>
<init-param>
<param-name>encoding</param-name>
<param-value>UTF-8</param-value>
</init-param>
</filter>
<filter-mapping>
<filter-name>encodingFilter</filter-name>
<url-pattern>/*</url-pattern>
</filter-mapping>
<welcome-file-list>
<welcome-file>index.jsp</welcome-file>
</welcome-file-list>
</web-app>
利用listener作为程序的入口。
package com.cff.study;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
public class TaskContextListener implements ServletContextListener{
private Context context = null;
public void contextDestroyed(ServletContextEvent arg0) {
context.close();
}
public void contextInitialized(ServletContextEvent arg0) {
// System.out.println("我是最伟大的人!!");
context = Context.getInstance();
try {
context.init();
} catch (ClassNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (InstantiationException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IllegalAccessException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
在Context中,定义多个线程,每个线程启动一个定时任务,这个小项目我只添加了一个项目,利用百度搜索的方法去网上搜索资源,并不完整,这里只稍做介绍。
package com.cff.study;
import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;
import java.util.TimerTask;
import org.apache.commons.logging.*;
import com.cff.study.util.FileUtil;
import com.cff.study.util.Logger;
public class Context {
private static Context instance = null;
private static Log log = LogFactory.getLog(Context.class);
Properties classConfig = new Properties();
synchronized public static Context getInstance() {
if(null == instance){
log.info("实例不存在!正在创建... ");
instance = new Context();
return instance;
}
else
return Context.getInstance();
}
public void init() throws ClassNotFoundException, InstantiationException, IllegalAccessException{
log.info("应用上下文初始化中...");
FileUtil.loadConfig(classConfig,"/class-config.properties");
int classNum = classConfig.size();
for(int i=0;i<classNum;i++){
TaskThread taskThread = new TaskThread();
Logger.info(classConfig.getProperty("class"+i));
Class<?> tempTask = Class.forName("com.cff.study."+classConfig.getProperty("class"+i));
TimerTask currentTask = (TimerTask)tempTask.newInstance();
taskThread.setCurrentTask(currentTask);
taskThread.setTimeDelay(5+i);
taskThread.start();
Logger.info(""+i+"个线程启动中...");
}
}
public void close(){
log.info("应用上下文正在注销...");
}
}
任务线程的启动
package com.cff.study;
import java.util.Calendar;
import java.util.Timer;
import java.util.TimerTask;
import com.cff.study.util.Logger;
public class TaskThread extends Thread {
private static Timer importDataTimer = null;
TimerTask currentTask = null;
int timeDelay = 5;
public int getTimeDelay() {
return timeDelay;
}
public void setTimeDelay(int timeDelay) {
this.timeDelay = timeDelay;
}
public TimerTask getCurrentTask() {
return currentTask;
}
public void setCurrentTask(TimerTask currentTask) {
this.currentTask = currentTask;
}
public TaskThread() {
super();
}
public void run(){
String importMins = "30";
if(importMins!=null&&!importMins.trim().equals("")&&!importMins.trim().equals("0")){
importDataTimer = new Timer();
Calendar date = Calendar.getInstance();
int sec = 60 * Integer.parseInt(importMins);
date.add(Calendar.SECOND, timeDelay);
importDataTimer.scheduleAtFixedRate(currentTask, date.getTime(),(long)(sec* 1000));
Logger.info("定时任务"+timeDelay+"秒钟后启动,扫描时间间隔为"+importMins+"分钟");
}
}
}
百度搜索任务
package com.cff.study;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.TimerTask;
import com.cff.study.search.ContentFromUrl;
import com.cff.study.search.JSoupBaiduSearcher;
import com.cff.study.search.SearchResult;
import com.cff.study.search.Searcher;
import com.cff.study.search.UrlOpen;
import com.cff.study.search.Webpage;
import com.cff.study.util.DateUtil;
import com.cff.study.util.Logger;
import com.cff.study.util.FileUtil;
public class BaiDuSearchTask extends TimerTask {
private int UrlLevel = 5; //Url最大层数
private int UrlStep = 10; //Url每层最大数量
private int ExecuteUrlNum=100;//一次定时任务最大处理量
private int MaxUrls = 100000;
Searcher searcher = null;
SearchResult searchResult = null;
List<Webpage> webpages = null;
String today = "";
List<String> keywords = new ArrayList<String>();
List<Integer> CurrentLocations = new ArrayList<Integer>();
int CurrentUrlIndex = 1;
private Properties contextConfig = new Properties();
private Properties urlCollectionFile = new Properties();
private Properties contentCollectionFile = new Properties();
String contextConfigFileName = "F:/Work_Study/TimerTask/context-config.properties";
String urlCollectionFileName = "F:/Work_Study/TimerTask/url-collect.properties";
String contentCollectionFileName = "F:/Work_Study/TimerTask/content-collect.properties";
public BaiDuSearchTask() {
}
@Override
public void run() {
today = DateUtil.format(DateUtil.now(), "yyyyMMdd");
Logger.info("今天是" + today.substring(0, 4) + "年" + today.substring(4, 6)
+ "月" + today.substring(6, 8) + "日,我启动了一个线程,叫百度搜索,嘿嘿,厉害吧?");
Logger.info("搜索正在启动,请不要关闭电脑!");
String searchDest = "陈付菲";
keywords.add("伟大的人");
keywords.add("华中科技大学");
FileUtil.loadAbsoluteConfig(contextConfig, contextConfigFileName);
FileUtil.loadAbsoluteConfig(urlCollectionFile, urlCollectionFileName);
FileUtil.loadAbsoluteConfig(contentCollectionFile, contentCollectionFileName);
doSearch(searchDest);
}
private void doSearch(String searchDest) {
int endRunning = Integer.parseInt(contextConfig.getProperty(
"endRunning").trim());
if (1 == endRunning)
return;
int SearchMethod = Integer.parseInt(contextConfig.getProperty(
"SearchMethod").trim());
//最大层数
UrlLevel = Integer.parseInt(contextConfig.getProperty("MaxUrlLevel").trim());
//最大Url数量
MaxUrls = Integer.parseInt(contextConfig.getProperty("MaxUrls").trim());
//最大步长
UrlStep = Integer.parseInt(contextConfig.getProperty("MaxUrlStep").trim());
if (1 == SearchMethod){
FileUtil.clearInfoForFile(urlCollectionFileName);
doSearchMethod(searchDest);
FileUtil.writeAbsoluteProperties(contextConfigFileName,
"SearchMethod", String.valueOf(0));
}
else {
int size = urlCollectionFile.size();
if (MaxUrls < size) {
int pageNo = Integer.parseInt(contextConfig.getProperty(
"pageNo").trim());
FileUtil.writeAbsoluteProperties(contextConfigFileName,
"pageNo", String.valueOf(pageNo + 1));
return;
}
readCurrentLocation(CurrentLocations);
int DsetUrlLevel = (int) (UrlLevel-1- Math.log(ExecuteUrlNum) / Math.log(UrlStep));; //递归开始的Url层数
int CurrentUrlLevel = Integer.parseInt(contextConfig.getProperty("CurrentUrlLevel").trim());//当前处理层
Logger.info("CurrentLocations size:"+CurrentLocations.size());
//比较如果DsetUrlLevel和CurrentUrlLevel,若相同则从DsetUrlLevel开始处理,不同则从CurrentUrlLevel开始处理
if(DsetUrlLevel==CurrentUrlLevel){
Logger.info("CurrentUrlLevel"+CurrentUrlLevel);
Logger.info("DsetUrlLevel:"+DsetUrlLevel);
int CurrentUrlNo=CurrentLocations.get(CurrentUrlLevel);
doUrlOpenMethod(DsetUrlLevel,CurrentUrlNo);
CurrentLocations.set(CurrentUrlLevel, CurrentUrlNo+1);
writeCurrentLocations(DsetUrlLevel,CurrentLocations);
}
else{
Logger.info("CurrentUrlLevel else"+CurrentUrlLevel);
int CurrentUrlNo=CurrentLocations.get(CurrentUrlLevel);
int result = doUrlOpenToDsetMethod(CurrentUrlNo,CurrentUrlLevel,DsetUrlLevel);
if(-1==result){
FileUtil.writeAbsoluteProperties(contextConfigFileName,
"endRunning", String.valueOf(1));
}
else if(DsetUrlLevel==result){
for(int i=CurrentUrlLevel+1;i<DsetUrlLevel;i++)
CurrentLocations.set(i, 0);
doUrlOpenMethod(DsetUrlLevel,0);
CurrentLocations.set(DsetUrlLevel, 1);
}
else if(CurrentUrlLevel==result){
for(int i=CurrentUrlLevel+1;i<DsetUrlLevel;i++)
CurrentLocations.set(i, -1);
CurrentLocations.set(CurrentUrlLevel, CurrentUrlNo+1);
}
else{
if(CurrentUrlLevel!=result){
for(int k=CurrentUrlLevel+1;k<result;k++ ){
CurrentLocations.set(k,0);
}
CurrentLocations.set(result, 1);
for(int i=result+1;i<DsetUrlLevel;i++)
CurrentLocations.set(i, -1);
}
}
writeCurrentLocations(DsetUrlLevel,CurrentLocations);
}
}
}
/**
* 写入当前处理位置
* @param dsetUrlLevel 目标处理层
* @param currentLocations 位置数组
*/
private void writeCurrentLocations(int dsetUrlLevel, List<Integer> currentLocations) {
int size = currentLocations.size()-1;
int CurrentUrlLevel = dsetUrlLevel;
for(int i=size;i>0;i--){
if(currentLocations.get(i)>9){
currentLocations.set(i, -1);
int LevelUpNo = currentLocations.get(i-1);
currentLocations.set(i-1, LevelUpNo+1);
CurrentUrlLevel = i-1;
}
}
if(currentLocations.get(0)>9){
FileUtil.writeAbsoluteProperties(contextConfigFileName,
"endRunning", String.valueOf(1));
return;
}
StringBuffer location = new StringBuffer();
location.append(currentLocations.get(0));
for(int i=1;i<=size;i++){
location.append("#");
location.append(currentLocations.get(i));
}
FileUtil.writeAbsoluteProperties(contextConfigFileName,
"CurrentLocation", location.toString());
FileUtil.writeAbsoluteProperties(contextConfigFileName,
"CurrentUrlLevel", String.valueOf(CurrentUrlLevel));
}
/**
* 读取当前处理位置
* @param currentLocations 位置数组
*/
public static void main(String[] args){
//readCurrentLocation(CurrentLocations);
}
private void readCurrentLocation(List<Integer> currentLocations) {
//FileUtil.loadAbsoluteConfig(contextConfig, contextConfigFileName);
String location = contextConfig.getProperty("CurrentLocation").trim();
String[] locations = location.split("#", -1);
Logger.info(locations.length);
for(int i=0;i<locations.length;i++){
currentLocations.add(Integer.parseInt(locations[i].trim()));
}
}
/**
* 到指定Url层数的解析过程
* @param CurrentUrlNo 当前url维数位置
* @param tempUrlNo 当前url层数位置
* @param CurrentUrlLevel 当前层
*/
private int doUrlOpenToDsetMethod(int CurrentUrlNo, int CurrentUrlLevel, int dsetUrlLevel) {
if(CurrentUrlNo>UrlStep||CurrentUrlLevel>UrlLevel){
return -1;
}
Logger.info("当前处理层:"+CurrentUrlLevel);
Logger.info("当前第"+CurrentUrlNo+"个Url");
Logger.info("Url标识为:"+"Url"+(CurrentUrlNo+CurrentUrlLevel*UrlStep));
String url = urlCollectionFile.getProperty("Url"+(CurrentUrlNo+CurrentUrlLevel*UrlStep));
Logger.info("当前Url为:"+url);
if(!CheckUrlIllegalOrNot(url)){
Logger.info("当前Url不合法!");
return CurrentUrlLevel;
}
Logger.info("当前Url合法,可以继续执行! ");
doUrlOpenMethodDetail(url,0,CurrentUrlLevel);
CurrentUrlLevel++;
while(CurrentUrlLevel!=dsetUrlLevel){
url = urlCollectionFile.getProperty("Url"+CurrentUrlLevel*UrlStep);
Logger.info("当前处理层:"+CurrentUrlLevel);
Logger.info("当前第"+0+"个Url");
Logger.info("Url标识为:"+"Url"+CurrentUrlLevel*UrlStep);
Logger.info("当前Url为:"+url);
if(!CheckUrlIllegalOrNot(url)){
Logger.info("当前Url不合法!");
return CurrentUrlLevel+1;
}
Logger.info("当前Url合法,可以继续执行! ");
doUrlOpenMethodDetail(url,0,CurrentUrlLevel);
CurrentUrlLevel++;
}
return dsetUrlLevel;
}
/**
* 打开url链接的方法
* @param CurrentUrlLevel
* @param CurrentUrlNo
*/
private void doUrlOpenMethod(int CurrentUrlLevel,int CurrentUrlNo) {
if(CurrentUrlNo>UrlStep||CurrentUrlLevel>UrlLevel){
return;
}
String url = urlCollectionFile.getProperty("Url"+(CurrentUrlNo+CurrentUrlLevel*UrlStep));
Logger.info("当前处理目标层:"+CurrentUrlLevel);
Logger.info("当前第"+0+"个Url");
Logger.info("Url标识为:"+"Url"+(CurrentUrlNo+CurrentUrlLevel*UrlStep));
Logger.info("当前Url为:"+url);
if(!CheckUrlIllegalOrNot(url)){
Logger.info("当前Url不合法!");
return;
}
Logger.info("当前Url合法,可以继续执行! ");
doUrlOpenMethodDetail(url,CurrentUrlNo,CurrentUrlLevel);
for(int i=0;i<UrlStep;i++){
doUrlOpenMethod(CurrentUrlLevel+1,i);
}
}
/**
* 搜索方法主体
* @param searchDest 搜索词
*/
private void doSearchMethod(String searchDest) {
int pageNo = Integer.parseInt(contextConfig.getProperty("pageNo")
.trim());
Logger.info("当前搜索页码:" + pageNo);
if (UrlStep < pageNo) {
FileUtil.writeAbsoluteProperties(contextConfigFileName,
"endRunning", "1");
return;
}
doSearchMethodDetail(searchDest, pageNo);
FileUtil.writeAbsoluteProperties(contextConfigFileName, "pageNo",
String.valueOf(pageNo + 1));
}
/**
* 搜索方法详细
* @param searchDest 百度搜索词
* @param pageNo 显示页码
*/
private void doSearchMethodDetail(String searchDest, int pageNo) {
searcher = new JSoupBaiduSearcher();
searchResult = searcher.search(searchDest, pageNo);
webpages = searchResult.getWebpages();
Map<String, String> map = new HashMap();
int size = urlCollectionFile.size();
if (webpages != null) {
int i = 0;
Logger.info("搜索结果 当前第 " + searchResult.getPage() + " 页,页面大小为:"
+ searchResult.getPageSize() + " 共有结果数:"
+ searchResult.getTotal());
for (Webpage webpage : webpages) {
// Logger.info("搜索结果 "+ i + " :");
// Logger.info("标题:" + webpage.getTitle());
Logger.info("URL:" + webpage.getUrl());
// Logger.info("摘要:" + webpage.getSummary());
// Logger.info("正文:" + webpage.getContent());
// Logger.info("");
map.put("Url" + i, webpage.getUrl());
FileUtil.writeAbsolutePropertiesFile(urlCollectionFileName, map);
i++;
if(i>UrlStep)break;
}
if(map.size()<UrlStep){
for(int j=map.size();j<UrlStep;j++){
map.put("Url" + j, " ");
FileUtil.writeAbsolutePropertiesFile(urlCollectionFileName, map);
}
}
} else {
Logger.error("没有搜索到结果");
}
}
/**
* 处理url地址的内容并添加urls
* @param url url链接
* @param currentUrlNo Url当前位置
* @param currentUrlLevel Url层数
*/
private void doUrlOpenMethodDetail(String url, int currentUrlNo, int currentUrlLevel) {
if(currentUrlNo>UrlStep||currentUrlLevel>UrlLevel){
return;
}
UrlOpen urlOpen = new ContentFromUrl();
SearchResult searchResult = urlOpen.urlOpen(url);
List<Webpage> webpages = searchResult.getWebpages();
Map<String, String> map = new HashMap();
int size = urlCollectionFile.size();
if (webpages != null) {
int i = 0;
Logger.info("搜索结果 当前第 " + searchResult.getPage() + " 页,页面大小为:"
+ searchResult.getPageSize() + " 共有结果数:"
+ searchResult.getTotal());
for (Webpage webpage : webpages) {
// Logger.info("搜索结果 "+ i + " :");
// Logger.info("标题:" + webpage.getTitle());
for (int k = 0; k < webpage.getUrls().size(); k++) {
Logger.info("URL:" + webpage.getUrls().get(k));
if (k > UrlStep)
break;
map.put("Url" + ((currentUrlLevel+1) * UrlStep + k), webpage.getUrls().get(k));
FileUtil.writeAbsolutePropertiesFile(urlCollectionFileName,
map);
}
// Logger.info("摘要:" + webpage.getSummary());
// Logger.info("正文:" + webpage.getContent());
// Logger.info("");
int sizeOfcontentCollectionFile = contentCollectionFile.size();
FileUtil.writeAbsoluteProperties(contentCollectionFileName,
"内容" + (sizeOfcontentCollectionFile + i + 1),
webpage.getContent());
if (webpage.getUrls().size() < UrlStep) {
for (int k = webpage.getUrls().size(); k < UrlStep; k++) {
map.put("Url" + ((currentUrlLevel+1) * UrlStep + k), " ");
FileUtil.writeAbsolutePropertiesFile(
urlCollectionFileName, map);
}
}
i++;
}
} else {
Logger.error("没有搜索到结果");
}
}
/**
* 检查url合法
* @param Url
* @return
*/
public boolean CheckUrlIllegalOrNot(String Url){
Logger.info(Url);
if("".equals(Url))return false;
if(null==Url)return false;
if(!Url.contains("http:"))return false;
if(Url.length()<10)return false;
if (Url.contains(".js") || Url.contains(".jpg")
|| Url.contains(".jpeg") || Url.contains(".mp4")
|| Url.contains(".avi") || Url.contains(".flv")
|| Url.contains("ico") || Url.contains(".css"))
return false;
return true;
}
}
这个其实没写完整,但是不想花心思搞了,百度搜索利用的jsoup的工具。
如果有人感兴趣可以留下邮箱,我把源代码发给他。