页面访问数据统计[初步]

问题：项目中加一个数据统计模块，主要统计页面访问信息，用户uv,pv,ip等做趋势分析，然后定时将访问数据批量存储到数据库。第一次写，一些代码或者说想法不太好，主要是实现缓存数据定时存储这个过程。

涉及：JFinal框架、quartz、ehcache、cookie

解决办法：

1、config中：

//1.配置ehcache缓存
		EhCachePlugin ecp = new EhCachePlugin();
		me.add(ecp);
//2.定时任务
		QuartzPlugin quartzPlugin = new QuartzPlugin("jobs.properties");
		me.add(quartzPlugin);

//数据库连接等...

2、jobs.properties：

定义一个工具任务 job为执行任务所在路径

                                 cron为定义工作执行时间 一般是定义到凌晨执行 可参照：http://www.cnblogs.com/sunjie9606/archive/2012/03/15/2397626.html

                                enable是否执行

job=com.cloud.main.stats.model.HelloJob
cron=0 0/30 * * * ?
enable=true

3、ehcache.xml

缓存配置文件

<ehcache name="lingshi-protal">
    <diskStore path="java.io.tmpdir/lingshi-protal-ehcache"/>

	<!-- 默认缓存，不指定缓存名称时使用 -->
    <defaultCache
		maxElementsInMemory="10000"
		eternal="false"
		timeToIdleSeconds="120"
		timeToLiveSeconds="120"
		overflowToDisk="false"
		diskPersistent="false"
		diskExpiryThreadIntervalSeconds="120">
    </defaultCache>
    
    <!-- 数据统计缓存 -->
    <cache name="stats"  
           maxEntriesLocalHeap="10000"  
           maxEntriesLocalDisk="1000"  
           eternal="false"  
           overflowToDisk="true"  
           diskSpoolBufferSizeMB="20"  
           timeToIdleSeconds="300"  
           timeToLiveSeconds="100"  
           memoryStoreEvictionPolicy="LFU"  
           transactionalMode="off"  
            />  

</ehcache>

4、statsController

获取数据

package com.cloud.main.stats.controller;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.UUID;
import javax.servlet.http.Cookie;
import org.apache.commons.lang3.StringUtils;
import com.cloud.core.BaseController;
import com.jfinal.plugin.activerecord.Record;
import com.jfinal.plugin.ehcache.CacheKit;
import com.jfinal.plugin.ehcache.IDataLoader;
/**
 * 页面数据统计：
 * 将页面访问数据置于记录中 ，添加到缓存
 * @author MN
 */
public class StatsController extends BaseController {
	
	//客户段唯一标识
	private static final String CUSTOMER_IDENTIFICATION = "CusID";
	private String accessorid ;
	//页面id
	private String pid;
	//页面路径
	private String referer;
	//访问者ip
	private String accessor;
	//获取访问设备
	private String device;

	public void index(){
		//当前页码
		pid=getPara("pid");
		//当前访问路径
		referer=getRequest().getHeader("Referer");
		// 判断 Referer      PropKit.get("domain")
		if(referer!=null&&(referer.trim().startsWith("http://localhost:8080/"))){  
			//当前访问者ip
			accessor=getAccessIp();
			//当前访问设备
			device=checkDevice();
			//获取cookie
			Cookie[] cookies = getRequest().getCookies(); 
			List<Record> pageval = null;
			if(cookies==null){
				//缓存中添加一笔访问记录  
				pageval=new ArrayList<Record>();
				//设置唯一标识cookie【cusID】
				accessorid = UUID.randomUUID().toString().replaceAll("[-]", "");
				Cookie cidCookie = new Cookie(CUSTOMER_IDENTIFICATION, accessorid);
				cidCookie.setMaxAge(24*60*60);
				cidCookie.setPath("/");
				getResponse().addCookie(cidCookie);
			}else{
				for(Cookie cookie:cookies){
					//取得用户唯一标识码
		    		if(StringUtils.equals(cookie.getName(),CUSTOMER_IDENTIFICATION)){  
		    			accessorid =cookie.getValue();
		    			pageval =CacheKit.get("stats",accessorid,new IDataLoader(){
							@Override
							public Object load() {
								return new ArrayList<Record>();
							}
						});
		    			break;
	    			}
	    		}
			}
			pageval.add(new Record()
					.set("pid", pid)
					.set("time", new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS").format(new Date()))
					.set("device", device)
					.set("accessor", accessor));
	        CacheKit.put("stats", accessorid, pageval);
		}
		//避免出现模板加载失败错误
		renderNull();
    }
    /**
     * 获取访问ip
     */
    public String getAccessIp(){
    	String ip=getRequest().getHeader("X-Real-IP");
    	if (!StringUtils.isBlank(ip) && !"unknown".equalsIgnoreCase(ip)) {
    		return ip;
    	}
    	ip = getRequest().getHeader("X-Forwarded-For");
    	if (!StringUtils.isBlank(ip) && !"unknown".equalsIgnoreCase(ip)) {
    		// 多次反向代理后会有多个IP值，第一个为真实IP。
    		int index = ip.indexOf(',');
    		if (index != -1) {
    			return ip.substring(0, index); 
    		} else {
    			return ip;
    		}
    	} else {
    		return getRequest().getRemoteAddr() ;
    	}
    }
    /**
     * 判断设备类型  
     * 这里写的很简单 网上有正则表达式验证的准确
     */
    public String  checkDevice(){  
    	String device=getRequest().getHeader("USER-AGENT").toLowerCase().substring(0,7);
    	if(device.equals("mozilla")){
    		return "PC";
    	}
    	return "手持设备";
    }  
    
}

5、quartzPlugin

/* 
 * All content copyright Terracotta, Inc., unless otherwise indicated. All rights reserved. 
 * 
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not 
 * use this file except in compliance with the License. You may obtain a copy 
 * of the License at 
 * 
 *   http://www.apache.org/licenses/LICENSE-2.0 
 *   
 * Unless required by applicable law or agreed to in writing, software 
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 
 * License for the specific language governing permissions and limitations 
 * under the License.
 * 
 */
 
package com.cloud.main.stats.model;

import java.io.IOException;





import java.io.InputStream;
import java.util.Enumeration;
import java.util.Properties;
import static org.quartz.CronScheduleBuilder.cronSchedule;
import static org.quartz.JobBuilder.newJob;
import static org.quartz.TriggerBuilder.newTrigger;
import org.quartz.CronTrigger;
import org.quartz.JobDetail;
import org.quartz.Scheduler;

import org.quartz.SchedulerException;
import org.quartz.SchedulerFactory;
import org.quartz.impl.StdSchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.jfinal.plugin.IPlugin;

/**
 * 复写QuartzPlugin 
 * @author MN
 *
 */
public class QuartzPlugin implements IPlugin {

	
	
	private Logger logger = LoggerFactory.getLogger(getClass());

	private static SchedulerFactory sf = null ;

	private static Scheduler sched=null;

	private String config = "jobs.properties";

	private Properties properties;

	public QuartzPlugin(String config) {
		this.config = config;
	}
	public QuartzPlugin() {

	}
	@Override
	public boolean start() {
		
		sf = new  StdSchedulerFactory();
		try {
			sched = sf.getScheduler();
		} catch (SchedulerException e1) {
			e1.printStackTrace();
		}
		loadProperties();

		Enumeration enums = properties.keys(); 

		while (enums.hasMoreElements()) {
			String key = enums.nextElement() + "";
			if (!key.endsWith("job")) {
				continue ;
			}
			String cronKey = key.substring(0, key.indexOf("job")) + "cron";
			String enable = key.substring(0, key.indexOf("job")) + "enable";
			if (isDisableJob(enable)) {

				continue;

			}

			String jobClassName = properties.get(key) + "";
			String jobCronExp = properties.getProperty(cronKey) + "";
			Class clazz;
			try {
				clazz = Class.forName(jobClassName);

			} catch (ClassNotFoundException e) {

				throw new RuntimeException(e);

			}
			//向Scheduler加入Job
			JobDetail job = newJob(clazz)
					.withIdentity(jobClassName,jobClassName)
					.build();
			CronTrigger trigger = newTrigger()
					.withIdentity(jobClassName, jobClassName)
					.withSchedule(cronSchedule(jobCronExp))
					.build();
              try {
					sched.scheduleJob(job, trigger);
					sched.start();
					
				} catch (SchedulerException e) {
					e.printStackTrace();
				}
				logger.info(job.getKey()
						+ " has been scheduled to run at: " 
						+ " and repeat based on expression: "
						+ trigger.getCronExpression());


		}
		return true;

	}



	private boolean isDisableJob(String enable) {
		return Boolean.valueOf(properties.get(enable) + "") == false;

	}



	private void loadProperties() {

		properties = new Properties();

		InputStream is = QuartzPlugin.class.getClassLoader()

				.getResourceAsStream(config);

		try {

			properties.load(is);

		} catch (IOException e) {

			throw new RuntimeException(e);

		}

	}



	@Override

	public boolean stop() {

		try {

			sched.shutdown();

		} catch (SchedulerException e) {

			logger.error("shutdown error", e);

			return false;

		}

		return true;

	}
	public static void main(String []args)  {
		  QuartzPlugin plugin = new QuartzPlugin();
		  plugin.start();
		  System.out.println("执行成功！！！");
		  
		 }
    	


}

6、HelloJob

package com.cloud.main.stats.model;

import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.jfinal.plugin.activerecord.Db;
import com.jfinal.plugin.activerecord.Record;
import com.jfinal.plugin.ehcache.CacheKit;
import org.quartz.Job;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;

/**
 * jobs.properties
 * 0/30 * * * * ?  
 * 0 0/30 * * * ?  
 * @author lixiang
 */
public class HelloJob implements Job {
	
	private static Logger _log = LoggerFactory.getLogger(HelloJob.class);
	private static boolean executing = false;
	
	public HelloJob() {
	
	}
	public void execute(JobExecutionContext context)  throws JobExecutionException {
		if(executing)return;
		executing = true;
		_log.info("\n"+"-------------------"+"\n"+"Test " + new Date());
		
		@SuppressWarnings("unchecked")
		List<String>  list=CacheKit.getKeys("stats");
		List<Record> saveval= new ArrayList<Record>();
		saveval.add(new Record().set("pid", 0).set("count", 0)); 
		for (String  accessorid: list) {      
			List<Record> pageval =CacheKit.get("stats",accessorid); 
			String  startTime = pageval.get(0).get("time");
			String  accessor=pageval.get(0).get("accessor");
			String device=pageval.get(0).get("device");
			HashMap<String, Integer> hs = new HashMap<String, Integer>();
			for (Record page : pageval) {
				Integer count = 1;
				if(hs.get(page.get("pid")) != null) {
					count = hs.get(page.get("pid")) + 1;
				}
				hs.put(page.get("pid"), count);
			}
			Iterator iter = hs.entrySet().iterator();
			List<String> sqlList=new ArrayList<String>();
			while (iter.hasNext()) {           
				Map.Entry entry = (Map.Entry) iter.next();
				String key = (String) entry.getKey();
				int val = (int) entry.getValue();
				String sql="INSERT INTO	t_stats_accesslog"
						+ "(accessor,startTime,pid,pagecount,accessorid,device)"
						+ "VALUES("+"'"+accessor+"'"+","
						+"'"+startTime+"'"+","
						+key+","
						+val+","
						+"'"+accessorid+"'"+","
						+"'"+device+"'"+")";
				sqlList.add(sql);
			}
			Db.batch(sqlList, sqlList.size());
			try{
				Thread.sleep(1000);
				CacheKit.remove("stats", accessorid);
			} catch (InterruptedException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
			executing = false;
		}
	}
}

最后附上quartz包和官方的例子

http://download.csdn.net/detail/qq_21544619/9805944