一个Spark程序，处理网络日志

package com.shibazhang.sparkapplog;


import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.List;


import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFunction;


import scala.Tuple2;




/**
 * 移动端app访问流量日志分析
 * @author zhang
 *
 */
public class AppLogSpark {
	static DBHelper db1 = null;
	
	public static void main(String[] args) {
		
		
		
		//1.创建spark配置文件和上下文对象
		SparkConf conf = new SparkConf().setAppName("AppLogSpark");
				//.setMaster("local");
		
		JavaSparkContext sc = new JavaSparkContext(conf);
		
		//2.读取日志文件，并创建一个RDD，使用sparkContext的textFile()方法，可以读取本地磁盘文件，或者读取
		//HDFS的文件，接下来我们就创建一个RDD，这个RDD包含所有日志文件中数据hdfs://spark1:9000/
		//JavaRDD<String> accessLogRDD = sc.textFile("C:\\Users\\zhang\\Desktop\\app_log.txt");
		JavaRDD<String> accessLogRDD = sc.textFile("hdfs://hadoop.namenode:9000/data/app_log.txt");
        //将RDD映射成为key-value格式，为后面的ruduceByKey聚合做准备
		JavaPairRDD<String, AccessLogInfo>  accessLogPairRDD=	mapAccessLogRDD2Pair(accessLogRDD);
		
		//根据deviceID进行聚合操作，获取每个deviceID当天的总上行流量和总下行流量 和最早访问时间戳
		JavaPairRDD<String, AccessLogInfo> aggrAccessLogPairRDD = aggregateByDeviceId(accessLogPairRDD);
		
		//将按照deviceID聚合的key映射为二次排序的key，value映射为deviceID
		JavaPairRDD<AceessLogSortKey,String> accesslogSortRDD =mapRDDKey2SortKey(aggrAccessLogPairRDD);
		
		//实现降序排序 
		
		JavaPairRDD<AceessLogSortKey,String> sortedAcessLogRDD = accesslogSortRDD.sortByKey(false);
		
		//获取前top10
		List<Tuple2<AceessLogSortKey, String>> top10DataList = 	sortedAcessLogRDD.take(10);
		db1 = new DBHelper();
		
		String sql = "insert into acceslog(deviceId,upTraffic,downTraffic,timeStamp) values (?,?,?,?)";
		//mysql数据里面。需要在本地上面装一个mysql，然后我会通过php读取mysql里面数据mysql  javaweb php
		//System.out.println(data._2 + "   " +data._1.getUpTraffic());
		for(Tuple2<AceessLogSortKey, String> data: top10DataList){
			try {
				PreparedStatement pt = db1.conn.prepareStatement(sql);
				pt.setString(1, data._2);
				pt.setString(2, data._1.getUpTraffic() + "");
				pt.setString(3, data._1.getDownTraffic() + "");
				pt.setString(4, data._1.getTimestamp() + "");
				pt.executeUpdate();
			} catch (SQLException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
			
			


			
		}
		//关闭spark上下文
		sc.close();
		
	}
	
	
	/**
	 * 将日志的RDD映射为key-value 的格式
	 */
	private static JavaPairRDD<String, AccessLogInfo> mapAccessLogRDD2Pair(JavaRDD<String> accessLogRDD){
		
		return accessLogRDD.mapToPair(new PairFunction<String, String, AccessLogInfo>() {


			private static final long serialVersionUID = 1L;


			@Override
			public Tuple2<String, AccessLogInfo> call(String accessLog)
					throws Exception {
				//1根据\t对日志进行切分
			String[] accessLogSplited =	accessLog.split("\t");
				
			   //2.获取四个自段
			 long timestamp = Long.valueOf(accessLogSplited[0]);		 
			 String deviceID = accessLogSplited[1];		 
			 long upTraffic = Long.valueOf(accessLogSplited[2]);		 
			 long downTraffic = Long.valueOf(accessLogSplited[3]);
				
			   //3.将时间戳，上行流量，下行流量封装为自定义的可序列化对象	
			 AccessLogInfo accessLogInfo = new AccessLogInfo(timestamp, upTraffic, downTraffic);
			 
				
			   //4.直接返回一个Tuple<String,AccessLogInfo>
               
				return new Tuple2<String, AccessLogInfo>(deviceID, accessLogInfo);
			}
		});
		
	} 
	
	/**
	 * 根据deviceID进行聚合操作
	 * 计算出每个deviceID的总上行流量，总下行流量已经最早访问时间
	 * v1+ v2  =v1
	 * 
	 */
	
	private static JavaPairRDD<String, AccessLogInfo> aggregateByDeviceId(JavaPairRDD<String, AccessLogInfo>  accessLogPairRDD){
	 return accessLogPairRDD.reduceByKey(new Function2<AccessLogInfo, AccessLogInfo, AccessLogInfo>() {
		
		private static final long serialVersionUID = 1L;
		


		@Override
		public AccessLogInfo call(AccessLogInfo accessLogInfo1, AccessLogInfo accessLogInfo2)
				throws Exception {
		    long timestamp = accessLogInfo1.getTimestamp() < accessLogInfo2.getTimestamp() ?
		    		accessLogInfo1.getTimestamp() : accessLogInfo2.getTimestamp();
		    
		   long upTraffic = accessLogInfo1.getUpTraffic() + accessLogInfo2.getUpTraffic();
		   long downTraffic = accessLogInfo1.getDownTraffic() + accessLogInfo2.getDownTraffic();
		   
		   AccessLogInfo accessLogInfo = new AccessLogInfo(timestamp, upTraffic, downTraffic);
		   
			
			return accessLogInfo;
		}
	});
	}
	
	
	/**
	 * 将RDD的key映射为二次排序的key
	 */
	
	private static JavaPairRDD<AceessLogSortKey,String> mapRDDKey2SortKey(JavaPairRDD<String, AccessLogInfo> aggrAccessLogPairRDD){
		return aggrAccessLogPairRDD.mapToPair(new PairFunction<Tuple2<String,AccessLogInfo>,AceessLogSortKey,String>() {


			private static final long serialVersionUID = 1L;


			@Override
			public Tuple2<AceessLogSortKey, String> call(
					Tuple2<String, AccessLogInfo> tuple) throws Exception {
				String deviceID = tuple._1;
				AccessLogInfo accessLogInfo = tuple._2;
				AceessLogSortKey aceessLogSortKey = new AceessLogSortKey(accessLogInfo.getTimestamp(), accessLogInfo.getUpTraffic(), accessLogInfo.getDownTraffic());
				
				
				return new Tuple2<AceessLogSortKey, String>(aceessLogSortKey, deviceID);
			}
		});
		
	}


}