import java.text.SimpleDateFormat
import java.util.Calendar
var cal:Calendar=Calendar.getInstance()
cal.add(Calendar.DATE,-4)
var format1:SimpleDateFormat = new SimpleDateFormat("yyyyMM")
var format2:SimpleDateFormat = new SimpleDateFormat("yyyyMMdd")
var yearmonth = format1.format(cal.getTime())
var logdate = format2.format(cal.getTime())
val logpath = "hdfs://ksdc-cluster/" + yearmonth + "/" + + "/*"
val outpath = "hdfs://ksdc-cluster/logs/" + logdate
def myfilter(line:String)={
val fields = line.split(" ")
if(fields.length >= 25){
val req_uri = fields(9)
val req_method = fields(10)
val regex="login".r
val method = "POST".r
((regex.findFirstMatchIn(req_uri) != None) && (method.findFirstMatchIn(req_method)!=None))
}
else{
false
}
}
def maptest(line:String) = {
val fields = line.split(" ")
val host = fields(8)
val uri = fields(9)
val ip = fields(5)
val url = ip + " " + host + uri
(url, 1)
}
sc.textFile(logpath).filter(myfilter).map(maptest).reduceByKey((x,y) => x+y).saveAsTextFile(outpath)
spark大数据平台导出特定IP日志 scala脚本
最新推荐文章于 2024-08-24 17:05:29 发布