spark大数据平台导出特定IP日志 scala脚本

import java.text.SimpleDateFormat
import java.util.Calendar


var cal:Calendar=Calendar.getInstance()
cal.add(Calendar.DATE,-4)
var format1:SimpleDateFormat = new SimpleDateFormat("yyyyMM")
var format2:SimpleDateFormat = new SimpleDateFormat("yyyyMMdd")
var yearmonth = format1.format(cal.getTime())
var logdate = format2.format(cal.getTime())
val logpath = "hdfs://ksdc-cluster/" + yearmonth + "/" +  + "/*"
val outpath = "hdfs://ksdc-cluster/logs/" + logdate 


def myfilter(line:String)={
  val fields = line.split(" ")
  if(fields.length >= 25){
      val req_uri = fields(9)
      val req_method = fields(10)
      val regex="login".r
      val method =  "POST".r
      ((regex.findFirstMatchIn(req_uri) != None) && (method.findFirstMatchIn(req_method)!=None))
  }
  else{
      false
  }
}
def maptest(line:String) = {
  val fields = line.split(" ")
  val host = fields(8)
  val uri = fields(9)
  val ip =  fields(5)
  val url = ip + " " + host + uri
  (url, 1)
}

sc.textFile(logpath).filter(myfilter).map(maptest).reduceByKey((x,y) => x+y).saveAsTextFile(outpath)