功能要求:实现在Spark解释器下,通过自定义功能,调用sendMail(subject,recipients,sql)实现将DataFrame查询结果保存到Hdfs上,并且通过邮件的形式发送给某人,以供后端人员使用。
环境:Mac操作系统,Zeppelin0.74;Spark2.1.1,Hadoop2.52,Scala2.11.,Java1.80
实现分了三部分很简单,代码上有备注,大家可以直接看备注,Zeppelin初始化sc(SparkContext),spark(SparkSession),zc(ZeppelinContext),所有不用自己去创建相关对象直接调用即可。
%spark
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileSystem
import org.apache.hadoop.fs.Path
import org.apache.hadoop.fs.FileUtil
import org.apache.hadoop.io.IOUtils
import java.net.URI
import java.io._
import javax.mail.Session
import javax.mail.Transport
import javax.mail.PasswordAuthentication
import java.text.SimpleDateFormat
import java.util.{Date, Properties}
import javax.mail.Authenticator
import javax.mail.Message
import javax.mail.internet.InternetAddress
import javax.mail.internet.MimeBodyPart
import javax.mail.internet.MimeMessage
import javax.mail.internet.MimeMultipart
import javax.mail.internet.MimeUtility
import javax.activation.DataHandler
import javax.activation.FileDataSource
// private var hdfsFileName :String =""
// private var from : String =""
// private var fromPassword :String =""
// private var localFileName : String=""
// private var fileName=""
// private val now = new java.util.Date()
// private val dateFormat = new java.text.SimpleDateFormat("yyyyMMddHHmmsss")
// private val time = dateFormat.format(now)
/**
* 生成sql查询 的DF,将其保存在hdfs中
* @param subject 主题
* @param sql sql语句
*/
def saveToHDFS(subject: String, sql: String,time:String) :Unit={
//文件名
val fileName=subject + time
//文件在hdfs的绝对路径
var hdfsFileName = "hdfs://localhost:9000/tmp/" + fileName
val df=spark.sql(sql).repartition(1).write.format("org.apache.spark.sql.execution.datasources.csv.CSVFileFormat").option("header", "true").save(hdfsFileName)
// val df= spark.sql(sql).persist()
// df.repartition(1).write.format("csv").option("header", "true").save(hdfsFileName)
}
/**
* 从hdfs中拉取数据到本地
*
*
*/
def copyFileFromHDFS(subject: String,time:String): Unit ={
//文件名
val fileName=subject + time
val localFileName = "/Users/growingio/tmp/" + fileName+".csv"
var hdfsFilePath = "hdfs://localhost:9000/tmp/" + fileName
val out =new FileOutputStream(localFileName)
val fs = FileSystem.get(new URI("hdfs://localhost:9000"),new Configuration())
val files=fs.listStatus(new Path(hdfsFilePath))
val fileNames=FileUtil.stat2Paths(files)
for (l <- fileNames){
if(l.getName.endsWith(".csv")){
val hdfsFileName= hdfsFilePath + "/"+ l.getName
val in= fs.open(new Path(hdfsFileName))
IOUtils.copyBytes(in, out, 4096, true)
}
/**
* IOUtils对io的一个封装
* BufferedReader封装一个InputStream,默认private val defaultCharBufferSize: Int = 8192,数据缓存大小为8192个字符
* 4096是对buffersize进行调整
* 第四个参数,是否关闭流
*
*/
}
}
/**
* 将查询结果,发送给指定的人:邮件发送
* @param from:发件人邮箱
* @param fromPassword:发件人邮箱密码
* @param subject:发件主题
*
* @param to:收件人的邮箱
*/
def sendEmai(from: String, fromPassword: String, subject: String, recipients: String,time:String): Unit = {
val fileName=subject + time + ".csv"
val localFileName = "/Users/growingio/tmp/" + fileName
// 指定发送邮件的主机为 smtp.qq.com
val host = "smtp.qiye.163.com"
//163企业 邮件服务器
// 获取系统属性
var properties = new Properties()
// 设置邮件服务器,主机地址
properties.setProperty("mail.smtp.host", host)
//设置邮件协议
properties.setProperty("mail.transport.protocol", "smtp");
// 认证
properties.setProperty("mail.smtp.auth", "true")
// 端口
properties.setProperty("mail.smtp.port", "25")
properties.setProperty("mail.smtp.starttls.enable", "true")
//2: 创建session,获取默认session对象
var session = Session.getInstance(properties, new Authenticator() {
override def getPasswordAuthentication: PasswordAuthentication = {
//qq邮箱服务器账户、第三方登录授权码
return new PasswordAuthentication(from, fromPassword) //发件人邮件用户名、密码
}
})
//开启debug模式,这样就可以查看到程序发送Email的运行状态
session.setDebug(true)
//创建邮件
//创建邮件对象
var message = new MimeMessage(session)
// Set Subject: 设置邮件标题
message.setSubject(subject)
//邮件发送日期
message.setSentDate(new Date())
// message.setText("Dear Mail Crawler," + "\n\n No spam to my email, please!")
// Set From: 头部头字段 指明邮件的发件人
message.setFrom(new InternetAddress(from))
// Set To: 头部头字段,指明收件人
// message.setRecipient(Message.RecipientType.TO, InternetAddress.parse(recipients))
message.setRecipient(Message.RecipientType.TO, new InternetAddress(recipients))
// 向multipart对象中添加邮件的各个部分内容,包括文本内容和附件
// 创建多重消息
var multipart = new MimeMultipart()
// 创建消息部分,正文部分
var contentBodyPart = new MimeBodyPart()
// 消息
// contentBodyPart.setText("UserAction")
// 设置文本消息部分
val result = "UserActionResult"
contentBodyPart.setContent(result, "text/html;charset=UTF-8")
multipart.addBodyPart(contentBodyPart)
// 附件部分
if (localFileName != null && !"".equals(localFileName)) {
var attachmentBodyPart = new MimeBodyPart();
// 根据附件路径获取文件,
val dataSource = new FileDataSource(localFileName);
attachmentBodyPart.setDataHandler(new DataHandler(dataSource));
//MimeUtility.encodeWord可以避免文件名乱码
attachmentBodyPart.setFileName(MimeUtility.encodeWord(dataSource.getFile().getName()));
multipart.addBodyPart(attachmentBodyPart);
}
// 邮件的文本内容
message.setContent(multipart);
// 4. 发送邮件,Transport每次发送成功程序帮忙关闭
Transport.send(message, message.getAllRecipients())
}
def sendMail(subject: String, recipients: String, sql: String): Unit ={
val now = new Date()
val dateFormat = new java.text.SimpleDateFormat("yyyyMMddHHmmsss")
val time = dateFormat.format(now)
saveToHDFS(subject, sql,time)
copyFileFromHDFS(subject,time)
sendEmai("wangxiaodong@growingio.com", "xxxxxxx", subject,recipients,time)
}