Zeppelin在求学之路----在Zeppelin上开发SendMai功能

功能要求：实现在Spark解释器下，通过自定义功能，调用sendMail（subject，recipients，sql）实现将DataFrame查询结果保存到Hdfs上，并且通过邮件的形式发送给某人，以供后端人员使用。

环境：Mac操作系统，Zeppelin0.74；Spark2.1.1,Hadoop2.52，Scala2.11.，Java1.80

实现分了三部分很简单，代码上有备注，大家可以直接看备注，Zeppelin初始化sc（SparkContext），spark（SparkSession），zc（ZeppelinContext），所有不用自己去创建相关对象直接调用即可。

%spark


import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileSystem
import org.apache.hadoop.fs.Path
import org.apache.hadoop.fs.FileUtil
import org.apache.hadoop.io.IOUtils
import java.net.URI
import java.io._
import javax.mail.Session
import javax.mail.Transport
import javax.mail.PasswordAuthentication
import java.text.SimpleDateFormat
import java.util.{Date, Properties}
import javax.mail.Authenticator
import javax.mail.Message
import javax.mail.internet.InternetAddress
import javax.mail.internet.MimeBodyPart
import javax.mail.internet.MimeMessage
import javax.mail.internet.MimeMultipart
import javax.mail.internet.MimeUtility
import javax.activation.DataHandler
import javax.activation.FileDataSource
    // private var hdfsFileName :String =""
    // private var from : String =""
    // private var fromPassword :String =""
    // private var localFileName : String=""
    // private var fileName=""
    // private val now = new java.util.Date()
    // private val dateFormat = new java.text.SimpleDateFormat("yyyyMMddHHmmsss")
    // private val time = dateFormat.format(now)

    /**
    * 生成sql查询 的DF，将其保存在hdfs中
    * @param subject     主题

    * @param sql         sql语句
    */
    def saveToHDFS(subject: String, sql: String,time:String) :Unit={
       
        //文件名
        val fileName=subject + time
        //文件在hdfs的绝对路径
        var hdfsFileName = "hdfs://localhost:9000/tmp/" + fileName
        val df=spark.sql(sql).repartition(1).write.format("org.apache.spark.sql.execution.datasources.csv.CSVFileFormat").option("header", "true").save(hdfsFileName)
        // val  df=  spark.sql(sql).persist()                                               
        // df.repartition(1).write.format("csv").option("header", "true").save(hdfsFileName)
    
  } 
        
    /**
    * 从hdfs中拉取数据到本地
    *
    *
    */
    def copyFileFromHDFS(subject: String,time:String): Unit ={
         //文件名
        val fileName=subject + time
        val localFileName = "/Users/growingio/tmp/" + fileName+".csv"
        var hdfsFilePath = "hdfs://localhost:9000/tmp/" + fileName
        val out =new FileOutputStream(localFileName)
        val fs = FileSystem.get(new URI("hdfs://localhost:9000"),new Configuration())
        val files=fs.listStatus(new Path(hdfsFilePath))    
        val fileNames=FileUtil.stat2Paths(files)      
        for (l <- fileNames){                          
          if(l.getName.endsWith(".csv")){                  
            val hdfsFileName= hdfsFilePath + "/"+ l.getName     
            val in= fs.open(new Path(hdfsFileName))           
               
            IOUtils.copyBytes(in, out, 4096, true)         
                                                           
                                                           
          }                                                
       
    
        /**
          * IOUtils对io的一个封装
          * BufferedReader封装一个InputStream，默认private val defaultCharBufferSize: Int = 8192，数据缓存大小为8192个字符
          * 4096是对buffersize进行调整
          * 第四个参数，是否关闭流
          *
          */
       
        }
  }
    /**
    * 将查询结果，发送给指定的人：邮件发送
    * @param from：发件人邮箱
    * @param fromPassword：发件人邮箱密码
    * @param subject：发件主题
    *
    * @param to：收件人的邮箱
    */

    def sendEmai(from: String, fromPassword: String, subject: String, recipients: String,time:String): Unit = {
          val fileName=subject + time + ".csv"
          val localFileName = "/Users/growingio/tmp/" + fileName
          // 指定发送邮件的主机为 smtp.qq.com
          val host = "smtp.qiye.163.com"
          //163企业 邮件服务器
          // 获取系统属性
          var properties = new Properties()
          // 设置邮件服务器,主机地址
          properties.setProperty("mail.smtp.host", host)
          //设置邮件协议
          properties.setProperty("mail.transport.protocol", "smtp");
          // 认证
          properties.setProperty("mail.smtp.auth", "true")
          // 端口
          properties.setProperty("mail.smtp.port", "25")
    
          properties.setProperty("mail.smtp.starttls.enable", "true")
    
          //2: 创建session，获取默认session对象
          var session = Session.getInstance(properties, new Authenticator() {
            override def getPasswordAuthentication: PasswordAuthentication = {
              //qq邮箱服务器账户、第三方登录授权码
              return new PasswordAuthentication(from, fromPassword) //发件人邮件用户名、密码
    
            }
          })
          //开启debug模式，这样就可以查看到程序发送Email的运行状态
          session.setDebug(true)
         
          //创建邮件
          //创建邮件对象
          var message = new MimeMessage(session)
          // Set Subject: 设置邮件标题
          message.setSubject(subject)
          //邮件发送日期
          message.setSentDate(new Date())
          // message.setText("Dear Mail Crawler," + "\n\n No spam to my email, please!")
          // Set From: 头部头字段 指明邮件的发件人
          message.setFrom(new InternetAddress(from))
          // Set To: 头部头字段,指明收件人
          //        message.setRecipient(Message.RecipientType.TO, InternetAddress.parse(recipients))
          message.setRecipient(Message.RecipientType.TO, new InternetAddress(recipients))
    
    
          // 向multipart对象中添加邮件的各个部分内容，包括文本内容和附件
          // 创建多重消息
          var multipart = new MimeMultipart()
          // 创建消息部分，正文部分
          var contentBodyPart = new MimeBodyPart()
          // 消息
          // contentBodyPart.setText("UserAction")
          // 设置文本消息部分
          val result = "UserActionResult"
          contentBodyPart.setContent(result, "text/html;charset=UTF-8")
          multipart.addBodyPart(contentBodyPart)
    
    
          // 附件部分
          
          if (localFileName != null && !"".equals(localFileName)) {
           
            var attachmentBodyPart = new MimeBodyPart();
            // 根据附件路径获取文件,
            val dataSource = new FileDataSource(localFileName);
            attachmentBodyPart.setDataHandler(new DataHandler(dataSource));
            //MimeUtility.encodeWord可以避免文件名乱码
            attachmentBodyPart.setFileName(MimeUtility.encodeWord(dataSource.getFile().getName()));
            multipart.addBodyPart(attachmentBodyPart);
        
          }
          // 邮件的文本内容
          message.setContent(multipart);
        
    
          // 4. 发送邮件,Transport每次发送成功程序帮忙关闭
          Transport.send(message, message.getAllRecipients())
    
          }
    def sendMail(subject: String, recipients: String, sql: String): Unit ={
        val now = new Date()             
        val dateFormat = new java.text.SimpleDateFormat("yyyyMMddHHmmsss")
        val time = dateFormat.format(now)
        saveToHDFS(subject, sql,time)
        copyFileFromHDFS(subject,time)
        sendEmai("wangxiaodong@growingio.com", "xxxxxxx", subject,recipients,time)


    }