1,读取本地图片,并上传至SFTP
val file = new File("/mnt/data/hdd4/yarn/local/usercache/IT_bigdata/appcache/tag1200.jpg")
val img = ImageIO.read(file)
val baos = new ByteArrayOutputStream()
ImageIO.write(img, "jpg", baos)
val byte = baos.toByteArray
val input01 = new ByteArrayInputStream(byte)
channel.put(input01, s"/bigdata/B3I_D63" + "/" + "yy123254.jpg")
二:spark读取卡夫卡中数据写入SFTP,图片类型为BASE64转码后的格式
try {
df.collect().foreach {
row => {
logger.info("begining----row----")
val picName = row.getString(0)
val filedata64 = row.getString(1)
val sftpFile = s"/dir" + "/" + picName
val input = new ByteArrayInputStream(Base64.getDecoder.decode(filedata64))
channel.put(input, sftpFile)
input.close()
}
}
} catch {
case e: Exception =>
e.printStackTrace()
} finally {
session.disconnect()
channel.disconnect()
logger.info("-------close----")
}
三:非图片格式
df.write
.format("com.springml.spark.sftp")
.option("host", host)
.option("username", username)
.option("password", password)
.option("header", false)
.option("fileType", "txt") // Text data source supports only a single column, and you have 3 columns.
.mode("append")
.save(sftpFile)