# -*- coding: UTF-8 -*-
#!/usr/local/bin/python2.7
##/usr/bin/python
#import codecs
import os
import shutil
import json
import sys
import paramiko
import hashlib
import requests
import socket
import struct
from flask import Flask, request
from flask_api import status
import logging
opApp = Flask(__name__)
# 日志系统配置
logging.basicConfig(filename='http_op.log', level=logging.INFO)
remoteAliHostIp="10.10.10.10"
remoteAliHostPort=22
remoteAliHostUser="root"
rsakey="/home/hadoop/.ssh/id_rsa"
remoteBasePath="/data/file/user"
from hdfs.client import Client
def hdfs(hdfsPath):
# hdfsPath='/tables/t/userlabel/test/' #sys.argv[1]
localPath=hdfsPath #sys.argv[2]
client = Client("http://192.168.0.1:50070")
dirList = client.list(localPath ,status=False )
#print dirList
for i in dirList:
#print( "/datalog/nasLZO/2018/05/21/"+ i )
if( i != '_SUCCESS' ):
hdfsLeafPath=hdfsPath+"/"+i
localPathExistsFlag=os.path.exists(localPath)
if( not localPathExistsFlag ):
os.makedirs( localPath )
localLeafPath=localPath+"/"+i
fileFlag=os.path.exists( localLeafPath )
logging.info("localLeafPath:" + localLeafPath)
logging.info(fileFlag)
if(fileFlag):
os.remove(localLeafPath)
logging.info('The File Is Exists , Remove OK ! ')
client.download(hdfsLeafPath, localPath )
logging.info( "#######################OP : " + hdfsPath + "<Migrate Nas OK>##########################")
return True
# ssh
def sshAndtransport(localFolder,remoteFolder, localFile):
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
# 计算本地md5值
localmd5 = None
logging.info( "local: {0}/{1}".format(localFolder, localFile))
logging.info( "remote: {0}/{1}/{2}".format(remoteBasePath, remoteFolder, localFile))
f = open("{0}/{1}".format(localFolder, localFile),'rb')
md5_obj = hashlib.md5()
md5_obj.update(f.read())
hash_code = md5_obj.hexdigest()
f.close()
localmd5 = str(hash_code).lower()
try:
ssh.connect(remoteAliHostIp, port=remoteAliHostPort, username=remoteAliHostUser, key_filename=rsakey, timeout=30)
# ssh.connect('10.10.10.10',port='22',username='root',key_filename="/home/bergXu/.ssh/id_rsa")
except paramiko.SSHException as e:
logging.error("ssh connect error: {0}".format(e))
else:
# 先创建远程目录
stdin, stdout, stderr = ssh.exec_command("mkdir -p {0}/{1}".format(remoteBasePath, remoteFolder))
# 远程调用netcat开启端口监听
stdin, stdout, stderr = ssh.exec_command("/usr/bin/ncat -n -4 -p 19898 -l -o {0}/{1}/{2} 1> /dev/null &".format(remoteBasePath, remoteFolder, localFile))
# 传输文件
SendFile(remoteAliHostIp, 19898, "{0}/{1}".format(localFolder, localFile))
# 远程文件md5值
stdin,stdout,stderr = ssh.exec_command("md5sum {0}/{1}/{2}".format(remoteBasePath, remoteFolder, localFile))
remoteMd5=stdout.readlines()
remoteMd5=str(remoteMd5[0]).split()[0]
logging.info( "{0}----{1}".format(remoteMd5,localmd5))
# md5比对
if localmd5 == remoteMd5 :
logging.info( "md5 check ok!")
return True
else:
logging.info( "md5 check failed!")
return False
finally:
# ssh关闭
ssh.close()
# send file
def SendFile(ip, port, filename):
BUFSIZE = 1024
FILEINFO_SIZE=struct.calcsize('128s32sI8s')
sendSock = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
sendSock.connect((ip, port))
logging.info( "socket connected")
fhead=struct.pack('128s11I',filename,0,0,0,0,0,0,0,0,os.stat(filename).st_size,0,0)
# sendSock.send(fhead)
fp = open(filename,'rb')
while 1:
filedata = fp.read(BUFSIZE)
if not filedata:
break
sendSock.send(filedata)
logging.info( "file upload access.")
fp.close()
sendSock.close()
logging.info( "socket disconnected.")
@opApp.route("/upload", methods=['POST'])
def main():
if request.method == "POST":
hdfsPath=request.form.get("path")
key=request.form.get("key")
localFolder=hdfsPath=os.path.join(hdfsPath, key)
# 从hdfs下载文件
hdfsState=hdfs(hdfsPath)
remoteFolder = localFolder.replace('/tables/r/bigdata/http_op_user/','')
#2019/05/17/type
logging.info(remoteFolder)
#上传到阿里云
for file in os.listdir(localFolder):
i=0
while i<5:
uploadState=sshAndtransport(localFolder=localFolder,remoteFolder=remoteFolder, localFile=file)
if uploadState:
logging.info( "upload file ok, md5 check ok, script exit!")
# 接口调用
templatePath = os.path.join(remoteFolder, file)
json_data = {"statue": 1,"templateKey": key, "templatePath": templatePath}
logging.info(json_data)
r=requests.post('http://10.10.10.10/dataSync/userTemplateNotice',json=json_data)
#r11 = requests.post("http://192.168.1.147:8899/dataSync/userTemplateNotice",json=json_data)
logging.info(r)
break
else:
i+=1
logging.info( "upload file ok, md5 check failed, try {0}".format(i))
uploadState=sshAndtransport(localFolder=localFolder, remoteFolder=remoteFolder,localFile=file)
# print hdfsPath
return "ok", status.HTTP_200_OK
if __name__ == '__main__':
opApp.run(host="0.0.0.0", port=5000, debug=True)