使用python-sftp协议下载远程主机数据

# -*- coding: utf-8 -*-
'''
Created on 2015年8月30日
此程序运行在10.5.226.32
不断去10.0.0.3下载原始数据,长时间运行如有报错,请重新启动
程序启动默认先下载远程的1x数据,然后下载do数据
@author: ZHOU
'''
import paramiko
import time
import os
import traceback
import datetime

workdir_1x='/data/initial_pcmd1x/'
workdir_do='/data/initial_pcmddo/'

remotedir_1x='/yixing/pcmd/pcmd1x/'
remotedir_do='/yixing/pcmd/pcmddo/'
print('The download_data programe is running...')

def toString(str):
    return str.strip()

def writelog(logcontent):
    with open("/data/peoplestat_work/download_initaildata.log","a") as log_file:
        dt=datetime.datetime.now()
        timelabel=dt.strftime("%Y-%m-%d %H:%M:%S %p ")
        log_file.write(timelabel+str(logcontent)+"\n")
        
def getsftpclient():
    username = "yixing"
    password = "yixing"
    hostname = "10.0.0.3"
    port = 22
    try:
        t = paramiko.Transport((hostname, port))
        t.connect(username=username, password=password)
        return t
    
    except Exception as e:
        writelog("getsftpclient exception!")
        traceback.print_exc()
        try:
            t.close()
        except:
            pass
        
if __name__ == '__main__':            
    while True:
        #建立传输通道           
        try:
            transport=getsftpclient()
            sftp =paramiko.SFTPClient.from_transport(transport)
            
            remote1xfiles=sftp.listdir(remotedir_1x)
            remote1xfileset=set(remote1xfiles)
            local1xfiles=os.listdir(path=workdir_1x)
            local1xfileset=set(local1xfiles)
            #求差集,获得新文件列表
            new1xfiles=remote1xfileset-local1xfileset
            new1xfileslist=list(new1xfiles)
            new1xfileslist.sort()#按顺序下载
            
            remotedofiles=sftp.listdir(remotedir_do)
            remotedofileset=set(remotedofiles)
            localdofiles=os.listdir(path=workdir_do)
            localdofileset=set(localdofiles)
            newdofiles=remotedofileset-localdofileset
            newdofileslist=list(newdofiles)
            newdofileslist.sort()#按顺序下载
            
            time.sleep(10)#10s执行一次,顺便等待文件写完整
            
            if len(new1xfileslist)>0:
                for item in new1xfileslist:
                    if item.startswith('.'):#过滤没有写完整的文件
                        continue
                    remote1xfile=remotedir_1x+str(item)
                    local1xfile=workdir_1x+str(item)
                    sftp.get(remote1xfile,local1xfile)
                    writelog(remote1xfile+" downloaded!")
                    
            if len(newdofileslist)>0:    
                for item in newdofileslist:
                    if item.startswith('.'):#过滤没有写完整的文件
                        continue
                    remotedofile=remotedir_do+str(item)
                    localdofile=workdir_do+str(item)
                    sftp.get(remotedofile,localdofile)
                    writelog(remotedofile+" downloaded!")   
                     
            transport.close()
            
        except Exception as e:
            writelog("sftpload exception!")
            traceback.print_exc()
            try:
                transport.close()
            except:
                    pass