python sftp传输文件总是报文件不存在_python sftp.get 同步远程文件夹 实践二

主要用途，从本机获取远端机器的文件夹#!/usr/bin/python

import pexpect

import paramiko

import os

import sys

import time

import multiprocessing

import datetime

import crash_on_ipy

from stat import S_ISDIR

ip_list = []

#room_id = sys.argv[1]

cur_time = datetime.datetime.now()

class get_thread():

def __init__(self,host_ip='ip'):

self.host_ip='ip'

self.scp = paramiko.SSHClient()

self.scp.set_missing_host_key_policy(paramiko.AutoAddPolicy())

self.scp = paramiko.Transport((host_ip, 22))

self.scp.connect(username='tomcat', password='xx')

self.sftp = paramiko.SFTPClient.from_transport(self.scp)

def _walk_remote(self, dirpath):

dirnames = []

filenames = []

for fd in self.sftp.listdir_attr(dirpath):

if S_ISDIR(fd.st_mode):

dirnames.append(fd.filename)

else:

filenames.append(fd.filename)

yield dirpath, dirnames, filenames

for dirname in dirnames:

new_dirpath = os.path.join(dirpath, dirname)

# yield from self._walk_remote(new_dirpath)

for walk in self._walk_remote(new_dirpath):

yield walk

def getall(self,local,remote):

st_mode = self.sftp.stat(remote).st_mode

if not S_ISDIR(st_mode):

filename = os.path.basename(remote)

self.sftp.get(remote, os.path.join(local, filename))

else:

parent, child = os.path.split(remote)

for dirpath, dirnames, filenames in self._walk_remote(remote):

dirpath = dirpath.replace(parent, '.')

for dirname in dirnames:

try:

os.makedirs(os.path.join(local, dirpath, dirname))

except:

pass

for filename in filenames:

localpath = os.path.join(local, dirpath, filename)

remotepath = os.path.join(parent, dirpath, filename)

self.sftp.get(remotepath, localpath)

self.scp.close()

if __name__=='__main__':

print "-"*50

getthread=get_thread()

room_pathd = '/opt/src/logs/'

room_paths = '/home/python/'

getthread.getall(room_paths,room_pathd)

因为想作为类，然后读取文件，批量传输 改进版#!/usr/bin/python

import pexpect

import paramiko

import os

import sys

import time

import multiprocessing

import datetime

import crash_on_ipy

from stat import S_ISDIR

ip_list = []

#room_id = sys.argv[1]

cur_time = datetime.datetime.now()

class get_thread():

def __init__(self,hostname,password,username,port=None):

#def __init__(self,hostname,username='root',key_file=None,password=None):

#def __init__(self,hostname=None,password=None,username=None,port=None,local_dir=None,remote_dir=None):

self.hostname = hostname

self.username = username

self.password = password

self.scp = paramiko.SSHClient()

self.scp.set_missing_host_key_policy(paramiko.AutoAddPolicy())

self.scp = paramiko.Transport((hostname, 22))

self.scp.connect(username=username, password=password)

self.sftp = paramiko.SFTPClient.from_transport(self.scp)

def _walk_remote(self, dirpath):

dirnames = []

filenames = []

for fd in self.sftp.listdir_attr(dirpath):

if S_ISDIR(fd.st_mode):

dirnames.append(fd.filename)

else:

filenames.append(fd.filename)

yield dirpath, dirnames, filenames

for dirname in dirnames:

new_dirpath = os.path.join(dirpath, dirname)

# yield from self._walk_remote(new_dirpath)

for walk in self._walk_remote(new_dirpath):

yield walk

def getall(self,local,remote):

st_mode = self.sftp.stat(remote).st_mode

if not S_ISDIR(st_mode):

filename = os.path.basename(remote)

self.sftp.get(remote, os.path.join(local, filename))

else:

parent, child = os.path.split(remote)

for dirpath, dirnames, filenames in self._walk_remote(remote):

dirpath = dirpath.replace(parent, '.')

for dirname in dirnames:

try:

os.makedirs(os.path.join(local, dirpath, dirname))

except:

pass

for filename in filenames:

localpath = os.path.join(local, dirpath, filename)

remotepath = os.path.join(parent, dirpath, filename)

self.sftp.get(remotepath, localpath)

self.scp.close()

if __name__=='__main__':

print "-"*50

#getthread=get_thread()

#room_pathd = '/opt/src/logs/'

#room_paths = '/home/python/'

f = file('/home/filelist','r')

c = f.readlines()

for x in c:

hostname = x.split('::')[0]

password = x.split('::')[1]

username = x.split('::')[2]

local= x.split('::')[3].strip('\n')

remote = x.split('::')[4].strip('\n')

print local,remote

getthread=get_thread(hostname,password,username)

getthread.getall(local,remote)

f.close()

#getthread.getall(room_paths,room_pathd)

最后，又想实现多进程，但是碰到困难了，没有实现，报错

UnpickleableError: Cannot pickle objects

报错版 不可用，待改进#!/usr/bin/python

import pexpect

import paramiko

import os

import sys

import time

import multiprocessing

import datetime

import crash_on_ipy

from stat import S_ISDIR

import copy_reg

import types

ip_list = []

#room_id = sys.argv[1]

cur_time = datetime.datetime.now()

class get_thread():

def __init__(self,hostname='ip'):

self.hostname='ip'

self.scp = paramiko.SSHClient()

self.scp.set_missing_host_key_policy(paramiko.AutoAddPolicy())

self.scp = paramiko.Transport((hostname, 22))

self.scp.connect(username='tomcat', password='xx')

self.sftp = paramiko.SFTPClient.from_transport(self.scp)

def walk_remote(self, dirpath):

dirnames = []

filenames = []

for fd in self.sftp.listdir_attr(dirpath):

if S_ISDIR(fd.st_mode):

dirnames.append(fd.filename)

else:

filenames.append(fd.filename)

yield dirpath, dirnames, filenames

for dirname in dirnames:

new_dirpath = os.path.join(dirpath, dirname)

# yield from self.walk_remote(new_dirpath)

for walk in self.walk_remote(new_dirpath):

yield walk

def __call__(self,local,remote):

return self.getall(local,remote)

def getall(self,local,remote):

st_mode = self.sftp.stat(remote).st_mode

if not S_ISDIR(st_mode):

filename = os.path.basename(remote)

self.sftp.get(remote, os.path.join(local, filename))

else:

parent, child = os.path.split(remote)

for dirpath, dirnames, filenames in self.walk_remote(remote):

dirpath = dirpath.replace(parent, '.')

for dirname in dirnames:

try:

os.makedirs(os.path.join(local, dirpath, dirname))

except:

pass

for filename in filenames:

localpath = os.path.join(local, dirpath, filename)

remotepath = os.path.join(parent, dirpath, filename)

self.sftp.get(remotepath, localpath)

self.scp.close()

def run(self):

room_pathd = '/opt/src/logs/'

room_paths = '/home/python/'

pool = multiprocessing.Pool(processes=1)

pool.apply_async(self, args=(room_paths,room_pathd))

#pool.apply_async(self.getall,args=(room_paths,room_pathd,))

#pool.apply_async(proxy, args=(self, num,))

#pool.apply_async(self.func, args=(num,))

pool.close()

pool.join()

if __name__=='__main__':

print "-"*50

getthread=get_thread()

getthread.run()