python sftp.get 同步远程文件夹 实践二

主要用途，从本机获取远端机器的文件夹

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66

#!/usr/bin/python    import  pexpect import  paramiko   import  os import  sys import  time import  multiprocessing import  datetime import  crash_on_ipy from  stat  import  S_ISDIR     ip_list  =  [] #room_id = sys.argv[1] cur_time  =  datetime.datetime.now() class  get_thread():      def  __init__( self ,host_ip = 'ip' ):          self .host_ip = 'ip'          self .scp  =  paramiko.SSHClient()          self .scp.set_missing_host_key_policy(paramiko.AutoAddPolicy())          self .scp  =  paramiko.Transport((host_ip,  22 ))          self .scp.connect(username = 'tomcat' , password = 'xx' )          self .sftp  =  paramiko.SFTPClient.from_transport( self .scp)      def  _walk_remote( self , dirpath):          dirnames  =  []          filenames  =  []            for  fd  in  self .sftp.listdir_attr(dirpath):              if  S_ISDIR(fd.st_mode):                  dirnames.append(fd.filename)              else :                  filenames.append(fd.filename)          yield  dirpath, dirnames, filenames            for  dirname  in  dirnames:              new_dirpath  =  os.path.join(dirpath, dirname)              # yield from self._walk_remote(new_dirpath)              for  walk  in  self ._walk_remote(new_dirpath):                  yield  walk        def  getall( self ,local,remote):                    st_mode  =  self .sftp.stat(remote).st_mode          if  not  S_ISDIR(st_mode):              filename  =  os.path.basename(remote)              self .sftp.get(remote, os.path.join(local, filename))          else :              parent, child  =  os.path.split(remote)                for  dirpath, dirnames, filenames  in  self ._walk_remote(remote):                  dirpath  =  dirpath.replace(parent,  '.' )                  for  dirname  in  dirnames:                          try :                             os.makedirs(os.path.join(local, dirpath, dirname))                                          except :                                                 pass                  for  filename  in  filenames:                      localpath  =  os.path.join(local, dirpath, filename)                      remotepath  =  os.path.join(parent, dirpath, filename)                      self .sftp.get(remotepath, localpath)          self .scp.close() if  __name__ = = '__main__' :      print  "-" * 50      getthread = get_thread()      room_pathd  =  '/opt/src/logs/'      room_paths  =  '/home/python/'      getthread.getall(room_paths,room_pathd)

因为想作为类，然后读取文件，批量传输 改进版

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84

#!/usr/bin/python    import  pexpect import  paramiko   import  os import  sys import  time import  multiprocessing import  datetime import  crash_on_ipy from  stat  import  S_ISDIR     ip_list  =  [] #room_id = sys.argv[1] cur_time  =  datetime.datetime.now() class  get_thread():      def  __init__( self ,hostname,password,username,port = None ):     #def __init__(self,hostname,username='root',key_file=None,password=None):      #def __init__(self,hostname=None,password=None,username=None,port=None,local_dir=None,remote_dir=None):          self .hostname  =  hostname          self .username  =  username          self .password  =  password          self .scp  =  paramiko.SSHClient()          self .scp.set_missing_host_key_policy(paramiko.AutoAddPolicy())          self .scp  =  paramiko.Transport((hostname,  22 ))          self .scp.connect(username = username, password = password)                    self .sftp  =  paramiko.SFTPClient.from_transport( self .scp)      def  _walk_remote( self , dirpath):          dirnames  =  []          filenames  =  []            for  fd  in  self .sftp.listdir_attr(dirpath):              if  S_ISDIR(fd.st_mode):                  dirnames.append(fd.filename)              else :                  filenames.append(fd.filename)          yield  dirpath, dirnames, filenames            for  dirname  in  dirnames:              new_dirpath  =  os.path.join(dirpath, dirname)              # yield from self._walk_remote(new_dirpath)              for  walk  in  self ._walk_remote(new_dirpath):                  yield  walk        def  getall( self ,local,remote):                    st_mode  =  self .sftp.stat(remote).st_mode          if  not  S_ISDIR(st_mode):              filename  =  os.path.basename(remote)              self .sftp.get(remote, os.path.join(local, filename))          else :              parent, child  =  os.path.split(remote)                for  dirpath, dirnames, filenames  in  self ._walk_remote(remote):                  dirpath  =  dirpath.replace(parent,  '.' )                  for  dirname  in  dirnames:                      try :                          os.makedirs(os.path.join(local, dirpath, dirname))                      except :                          pass                    for  filename  in  filenames:                      localpath  =  os.path.join(local, dirpath, filename)                      remotepath  =  os.path.join(parent, dirpath, filename)                      self .sftp.get(remotepath, localpath)          self .scp.close() if  __name__ = = '__main__' :      print  "-" * 50      #getthread=get_thread()      #room_pathd = '/opt/src/logs/'      #room_paths = '/home/python/'      f  =  file ( '/home/filelist' , 'r' )      c  =  f.readlines()      for  x  in  c:          hostname  =  x.split( '::' )[ 0 ]          password  =  x.split( '::' )[ 1 ]          username  =  x.split( '::' )[ 2 ]          local =  x.split( '::' )[ 3 ].strip( '\n' )          remote  =  x.split( '::' )[ 4 ].strip( '\n' )          print  local,remote                  getthread = get_thread(hostname,password,username)          getthread.getall(local,remote)      f.close()     #getthread.getall(room_paths,room_pathd)

最后，又想实现多进程，但是碰到困难了，没有实现，报错

UnpickleableError: Cannot pickle <type 'thread.lock'> objects

报错版 不可用，待改进

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84

#!/usr/bin/python    import  pexpect import  paramiko   import  os import  sys import  time import  multiprocessing import  datetime import  crash_on_ipy from  stat  import  S_ISDIR import  copy_reg import  types     ip_list  =  [] #room_id = sys.argv[1] cur_time  =  datetime.datetime.now() class  get_thread():      def  __init__( self ,hostname = 'ip' ):          self .hostname = 'ip'          self .scp  =  paramiko.SSHClient()          self .scp.set_missing_host_key_policy(paramiko.AutoAddPolicy())          self .scp  =  paramiko.Transport((hostname,  22 ))          self .scp.connect(username = 'tomcat' , password = 'xx' )          self .sftp  =  paramiko.SFTPClient.from_transport( self .scp)      def  walk_remote( self , dirpath):          dirnames  =  []          filenames  =  []            for  fd  in  self .sftp.listdir_attr(dirpath):              if  S_ISDIR(fd.st_mode):                  dirnames.append(fd.filename)              else :                  filenames.append(fd.filename)          yield  dirpath, dirnames, filenames            for  dirname  in  dirnames:              new_dirpath  =  os.path.join(dirpath, dirname)              # yield from self.walk_remote(new_dirpath)              for  walk  in  self .walk_remote(new_dirpath):                  yield  walk          def  __call__( self ,local,remote):          return  self .getall(local,remote)        def  getall( self ,local,remote):                    st_mode  =  self .sftp.stat(remote).st_mode          if  not  S_ISDIR(st_mode):              filename  =  os.path.basename(remote)              self .sftp.get(remote, os.path.join(local, filename))          else :              parent, child  =  os.path.split(remote)                for  dirpath, dirnames, filenames  in  self .walk_remote(remote):                  dirpath  =  dirpath.replace(parent,  '.' )                  for  dirname  in  dirnames:                      try :                          os.makedirs(os.path.join(local, dirpath, dirname))                      except :                          pass                    for  filename  in  filenames:                      localpath  =  os.path.join(local, dirpath, filename)                      remotepath  =  os.path.join(parent, dirpath, filename)                      self .sftp.get(remotepath, localpath)          self .scp.close()          def  run( self ):          room_pathd  =  '/opt/src/logs/'          room_paths  =  '/home/python/'          pool  =  multiprocessing.Pool(processes = 1 )          pool.apply_async( self , args = (room_paths,room_pathd))          #pool.apply_async(self.getall,args=(room_paths,room_pathd,))          #pool.apply_async(proxy, args=(self, num,))          #pool.apply_async(self.func, args=(num,))          pool.close()          pool.join()   if  __name__ = = '__main__' :      print  "-" * 50      getthread = get_thread()      getthread.run()

本文转自 liqius 51CTO博客，原文链接：http://blog.51cto.com/szgb17/1860408，如需转载请自行联系原作者