条件:
有一台跳板机, 可以ssh到所有机器
知道其他机器的用户名和密码
思路:
1. 在跳板机先简单for循环把文件从本地拷贝到远程10台机器( 因为我们设置了并发Pool(10))
2. 然后 调用 do_peer_copy 并发10台同时拷贝
#!/usr/bin/env python # -*- coding:utf-8 -*- import logging import multiprocessing import invoke import redis from fabric import Connection logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) fh = logging.FileHandler('distributeCopy.log') fh.setLevel(logging.INFO) ch = logging.StreamHandler() ch.setLevel(logging.ERROR) formatter = logging.Formatter(fmt='%(asctime)s %(processName)s-%(threadName)s - %(name)s - %(levelname)s - %(message)s', datefmt='%Y-%m-%d %H:%M:%S') fh.setFormatter(formatter) ch.setFormatter(formatter) logger.addHandler(fh) logger.addHandler(ch) passwd = 'ssh_password'
def get_server_ip(): """
获取所有机器列表
"""
pass
def get_remote_model(): remote_path = '/data1/online/model/final_model/model' local_path = '/data1/rsync_data/ctr_model_webapi' invoke.run('rsync -a 192.168.10.100:%s %s' % (remote_path, local_path)) def do_local_copy(dst_ip, dst_path, queue):
"""
脚本运行在跳板机上(可以免密码登录其它机器的), 因为设置了Pool(10), 所以事先需要先准备10台已经同步好了文件
""" src_path = '/data1/rsync_data/ctr_model_webapi/model' result = Connection(dst_ip).put(src_path, remote=dst_path) queue.put(dst_ip) logger.info('do_local_copy %s: %s', dst_ip, result) def do_peer_copy(args):
"""
从跳板机ssh到server-Y, 在server-Y上 rsync 文件
""" dst_ip = args[0] queue = args[1] print 'before qsize: %s' % queue.qsize() src_ip = queue.get() cmd = """sshpass -p "%s" rsync -e 'ssh -o "StrictHostKeyChecking no"' -a username@%s:%s %s""" % \ (passwd, src_ip, '/tmp/model', '/tmp') result = Connection(dst_ip).run(cmd) queue.put(dst_ip) print 'end qsize: %s' % queue.qsize() logger.info('do_peer_copy %s => %s, %s', src_ip, dst_ip, result)
#return result 此处不要return任何东西 因为会触发python的一个issue https://github.com/joblib/joblib/issues/818#issuecomment-445865581 if __name__ == '__main__': manager = multiprocessing.Manager() q = manager.Queue() pool = multiprocessing.Pool(10) ips = get_server_ip() for ip in ips[:10]: do_local_copy(ip, '/tmp', q) pool.map(do_peer_copy, [(ip, q) for ip in ips[10:]]) pool.close() pool.join() logger.info('finished')