Linux下文件夹的自动同步

Linux下文件夹的自动同步

问题：服务器a每天21：00给服务器B发送日志文件，服务器B接受到日志文件后，如果磁盘大小小于10G则选择创建时间最早的文件删除，保证磁盘剩余空间大于等于10G。

安装的python库

#coding=utf-8
import time
import os
import threading
import socket
import schedule
import hashlib
import shutil
from functools import cmp_to_key

#coding=utf-8在Linux系统中用来指定编码方式，否则程序中有中文会报错！
path_str = ‘./home1’ ##工作目录
ip = ‘127.0.0.1’
port = 6969
指定工作目录，目标IP，端口号等信息。

对文件夹下文件进行排序，按照时间排序

def compare(x,y): ##文件夹排序函数--sort()
    stat_x = os.stat(path_str + '/' + x)
    stat_y = os.stat(path_str + '/' + y)
    if stat_x.st_ctime < stat_y.st_ctime:
        return -1
    elif stat_x.st_ctime > stat_y.st_ctime: ##由操作系统的ctime统一
        return 1
    else:
        return 0

定义多线程函数

def run_threaded(job_func): ##多线程并发执行
    job_thread = threading.Thread(target=job_func)
    job_thread.start()

监听，每天定时执行，A和B错开一分钟确保不会掉线

#服务器B
    schedule.every().day.at("21:01").do(run_threaded,start)
    while True:
        schedule.run_pending()
        time.sleep(1)
#服务器A
    if not os.path.exists(path_str):
        os.mkdir(path_str)
    schedule.every().day.at("21:00").do(run_threaded,Serber_start)
    schedule.every().day.at("21:00").do(run_threaded,storage_space)
    while True:
        schedule.run_pending()
        time.sleep(1)

创建收发信息的日志文件

def journal(record):
    if os.path.exists('./abc.log'):
        with open('./abc.log','a') as f:
            f.write(record)
    else:
        with open('./abc.log','w+') as f:
            f.write(record)

监视磁盘空间，小于指定值则删除最早创建的文件

def storage_space(): ##监视磁盘空间，当磁盘空间少于10时，删除最先创建的文件夹
    global path_str
    disk = os.statvfs(path_str)
    disk_size = disk.f_bsize * disk.f_blocks / (1024 ** 3)  # 1G = 1024M  1M = 1024KB 1KB = 1024bytes
    disk_used = disk.f_bsize * (disk.f_blocks - disk.f_bfree) / (1024 ** 3)
    count = time.time()
    while True:
        if time.time() - count > 600:
            break
        iterms = os.listdir(path_str)
        iterms.sort(key=cmp_to_key(compare)) ##对文件夹的创建时间排序
        if disk_size-disk_used < 10:
            time.sleep(10)
            shutil.rmtree(path_str + '/' + iterms[0])
            journal('delete ' + iterms[0] + time.strftime(" %Y-%m-%d %H:%M:%S\n", time.localtime()))
        time.sleep(1)

服务器B完整代码

#coding=utf-8
import time
import os
import threading
import socket
import schedule
import hashlib
import shutil
from functools import cmp_to_key
path_str = './home1' ##工作目录
ip = '127.0.0.1'
port = 6969
def journal(record):
    if os.path.exists('./abc.log'):
        with open('./abc.log','a') as f:
            f.write(record)
    else:
        with open('./abc.log','w+') as f:
            f.write(record)
def server_bind():
    global ip
    global port
    server = socket.socket()
    server.bind((ip,port))
    server.listen(5)
    if os.path.exists(path_str + '/{}'.format(time.strftime('%Y-%m-%d'))):
        shutil.rmtree(path_str + '/{}'.format(time.strftime('%Y-%m-%d')))
    os.mkdir(path_str + '/{}'.format(time.strftime('%Y-%m-%d')))  ##创建日期文件夹
    flag = True
    count = 0
    count_1 = -1
    while True:
        if count_1 == count:
            break
        conn, addr = server.accept()
        if flag:
            count_1 = int(conn.recv(1024).decode('utf-8'))
            flag = False
            conn.close()
            continue
        Serber_start(conn)
        count += 1
def Serber_start(conn): ##接受数据
    ## os.makedirs('./{}'.format(time.strftime('%Y-%m-%d'))) ##创建日期文件夹
    filename,filesize= conn.recv(1024).decode('utf-8').split(':')
    filesize=int(filesize)
    received_size = 0
    with open(path_str + '/{}/{}'.format(time.strftime('%Y-%m-%d'), filename), 'wb') as f:
        m = hashlib.md5()
        while received_size < filesize:
            size = 0 ##准确接受的数据包大小
            if filesize - received_size > 1024:
                size = 1024
            else:
                size = filesize - received_size
            data=conn.recv(size)
            received_size += len(data)
            m.update(data)
            f.write(data)
    md5_server = conn.recv(1024)
    md5_client = m.hexdigest()
    if md5_server != md5_client:
        journal(filename + ':' + time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + '-md5校验失败\n')
def compare(x,y): ##文件夹排序函数--sort()
    stat_x = os.stat(path_str + '/' + x)
    stat_y = os.stat(path_str + '/' + y)
    if stat_x.st_ctime < stat_y.st_ctime:
        return -1
    elif stat_x.st_ctime > stat_y.st_ctime: ##由操作系统的ctime统一
        return 1
    else:
        return 0
def storage_space(): ##监视磁盘空间，当磁盘空间少于10时，删除最先创建的文件夹
    global path_str
    disk = os.statvfs(path_str)
    disk_size = disk.f_bsize * disk.f_blocks / (1024 ** 3)  # 1G = 1024M  1M = 1024KB 1KB = 1024bytes
    disk_used = disk.f_bsize * (disk.f_blocks - disk.f_bfree) / (1024 ** 3)
    count = time.time()
    while True:
        if time.time() - count > 600:
            break
        iterms = os.listdir(path_str)
        iterms.sort(key=cmp_to_key(compare)) ##对文件夹的创建时间排序
        if disk_size-disk_used < 10:
            time.sleep(10)
            shutil.rmtree(path_str + '/' + iterms[0])
            journal('delete ' + iterms[0] + time.strftime(" %Y-%m-%d %H:%M:%S\n", time.localtime()))
        time.sleep(1)
def run_threaded(job_func): ##多线程并发执行
    job_thread = threading.Thread(target=job_func)
    job_thread.start()
if __name__ == '__main__':
    # b=threading.Thread(target=server_bind())
    # a=threading.Thread(target=storage_space())
    # b.start()
    # a.start()
    # b.join()
    # a.join()
    # schedule.every(1).minutes.do(run_threaded,server_bind)
    # schedule.every(1).minutes.do(run_threaded,storage_space)
    if not os.path.exists(path_str):
        os.mkdir(path_str)
    schedule.every().day.at("21:00").do(run_threaded,Serber_start)
    schedule.every().day.at("21:00").do(run_threaded,storage_space)
    while True:
        schedule.run_pending()
        time.sleep(1)

服务器A完整代码

#coding=utf-8
import time
import os
import threading
import socket
import schedule
import hashlib
from functools import cmp_to_key
path_str = './home' ##working
ip = '127.0.0.1'
port = 6969
iterms_1 = '' ## name
def compare(x,y): ##文件夹排序函数--sort()
    stat_x = os.stat(path_str + '/' + x)
    stat_y = os.stat(path_str + '/' + y)
    if stat_x.st_ctime < stat_y.st_ctime:
        return 1
    elif stat_x.st_ctime > stat_y.st_ctime: ##由操作系统的ctime统一
        return -1
    else:
        return 0
def start():
    global path_str
    global iterms_1
    iterms = os.listdir(path_str)
    client = socket.socket()
    client.connect((ip,port))
    iterms.sort(key=cmp_to_key(compare))  ##对文件夹的创建时间排序
    iterms_1 = os.listdir((path_str + '/' +iterms[0]))
    client.send("{}".format(len(iterms_1)).encode('utf-8'))
    client.close()
    time.sleep(1)
    for iterm in iterms_1:
        client_start(iterms[0] + '/' + iterm)
def client_start(filename):
    global path_str
    global ip
    global port
    client = socket.socket()
    client.connect((ip,port))
    size = os.stat(path_str + '/' + filename).st_size
    name = filename.split('/')[-1] + ':' + str(size)
    time.sleep(0.5)
    client.send(name.encode('utf-8'))
    time.sleep(1)
    print(filename,size)
    with open(path_str + '/' + filename,'rb') as f:
        m = hashlib.md5()
        for line in f:
            client.send(line)
            m.update(line)
    md5 = m.hexdigest()
    client.send(md5.encode('utf-8'))
    client.close()
    time.sleep(1)
def run_threaded(job_func): ##多线程并发执行
    job_thread = threading.Thread(target=job_func)
    job_thread.start()
if __name__ == '__main__':
    # start()
    # schedule.every(0.1).minutes.do(run_threaded,Serber_start)
    # schedule.every(0.1).minutes.do(run_threaded,storage_space)
    schedule.every().day.at("21:01").do(run_threaded,start)
    while True:
        schedule.run_pending()
        time.sleep(1)