centos mysql备份工具_MySQL备份工具mydumper备份

MySQL备份工具mydumper备份

#!/usr/bin/phthon

import os

import time

import commands

import shutil

import threading

from os.path import join, getsize

import MySQLdb as mydb

# 备份目录

baseDir = "/data2/backup/backup_data/"

# ns 或 wx;  备份后是否要压缩(mydumper 自带压缩功能)，要压缩 True，否则 False.

idc = 'ns'; isZip = True

# 备份失败是否重试 ，True 则重试,不重试 设 False, retry_sleep 多久后才开始重试(秒)

is_errRetryBackup = True; retry_sleep = 300

# 备份日期

backup_date = time.strftime("%Y%m%d")

# 备份命令

cmd  = "/usr/local/bin/mydumper -h %s -u root -p password? -P %s %s -t 5 -o %s"

'''''

功能描述：

1. mydumper 远程批量备份, 备份列表由配置文件提供

2. 可按要求对备份是否压缩(mydumper 自动压缩)

3. 备份失败允许再尝试备份一次

4. 备份信息写入数据库

'''

def main():

thread_pool = []

# 是否启用压缩

zip = '-c' if isZip == True else ''

# 从配置文件读取 ip, 名称, 端口, 并拼凑备份语句

#f = open('/data2/backup/cnf/other_list.cnf', 'r')

f = open('/data/other_list.cnf', 'r')

for lines in f.readlines():

if (not lines.startswith('#') and len(lines.strip()) > 0):

str = lines.split()

host, businessName, port, isMaster = str[0], str[1], str[2], str[3]

# 业务文件夹不存在则创建

dir = baseDir + '/' + businessName;

if (not os.path.exists(dir)):

os.makedirs(dir)

dir += "/%s%s" % (businessName, backup_date)

# 业务目录: dir , 备份目录: dir/name+备份日期

strcmd  = cmd % (host, port, zip, dir)

th = threading.Thread(target = mydumper, args =(strcmd, dir, businessName, host, port, is_errRetryBackup, int(isMaster)))

thread_pool.append(th)

if (thread_pool):

for t in thread_pool:

t.daemon = True

t.start()

for t in thread_pool:

t.join()

def mydumper(sCmd, backupDir, businessName, host, port, is_Retry, isMaster):

master_host = ""; backup_host = host; name = businessName; port = port; backup_type = 1; file = "";

start_time = ""; stop_time = ""; returncode = 0; file_size = 0; slave_statement = ""; std_err = "";

start_time = time.strftime("%Y%m%d%H%M%S")

# 清除可能遗留的备份

if (os.path.exists(backupDir)):

shutil.rmtree(backupDir)

# 执行备份

returncode, std_err = execute(sCmd)

stop_time = time.strftime("%Y%m%d%H%M%S")

if (returncode == 0):

# 备份 std_err 返回不为空也视为出错。

if (std_err.strip() != ""):

returncode = 123456

else:

# 获取 change master to 信息，再次校验备份是否有效

returncode, std_err, master_host, slave_statement = statement(backupDir, backup_host, isMaster)

if (returncode == 0):

file = backupDir

if (returncode != 0):

# 异常备份标记为: 日期 + _ERR

errDir = backupDir + "_ERR"

os.rename(backupDir, errDir)

file = errDir

# 获取备份大小

file_size = getDirsize(file)

if (len(std_err) > 255):

std_err = std_err[:250] + "..."

my_args  = [idc, master_host, backup_host, name, port, backup_type, file, start_time, stop_time, returncode, file_size, slave_statement, std_err]

# 写入数据库

call_proc(my_args)

# 备份失败是否需要重备? 重备允许一次.

if (is_Retry == True and returncode != 0):

time.sleep(retry_sleep)

oldfile = sCmd.split('-o')[1]

pos = oldfile.rfind("/") + 1

# 获取备份全路径, 备份文件标记为重备 字样

retry_file = oldfile[:pos] + "ReBackup-" + oldfile[pos:]

retryCmd = sCmd.replace(oldfile,  retry_file)

# 重备开始

mydumper(retryCmd, retry_file.strip(), name, host, port, False, isMaster)

def getDirsize(path):

# 获取备份文件夹大小

size = 0L

for root, dirs, files in os.walk(path):

size += sum([getsize(join(root, name)) for name in files])

return (size)

def statement(path, backup_host, isMaster):

'''''

功能: 从 metadata 读取change master to 信息

1. 备份过程: 会先生成: metadata.partial, 完成后metadata.partial会重名为: metadata 并写入备份完成时间

2. metadata 分3段:

(1) Started dump: 备份开始时间.

(2) master 的log-file 和 log-pos 信息 (必有); slave 的host、log-file 和 log-pos 信息 (备机是slave 才有)

(3) Finished dump: 备份结束时间

3. 返回错码, master_host 和 change master to 信息

'''

path += "/metadata"; sMetadata = ""; master_host = ""; er_code = 654321; er_info = "%s not exists !!!" % (path)

if (os.path.exists(path)):

if (isMaster != 1):

# 备机是 slave

num = 3

sFinds = "SLAVE STATUS"

else:

num = 2

sFinds = "MASTER STATUS"

f = open(path, 'r')

rows = f.readlines(); i = 100; lst =[]

for s in rows:

if (s.find(sFinds) > 0):

i = 1; continue

if (i <= num):

lst.append(s.split(':')[1].strip())

i += 1

if (isMaster == 1):

# 备机是 master

master_host = backup_host

log_file, log_pos = lst;

else:

# 备机是 slave

master_host, log_file, log_pos = lst;

er_code = 0

er_info = ""

sMetadata = "CHANGE MASTER TO MASTER_HOST='%s',MASTER_LOG_FILE='%s',MASTER_LOG_POS=%s,MASTER_USER='rep_user',MASTER_PASSWORD='meizu.com'" % (master_host, log_file, log_pos )

return (er_code, er_info, master_host, sMetadata)

def execute(cmd):

'''''

1.执行 shell 命令

2.返回执行信息 (returncode = 0 则执行成功, std_err 为报错的错误信息)

'''

try:

returncode, std_err = commands.getstatusoutput(cmd)

return (returncode, std_err)

except os.error, e:

# 异常返回 1001 错误

return (1001, e)

def call_proc(my_args):

# 备份信息写入数据库

try:

conn = mydb.connect(host = '127.0.0.1', user = 'test', passwd = 'zxc/213?', db = 'meizu_item')

cur  = conn.cursor()

cur.callproc('sp_backup_i',[my_args[0], my_args[1], my_args[2], my_args[3], my_args[4], my_args[5], my_args[6], my_args[7], my_args[8], my_args[9], my_args[10], my_args[11], my_args[12]])

conn.commit()

except mydb.Error, e:

pass

# print "Mysql Error %d: %s" % (e.args[0], e.args[1])

finally:

cur.close(); conn.close()

if __name__ == '__main__':

main()

#!/usr/bin/phthon

import os

import time

import commands

import shutil

import threading

from os.path import join, getsize

import MySQLdb as mydb

# 备份目录

baseDir = "/data2/backup/backup_data/"

# ns 或 wx;  备份后是否要压缩(mydumper 自带压缩功能)，要压缩 True，否则 False.

idc = 'ns'; isZip = True

# 备份失败是否重试 ，True 则重试,不重试 设 False, retry_sleep 多久后才开始重试(秒)

is_errRetryBackup = True; retry_sleep = 300

# 备份日期

backup_date = time.strftime("%Y%m%d")

# 备份命令

cmd  = "/usr/local/bin/mydumper -h %s -u root -p password? -P %s %s -t 5 -o %s"

'''

功能描述：

1. mydumper 远程批量备份, 备份列表由配置文件提供

2. 可按要求对备份是否压缩(mydumper 自动压缩)

3. 备份失败允许再尝试备份一次

4. 备份信息写入数据库

'''

def main():

thread_pool = []

# 是否启用压缩

zip = '-c' if isZip == True else ''

# 从配置文件读取 ip, 名称, 端口, 并拼凑备份语句

#f = open('/data2/backup/cnf/other_list.cnf', 'r')

f = open('/data/other_list.cnf', 'r')

for lines in f.readlines():

if (not lines.startswith('#') and len(lines.strip()) > 0):

str = lines.split()

host, businessName, port, isMaster = str[0], str[1], str[2], str[3]

# 业务文件夹不存在则创建

dir = baseDir + '/' + businessName;

if (not os.path.exists(dir)):

os.makedirs(dir)

dir += "/%s%s" % (businessName, backup_date)

# 业务目录: dir , 备份目录: dir/name+备份日期

strcmd  = cmd % (host, port, zip, dir)

th = threading.Thread(target = mydumper, args =(strcmd, dir, businessName, host, port, is_errRetryBackup, int(isMaster)))

thread_pool.append(th)

if (thread_pool):

for t in thread_pool:

t.daemon = True

t.start()

for t in thread_pool:

t.join()

def mydumper(sCmd, backupDir, businessName, host, port, is_Retry, isMaster):

master_host = ""; backup_host = host; name = businessName; port = port; backup_type = 1; file = "";

start_time = ""; stop_time = ""; returncode = 0; file_size = 0; slave_statement = ""; std_err = "";

start_time = time.strftime("%Y%m%d%H%M%S")

# 清除可能遗留的备份

if (os.path.exists(backupDir)):

shutil.rmtree(backupDir)

# 执行备份

returncode, std_err = execute(sCmd)

stop_time = time.strftime("%Y%m%d%H%M%S")

if (returncode == 0):

# 备份 std_err 返回不为空也视为出错。

if (std_err.strip() != ""):

returncode = 123456

else:

# 获取 change master to 信息，再次校验备份是否有效

returncode, std_err, master_host, slave_statement = statement(backupDir, backup_host, isMaster)

if (returncode == 0):

file = backupDir

if (returncode != 0):

# 异常备份标记为: 日期 + _ERR

errDir = backupDir + "_ERR"

os.rename(backupDir, errDir)

file = errDir

# 获取备份大小

file_size = getDirsize(file)

if (len(std_err) > 255):

std_err = std_err[:250] + "..."

my_args  = [idc, master_host, backup_host, name, port, backup_type, file, start_time, stop_time, returncode, file_size, slave_statement, std_err]

# 写入数据库

call_proc(my_args)

# 备份失败是否需要重备? 重备允许一次.

if (is_Retry == True and returncode != 0):

time.sleep(retry_sleep)

oldfile = sCmd.split('-o')[1]

pos = oldfile.rfind("/") + 1

# 获取备份全路径, 备份文件标记为重备 字样

retry_file = oldfile[:pos] + "ReBackup-" + oldfile[pos:]

retryCmd = sCmd.replace(oldfile,  retry_file)

# 重备开始

mydumper(retryCmd, retry_file.strip(), name, host, port, False, isMaster)

def getDirsize(path):

# 获取备份文件夹大小

size = 0L

for root, dirs, files in os.walk(path):

size += sum([getsize(join(root, name)) for name in files])

return (size)

def statement(path, backup_host, isMaster):

'''

功能: 从 metadata 读取change master to 信息

1. 备份过程: 会先生成: metadata.partial, 完成后metadata.partial会重名为: metadata 并写入备份完成时间

2. metadata 分3段:

(1) Started dump: 备份开始时间.

(2) master 的log-file 和 log-pos 信息 (必有); slave 的host、log-file 和 log-pos 信息 (备机是slave 才有)

(3) Finished dump: 备份结束时间

3. 返回错码, master_host 和 change master to 信息

'''

path += "/metadata"; sMetadata = ""; master_host = ""; er_code = 654321; er_info = "%s not exists !!!" % (path)

if (os.path.exists(path)):

if (isMaster != 1):

# 备机是 slave

num = 3

sFinds = "SLAVE STATUS"

else:

num = 2

sFinds = "MASTER STATUS"

f = open(path, 'r')

rows = f.readlines(); i = 100; lst =[]

for s in rows:

if (s.find(sFinds) > 0):

i = 1; continue

if (i <= num):

lst.append(s.split(':')[1].strip())

i += 1

if (isMaster == 1):

# 备机是 master

master_host = backup_host

log_file, log_pos = lst;

else:

# 备机是 slave

master_host, log_file, log_pos = lst;

er_code = 0

er_info = ""

sMetadata = "CHANGE MASTER TO MASTER_HOST='%s',MASTER_LOG_FILE='%s',MASTER_LOG_POS=%s,MASTER_USER='rep_user',MASTER_PASSWORD='meizu.com'" % (master_host, log_file, log_pos )

return (er_code, er_info, master_host, sMetadata)

def execute(cmd):

'''

1.执行 shell 命令

2.返回执行信息 (returncode = 0 则执行成功, std_err 为报错的错误信息)

'''

try:

returncode, std_err = commands.getstatusoutput(cmd)

return (returncode, std_err)

except os.error, e:

# 异常返回 1001 错误

return (1001, e)

def call_proc(my_args):

# 备份信息写入数据库

try:

conn = mydb.connect(host = '127.0.0.1', user = 'test', passwd = 'zxc/213?', db = 'meizu_item')

cur  = conn.cursor()

cur.callproc('sp_backup_i',[my_args[0], my_args[1], my_args[2], my_args[3], my_args[4], my_args[5], my_args[6], my_args[7], my_args[8], my_args[9], my_args[10], my_args[11], my_args[12]])

conn.commit()

except mydb.Error, e:

pass

# print "Mysql Error %d: %s" % (e.args[0], e.args[1])

finally:

cur.close(); conn.close()

if __name__ == '__main__':

main()