python自动化部署平台_Python自动化部署

# -*- coding: utf-8 -*-

#!/bin/env python

'''

#Auth: karl

#Function: released version

#Date:2017/6/27

#Version:V1.0

'''

import  sys,re,os,time,datetime

import  paramiko

import logging

import socket

import ConfigParser

import traceback

from progressbar import *

import Auto_Mysql_release

import platform

import smtplib

import email.mime.multipart

import email.mime.text

import json

import os

import struct

import requests

receivers = "XXX@.com"

receiver = "XXX@com"

#reg 为0时正常执行命令,为1时开始检查服务启动是否正常,为3时不用再备份原文件

def ssh2(host, port, username, password, cmds,reg=0):

#  链接远程服务器并执行命令p

try:

paramiko.util.log_to_file('./../log/exec_cmd_' + time_str + '.log')

ssh = paramiko.SSHClient()

ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())

ssh.connect(host, int(port), username, password, timeout=5)

# 执行命令文件中的命令

if reg == 3:

stdin, stdout, stderr = ssh.exec_command('ls /tmp/backup/old/ADMIN/integration.properties')

if stdout.readline() != '':

pass

else:

for cmd in cmds:

logging.info('running: ' + cmd)

print 'running: ' + cmd

time.sleep(2)

stdin, stdout, stderr = ssh.exec_command(cmd)

for out_msg in stdout.readlines():

print out_msg

else:

for cmd in cmds:

print "=======>>",cmd

logging.info('running: ' + cmd)

print 'running: ' + cmd

time.sleep(5)

stdin, stdout, stderr = ssh.exec_command(cmd)

if reg == 1:

out_msg=str("".join(stdout.readlines()))

print "000000------>>",out_msg

if "[main] INFO  org.eclipse.jetty.server.Server.doStart(Server.java:379)" in out_msg and "error" not in out_msg and "Failed startup" not in out_msg:

print  "THIS IS ok..."

print "------->> %s"%(host)

else:

#回滚到当前版本

#                        repair_cmd_file = conf.get(section, 'repair_cmd_file')

#                        exec_file_cmd(conf, section, repair_cmd_file)

copy_cmd_file(conf, section, host,1)

for ip_f in host_ip[1:]:

num_f = host_ip.index(ip_f)

num_f += 1

process_cmd_file(conf, section,ip_f, num_f)

for ip_s in host_ip:

num_s = host_ip.index(ip_s)

num_s += 1

print "+++++++ %s" % (ip_s)

start_server(conf, section, ip_s, num_s)

exit(1)

else:

print "++++++++++++++",reg

for out_msg in stdout.readlines():

print "----", out_msg

for err_msg in stderr.readlines():

print err_msg

exit(1)

ssh.close()

print 'command execute successful!'

logging.info('command execute successful!')

except Exception, e:

print '%s\tError\n' % (host)

print traceback.format_exc()

__err_exit_show_msg(str(e))

def upload(conf, section):

'''

上传文件到远程服务器

'''

host = conf.get(section, 'host')

port = conf.get(section, 'port')

username = conf.get(section, 'username')

password = conf.get(section, 'password')

local_file = conf.get(section, 'local_file')

remote_file = conf.get(section, 'remote_file')

try:

paramiko.util.log_to_file('../log/upload_' + time_str + '.log')

logging.info('paramiko log created')

t = paramiko.Transport((host, int(port)))

t.connect(username=username, password=password)

logging.info('connected host <' + host + '> successful')

logging.info('Upload file SUCCESSFUL %s ' % datetime.datetime.now())

sftp = paramiko.SFTPClient.from_transport(t)

logging.info('Upload file SUCCESSFUL %s ' % datetime.datetime.now())

print 'Beginning to upload file to %s  %s ' % (host, datetime.datetime.now())

# 定义上传进度条样式

widgets = ['File: ', Percentage(), ' ',

Bar(marker='#', left='[', right=']'),

' ', ETA(), ' ', FileTransferSpeed()]

file_size = os.path.getsize(local_file)

pbar = ProgressBar(widgets=widgets, maxval=file_size)

# 开始进度条

pbar.start()

# 使用匿名方法接收上传返回值，并且显示进度条

progress_bar = lambda transferred, toBeTransferred: pbar.update(transferred)

sftp.put(local_file,remote_file, callback=progress_bar)

pbar.finish()

logging.info('Upload file SUCCESSFUL %s ' % datetime.datetime.now())

print 'Upload file SUCCESSFUL %s ' % datetime.datetime.now()

t.close()

logging.info('sftp closed!')

cmd="tar -xvf %s  -C /home/appdeploy/version/ >/dev/null " % local_file

if "Linux" == platform.system():

os.system(cmd)

report_cmd_file = __checke_conf_key_value_empty(conf, section, 'test_report')

except Exception, e:

logging.error('host: <' + host + '> connect error!')

print host, 'connect error!'

print traceback.format_exc()

__err_exit_show_msg(str(e))

def email_send(section,version):

evn=section

V=version

msg = email.mime.multipart.MIMEMultipart()

msg['Subject'] = '版本发布通知邮件'

msg['From'] = '18682147058@163.com'

msg['To'] = ','.join(receivers)

content = '''

你好，各位同事：

本次%s版本(%s)发布:   发布成功,祝贺！！！

'''%(evn,V)

txt = email.mime.text.MIMEText(content)

msg.attach(txt)

smtp = smtplib.SMTP()

smtp.connect('smtp.163.com', '25')

smtp.login('18682147058@163.com', 'passwd')

smtp.sendmail(msg['From'], receivers, msg.as_string())

smtp.quit()

print('邮件发送成功email has send out !')

def email_linux(receivers, subject=None, bodyhtml=None,attachments=None):

'''

对接统一通知平台，发邮件样例

receivers 收件人邮箱

subject 主题

bodyhtml 邮件内容

attachment即附件路径默认为空，如有附件传入文件路径'''

file_name = attachments.split("/")[-1]

lis = ''

time_str =str(time.strftime('%Y-%m-%d',time.localtime(time.time())))

# 把附件内容转换为字符列表

if attachments != None:

file_name = os.path.basename(attachments)

file = open(attachments, 'rb')

_content = file.read()

lis = struct.unpack('%db' % len(_content),_content)

# 对应渠道模板中`message`中参数

templateJson = {

'version'  : bodyhtml,

'time'     :time_str,

'question' :"详情请审阅附件，谢谢！"

}

data = {

'userId': receivers, #收件人邮箱,支持多收件人,分号隔离;如下面的抄送人格式

'ccId': receiver, #抄送人,如果没有可以屏蔽该语句

'templateCode': 'version_release_code', #业务模板code

'templateParam': templateJson, #如果对应渠道模板中没有类{{}}格式的参数，可以屏蔽该语句

'subject': subject, #邮件主题，默认是渠道模板名称

'attachmentName' : file_name, #邮件附件名称， 如果没有附件可以屏蔽该语句

'attachmentArray': lis, #邮件附件内容,如果没有附件可以屏蔽该语句

'msgType': 'txt', #消息类型，目前只支持txt

'accessId': '52HX1CYE', #通知平台接入Id

'accessToken': 'ebf1dd3140cf4f0abd79872d7d237c3d' #通知平台接入Token

}

json_str = json.dumps(data)

url = "http://public-int-gw.int.sfdc.com.cn:1080/unp/notice/single"

headers = {'content-type': 'application/json; charset=UTF-8'}

try:

response = requests.post(url, data=json_str, headers=headers)

print response.text

print "-------------"

result_json = json.loads(response.text)

print(result_json) #打印返回内容

except Exception as e:

print('调用统一通知平台接口失败:',str(e))

def copy_cmd_file(conf, section, ip,reg):

if reg == 0:

filep="/home/appdeploy/version/Version_3.0"

else:

filep="/tmp/backup/old"

host=ip

port = conf.get(section, 'port')

username = conf.get(section, 'username')

password = conf.get(section, 'password')

remote_file = conf.get(section, 'remote_file')

print "----------------------------copy files--- %s--------------------" % (host)

cmd_c = [

'\cp -vr {files}/ADMIN/*.war  {path}_ADMIN_01/deploy/webapps/'.format(files=filep, path=Filepath),

'\cp -vr {files}/ADMIN/integration.properties {path}_ADMIN_01/deploy/resources/'.format(files=filep,path=Filepath),

'\cp -vr {files}/TRAPP/*.war  {path}_TRAPP_01/deploy/webapps/'.format(files=filep, path=Filepath),

'\cp -vr {files}/TRAPP/integration.properties  {path}_TRAPP_01/deploy/resources/'.format(files=filep,path=Filepath),

'\cp -vr {files}/TRTS/*.war  {path}_TRTS_01/deploy/webapps/'.format(files=filep, path=Filepath),

'\cp -vr {files}/TRTS/integration.properties  {path}_TRTS_01/deploy/resources/'.format(files=filep, path=Filepath),

]

ssh2(host, port, username, password, cmd_c)

#针对脚本进行参数化设置

def process_cmd_file(conf, section,ip_f, num_f):

host = ip_f

port = conf.get(section, 'port')

username = conf.get(section, 'username')

password = conf.get(section, 'password')

remote_file = conf.get(section, 'remote_file')

print "----------------------------copy remote files--- %s--------------------" % (host)

print "-------->>>>>",host_ip[0],port,username,password,remote_file

cmd=[

'scp {path}_ADMIN_01/deploy/webapps/*.war  {user}@{ip}:{path}_ADMIN_0{num}/deploy/webapps/'.format(path=Filepath,user=username,ip=host,num=num_f),

#       'scp {path}_ADMIN_01/deploy/resources/integration.properties  {user}@{ip}:{path}_ADMIN_0{num}/deploy/resources/integration.properties'.format(path=Filepath,user=username,ip=host,num=num_f),

'scp {path}_TRTS_01/deploy/webapps/*.war  {user}@{ip}:{path}_TRTS_0{num}/deploy/webapps/'.format(path=Filepath,user=username,ip=host,num=num_f),

#       'scp {path}_TRTS_01/deploy/resources/integration.properties  {user}@{ip}:{path}_TRTS_0{num}/deploy/resources/integration.properties'.format(path=Filepath,user=username,ip=host,num=num_f),

'scp {path}_TRAPP_01/deploy/webapps/*.war  {user}@{ip}:{path}_TRAPP_0{num}/deploy/webapps/'.format(path=Filepath,user=username,ip=host,num=num_f),

#        'scp {path}_TRAPP_01/deploy/resources/integration.properties  {user}@{ip}:{path}_TRAPP_0{num}/deploy/resources/integration.properties'.format(path=Filepath,user=username,ip=host,num=num_f)

]

ssh2(host_ip[0], port, username, password, cmd)

def exec_file_cmd(conf, section, cmd_file):

'''

执行文件中的命令

'''

host = conf.get(section, 'host')

port = conf.get(section, 'port')

username = conf.get(section, 'username')

password = conf.get(section, 'password')

cmds = __get_cmds(cmd_file)

ssh2(host, port, username, password, cmds)

def backup_ori(conf, section):

'''

备份远程原文件

'''

host = conf.get(section, 'host')

port = conf.get(section, 'port')

username = conf.get(section, 'username')

password = conf.get(section, 'password')

remote_file = conf.get(section, 'remote_file')

remote_ori_backup_dir = conf.get(section, 'remote_ori_backup_dir')

# 获得备份后缀

suffix_time =  time.strftime('%Y-%m-%d',time.localtime(time.time()))

backup_ori_cmd = [

'mkdir -p {dir}/ADMIN {dir}/TRTS {dir}/TRAPP'.format(dir=remote_ori_backup_dir),

'cp -vr {path}_TRAPP_01/deploy/webapps/*.war  {dir}/TRAPP'.format(path=Filepath,dir=remote_ori_backup_dir),

'cp -vr {path}_ADMIN_01/deploy/webapps/*.war  {dir}/ADMIN'.format(path=Filepath,dir=remote_ori_backup_dir),

'cp -vr {path}_TRTS_01/deploy/webapps/*.war  {dir}/TRTS'.format(path=Filepath,dir=remote_ori_backup_dir),

'cp -vr {path}_TRAPP_01/deploy/resources/integration.properties  {dir}/TRAPP'.format(path=Filepath,dir=remote_ori_backup_dir),

'cp -vr {path}_ADMIN_01/deploy/resources/integration.properties  {dir}/ADMIN'.format(path=Filepath,dir=remote_ori_backup_dir),

'cp -vr {path}_TRTS_01/deploy/resources/integration.properties  {dir}/TRTS'.format(path=Filepath,dir=remote_ori_backup_dir)

]

ssh2(host, port, username, password, backup_ori_cmd,3)

def backup_new(conf, section):

'''

备份远程新上传的文件

'''

host = conf.get(section, 'host')

port = conf.get(section, 'port')

username = conf.get(section, 'username')

password = conf.get(section, 'password')

remote_file = conf.get(section, 'remote_file')

remote_backup_dir = conf.get(section, 'remote_backup_dir')

# 获得备份后缀

suffix_time = time.strftime('%Y-%m-%d_%H-%M-%S',time.localtime(time.time()))

backup_new_cmd = [

'mkdir -p {dir}'.format(dir=remote_backup_dir),

'cp -vr {new_file} {dir}/{new_bak_file}_{time}'.format(new_file=remote_file,

dir=remote_backup_dir,

new_bak_file=os.path.basename(remote_file),

time=str(suffix_time))

]

ssh2(host, port, username, password, backup_new_cmd)

def select_section(conf_file_name):

'''

选择指定读取的配置文件项

例如：*.conf配置文件中有多个配置项 a 和 b:

[a]

xxxxx

[b]

yyyyy

'''

# 检测指定的配置文件是否存在

__check_file_exists(conf_file_name)

# 读取配置文件

conf = ConfigParser.ConfigParser()

conf.read(conf_file_name)

sections = conf.sections()

# 选择配置文件选项界面

print 'please choose confit item:'

for index, value in enumerate(sections):

print  '  ', index, ':', value

while True:

sec_index = raw_input('please choose one item default [0]:')

if not sec_index.isdigit() or int(sec_index) >= len(sections):

print 'choose invalid!'

continue

return conf, sections[int(sec_index)]

return conf, sections[0]

def check_config(conf, section):

'''

检测配置文件的正确性

'''

logging.info('check config starting...')

print 'check config starting...'

# 检测配置文件中值是否都填写

host = __checke_conf_key_value_empty(conf, section, 'host')  # 检测配置文件中主机名

port = __checke_conf_key_value_empty(conf, section, 'port')  # 检测配置文件中端口

username = __checke_conf_key_value_empty(conf, section, 'username')  # 检测配置文件用户名

password = __checke_conf_key_value_empty(conf, section, 'password')  # 检测配置文件密码

local_file = __checke_conf_key_value_empty(conf, section, 'local_file')  # 检测配置文件本地需要上传文件

remote_file = __checke_conf_key_value_empty(conf, section, 'remote_file')  # 检测配置文件上传到远程的文件

remote_backup_dir = __checke_conf_key_value_empty(conf, section, 'remote_backup_dir')  # 检测配置文件远程备份目录

remote_ori_backup_dir = __checke_conf_key_value_empty(conf, section, 'remote_ori_backup_dir')  # 检测配置文件远程临时备份目录

start_cmd_file = __checke_conf_key_value_empty(conf, section, 'start_cmd_file')  # 检测配置文件启动服务文件

report_cmd_file = __checke_conf_key_value_empty(conf, section, 'test_report')  # 检测配置文件停止服务文件

# 检测配置文件中的网络是否可用

__check_network_ping(host)

# 检测ssh链接是否成功

__check_ssh(host, int(port), username, password)

# 检测本地需要上传的文件是否存在

__check_file_exists(local_file)

# 检测命令文件是否存在

__check_file_exists(start_cmd_file)

print 'check config successful!!'

logging.info('check config successful!!')

def __valid_ip(address):

'''

检测IP是否合法IP

'''

try:

socket.inet_aton(address)

return True

except:

print traceback.format_exc()

return False

def __check_file_exists(conf_file_name):

'''

检测指定的配置文件是否存在

'''

if not os.path.exists(conf_file_name):

logging.error('can not find config file: ' + conf_file_name)

__err_exit_show_msg('can not find config file: ' + conf_file_name)

return conf_file_name

def __checke_conf_key_value_empty(conf, section, key):

'''

检测配置文件的key是否存在

'''

try:

value = conf.get(section, key)

# 检测配置文件中的值是否为空

if value:

return value

else:

msg = '''

ERROR  The key:{key} value is empty in conf file

'''.format(key=key)

__err_exit_show_msg(msg)

except ConfigParser.NoOptionError:

print traceback.format_exc()

msg = '''

ERROR  cannot find key:{key} in conf file

'''.format(key=key)

__err_exit_show_msg(msg)

def __check_network_ping(host):

if not __valid_ip(host):

__err_exit_show_msg('host: ' + host + ' invalid')

if "Linux" == platform.system():

if 0 <> os.system('ping -c 3 ' + host):

__err_exit_show_msg('host: ' + host + ' cannot ping...')

else:

if 0 <> os.system('ping -n 1 -w 5 ' + host):

__err_exit_show_msg('host: ' + host + ' cannot ping...')

def __check_ssh(host, port, username, password):

try:

ssh = paramiko.SSHClient()

ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())

ssh.connect(host, port, username, password, timeout=5)

ssh.close()

except Exception as e:

print traceback.format_exc()

msg = '''

SSH connect failure.

please check your host/port/username/password

host    :  {host}

port    :  {port}

username:  {username}

password:  {password}

'''.format(host=host, port=port,

username=username,

password=password)

__err_exit_show_msg(msg)

def __get_cmds(cmd_file):

'''

文件中获取执行命令

'''

with open(cmd_file, 'r') as cmd_f:

pattern = re.compile('(^\s*#|^\s*$)')

func = lambda x: x if not re.match(pattern, x) else None

cmds = [cmd for cmd in cmd_f]

return filter(func, cmds)

def check_server(conf, section,ip,num):

host = ip

port = conf.get(section, 'port')

username = conf.get(section, 'username')

password = conf.get(section, 'password')

suffix_time = time.strftime('%Y%m%d', time.localtime(time.time()))

print "----------------------------check result--- %s--------------------"%(ip)

checkserver_cmd=[

'cat {path}_ADMIN_0{num}/logs/novatar_{time}.0.log |grep  "error\|org.eclipse.jetty.server.Server.doStart(Server.java:379)\|Failed startup"'.format(path=Filepath,time=suffix_time,num=num),

'cat {path}_TRAPP_0{num}/logs/novatar_{time}.0.log |grep  "error\|org.eclipse.jetty.server.Server.doStart(Server.java:379)\|Failed startup"'.format(path=Filepath,time=suffix_time,num=num),

'cat {path}_TRTS_0{num}/logs/novatar_{time}.0.log |grep  "error\|org.eclipse.jetty.server.Server.doStart(Server.java:379)\|Failed startup"'.format(path=Filepath,time=suffix_time,num=num)

]

ssh2(host, port, username, password, checkserver_cmd,1)

def start_server(conf, section, ip, num):

host = ip

port = conf.get(section, 'port')

username = conf.get(section, 'username')

password = conf.get(section, 'password')

print "----------------------------restart server --- %s--------------------" % (ip)

checkserver_cmd = [

'sh {path}_ADMIN_0{num}_run.sh stop; sh {path}_ADMIN_0{num}_run.sh start'.format(path=Restartfile,num=num),

'sh {path}_TRAPP_0{num}_run.sh stop; sh {path}_TRAPP_0{num}_run.sh start'.format(path=Restartfile,num=num),

'sh {path}_TRTS_0{num}_run.sh stop; sh {path}_TRTS_0{num}_run.sh start'.format(path=Restartfile,num=num)

]

ssh2(host, port, username, password, checkserver_cmd)

def __err_exit_show_msg(msg):

'''

发生错误的时候显示相关错误信息并且退出程序

'''

print 'ERROR:  ' + msg

logging.error('ERROR:  ' + msg)

os.system('pause')

sys.exit()

if __name__ == '__main__':

try:

start = time.clock()

# 设置日志文件

time_str = time.strftime('%Y-%m-%d',time.localtime(time.time()))

log_file = '../log/upload_distribution_' + str(time_str) + '.log'

logging.basicConfig(level=logging.INFO,

format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',

filename=log_file,

filemode='w',

datefmt='%Y-%m-%d %X')

# 定义配置文件路径

conf_file_name = '../conf/release.conf'

# 选择配置文件section

conf, section = select_section(conf_file_name)

if section == "STG":

Filepath="/app/jetty/server/SCS_ATP_CNSZ99_JETTY_APP"

Restartfile="/app/jetty/logs/SCS_ATP_CNSZ99_JETTY_APP"

host_ip=[]

elif section == "DEVTEST":

Filepath="/app/jetty/server/SCS_ATP_CORE_CNSZ22_JETTY_APP"

Restartfile="/app/jetty/logs/SCS_ATP_CORE_CNSZ22_JETTY_APP"

host_ip=[]

else :

Filepath="/app/jetty/server/SCS_ATP_CORE_CNSZ17_JETTY_APP"

Restartfile="/app/jetty/logs/SCS_ATP_CNSZ17_JETTY_APP"

host_ip=[]

# 检测配置文件正确性

check_config(conf, section)

print('\033[1;35m You can view the configuration file in the version number:\"/home/appdeploy/version/Version_\033[1;32m3.0\033[1;35m.tar" \033[0m!')

VERSION=raw_input("Please Enter Version: eg 3.0 \n")

# 备份原文件

backup_ori(conf, section)

# 上传文件

upload(conf, section)

# 备份新上传的文件

backup_new(conf, section)

# 解压文件

start_cmd_file = conf.get(section, 'start_cmd_file')

exec_file_cmd(conf, section, start_cmd_file)

# 上传新文件到应用目录

copy_cmd_file(conf, section, host_ip[0],0)

# 拷贝文件到远程服务器

if len(host_ip) > 1:

for ip_f in host_ip[1:]:

num_f = host_ip.index(ip_f)

num_f += 1

process_cmd_file(conf, section,ip_f, num_f)

# 执行拷贝excel 等文件,记录到数据库中

agre=["3306", "XXX", "root",host_ip[0],VERSION]

print "agre :",agre

stg=Auto_Mysql_release.Mysql_connect(*agre)

stg.File_get()

stg.File_deal()

#启动所有服务器的服务

for ip_s in host_ip:

num_s = host_ip.index(ip_s)

num_s += 1

start_server(conf, section,ip_s,num_s)

# 监听服务是否启动成功

time.sleep(90)

for ip in host_ip:

num=host_ip.index(ip)

num+=1

check_server(conf, section,ip,num)

# 实行完毕退出

cmd_f = ["rm -fr /tmp/backup/old/*"]

ssh2(host_ip[0],"22","mwopr","XXX",cmd_f)

os.system('exit')

end = time.clock()

print "version release SUCCESSFULL.......共耗时%f  s" %(end - start)

# 邮件群发到相关责任人

if section == "ONLINE":

testReport = conf.get(section, 'test_report')

text="%s版本发布通知邮件"% (VERSION)

email_linux(receivers,"版本发布通知邮件",VERSION,testReport)

elif section == "DEVTEST":

email_send(section,VERSION)

else:

pass

exit(0)

except Exception as e:

print traceback.format_exc()

__err_exit_show_msg(str(e))

具体流程如下：