pod及rds每日信息收集

1. 前言：

通过top命令收集pod的cpu及内存信息，每30秒收集一次，插入数据库进行数据存储。
通过阿里云接口来收集rds的相关信息

一、定义模型（表）

所有pod信息表
pod_all_info

pod实时数据表
pod_realtime_date_202012

pod数据计算邮件表
pod_day_info

所有rds信息表
rds_all_info

实时数据表
rds_*_info_202012

数据计算邮件表
rds_day_info

二、邮件及数据库连接池引用

send_email.py

#!/usr/bin/env python
# encoding=utf-8
from email.mime.image import MIMEImage
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
import smtplib


class SendEmail(object):
    # 构造函数：初始化基本信息
    def __init__(self, host, user, passwd):
        self._user = user
        self._account = user
        server = smtplib.SMTP_SSL(host, 465)
        #server.connect(host)
        server.login(self._account, passwd)
        self._server = server
        # 发送文件或html邮件

    def sendTxtMail(self, to_list, sub, content, subtype='html'):
        # 如果发送的是文本邮件，则_subtype设置为plain
        # 如果发送的是html邮件，则_subtype设置为html
        msg = MIMEText(content, _subtype=subtype, _charset='utf-8')
        msg['Subject'] = sub
        msg['From'] = self._user
        msg['To'] = ",".join(to_list)
        try:
            self._server.sendmail(self._user, to_list, msg.as_string())
            print("mail has been send successfully.")
            return True
        except Exception as e:
            print(str(e))
            return False

    # 发送带附件的文件或html邮件
    def sendAttachMail(self, to_list, sub, content ,file_ ,filename, subtype='html'):
        # 创建一个带附件的实例
        msg = MIMEMultipart()
        # 增加附件1
        att1 = MIMEText(open(r'%s'%file_, 'rb').read(), 'base64', 'utf-8')
        att1["Content-Type"] = 'application/octet-stream'
        # 这里的filename可以任意写，写什么名字，邮件中显示什么名字
        att1["Content-Disposition"] = 'attachment; filename="%s"'%filename
        msg.attach(att1)
        # 增加邮件内容
        msg.attach(MIMEText(content, _subtype=subtype, _charset='utf-8'))

        msg['Subject'] = sub
        msg['From'] = self._user
        msg['To'] = ";".join(to_list)

        try:
            self._server.sendmail(self._user, to_list, msg.as_string())
            print("mail has been send successfully.")
            return True
        except Exception as e:
            print(str(e))
            return False

    # 发送图片附件或html邮件
    def sendImageMail(self, to_list, sub, content, file_, subtype='html'):
        # 创建一个带附件的实例
        msg = MIMEMultipart()
        # 增加邮件内容
        msg.attach(MIMEText(content, _subtype=subtype, _charset='utf-8'))
        # 增加图片附件
        image = MIMEImage(open(r'%s'%file_, 'rb').read())
        # 附件列表中显示的文件名
        image.add_header('Content-Disposition', 'attachment;filename=p.jpg')
        msg.attach(image)
        msg['Subject'] = sub
        msg['From'] = self._user
        msg['To'] = ";".join(to_list)
        try:
            self._server.sendmail(self._user, to_list, msg.as_string())
            print("mail has been send successfully.")
            return True
        except Exception as e:
            print(str(e))
            return False

    # 析构函数：释放资源
    def __del__(self):
        self._server.close()


if __name__=="__main__":
    pass

数据库
mysql_thread.py

#!/usr/bin/env python
# encoding=utf-8
import pymysql
import threading
import re
import time,os
from queue import Queue
from DBUtils.PooledDB import PooledDB
import logging
from logging import handlers

class Logger(object):
    level_relations = {
        'debug': logging.DEBUG,
        'info': logging.INFO,
        'warning': logging.WARNING,
        'error': logging.ERROR,
        'crit': logging.CRITICAL
    }  # 日志级别关系映射

    def __init__(self, filename='/root/logs/server_all.log', level='info', when='D', backCount=30,
                 fmt='%(asctime)s -[line:%(lineno)d] - %(levelname)s: %(message)s'):
        (filepath, file) = os.path.split(filename)
        if not os.path.exists(filepath):  # 如果路径不存在
            os.makedirs(filepath)
        self.logger = logging.getLogger(filename)
        format_str = logging.Formatter(fmt)  # 设置日志格式
        self.logger.setLevel(self.level_relations.get(level))  # 设置日志级别
        sh = logging.StreamHandler()  # 往屏幕上输出
        sh.setFormatter(format_str)  # 设置屏幕上显示的格式
        th = handlers.TimedRotatingFileHandler(filename=filename, when=when, backupCount=backCount,encoding='utf-8')
        th.setFormatter(format_str)  # 设置文件里写入的格式
        self.logger.addHandler(sh)  # 把对象加到logger里
        self.logger.addHandler(th)

try:
    print_log = Logger()
except Exception as e:
    print("logging初始化异常：" + str(e))

class ThreadInsert(object):
    "多线程并发MySQL插入数据"
    def __init__(self,table_name,create_sql,argcount,lists):
        self.curdate = time.strftime("%Y-%m-%d %H:%M:%S")
        self.curdate_day = time.strftime("%Y%m%d")
        start_time = time.time()
        self.tablename = table_name
        self.create_sql = create_sql 
        self.arg_count = argcount
        self.lists = lists
        self.pool = self.mysql_connection()
        self.mysql_create()
        self.data = self.getData()
        self.task()
        print_log.logger.info("========= 数据插入,共耗时:{}'s =========".format(round(time.time() - start_time, 3)))

    def mysql_connection(self):
        maxconnections = 20  # 最大连接数
        pool = PooledDB(
            pymysql,
            maxconnections,
            host='127.0.0.1',
            user='ops',
            port=3306,
            passwd='',
            db='',
            use_unicode=True)
        return pool

    def getData(self):
        st = time.time()
        result = []
        line_lists = ['line["%s"]'%x for x in self.lists[0]]
        for line in self.lists:
            a = [eval(",".join(line_lists))]  
            #a = [(k["dname"],k["name"],k["replicas"],k["port"],k["request_cpu"],str(k["request_disk"]),k["request_mem"],k["limit_cpu"],str(k["limit_disk"]),k["limit_mem"],str(k["file_path"]))]
            result.append(a)
        print_log.logger.info("共获取{}组数据,=>> 耗时:{}'s".format(len(result),  round(time.time() - st, 3)))
        return result

    def mysql_create(self):
        st = time.time()
        con = self.pool.connection()
        cur = con.cursor()
        sql_del = "drop table if exists %s;"%self.tablename
        cur.execute(sql_del)
        cur.execute(self.create_sql)
        con.commit()
        cur.close()
        con.close()
        print("创建新表完成：.==>> 耗时:{}'s".format(round(time.time() - st, 3)))

    def mysql_insert(self, *args):
        con = self.pool.connection()
        cur = con.cursor()
        count_str = int(self.arg_count)*"%s ," 
        sql = "INSERT INTO `%s` VALUES (%s,now());"%(self.tablename,count_str[:-1])
        try:
            cur.executemany(sql, *args)
            con.commit()
        except Exception as e:
            con.rollback()  # 事务回滚
            print('SQL执行有误,原因:', e)
            print_log.logger.error('SQL执行有误,原因:', e)
        finally:
            cur.close()
            con.close()

    def task(self):
        q = Queue(maxsize=10)  # 设定最大队列数和线程数
        st = time.time()
        while self.data:
            content = self.data.pop()
            t = threading.Thread(target=self.mysql_insert, args=(content,))
            q.put(t)
            if (q.full() == True) or (len(self.data)) == 0:
                thread_list = []
                while q.empty() == False:
                    t = q.get()
                    thread_list.append(t)
                    t.start()
                for t in thread_list:
                    t.join()
        print_log.logger.info("数据插入完成.==>> 耗时:{}'s".format(round(time.time() - st, 3)))

mysql_thread_pod.py

#!/usr/bin/env python
# encoding=utf-8

import pymysql
import threading
import re
import time,os
from queue import Queue
from DBUtils.PooledDB import PooledDB
import logging
from logging import handlers

class Logger(object):
    level_relations = {
        'debug': logging.DEBUG,
        'info': logging.INFO,
        'warning': logging.WARNING,
        'error': logging.ERROR,
        'crit': logging.CRITICAL
    }  # 日志级别关系映射

    def __init__(self, filename='/root/logs/server_all.log', level='info', when='D', backCount=30,
                 fmt='%(asctime)s -[line:%(lineno)d] - %(levelname)s: %(message)s'):
        (filepath, file) = os.path.split(filename)
        if not os.path.exists(filepath):  # 如果路径不存在
            os.makedirs(filepath)
        self.logger = logging.getLogger(filename)
        format_str = logging.Formatter(fmt)  # 设置日志格式
        self.logger.setLevel(self.level_relations.get(level))  # 设置日志级别
        sh = logging.StreamHandler()  # 往屏幕上输出
        sh.setFormatter(format_str)  # 设置屏幕上显示的格式
        th = handlers.TimedRotatingFileHandler(filename=filename, when=when, backupCount=backCount,encoding='utf-8')
        th.setFormatter(format_str)  # 设置文件里写入的格式
        self.logger.addHandler(sh)  # 把对象加到logger里
        self.logger.addHandler(th)

try:
    print_log = Logger()
except Exception as e:
    print("logging初始化异常：" + str(e))

class ThreadInsert(object):
    "多线程并发MySQL插入数据"
    def __init__(self,table_name,create_sql,argcount,lists):
        self.curdate = time.strftime("%Y-%m-%d %H:%M:%S")
        self.curdate_day = time.strftime("%Y%m%d")
        start_time = time.time()
        self.tablename = table_name
        self.create_sql = create_sql 
        self.arg_count = argcount
        self.lists = lists
        self.pool = self.mysql_connection()
        self.mysql_create()
        self.data = self.getData()
        self.task()
        print_log.logger.info("========= 数据插入,共耗时:{}'s =========".format(round(time.time() - start_time, 3)))

    def mysql_connection(self):
        maxconnections = 20  # 最大连接数
        pool = PooledDB(
            pymysql,
            maxconnections,
            host='127.0.0.1',
            user='ops',
            port=3306,
            passwd='',
            db='',
            use_unicode=True)
        return pool

    def getData(self):
        st = time.time()
        result = []
        line_lists = ['line["%s"]'%x for x in self.lists[0]]
        for line in self.lists:
            a = [eval(",".join(line_lists))]  
            #a = [(k["dname"],k["name"],k["replicas"],k["port"],k["request_cpu"],str(k["request_disk"]),k["request_mem"],k["limit_cpu"],str(k["limit_disk"]),k["limit_mem"],str(k["file_path"]))]
            result.append(a)
        print_log.logger.info("共获取{}组数据,=>> 耗时:{}'s".format(len(result),  round(time.time() - st, 3)))
        return result

    def mysql_create(self):
        st = time.time()
        con = self.pool.connection()
        cur = con.cursor()
        #sql_del = "drop table if exists %s;"%self.tablename
        #cur.execute(sql_del)
        cur.execute(self.create_sql)
        con.commit()
        cur.close()
        con.close()
        print("创建新表完成：.==>> 耗时:{}'s".format(round(time.time() - st, 3)))

    def mysql_insert(self, *args):
        con = self.pool.connection()
        cur = con.cursor()
        count_str = int(self.arg_count)*"%s ," 
        sql = "INSERT INTO `%s` VALUES (%s,'%s');"%(self.tablename,count_str[:-1],self.curdate)
        try:
            cur.executemany(sql, *args)
            con.commit()
        except Exception as e:
            con.rollback()  # 事务回滚
            print('SQL执行有误,原因:', e)
            print_log.logger.error('SQL执行有误,原因:', e)
        finally:
            cur.close()
            con.close()

    def task(self):
        q = Queue(maxsize=10)  # 设定最大队列数和线程数
        st = time.time()
        while self.data:
            content = self.data.pop()
            t = threading.Thread(target=self.mysql_insert, args=(content,))
            q.put(t)
            if (q.full() == True) or (len(self.data)) == 0:
                thread_list = []
                while q.empty() == False:
                    t = q.get()
                    thread_list.append(t)
                    t.start()
                for t in thread_list:
                    t.join()
        print_log.logger.info("数据插入完成.==>> 耗时:{}'s".format(round(time.time() - st, 3)))

三、数据插入

脚本较多评论留联系方式我发吧

邮件展示类似