tornado远程访问_django+tornado实现实时查看远程日志的方法

大致思路：

1.利用tornado提供的websocket功能与浏览器建立长连接，读取实时日志并输出到浏览器

2.写一个实时读取日志的脚本，利用saltstack远程执行，并把实时日志发往redis中。

3.tornado读取redis中的信息，发往浏览器。

此过程用到了redis的发布和订阅功能。

先看一下tornado中是如何处理的：

import os

import sys

import tornado.websocket

import tornado.web

import tornado.ioloop

import redis

import salt.client

from tornado import gen

from tornado.escape import to_unicode

from logs.utility import get_last_lines

from logs import settings

class SubWebSocket(tornado.websocket.WebSocketHandler):

"""

此handler处理远程日志查看

"""

def open(self, *args, **kwargs):

print("opened")

@gen.coroutine

def on_message(self, message):

# 主机名，要查看的日志路径，运行脚本的命令这些信息从浏览器传过来

hostname, log_path, cmd = message.split("||")

local = salt.client.LocalClient()

r = redis.StrictRedis(host=settings.REDIS_HOST, port=settings.REDIS_PORT,

password=settings.REDIS_PASSWD, db=5)

# 订阅频道，服务器和日志路径确定一个频道

key = settings.LOG_KEY.format(server=hostname.strip(), log_path=log_path.strip())

channel = r.pubsub()

channel.subscribe(key)

# 异步方式执行命令，远程运行脚本

local.cmd_async(hostname, "cmd.run", [cmd])

try:

while True:

data = channel.get_message()

if not data:

# 如果读取不到消息，间隔一定时间，避免无谓的CPU消耗

yield gen.sleep(0.05)

continue

if data["type"] == "message":

line = format_line(data["data"])

self.write_message(line)

except tornado.websocket.WebSocketClosedError:

self.close()

def on_close(self):

global FLAG

FLAG = False

print("closed")

def format_line(line):

line = to_unicode(line)

if "INFO" in line:

color = "#46A3FF"

elif "WARN" in line:

color = "#FFFF37"

elif "ERROR" in line:

color = "red"

elif "CRITICAL" in line:

color = "red"

else:

color = "#FFFFFF"

return "{}".format(color, line)

class EchoWebSocket(tornado.websocket.WebSocketHandler):

def open(self):

print("WebSocket opened")

@gen.coroutine

def on_message(self, message):

log = message

print "log file: ", log

try:

with open(log, 'r') as f:

for line in get_last_lines(f):

line1 = format_line(line)

self.write_message(line1)

while True:

line = f.readline()

if not line:

yield gen.sleep(0.05)

continue

self.write_message(format_line(line.strip()))

except tornado.websocket.WebSocketClosedError as e:

print e

self.close()

# def check_origin(self, origin):

# print origin, self.request.headers.get("Host")

# # super(EchoWebSocket, self).check_origin()

# return True

def on_close(self):

print("WebSocket closed")

class Application(tornado.web.Application):

def __init__(self):

handlers = [

(r'/log/', MainHandler), # 提供浏览页面，页面中的JS与服务器建立连接

(r'/log/local', EchoWebSocket), # 处理本地日志实时查看，比较简单

(r'/log/remote', SubWebSocket), # 处理远程日志实时查看，稍微复杂

]

settings = {

"debug": True,

"template_path": os.path.join(os.path.dirname(__file__), "templates"),

"static_path": os.path.join(os.path.dirname(__file__), "static"),

}

super(Application, self).__init__(handlers, **settings)

class MainHandler(tornado.web.RequestHandler):

def get(self):

# 要查看的日志路径

log = self.get_argument("log", None)

# hostname实际上是saltstack中这台机器对应的minion id

hostname = self.get_argument("hostname", None)

# 本地日志还是远程日志

type = self.get_argument("type", "local")

# 运行读取实时日志的脚本，参数比较多，后面会有

cmd = self.get_argument("cmd", "")

context = {

"log": log,

"hostname": hostname,

"type": type,

"cmd": cmd,

}

self.render("index.html", **context)

配置文件中主要记录了redis服务器的地址等信息

# encoding: utf-8

LOG_KEY = "logs:{server}:{log_path}"

LOG_NAME = "catalina.out"

TAIL_LINE_NUM = 20

REDIS_HOST = "127.0.0.1"

REDIS_PORT = "6379"

REDIS_PASSWD = None

REDIS_EXPIRE = 300

try:

from local_settings import *

except ImportError:

pass

index.html的内容如下：

Stop

Start

var log_name = $("#id-log").val();

var type = $("#id-type").val();

var hostname = $("#id-hostname").val();

var cmd = $("#id-cmd").val();

// 初始化websocket对象

var ws = new WebSocket("ws://{{ request.host }}/log/" + type);

ws.onopen = function(){

if (type === "local"){

ws.send(log_name);

} else {

// 建立连接后把相关信息发往服务器，对应上面的SubWebSocket

ws.send(hostname + "||" + log_name + "||" + cmd);

}

};

var get_message = function(evt){

$("#id-content").append(evt.data + "\n");

document.getElementById("id-bottom").scrollIntoView()

};

ws.onmessage = get_message;

// 两个按钮控制日志的输出，如果看到需要的日志信息，可以暂停日志的输出，

// 之后可以继续启动日志的输出

$("#id-stop").click(function(){

ws.onmessage = function(){};

})

$("#id-start").click(function(){

ws.onmessage = get_message;

})

这个tornado仅仅是提供了实时日志的服务，实际项目使用的是django，django中要做的其实很简单，提供log_name,hostname,type,cmd等四个参数。

下面看一个实例：

class LogView(KylinView):

# 实时读取日志的脚本，事先使用saltstack批量传到各台服务器上

client_path = "/tmp/logtail.py"

def get(self, request):

minion_id = request.GET.get("minion_id")

context = {

"minion_id": minion_id,

"tail_log_url": settings.TAIL_LOG_URL,

}

return render(request, "cmdb/log_view.html", context)

def post(self, request):

minion_id = request.POST.get("minion_id")

log_path = request.POST.get("log_path")

if not log_path:

return JsonResponse({"success": False, "message": "请填写日志路径"})

try:

# 制定一开始读取的行数

line_count = request.POST.get("line_count")

except (TypeError, ValueError):

return JsonResponse({"success": False, "message": "请输入正确的行数"})

local = salt.client.LocalClient()

# 确保saltstack能连通并且日志文件存在

ret = local.cmd(minion_id, "file.file_exists", [log_path])

if minion_id not in ret:

return JsonResponse({"success": False, "message": "服务器无法连通"})

if not ret[minion_id]:

return JsonResponse({"success": False, "message": "日志文件不存在"})

# 组成命令的各个参数，redis信息需要和tornado配置文件中的redis信息一致

cmd = "{} {} {} {} {} {} {} {}".format(

settings.PYTHON_BIN, self.client_path, minion_id, log_path, line_count, settings.REDIS_HOST,

settings.REDIS_PORT, settings.REDIS_PASSWD)

# settings.TAIL_LOG_URL是tornado中MainHandler对应的url,把其它几个

# 参数组合成最终的URL，直接访问这个URL就可以在浏览器中实时读取日志了。

url = "{}?type=remote&log={}&hostname={}&cmd={}".format(

settings.TAIL_LOG_URL, log_path, minion_id, cmd)

# 这一步的操作确保同一个日志文件只有一个脚本在读取，避免日志信息重复，这一步

# 也很重要，必不可少

local.cmd(minion_id, "cmd.run",

["kill `ps aux|grep logtail.py|grep %s|grep -v grep|awk '{print $2}'`" % (log_path,)])

return JsonResponse({"success": True, "url": url})

下面来看看logtail.py的实现：

# encoding: utf-8

from __future__ import unicode_literals, division

import math

import time

import sys

import socket

import signal

import redis

FLAG = True

def get_last_lines(f, num=10):

"""读取文件的最后几行

"""

size = 1000

try:

f.seek(-size, 2)

except IOError: # 文件内容不足size

f.seek(0)

return f.readlines()[-num:]

data = f.read()

lines = data.splitlines()

n = len(lines)

while n < num:

size *= int(math.ceil(num / n))

try:

f.seek(-size, 2)

except IOError:

f.seek(0)

return f.readlines()[-num:]

data = f.read()

lines = data.splitlines()

n = len(lines)

return lines[-num:]

def process_line(r, channel, line):

r.publish(channel, line.strip())

def sig_handler(signum, frame):

global FLAG

FLAG = False

# 收到退出信号后，以比较优雅的方式终止脚本

signal.signal(signal.SIGTERM, sig_handler)

# 为了避免日志输出过多，浏览器承受不住，设置5分钟后脚本自动停止

signal.signal(signal.SIGALRM, sig_handler)

signal.alarm(300)

def get_hostname():

return socket.gethostname()

def force_str(s):

if isinstance(s, unicode):

s = s.encode("utf-8")

return s

def tail():

password = sys.argv[6]

if password == "None":

password = None

r = redis.StrictRedis(host=sys.argv[4], port=sys.argv[5], password=password, db=5)

log_path = sys.argv[2]

line_count = int(sys.argv[3])

# 往redis频道发送实时日志

channel = "logs:{hostname}:{log_path}".format(hostname=sys.argv[1], log_path=log_path)

with open(log_path, 'r') as f:

last_lines = get_last_lines(f, line_count)

for line in last_lines:

process_line(r, channel, force_str(line))

try:

while FLAG: # 通过信号控制这个变量，实现优雅退出循环

line = f.readline()

if not line:

time.sleep(0.05)

continue

process_line(r, channel, line)

except KeyboardInterrupt:

pass

print("Exiting...")

if __name__ == "__main__":

if len(sys.argv) < 6:

print "Usage: %s minion_id log_path host port redis_pass"

exit(1)

tail()

到此为止，整个实时读取远程日志的流程就讲完了。

以上这篇django+tornado实现实时查看远程日志的方法就是小编分享给大家的全部内容了，希望能给大家一个参考，也希望大家多多支持脚本之家。