android 联机，在线分析频繁打印日志问题

 

#coding=utf-8
import threading

import subprocess

import sys
from queue import Queue

import re

import time

__author__='ao.deng'
'''
该工具主要用于在线分析频繁打印日志问题

主要的衡量指标是一秒钟和一分钟相同tag打印的次数
'''
ADB_STDOUT_READ_TIMEOUT = 0.2
# Minimum number of seconds between displaying status updates.
MIN_TIME_BETWEEN_STATUS_UPDATES = 1

class FileReaderThread(threading.Thread):
    """Reads data from a file/pipe on a worker thread.

    Use the standard threading. Thread object API to start and interact with the
    thread (start(), join(), etc.).
    """

    def __init__(self, file_object, output_queue, text_file, chunk_size=-1):
        """Initializes a FileReaderThread.

        Args:
          file_object: The file or pipe to read from.
          output_queue: A Queue.Queue object that will receive the data
          text_file: If True, the file will be read one line at a time, and
              chunk_size will be ignored.  If False, line breaks are ignored and
              chunk_size must be set to a positive integer.
          chunk_size: When processing a non-text file (text_file = False),
              chunk_size is the amount of data to copy into the queue with each
              read operation.  For text files, this parameter is ignored.
        """
        threading.Thread.__init__(self)
        self._file_object = file_object
        self._output_queue = output_queue
        self._text_file = text_file
        self._chunk_size = chunk_size
        assert text_file or chunk_size > 0

    def run(self):
        """Overrides Thread's run() function.

        Returns when an EOF is encountered.
        """
        if self._text_file:
            # Read a text file one line at a time.
            for line in self._file_object:
                self._output_queue.put(line)
        else:
            # Read binary or text data until we get to EOF.
            while True:
                chunk = self._file_object.read(self._chunk_size)
                if not chunk:
                    break
                self._output_queue.put(chunk)

    def set_chunk_size(self, chunk_size):
        """Change the read chunk size.

        This function can only be called if the FileReaderThread object was
        created with an initial chunk_size > 0.
        Args:
          chunk_size: the new chunk size for this file.  Must be > 0.
        """
        # The chunk size can be changed asynchronously while a file is being read
        # in a worker thread.  However, type of file can not be changed after the
        # the FileReaderThread has been created.  These asserts verify that we are
        # only changing the chunk size, and not the type of file.
        assert not self._text_file
        assert chunk_size > 0
        self._chunk_size = chunk_size


class OnlineCheckLogLines():

    def __init__(self):
        self._adb = None
        self.tagMap = {}
        self.allLogLines=0
        self.logExit=[]
        self._log_args =["adb","shell","logcat","-b","all"]
    def start(self):
        self._adb = do_popen(self._log_args)

    def _collect_log_data(self):
        # Read the output from ADB in a worker thread.  This allows us to monitor
        # the progress of ADB and bail if ADB becomes unresponsive for any reason.

        # Limit the stdout_queue to 128 entries because we will initially be reading
        # one byte at a time.  When the queue fills up, the reader thread will
        # block until there is room in the queue.  Once we start downloading the
        # trace data, we will switch to reading data in larger chunks, and 128
        # entries should be plenty for that purpose.
        stdout_queue = Queue(maxsize=128)
        stderr_queue = Queue()


        # Use a chunk_size of 1 for stdout so we can display the output to
        # the user without waiting for a full line to be sent.
        stdout_thread = FileReaderThread(self._adb.stdout, stdout_queue,
                                         text_file=True)
        stderr_thread = FileReaderThread(self._adb.stderr, stderr_queue,
                                         text_file=True)
        stdout_thread.start()
        stderr_thread.start()

        # Holds the trace data returned by ADB.
        log_data = []
        last_status_update_time = time.time()
        while (stdout_thread.isAlive() or stderr_thread.isAlive() or
                   not stdout_queue.empty() or not stderr_queue.empty()):
            last_status_update_time = self.status_update(last_status_update_time)

            while not stderr_queue.empty():
                # Pass along errors from adb.
                line = stderr_queue.get()
                line = bytes.decode(line, errors="ignore")
                sys.stderr.write(line)




            # Read stdout from adb.  The loop exits if we don't get any data for
            # ADB_STDOUT_READ_TIMEOUT seconds.
            while True:
                try:
                    chunk = stdout_queue.get(True, ADB_STDOUT_READ_TIMEOUT)
                except Exception as e:
                    # Didn't get any data, so exit the loop to check that ADB is still
                    # alive and print anything sent to stderr.
                    break


                # Save, but don't print, the trace data.
                chunk = bytes.decode(chunk,errors="ignore")
                #sys.stdout.write(chunk)
                self.extractTag(chunk)
                last_status_update_time = self.status_update(last_status_update_time)
                #log_data.append(chunk)



        # The threads should already have stopped, so this is just for cleanup.
        stdout_thread.join()
        stderr_thread.join()

        self._adb.stdout.close()
        self._adb.stderr.close()

        # The adb process should be done since it's io pipes are closed.  Call
        # poll() to set the returncode.
        self._adb.poll()

        if self._adb.returncode != 0:
            print(">> {} {}".format(sys.stderr, ('The command "%s" returned error code %d.' %
                                  (' '.join(self._log_args), self._adb.returncode))))
            sys.exit(1)

        return log_data

    def extractTag(self,line):
        pattern = "\d{2}-\d{2} (\d{2}:\d{2}:\d{2}.\d+)\s+\d+\s+\d+\s+\w (.*?)(\s+)?:"
        timeAndTagList = re.findall(pattern, line)
        if len(timeAndTagList)==0:
            return
        #print(timeAndTagList)
        time = timeAndTagList[0][0]
        tag = timeAndTagList[0][1]
        self.allLogLines+=len(timeAndTagList)
        if self.tagMap.keys().__contains__(tag):
            self.tagMap[tag].append(time)
        else:
            timeList = [time]
            self.tagMap[tag] = timeList

    def countOneSecondOfData(self,timeList):
        secondMap = {}
        for time in timeList:
            second = time.split(".")[0]
            if secondMap.keys().__contains__(second):
                secondMap[second] += 1
            else:
                secondMap[second] = 1

        return secondMap

    def status_update(self, last_update_time):
        current_time = time.time()
        if (current_time - last_update_time) >= MIN_TIME_BETWEEN_STATUS_UPDATES:
            # Gathering a trace may take a while.  Keep printing something so users
            # don't think the script has hung.
            for tag, timeList in self.tagMap.items():
                secondMap = self.countOneSecondOfData(timeList)
                for second, count in secondMap.items():
                    if count > 100:

                        line_str ="tag={}, lines={}, all log %={:.2f}, time={}, count={},".format(tag,len(timeList),len(timeList)/self.allLogLines, second, count)
                        line_str_key = "tag={}, time={}, count={},".format(tag,second, count)
                        if not self.logExit.__contains__(line_str_key):
                            print(line_str)

                            self.logExit.append(line_str_key)
            # sys.stdout.write("\r{0}".format(11))
            # sys.stdout.flush()
            return current_time

        return last_update_time

def do_popen(args):
    print(args)
    try:
        adb = subprocess.Popen(args, stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE)
    except OSError as error:
        print (">> {} {}".format(sys.stderr, (
            'The command "%s" failed with the following error:' %
            ' '.join(args))))
        print (">> {} {}".format(sys.stderr, '    ', error))
        sys.exit(1)

    return adb



OnlineCheckLogLinesWrapper  = OnlineCheckLogLines()
OnlineCheckLogLinesWrapper.start()
OnlineCheckLogLinesWrapper._collect_log_data()