优化的下载Android源码的Python脚本

在上一篇文章中，提到如何过滤掉一些git库、如何仅下载指定的git库，且给了部分脚本示例。

在这一篇文章中，我们给出完整的代码，如下：

#! /usr/bin/env python  
  
import xml.dom.minidom    
import os    
import sys
from subprocess import call    

def usage():
    print sys.argv[0], "[ignore|select|all]"
    
def is_ignored_git(cmd):
    ignored_list = ["device/lge/mako-kernel", 
                   "device/samsung/manta",
                   "platform/prebuilts/eclipse-build-deps"]
    for i in ignored_list:
        if cmd.find(i) == -1:
            continue
        else:
            print "Ignored: ", cmd
            return True

    return False

def is_selected_git(cmd):
    selected_list = ["platform/system/core"]
    for i in selected_list:
        if cmd.find(i) > 0: # could not equals 0
            return True

    return False

def is_download_git(cmd, mode):
    if mode=="ignore":
        return not is_ignored_git(cmd)

    if mode=="select":
        return is_selected_git(cmd)

    return True # we have validate the mode in main.

def download_android_source_code(mode):
    # downloaded source path    
    # if the dir does not exist, it will be created automatically as below.    
    rootdir = "/material/android/android_4.4.2_r2"    
    if not os.path.exists(rootdir):
        os.mkdir(rootdir)
    
    #git program path    
    git="git"    
    
    dom = xml.dom.minidom.parse("/material/android/manifest/default.xml")    
    root = dom.documentElement    
    
    prefix = git + " clone http://android.googlesource.com/"    
    suffix = ".git"    
    
    for node in root.getElementsByTagName("project"):    
        os.chdir(rootdir)
        
        d = node.getAttribute("path")    
        last = d.rfind("/")

        if last != -1:
            d = rootdir + "/" + d[:last]
            if not os.path.exists(d):
                os.makedirs(d) # recursively if requires.   
            os.chdir(d)
        
        cmd = prefix + node.getAttribute("name") + suffix
        print cmd
        if is_download_git(cmd, mode):
            print "Downloading: ", cmd
            call(cmd, shell=True)
        else:
            print "Skipped: ", cmd

def is_valid_mode(mode):
    modes = ["ignore", "select", "all"]
    return mode in modes

if __name__=="__main__":
    argc = len(sys.argv)
    if argc == 1:
        mode = "all"
    elif argc == 2:
        mode = sys.argv[1]
    else:
        usage()
        sys.exit(-1)
    
    if is_valid_mode(mode):
        download_android_source_code(mode)
        sys.exit(0)
    else:
        usage()
        sys.exit(-1)

注意在这个代码中，过滤或选择准则是以manifest/default.xml的name字段来比较的：

  <project path="device/lge/hammerhead-kernel" name="device/lge/hammerhead-kernel" groups="device,hammerhead" />
  <project path="device/lge/mako" name="device/lge/mako" groups="device,mako" />
  <project path="device/lge/mako-kernel" name="device/lge/mako-kernel" groups="device,mako" />

对于以上代码，可以看到对于过滤或选择的git库列表直接hard code到代码中，这会造成使用的不便、以及代码维护上的成本。即要遵循开闭设计原则，把这些变化的列表从不变的处理流程中分离出来。为此我们增加一个命令行参数，传入存放这些列表的文本文件。以下是优化后的代码：

#! /usr/bin/env python    
    
import xml.dom.minidom      
import os      
import sys  
from subprocess import call      
import global_vars  

  
def usage():  
    print sys.argv[0], "[ignore|select|all] items_file"  
    print "For 'all' option, the 'items_file' is only a placeholder."  
      
def is_ignored_git(cmd):  
    for i in global_vars.items_list:  
        if cmd.find(i) == -1:  
            continue  
        else:  
            print "Ignored: ", cmd  
            return True  
  
    return False  
  
def is_selected_git(cmd):
    for i in global_vars.items_list:
        if cmd.find(i) > 0: # could not equals 0  
            return True  
  
    return False  
  
def is_download_git(cmd, mode):  
    if mode=="ignore":  
        return not is_ignored_git(cmd)  
  
    if mode=="select":
        return is_selected_git(cmd)  
  
    return True # we have validate the mode in main.  
  
def download_android_source_code(mode):  
    # ATTENTION PLS: SET THE TWO DIRECTORIES BELOW FIRST.
    rootdir = "/home/flying-bird/android/android_4.4.2_r2"
    default_xml = "/home/flying-bird/android/manifest/default.xml"

    if not os.path.exists(rootdir):  
        os.mkdir(rootdir)  
      
    #git program path      
    git="git"
      
    dom = xml.dom.minidom.parse(default_xml)      
    root = dom.documentElement      
      
    prefix = git + " clone http://android.googlesource.com/"      
    suffix = ".git"      
      
    for node in root.getElementsByTagName("project"):      
        os.chdir(rootdir)  
          
        d = node.getAttribute("path")      
        last = d.rfind("/")  
  
        if last != -1:  
            d = rootdir + "/" + d[:last]  
            if not os.path.exists(d):  
                print "debug, makedirs: ", d
                os.makedirs(d) # recursively if requires.     
            os.chdir(d)  
          
        cmd = prefix + node.getAttribute("name") + suffix  
        #print cmd
  
        if is_download_git(cmd, mode):  
            print "Downloading: ", cmd  
            print "curdir: ", os.getcwd()
            call(cmd, shell=True)  
        else:  
            aaaa = 1#print "Skipped: ", cmd  #debug, not to print too many logs. 
  
def is_valid_mode(mode):  
    modes = ["ignore", "select", "all"]  
    return mode in modes  
  
# Each item is in a single line.  
# Comment syntax is not supported yet. So you will add this feature if you want.  
def initialize_item_list(file_name):
    f = open(file_name, "r")  
    items = f.read()  
    f.close()  
      
    items = items.split("\n")  
    # trim all the spaces  
    items_list = [i.strip() for i in items]
    items_set = set(items_list)
    items_list = list(items_set)
    items_list.remove('')

    global_vars.items_list = items_list
  
if __name__=="__main__":  
    argc = len(sys.argv)  
    if argc != 3:  
        usage()  
        sys.exit(-1)  
  
    mode = sys.argv[1]  
    file_name = sys.argv[2]  
      
    if not is_valid_mode(mode):  
        print "mode must be 'ignore', 'select', or 'all'."  
        usage()  
        sys.exit(-1)  
  
    if mode != "all":  
        if os.path.exists(file_name):  
            initialize_item_list(file_name)
        else:  
            print "file (%s) does not exist." % (file_name,)  
            usage()  
            sys.exit(-1)  
  
    download_android_source_code(mode)  
    sys.exit(0)  

另外一个定义全局变量的文件(global_vars.py)：

items_list = [] # item list to be ignored or selected.  

一个设置待下载列表的文本文件：

flying-bird@flying-bird:~/android$ cat to_be_download_items.txt
platform/frameworks/base
platform/bootable/recovery
flying-bird@flying-bird:~/android$ 

调用示例：

flying-bird@flying-bird:~/android$ ./download_android_source_code_new.py select ./to_be_download_items.txt
Downloading:  git clone http://android.googlesource.com/platform/bootable/recovery.git
curdir:  /home/flying-bird/android/android_4.4.2_r2/bootable
Cloning into 'recovery'...
remote: Counting objects: 104, done
remote: Finding sources: 100% (104/104)
Receiving objects:  82% (2128/2595), 3.00 MiB | 6 KiB/s     

如果做过性能优化方面的工作，会看到以上算法在效率方面的问题。比如对于default.xml中的每一个节点，都需要判断一遍mode是属于何种类型，且对于每种类型，都需要重复判断是否需要ignore或select。

从另一方面来讲，对于这个源码下载，性能瓶颈并不在于这个脚本本身的执行效率，而是远程网络的访问性能、以及写本地磁盘文件的性能。相比较起来，Python脚本实在没有优化的必要了。