搭建opengork

Tools:
        apache-tomcat-9.0.8(github下载地址)
        java-8-openjdk-amd64(官网下载地址)
        opengrok-1.1-rc36(github下载地址)
        ctags(github下载地址)
 python UpdateProject.py -p "projectname" -s 'repo init -u xxxxxxx"

或者

 python UpdateProject.py -p "projectname" -s 'git clone xxxxx"

脚本内容如下:

#! /usr/bin/env python
# -*- coding: utf-8 -*-

import argparse

try:
    import configparser as configparser
except Exception:
    import ConfigParser as configparser
import logging
import os

import shutil

import time
from xml.dom.minidom import parse
import xml.dom.minidom
import subprocess

LOG_FORMAT = "%(asctime)s - %(levelname)s - %(message)s"
DATE_FORMAT = "%m/%d/%Y %H:%M:%S %p"

CONFIG_FILE = os.path.join("/home/liubin/code/Projects", "Config.ini")


OPEN_GROK_VERSION = 'opengrok-1.1-rc36'
APACHE_TOMCAT_VERSION = 'apache-tomcat-8.0.32.0'
JAVA_VERSION = 'java-8-openjdk-amd64'


def get_all_projects():
    cf = configparser.ConfigParser()
    cf.read(CONFIG_FILE)
    secs = cf.sections()
    return secs


def get_project_sync_cmd(project):
    cf = configparser.ConfigParser()
    cf.read(CONFIG_FILE)
    if cf.has_section(project):
        cf.has_option(project, 'sync_cmd')
        return cf.get(project, 'sync_cmd')
    return None


def set_project_sync_cmd(project, sync_cmd):
    if sync_cmd and (sync_cmd.startswith("git clone") or sync_cmd.startswith("repo init")):
        cf = configparser.ConfigParser()
        cf.read(CONFIG_FILE)
        if not cf.has_section(project):
            cf.add_section(project)
        cf.set(project, 'sync_cmd', sync_cmd)
        cf.write(open(CONFIG_FILE, 'w'))


def restart_apache_tomcat(work_path):
    cmd = "export JRE_HOME=%s; %s; %s" % (
        os.path.join("/usr/lib/jvm/java-8-openjdk-amd64", "jre"),
        os.path.join("/usr/share/tomcat8", "bin/shutdown.sh"),
        os.path.join("/usr/share/tomcat8", "bin/startup.sh")
    )
    exe_cmd(cmd)


def exe_cmd(cmd):
    logging.debug("Exe cmd: %s" % cmd)
    # result = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
    # for info in result.communicate():
    #     logging.debug(info)
    os.system(cmd)


def index_project(work_path, project_name, sync_cmd):
    if work_path is None or project_name is None or sync_cmd is None:
        logging.debug("Error Project or cmd %s %s" % (project_name, sync_cmd))
    else:
        # init code
        project_path = os.path.join("/home/liubin/code/Projects", project_name)
        if not os.path.exists(project_path):
            os.makedirs(project_path)
        if sync_cmd.startswith("git clone"):
            sync = "git pull"
        else:
            sync = "repo sync"
        cmd = "cd %s; %s " % (project_path, sync_cmd)
        exe_cmd(cmd)

        # sync code
        if sync.startswith("git pull"):
            cmd = os.listdir(project_path)[0]
            cmd = "cd %s; git pull" % (os.path.join(project_path, cmd))
        elif sync.startswith("repo sync"):
            cmd = "cd %s; repo sync" % project_path
        exe_cmd(cmd)

        # open grok indexer
        cmd = "%s \
               -j=%s \
               -a=%s -- \
               -c %s \
               -i %s \
               -H -P \
               -s %s \
               -d %s \
               -W %s" % (
            os.path.join("/home/liubin/tools/opengrok/opengrok-1.1-rc36", "bin/indexer.py"),
            os.path.join("/usr/lib/jvm/java-8-openjdk-amd64/jre", "bin/java"),  # -j java path
            os.path.join("/home/liubin/tools/opengrok/opengrok-1.1-rc36", "lib/opengrok.jar"),  # -a opengrok jar path
            "/usr/bin/ctags",  # -c ctags path
            "d:*.git -i d:*.repo -i f:*.jar",  # -i filter files and dirs
            project_path,  # -s Project source path
            os.path.join("/home/liubin/code/Projects/data", project_name),  # -d opengrok generate data
            os.path.join("/home/liubin/code/Projects/etc", project_name, "configuration.xml")
            # -W cfg for apache
        )
        exe_cmd(cmd)

        # create project in apache tomcat
        apache_app_path = os.path.join("/var/lib/tomcat8", "webapps",project_name + ".war")
        if not os.path.exists(apache_app_path):
            war_path = OPEN_GROK_VERSION + "/lib/source.war"
            shutil.copyfile(war_path, apache_app_path)
            restart_apache_tomcat(work_path)
            time.sleep(3)

        # set the project configuration.xml to apache tomcat
        apache_app_config = os.path.join(
            os.path.join("/var/lib/tomcat8", "webapps", project_name),
            "WEB-INF", "web.xml")

        wait_time = 0
        while not os.path.exists(apache_app_config):
            if wait_time > 10:
                logging.debug(
                    "Cannot find apache tomcat app=%s config_path=%s error!" % (project_name, apache_app_config))
                return
            wait_time = wait_time + 1
            time.sleep(3)

        tree = xml.dom.minidom.parse(apache_app_config)
        nodes = tree.documentElement.getElementsByTagName('context-param')
        for node in nodes:
            cfg_node = node.getElementsByTagName('param-value')
            if cfg_node:
                value = os.path.join("/home/liubin/code/Projects/etc", project_name, "configuration.xml")
                cfg_node[0].childNodes[0].data = value
                logging.debug("Update %s config %s success" % (project_name, value))
                with open(apache_app_config, 'w') as fh:
                    tree.writexml(fh, indent='    ', encoding='utf-8')  # writexml(fh)
                break

        restart_apache_tomcat(work_path)


def start_lock():
    lock = os.path.join("/home/liubin/code/Projects/lock", project_name,"lock.lock")
    logging.debug("Lock file %s" % lock)
    if os.path.exists(lock):
        return False
    exe_cmd("touch %s" % lock)
    return True


def end_lock(code):
    lock = os.path.join("/home/liubin/code/Projects/lock", project_name,"lock.lock")
    exe_cmd("rm %s" % lock)
    exit(code)


if __name__ == '__main__':
    _parser = argparse.ArgumentParser(description='Update or Config a Project for Source Search')
    _parser.add_argument("-u", "--update", help="Update All exist Project in Config.ini")
    _parser.add_argument("-p", "--project_name", help="Project Name")
    _parser.add_argument("-s", "--sync_cmd", help="Sync cmd, like git clone xxx.git, repo sync xxx")
    _args = _parser.parse_args()

    _work_path = os.path.dirname(os.path.realpath(__file__))
    _now = time.strftime("%Y-%m-%d-%H_%M_%S", time.localtime(time.time()))
    _log_file_name = os.path.join("/home/liubin/code/Projects/logs", project_name, 'log' + _now + ".log")
    if not os.path.exists(os.path.dirname(_log_file_name)):
        os.mkdir(os.path.dirname(_log_file_name))
    logging.basicConfig(level=logging.DEBUG,
                        format=LOG_FORMAT,
                        datefmt=DATE_FORMAT,
                        filename=_log_file_name
                        )

    logging.debug("This is a debug log.")
    logging.info("This is a info log.")
    logging.warning("This is a warning log.")
    logging.error("This is a error log.")
    logging.critical("This is a critical log.")

    if not start_lock():
        print("Cannot create lock file")
        logging.debug("Cannot create lock file")
        exit(1)

    if _args.update == "True" or _args.update == "1":
        _projects = get_all_projects()
        if len(_projects) > 0:
            for _project_name in _projects:
                logging.debug("Start Update Project %s" % _project_name)
                index_project(_work_path, _project_name, get_project_sync_cmd(_project_name))
                logging.debug("End Update Project %s" % _project_name)
            end_lock(0)
        else:
            logging.debug("No Exist Project Exit")
            end_lock(1)
    else:
        _project_name = _args.project_name
        if not _project_name:
            _projects = get_all_projects()
            if len(_projects) <= 0:
                logging.debug("Pls set project with options -p")
                end_lock(1)
            else:
                _index = 0
                for _project in _projects:
                    print("%s: %s" % (_index, _project))
                    _index = _index + 1
                _select = int(input("Please Select project:"))
                if 0 <= _select < _index:
                    _project_name = _projects[_select]
                else:
                    logging.debug("Project index error")
                    end_lock(2)

        logging.debug("Cur Project is: %s" % _project_name)
        _sync_cmd = _args.sync_cmd
        set_project_sync_cmd(_project_name, _sync_cmd)

        _project_path = os.path.join("/home/liubin/code/Projects", _project_name)
        _sync_cmd = get_project_sync_cmd(_project_name)
        if not _sync_cmd:
            _sync_cmd = input("Please Set %s Sync cmd (only support git clone xxx.git or repo sync xxx):")
            if _sync_cmd:
                if _sync_cmd.startswith("git clone") or _sync_cmd.startswith("repo sync"):
                    set_project_sync_cmd(_project_name, _sync_cmd)
                else:
                    logging.debug("Sync cmd is Error!")
                    end_lock(3)
            else:
                logging.debug("Sync cmd is empty!")
                end_lock(4)
        index_project(_work_path, _project_name, _sync_cmd)
        end_lock(0)

如果想索引某个文件夹的多个项目:

脚本改良后:

使用方法:python UpdateProject_new.py -p "source1" -c "/home/liubin/code/project" -a "/home/liubin/tools/apache-tomcat-9.0.54" -o "/home/liubin/tools/opengrok/opengrok-1.1-rc36"

代码都放在/home/liubin/code/project 

定时更新代码和索引需要

Ubuntu下面添加定时任务(cron):

  1. android@ubuntu:~$ crontab -e

  2. 比如每天23:59开始同步

  3. 59 23 * * *  cd /home/xxx/code/xxx ; repo sync -d -j8;cd /home/android/OpenGrok; python UpdateProject_new.py -p "source1" -c "/home/liubin/code/project" -a "/home/liubin/tools/apache-tomcat-9.0.54" -o "/home/liubin/tools/opengrok/opengrok-1.1-rc36"

#! /usr/bin/env python
# -*- coding: utf-8 -*-

import argparse

try:
    import configparser as configparser
except Exception:
    import ConfigParser as configparser
import logging
import os

import shutil

import time
from xml.dom.minidom import parse
import xml.dom.minidom
import subprocess

LOG_FORMAT = "%(asctime)s - %(levelname)s - %(message)s"
DATE_FORMAT = "%m/%d/%Y %H:%M:%S %p"

#CONFIG_FILE = os.path.join("/home/liubin/code", "Config.ini")


OPEN_GROK_VERSION = 'opengrok-1.1-rc36'
APACHE_TOMCAT_VERSION = 'apache-tomcat-9.0.54'
JAVA_VERSION = 'java-8-openjdk-amd64'


#def get_all_projects():
#    cf = configparser.ConfigParser()
#    cf.read(CONFIG_FILE)
#    secs = cf.sections()
#    return secs
#
#
#def get_project_sync_cmd(project):
#    cf = configparser.ConfigParser()
#    cf.read(CONFIG_FILE)
#    if cf.has_section(project):
#        cf.has_option(project, 'sync_cmd')
#        return cf.get(project, 'sync_cmd')
#    return None
#
#
#def set_project_sync_cmd(project, sync_cmd):
#    if sync_cmd and (sync_cmd.startswith("git clone") or sync_cmd.startswith("repo init")) or True:
#        cf = configparser.ConfigParser()
#        cf.read(CONFIG_FILE)
#        if not cf.has_section(project):
#            cf.add_section(project)
#        cf.set(project, 'sync_cmd', sync_cmd)
#        cf.write(open(CONFIG_FILE, 'w'))
#

def restart_apache_tomcat(apache_dir):
    cmd = "export JRE_HOME=%s; %s; %s" % (
        os.path.join("/usr/lib/jvm/java-8-openjdk-amd64", "jre"),
        os.path.join(apache_dir, "bin/shutdown.sh"),
        os.path.join(apache_dir, "bin/startup.sh")
    )
    exe_cmd(cmd)


def exe_cmd(cmd):
    logging.debug("Exe cmd: %s" % cmd)
    # result = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
    # for info in result.communicate():
    #     logging.dapache_dir)
    os.system(cmd)


def index_project(code_dir, opengork_dir,apache_dir,project):
    #if work_path is None or project_name is None or sync_cmd is None:
    #    logging.error("Error Project or cmd %s %s" % (project_name, sync_cmd))
    #else:
        # init code
    #project_path = os.path.join(project_name)
    #if not os.path.exists(project_path):
    #os.makedirs(project_path)
        #if sync_cmd.startswith("git clone") and False:
        #    sync = "git pull"
        #else:
        #    sync = "repo sync"
        #cmd = "cd %s; %s " % ("/home/liubin/code/Projects/rk88", sync_cmd)
        #exe_cmd(cmd)
        #cmd = "cd %s; %s " % ("/home/liubin/code/Projects/5073_10", sync_cmd)
        #exe_cmd(cmd)

        # sync code
        #if sync.startswith("git pull"):
        #    cmd = os.listdir(project_path)[0]
        #    cmd = "cd %s; git pull" % (os.path.join(project_path, cmd))
        #elif sync.startswith("repo sync"):
        #cmd = "cd %s; %s " % ("/home/liubin/code/Projects/rk88", sync_cmd)
        #exe_cmd(cmd)
        #cmd = "cd %s; %s " % ("/home/liubin/code/Projects/5073_10", sync_cmd)
        #exe_cmd(cmd)
        #cmd = "cd %s;repo sync -j8" % ("/home/liubin/code/Projects/rk88")
        #exe_cmd(cmd)
        #cmd = "cd %s;repo sync -j8" % ("/home/liubin/code/Projects/5073_10")
        #exe_cmd(cmd)

        # open grok indexer
        cmd = "%s \
               -j=%s \
               -a=%s -- \
               -c %s \
               -i %s \
               -H -P \
               -s %s \
               -d %s \
               -W %s" % (
            #os.path.join("/home/liubin/tools/opengrok/opengrok-1.1-rc36", "bin/indexer.py"),
            os.path.join(opengork_dir, "bin/indexer.py"),
            os.path.join("/usr/lib/jvm/java-8-openjdk-amd64/jre", "bin/java"),  # -j java path
            #os.path.join("/home/liubin/tools/opengrok/opengrok-1.1-rc36", "lib/opengrok.jar"),  # -a opengrok jar path
            os.path.join(opengork_dir, "lib/opengrok.jar"),  # -a opengrok jar path
            "/usr/bin/ctags",  # -c ctags path
            "d:*.git -i d:*.repo -i d:out",  # -i filter files and dirs
            os.path.join(code_dir),  # -s Project source path
            os.path.join(code_dir,"data"),  # -d opengrok generate data
            os.path.join(code_dir, "etc/configuration.xml")
            # -W cfg for apache
        )

        exe_cmd("mkdir " + code_dir +  "/data")
        exe_cmd("mkdir " + code_dir +  "/etc")
        #exe_cmd("mkdir /home/liubin/code/Projects/etc")
        exe_cmd(cmd)

        # create project in apache tomcat
        apache_app_path = os.path.join(apache_dir, "webapps", project + ".war")
        if not os.path.exists(apache_app_path):
            war_path = opengrok_dir + "/lib/source.war"
            shutil.copyfile(war_path, apache_app_path)
            restart_apache_tomcat(apache_dir)
            time.sleep(3)

        # set the project configuration.xml to apache tomcat
        apache_app_config = os.path.join(
            os.path.join(apache_dir, "webapps", project),
            "WEB-INF", "web.xml")

        wait_time = 0
        while not os.path.exists(apache_app_config):
            if wait_time > 10:
                logging.debug(
                    "Cannot find apache tomcat app=%s config_path=%s error!" % ("source", apache_app_config))
                return
            wait_time = wait_time + 1
            time.sleep(3)

        tree = xml.dom.minidom.parse(apache_app_config)
        nodes = tree.documentElement.getElementsByTagName('context-param')
        for node in nodes:
            cfg_node = node.getElementsByTagName('param-value')
            if cfg_node:
                value = os.path.join(code_dir, "etc/configuration.xml")
                cfg_node[0].childNodes[0].data = value
                logging.debug("Update  config %s success" % (value))
                with open(apache_app_config, 'w') as fh:
                    tree.writexml(fh, indent='    ', encoding='utf-8')  # writexml(fh)
                break

        restart_apache_tomcat(apache_dir)


#def start_lock():
#    lock = os.path.join("/home/liubin/code/Projects/lock","lock/lock.lock")
#    logging.debug("Lock file %s" % lock)
#    if os.path.exists(lock):
#        return False
#    exe_cmd("touch %s" % lock)
#    return True


#def end_lock(code):
#    lock = os.path.join("/home/liubin/code/Projects/lock","lock.lock")
#    exe_cmd("rm %s" % lock)
#    exit(code)


if __name__ == '__main__':
    _parser = argparse.ArgumentParser(description='Update or Config a Project for Source Search')
    _parser.add_argument("-p", "--project", help="http://localhost/project")
    _parser.add_argument("-c", "--code_dir", help="code dir,for example ~/code")
    #_parser.add_argument("-s", "--sync_cmd", help="Sync cmd, like git clone xxx.git, repo sync xxx")
    _parser.add_argument("-a", "--apache", help="apache dir,for example ")
    _parser.add_argument("-o", "--opengrok", help="opengrok dir,for example ")
    _args = _parser.parse_args()

    _work_path = os.path.dirname(os.path.realpath(__file__))
    _logs_dir = os.path.join(_work_path, "logs")
    exe_cmd("mkdir " + _logs_dir)
    ##_work_path = os.path.dirname("/home/liubin/code/Projects")
    _now = time.strftime("%Y-%m-%d-%H_%M_%S", time.localtime(time.time()))
    _log_file_name = os.path.join(_logs_dir, 'log' + _now + ".log")
    if not os.path.exists(os.path.dirname(_log_file_name)):
        os.mkdir(os.path.dirname(_log_file_name))
    logging.basicConfig(level=logging.DEBUG,
                        format=LOG_FORMAT,
                        datefmt=DATE_FORMAT,
                        filename=_log_file_name
                        )

    #logging.debug("This is a debug log.")
    #logging.info("This is a info log.")
    #logging.warning("This is a warning log.")
    #logging.error("This is a error log.")
    #logging.critical("This is a critical log.")

    #if not start_lock():
    #    print("Cannot create lock file")
    #    logging.error("Cannot create lock file")
    #    exit(1)
    code_dir = _args.code_dir
    opengrok_dir = _args.opengrok
    apache_dir = _args.apache
    project = _args.project
    #if _args.update == "True" or _args.update == "1":
        #_projects = get_all_projects()
        #if len(_projects) > 0:
        #for _project_name in _projects:
    logging.error("Start Update Project %s" % project)
    index_project(code_dir, opengrok_dir, apache_dir,project)
    logging.error("End Update Project %s" % project)
        #end_lock(0)
        #else:
        #    logging.error("No Exist Project Exit")
        #    end_lock(1)
    #else:
    #    _project_name = _args.project_name
    #    if not _project_name:
    #        _projects = get_all_projects()
    #        if len(_projects) <= 0:
    #            logging.debug("Pls set project with options -p")
    #            end_lock(1)
    #        else:
    #            _index = 0
    #            for _project in _projects:
    #                print("%s: %s" % (_index, _project))
    #                _index = _index + 1
    #            _select = int(input("Please Select project:"))
    #            if 0 <= _select < _index:
    #                _project_name = _projects[_select]
    #            else:
    #                logging.debug("Project index error")
    #                end_lock(2)

    #    logging.debug("Cur Project is: %s" % _project_name)
    #    _sync_cmd = _args.sync_cmd
    #    set_project_sync_cmd(_project_name, _sync_cmd)

    #    _project_path = os.path.join(_project_name)
    #    _sync_cmd = get_project_sync_cmd(_project_name)
    #    if not _sync_cmd:
    #        _sync_cmd = input("Please Set %s Sync cmd (only support git clone xxx.git or repo sync xxx):")
    #        if _sync_cmd:
    #            if _sync_cmd.startswith("git clone") or _sync_cmd.startswith("repo sync"):
    #                set_project_sync_cmd(_project_name, _sync_cmd)
    #            else:
    #                logging.debug("Sync cmd is Error!")
    #                end_lock(3)
    #        else:
    #            logging.debug("Sync cmd is empty!")
    #            end_lock(4)
    #    index_project(_work_path, _project_name, _sync_cmd)
    #    logging.error("sync cmd  completed!")
    #    end_lock(0)

评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值