Tools:
apache-tomcat-9.0.8(github下载地址)
java-8-openjdk-amd64(官网下载地址)
opengrok-1.1-rc36(github下载地址)
ctags(github下载地址)
python UpdateProject.py -p "projectname" -s 'repo init -u xxxxxxx"
或者
python UpdateProject.py -p "projectname" -s 'git clone xxxxx"
脚本内容如下:
#! /usr/bin/env python
# -*- coding: utf-8 -*-
import argparse
try:
import configparser as configparser
except Exception:
import ConfigParser as configparser
import logging
import os
import shutil
import time
from xml.dom.minidom import parse
import xml.dom.minidom
import subprocess
LOG_FORMAT = "%(asctime)s - %(levelname)s - %(message)s"
DATE_FORMAT = "%m/%d/%Y %H:%M:%S %p"
CONFIG_FILE = os.path.join("/home/liubin/code/Projects", "Config.ini")
OPEN_GROK_VERSION = 'opengrok-1.1-rc36'
APACHE_TOMCAT_VERSION = 'apache-tomcat-8.0.32.0'
JAVA_VERSION = 'java-8-openjdk-amd64'
def get_all_projects():
cf = configparser.ConfigParser()
cf.read(CONFIG_FILE)
secs = cf.sections()
return secs
def get_project_sync_cmd(project):
cf = configparser.ConfigParser()
cf.read(CONFIG_FILE)
if cf.has_section(project):
cf.has_option(project, 'sync_cmd')
return cf.get(project, 'sync_cmd')
return None
def set_project_sync_cmd(project, sync_cmd):
if sync_cmd and (sync_cmd.startswith("git clone") or sync_cmd.startswith("repo init")):
cf = configparser.ConfigParser()
cf.read(CONFIG_FILE)
if not cf.has_section(project):
cf.add_section(project)
cf.set(project, 'sync_cmd', sync_cmd)
cf.write(open(CONFIG_FILE, 'w'))
def restart_apache_tomcat(work_path):
cmd = "export JRE_HOME=%s; %s; %s" % (
os.path.join("/usr/lib/jvm/java-8-openjdk-amd64", "jre"),
os.path.join("/usr/share/tomcat8", "bin/shutdown.sh"),
os.path.join("/usr/share/tomcat8", "bin/startup.sh")
)
exe_cmd(cmd)
def exe_cmd(cmd):
logging.debug("Exe cmd: %s" % cmd)
# result = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
# for info in result.communicate():
# logging.debug(info)
os.system(cmd)
def index_project(work_path, project_name, sync_cmd):
if work_path is None or project_name is None or sync_cmd is None:
logging.debug("Error Project or cmd %s %s" % (project_name, sync_cmd))
else:
# init code
project_path = os.path.join("/home/liubin/code/Projects", project_name)
if not os.path.exists(project_path):
os.makedirs(project_path)
if sync_cmd.startswith("git clone"):
sync = "git pull"
else:
sync = "repo sync"
cmd = "cd %s; %s " % (project_path, sync_cmd)
exe_cmd(cmd)
# sync code
if sync.startswith("git pull"):
cmd = os.listdir(project_path)[0]
cmd = "cd %s; git pull" % (os.path.join(project_path, cmd))
elif sync.startswith("repo sync"):
cmd = "cd %s; repo sync" % project_path
exe_cmd(cmd)
# open grok indexer
cmd = "%s \
-j=%s \
-a=%s -- \
-c %s \
-i %s \
-H -P \
-s %s \
-d %s \
-W %s" % (
os.path.join("/home/liubin/tools/opengrok/opengrok-1.1-rc36", "bin/indexer.py"),
os.path.join("/usr/lib/jvm/java-8-openjdk-amd64/jre", "bin/java"), # -j java path
os.path.join("/home/liubin/tools/opengrok/opengrok-1.1-rc36", "lib/opengrok.jar"), # -a opengrok jar path
"/usr/bin/ctags", # -c ctags path
"d:*.git -i d:*.repo -i f:*.jar", # -i filter files and dirs
project_path, # -s Project source path
os.path.join("/home/liubin/code/Projects/data", project_name), # -d opengrok generate data
os.path.join("/home/liubin/code/Projects/etc", project_name, "configuration.xml")
# -W cfg for apache
)
exe_cmd(cmd)
# create project in apache tomcat
apache_app_path = os.path.join("/var/lib/tomcat8", "webapps",project_name + ".war")
if not os.path.exists(apache_app_path):
war_path = OPEN_GROK_VERSION + "/lib/source.war"
shutil.copyfile(war_path, apache_app_path)
restart_apache_tomcat(work_path)
time.sleep(3)
# set the project configuration.xml to apache tomcat
apache_app_config = os.path.join(
os.path.join("/var/lib/tomcat8", "webapps", project_name),
"WEB-INF", "web.xml")
wait_time = 0
while not os.path.exists(apache_app_config):
if wait_time > 10:
logging.debug(
"Cannot find apache tomcat app=%s config_path=%s error!" % (project_name, apache_app_config))
return
wait_time = wait_time + 1
time.sleep(3)
tree = xml.dom.minidom.parse(apache_app_config)
nodes = tree.documentElement.getElementsByTagName('context-param')
for node in nodes:
cfg_node = node.getElementsByTagName('param-value')
if cfg_node:
value = os.path.join("/home/liubin/code/Projects/etc", project_name, "configuration.xml")
cfg_node[0].childNodes[0].data = value
logging.debug("Update %s config %s success" % (project_name, value))
with open(apache_app_config, 'w') as fh:
tree.writexml(fh, indent=' ', encoding='utf-8') # writexml(fh)
break
restart_apache_tomcat(work_path)
def start_lock():
lock = os.path.join("/home/liubin/code/Projects/lock", project_name,"lock.lock")
logging.debug("Lock file %s" % lock)
if os.path.exists(lock):
return False
exe_cmd("touch %s" % lock)
return True
def end_lock(code):
lock = os.path.join("/home/liubin/code/Projects/lock", project_name,"lock.lock")
exe_cmd("rm %s" % lock)
exit(code)
if __name__ == '__main__':
_parser = argparse.ArgumentParser(description='Update or Config a Project for Source Search')
_parser.add_argument("-u", "--update", help="Update All exist Project in Config.ini")
_parser.add_argument("-p", "--project_name", help="Project Name")
_parser.add_argument("-s", "--sync_cmd", help="Sync cmd, like git clone xxx.git, repo sync xxx")
_args = _parser.parse_args()
_work_path = os.path.dirname(os.path.realpath(__file__))
_now = time.strftime("%Y-%m-%d-%H_%M_%S", time.localtime(time.time()))
_log_file_name = os.path.join("/home/liubin/code/Projects/logs", project_name, 'log' + _now + ".log")
if not os.path.exists(os.path.dirname(_log_file_name)):
os.mkdir(os.path.dirname(_log_file_name))
logging.basicConfig(level=logging.DEBUG,
format=LOG_FORMAT,
datefmt=DATE_FORMAT,
filename=_log_file_name
)
logging.debug("This is a debug log.")
logging.info("This is a info log.")
logging.warning("This is a warning log.")
logging.error("This is a error log.")
logging.critical("This is a critical log.")
if not start_lock():
print("Cannot create lock file")
logging.debug("Cannot create lock file")
exit(1)
if _args.update == "True" or _args.update == "1":
_projects = get_all_projects()
if len(_projects) > 0:
for _project_name in _projects:
logging.debug("Start Update Project %s" % _project_name)
index_project(_work_path, _project_name, get_project_sync_cmd(_project_name))
logging.debug("End Update Project %s" % _project_name)
end_lock(0)
else:
logging.debug("No Exist Project Exit")
end_lock(1)
else:
_project_name = _args.project_name
if not _project_name:
_projects = get_all_projects()
if len(_projects) <= 0:
logging.debug("Pls set project with options -p")
end_lock(1)
else:
_index = 0
for _project in _projects:
print("%s: %s" % (_index, _project))
_index = _index + 1
_select = int(input("Please Select project:"))
if 0 <= _select < _index:
_project_name = _projects[_select]
else:
logging.debug("Project index error")
end_lock(2)
logging.debug("Cur Project is: %s" % _project_name)
_sync_cmd = _args.sync_cmd
set_project_sync_cmd(_project_name, _sync_cmd)
_project_path = os.path.join("/home/liubin/code/Projects", _project_name)
_sync_cmd = get_project_sync_cmd(_project_name)
if not _sync_cmd:
_sync_cmd = input("Please Set %s Sync cmd (only support git clone xxx.git or repo sync xxx):")
if _sync_cmd:
if _sync_cmd.startswith("git clone") or _sync_cmd.startswith("repo sync"):
set_project_sync_cmd(_project_name, _sync_cmd)
else:
logging.debug("Sync cmd is Error!")
end_lock(3)
else:
logging.debug("Sync cmd is empty!")
end_lock(4)
index_project(_work_path, _project_name, _sync_cmd)
end_lock(0)
如果想索引某个文件夹的多个项目:
脚本改良后:
使用方法:python UpdateProject_new.py -p "source1" -c "/home/liubin/code/project" -a "/home/liubin/tools/apache-tomcat-9.0.54" -o "/home/liubin/tools/opengrok/opengrok-1.1-rc36"
代码都放在/home/liubin/code/project
定时更新代码和索引需要
Ubuntu下面添加定时任务(cron):
-
android@ubuntu:~$ crontab -e
-
比如每天23:59开始同步
-
59 23 * * * cd /home/xxx/code/xxx ; repo sync -d -j8;cd /home/android/OpenGrok; python UpdateProject_new.py -p "source1" -c "/home/liubin/code/project" -a "/home/liubin/tools/apache-tomcat-9.0.54" -o "/home/liubin/tools/opengrok/opengrok-1.1-rc36"
#! /usr/bin/env python
# -*- coding: utf-8 -*-
import argparse
try:
import configparser as configparser
except Exception:
import ConfigParser as configparser
import logging
import os
import shutil
import time
from xml.dom.minidom import parse
import xml.dom.minidom
import subprocess
LOG_FORMAT = "%(asctime)s - %(levelname)s - %(message)s"
DATE_FORMAT = "%m/%d/%Y %H:%M:%S %p"
#CONFIG_FILE = os.path.join("/home/liubin/code", "Config.ini")
OPEN_GROK_VERSION = 'opengrok-1.1-rc36'
APACHE_TOMCAT_VERSION = 'apache-tomcat-9.0.54'
JAVA_VERSION = 'java-8-openjdk-amd64'
#def get_all_projects():
# cf = configparser.ConfigParser()
# cf.read(CONFIG_FILE)
# secs = cf.sections()
# return secs
#
#
#def get_project_sync_cmd(project):
# cf = configparser.ConfigParser()
# cf.read(CONFIG_FILE)
# if cf.has_section(project):
# cf.has_option(project, 'sync_cmd')
# return cf.get(project, 'sync_cmd')
# return None
#
#
#def set_project_sync_cmd(project, sync_cmd):
# if sync_cmd and (sync_cmd.startswith("git clone") or sync_cmd.startswith("repo init")) or True:
# cf = configparser.ConfigParser()
# cf.read(CONFIG_FILE)
# if not cf.has_section(project):
# cf.add_section(project)
# cf.set(project, 'sync_cmd', sync_cmd)
# cf.write(open(CONFIG_FILE, 'w'))
#
def restart_apache_tomcat(apache_dir):
cmd = "export JRE_HOME=%s; %s; %s" % (
os.path.join("/usr/lib/jvm/java-8-openjdk-amd64", "jre"),
os.path.join(apache_dir, "bin/shutdown.sh"),
os.path.join(apache_dir, "bin/startup.sh")
)
exe_cmd(cmd)
def exe_cmd(cmd):
logging.debug("Exe cmd: %s" % cmd)
# result = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
# for info in result.communicate():
# logging.dapache_dir)
os.system(cmd)
def index_project(code_dir, opengork_dir,apache_dir,project):
#if work_path is None or project_name is None or sync_cmd is None:
# logging.error("Error Project or cmd %s %s" % (project_name, sync_cmd))
#else:
# init code
#project_path = os.path.join(project_name)
#if not os.path.exists(project_path):
#os.makedirs(project_path)
#if sync_cmd.startswith("git clone") and False:
# sync = "git pull"
#else:
# sync = "repo sync"
#cmd = "cd %s; %s " % ("/home/liubin/code/Projects/rk88", sync_cmd)
#exe_cmd(cmd)
#cmd = "cd %s; %s " % ("/home/liubin/code/Projects/5073_10", sync_cmd)
#exe_cmd(cmd)
# sync code
#if sync.startswith("git pull"):
# cmd = os.listdir(project_path)[0]
# cmd = "cd %s; git pull" % (os.path.join(project_path, cmd))
#elif sync.startswith("repo sync"):
#cmd = "cd %s; %s " % ("/home/liubin/code/Projects/rk88", sync_cmd)
#exe_cmd(cmd)
#cmd = "cd %s; %s " % ("/home/liubin/code/Projects/5073_10", sync_cmd)
#exe_cmd(cmd)
#cmd = "cd %s;repo sync -j8" % ("/home/liubin/code/Projects/rk88")
#exe_cmd(cmd)
#cmd = "cd %s;repo sync -j8" % ("/home/liubin/code/Projects/5073_10")
#exe_cmd(cmd)
# open grok indexer
cmd = "%s \
-j=%s \
-a=%s -- \
-c %s \
-i %s \
-H -P \
-s %s \
-d %s \
-W %s" % (
#os.path.join("/home/liubin/tools/opengrok/opengrok-1.1-rc36", "bin/indexer.py"),
os.path.join(opengork_dir, "bin/indexer.py"),
os.path.join("/usr/lib/jvm/java-8-openjdk-amd64/jre", "bin/java"), # -j java path
#os.path.join("/home/liubin/tools/opengrok/opengrok-1.1-rc36", "lib/opengrok.jar"), # -a opengrok jar path
os.path.join(opengork_dir, "lib/opengrok.jar"), # -a opengrok jar path
"/usr/bin/ctags", # -c ctags path
"d:*.git -i d:*.repo -i d:out", # -i filter files and dirs
os.path.join(code_dir), # -s Project source path
os.path.join(code_dir,"data"), # -d opengrok generate data
os.path.join(code_dir, "etc/configuration.xml")
# -W cfg for apache
)
exe_cmd("mkdir " + code_dir + "/data")
exe_cmd("mkdir " + code_dir + "/etc")
#exe_cmd("mkdir /home/liubin/code/Projects/etc")
exe_cmd(cmd)
# create project in apache tomcat
apache_app_path = os.path.join(apache_dir, "webapps", project + ".war")
if not os.path.exists(apache_app_path):
war_path = opengrok_dir + "/lib/source.war"
shutil.copyfile(war_path, apache_app_path)
restart_apache_tomcat(apache_dir)
time.sleep(3)
# set the project configuration.xml to apache tomcat
apache_app_config = os.path.join(
os.path.join(apache_dir, "webapps", project),
"WEB-INF", "web.xml")
wait_time = 0
while not os.path.exists(apache_app_config):
if wait_time > 10:
logging.debug(
"Cannot find apache tomcat app=%s config_path=%s error!" % ("source", apache_app_config))
return
wait_time = wait_time + 1
time.sleep(3)
tree = xml.dom.minidom.parse(apache_app_config)
nodes = tree.documentElement.getElementsByTagName('context-param')
for node in nodes:
cfg_node = node.getElementsByTagName('param-value')
if cfg_node:
value = os.path.join(code_dir, "etc/configuration.xml")
cfg_node[0].childNodes[0].data = value
logging.debug("Update config %s success" % (value))
with open(apache_app_config, 'w') as fh:
tree.writexml(fh, indent=' ', encoding='utf-8') # writexml(fh)
break
restart_apache_tomcat(apache_dir)
#def start_lock():
# lock = os.path.join("/home/liubin/code/Projects/lock","lock/lock.lock")
# logging.debug("Lock file %s" % lock)
# if os.path.exists(lock):
# return False
# exe_cmd("touch %s" % lock)
# return True
#def end_lock(code):
# lock = os.path.join("/home/liubin/code/Projects/lock","lock.lock")
# exe_cmd("rm %s" % lock)
# exit(code)
if __name__ == '__main__':
_parser = argparse.ArgumentParser(description='Update or Config a Project for Source Search')
_parser.add_argument("-p", "--project", help="http://localhost/project")
_parser.add_argument("-c", "--code_dir", help="code dir,for example ~/code")
#_parser.add_argument("-s", "--sync_cmd", help="Sync cmd, like git clone xxx.git, repo sync xxx")
_parser.add_argument("-a", "--apache", help="apache dir,for example ")
_parser.add_argument("-o", "--opengrok", help="opengrok dir,for example ")
_args = _parser.parse_args()
_work_path = os.path.dirname(os.path.realpath(__file__))
_logs_dir = os.path.join(_work_path, "logs")
exe_cmd("mkdir " + _logs_dir)
##_work_path = os.path.dirname("/home/liubin/code/Projects")
_now = time.strftime("%Y-%m-%d-%H_%M_%S", time.localtime(time.time()))
_log_file_name = os.path.join(_logs_dir, 'log' + _now + ".log")
if not os.path.exists(os.path.dirname(_log_file_name)):
os.mkdir(os.path.dirname(_log_file_name))
logging.basicConfig(level=logging.DEBUG,
format=LOG_FORMAT,
datefmt=DATE_FORMAT,
filename=_log_file_name
)
#logging.debug("This is a debug log.")
#logging.info("This is a info log.")
#logging.warning("This is a warning log.")
#logging.error("This is a error log.")
#logging.critical("This is a critical log.")
#if not start_lock():
# print("Cannot create lock file")
# logging.error("Cannot create lock file")
# exit(1)
code_dir = _args.code_dir
opengrok_dir = _args.opengrok
apache_dir = _args.apache
project = _args.project
#if _args.update == "True" or _args.update == "1":
#_projects = get_all_projects()
#if len(_projects) > 0:
#for _project_name in _projects:
logging.error("Start Update Project %s" % project)
index_project(code_dir, opengrok_dir, apache_dir,project)
logging.error("End Update Project %s" % project)
#end_lock(0)
#else:
# logging.error("No Exist Project Exit")
# end_lock(1)
#else:
# _project_name = _args.project_name
# if not _project_name:
# _projects = get_all_projects()
# if len(_projects) <= 0:
# logging.debug("Pls set project with options -p")
# end_lock(1)
# else:
# _index = 0
# for _project in _projects:
# print("%s: %s" % (_index, _project))
# _index = _index + 1
# _select = int(input("Please Select project:"))
# if 0 <= _select < _index:
# _project_name = _projects[_select]
# else:
# logging.debug("Project index error")
# end_lock(2)
# logging.debug("Cur Project is: %s" % _project_name)
# _sync_cmd = _args.sync_cmd
# set_project_sync_cmd(_project_name, _sync_cmd)
# _project_path = os.path.join(_project_name)
# _sync_cmd = get_project_sync_cmd(_project_name)
# if not _sync_cmd:
# _sync_cmd = input("Please Set %s Sync cmd (only support git clone xxx.git or repo sync xxx):")
# if _sync_cmd:
# if _sync_cmd.startswith("git clone") or _sync_cmd.startswith("repo sync"):
# set_project_sync_cmd(_project_name, _sync_cmd)
# else:
# logging.debug("Sync cmd is Error!")
# end_lock(3)
# else:
# logging.debug("Sync cmd is empty!")
# end_lock(4)
# index_project(_work_path, _project_name, _sync_cmd)
# logging.error("sync cmd completed!")
# end_lock(0)