jython executor
# -*- coding: utf-8 -*-
try:
from webx import settings
except ImportError:
import os,sys
p = os.path.dirname(__file__)
for x in xrange(5):
p=os.path.split(p)[0]
os.sys.path.append(p+'/src/main/webapp')
"""
由于需要依赖 java concurrent框架, 所以只能通过jython启动
"""
from java.util.concurrent import ThreadFactory,Executors,ThreadPoolExecutor,TimeUnit,ArrayBlockingQueue
from java.lang import Thread,Runnable
from java.util.concurrent.atomic import AtomicInteger
from task.index.solr import Solr
from people.service import userQueryService
from blog.service import blogQueryService
from tag.service import tagQueryService
from shopping.service import shoppingItemService
import logging
logger = logging.getLogger("opener")
'''
负责创建Solr Client
'''
class SolrFactory():
def __init__(self,server,timeout):
self.cores = ['people','feed','feed2','album']
self.solrs = {}
for core in self.cores:
logger.info("init solr client "+core)
self.solrs[core] = Solr(server + core + "/", timeout)
def get_solr(self,core):
return self.solrs.get(core)
write = 'http://192.168.173.6:7100/solr/'
solrFactory = SolrFactory(write,20)
'''
为 Executors中的线程增加名字,便于jstack的时候查看Thread
参考:java concurrent包的 Executors.DefaultThreadFactory
'''
class NamedThreadFactory(ThreadFactory):
def __init__(self,prefix,daemo=False):
self.mPrefix = prefix + "-thread-";
self.mDaemo = daemo;
self.mGroup = Thread.currentThread().getThreadGroup()
self.mThreadNum = AtomicInteger(1);
def newThread(self, runnable):
name = '%s%s ' %(self.mPrefix , self.mThreadNum.getAndIncrement())
ret = Thread(self.mGroup, runnable, name, 0)
ret.setDaemon(self.mDaemo)
return ret;
rejectedExecutionHandler = ThreadPoolExecutor.CallerRunsPolicy() #使用CallerRuns策略,变相的减缓新任务的提交速度
buildExecutor = ThreadPoolExecutor(5,5, 0L, TimeUnit.MILLISECONDS,ArrayBlockingQueue(1000), NamedThreadFactory("build"),rejectedExecutionHandler );
commitExecutor = ThreadPoolExecutor(5,5, 0L, TimeUnit.MILLISECONDS,ArrayBlockingQueue(1000), NamedThreadFactory("commit"),rejectedExecutionHandler );
'''
负责把模型build成solr document,分别有:
BlogBuilder
PeopleBuilder
SolrBuilder
'''
class BaseBuilder(Runnable):
def _solr_str(self,date):
"""1995-12-31T23:59:59Z"""
return date.strftime("%Y-%m-%dT%H:%M:%SZ")
def run(self):
raise NotImplementedError
class BlogBuilder(BaseBuilder):
def __init__(self, ids,executor):
self.ids = ids
self.executor = executor
self.solr = solrFactory.get_solr("feed")
def _extra_tags(self,blog):
iphone_wall_paper = blog.photo_width>=320 and blog.photo_height>=480 and blog.photo_height/float(blog.photo_width) == 1.5 and "INNERTAGIPHWP" or ""
iphone_wall_paper = blog.photo_width>=320 and blog.photo_height>=568 and blog.photo_height/float(blog.photo_width) == 1.775 and "INNERTAGIPHWP5" or iphone_wall_paper
item_cat_id = ''
item_cat_pid = ''
# if blog.is_buyable():
# item_id = shoppingItemService.find_item_id(blog)
# item_cat = shoppingItemService.find_or_create_taobao_category(None,item_id)
# if item_cat:
# item_cat_id = item_cat.get("cid") and 'ITEMCATID_%s'%item_cat.get("cid")
# item_cat_pid = item_cat.get("parent_id") and 'ITEMCATPID_%s'%item_cat.get("parent_id")
# else:
# logger.error('Can not include taobao category info when indexing %s'%blog.id)
return (iphone_wall_paper, item_cat_id, item_cat_pid)
def run(self):
#print 'build %s'%len(self.ids)
blog_list = blogQueryService.queryBlogDetail(self.ids)
doc = []
for blog in blog_list:
if not blog.is_normal():
continue
if blog.is_in_audit():
continue
blog_tags = tagQueryService.queryTagStrByBlogId(blog.id)
extra_tags = self._extra_tags(blog) #这里可以扩展内部的其他各种查询条件, 比增加字段靠谱, 性能更好, 更灵活
data = {"id": blog.id,
"sender_id": blog.sender_id,
"msg": blog.msg or "" +" " +"".join(blog_tags)+" " +"".join(extra_tags),
"buyable": blog.buyable,
"add_datetime": self._solr_str(blog.add_datetime),
"last_replied_datetime":self._solr_str(blog.last_replied_datetime),
"photo_id":blog.photo_id,
"photo_width": blog.photo_width,
"photo_height": blog.photo_height,
"favorite_count":blog.favorite_count,
"category":blog.category,
"status":blog.status,
"group_id": blog.group_id,
"album_id":blog.album_id,
"pop_score":1, #已经废弃
"shopping_score":1,#已经废弃
"src_domain":blog.source_link,
"price":blog.price,
}
doc.append(data)
self.executor.execute(Commiter(doc,self.solr)) #提交task
class Commiter(Runnable):
def __init__(self,doc,solr):
self.doc = doc
self.solr = solr
def run(self):
#print 'commit %s'%len(self.doc)
if self.doc:
try:
self.solr.add_many(self.doc)
self.solr.commit()
except Exception,e:
logger.exception(e)
if __name__ == "__main__":
buildExecutor.execute(BlogBuilder([1,2,3,4,5,6,7,8,9,10,11],commitExecutor))
import time
while True:
time.sleep(2)