展开全部
# -*- coding: utf-8 -*-
import jieba
'''''
Created on 2015-11-23
'''
def word_split(text):
"""
Split a text in words. Returns a list of tuple that contains
(word, location) location is the starting byte position of the word.
"""
word_list = []
windex = 0
word_primitive = jieba.cut(text, cut_all = True)
for word in word_primitive:
if len(word) > 0:
word_list.append((windex, word))
windex += 1
return word_list
def inverted_index(text):
"""
Create an Inverted-Index of the specified text document.
{word:[locations]}
"""
inverted = {}
for index, word in word_split(text):
locations = inverted.setdefault(word, [])
locations.append(index)
return inverted
def inverted_index_add(inverted, doc_id, doc_index):
"""
Add Invertd-Index doc_index of the document doc_id to the
Multi-Document Inverted-Index (inverted),
using doc_id as document identifier.
{word:{doc_id:[locations]}}
"""
for word, locations in doc_index.iteritems():
indices = inverted.setdefault(word, {})
indices[doc_id] = locations
return inverted
def search_a_word(inverted, word):
"""
search one word
"""
word = word.decode('utf-8')
if word not in inverted:
return None
else:
word_index = inverted[word]
return word_index
def search_words(inverted, wordList):
"""
search more than one word
"""
wordDic = []
docRight = []
for word in wordList:
if isinstance(word, str):
word = word.decode('utf-8')
if word not in inverted:</
最低0.47元/天 解锁文章
2758
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
