利用百度构造拼音词表[python]

做中文查询纠错,需要一个拼音到中文词汇的对应表。网上找不到,只要利用baidu来构建一个。

原理:

难点是多音字。输入一个词,根据拼音汉字表,可以得到其搜有发音,比如“银行”的发音有可能有 “yinhang”, “yinxing”,哪一个才是对的呢?

发送 GET 请求到 to baidu.com,
"http://www.baidu.com/s?bs=zhongguo&f=8&wd=yinhang"

从搜索结果可以得到,

您要找的是不是: 银行 ,
百度一下,找到相关网页约170,000篇,用时 0.070秒

这样就可以知道” yinhang“是”银行“正确的发音。


步骤
1. 利用拼音汉字表, 构造 汉字->拼音 hash 表。
2. 对词表的每一个词汇,得到所有的拼音候选。如果只有一个,就是得到一个结果。
如果有多个发音,到步骤3。
3. 发送 GET 请求到 to baidu.com, "http://www.baidu.com/s?bs=zhongguo&f=8&wd=*****" (候选拼音)
4. 分析返回结果,得到正确的发音。

注意,由于发送GET到baidu太慢,采用用200个线程,
下面是python代码


#!/usr/bin/python

import sys
import chardet
import re
import httplib
from urllib2 import Request, HTTPError, URLError, urlopen
from httplib import InvalidURL
import urlparse, re, urllib, logging, StringIO, logging
import htmllib
from os.path import join, getsize
import threading


verbose = False

#编码转换
def transcode(s):
try:
encode_dict = chardet.detect(s)
if encode_dict["encoding"] == "GB2312" :
s = unicode(s,"gb18030").encode("utf-8")
if encode_dict["encoding"] == "gbk" :
s = unicode(s,"gb18030").encode("utf-8")
if isinstance(s, unicode):
s = s.encode('utf-8')
return s
except:
return s

pinyin_map = dict();

#构造拼音词汇表,因为拼音词汇表格式可能不一样,所以省略
def generate_pinyin_map():
#省略

def generate_synonym_worker( input_file, out_file, idx ):
f = open(input_file+"_"+str(idx), 'r')
of = open(out_file+"_"+str(idx), 'w+')
of.seek(0, 2)
idx = 0
for eachline in f:
idx += 1
try:
eachline = unicode(eachline,'utf-8')
eachline = eachline[:-1]
words = re.split(u' +', eachline)
pinyin = generate_pinyin_canadiate(words[0], of)
except:
pass
f.close()
of.close()

#两百个线程
num_worker = 200

#map
def split(input_file):
f = open(input_file, 'r')
files = []
for idx in range(num_worker):
files.append( open(input_file+"_"+str(idx), 'w') )
idx = 0
for eachline in f:
idx += 1
files[idx%num_worker].write(eachline)
for onefile in files:
onefile.close()

#reduce
def merge(out_file):
outf = open(out_file, 'w+')
files = []
for idx in range(num_worker):
file_name = out_file+"_"+str(idx)
f = open(file_name, 'r')
for eachline in f:
outf.write(eachline)
f.close()
outf.close

#产生拼音词汇表
def generate_synonym(input_file, out_file):
split( input_file )
thread_pool = []
for i in range(num_worker):
th = threading.Thread(target=generate_synonym_worker, args=(input_file, out_file ,i) ) ;
thread_pool.append(th)

# start threads one by one
for i in range(num_worker):
thread_pool[i].start()

#collect all threads
for i in range(num_worker):
threading.Thread.join(thread_pool[i])
merge(out_file)

#枚举所有的拼音组合
def listlist_enumerate(listlist, count):
result = []
if count <= 0:
return result
if count == 1:
for item in listlist[0]:
aa = []
aa.append(item)
result.append(aa)
else:
pre_result = listlist_enumerate(listlist, count-1)
result = []
for one in pre_result:
for item in listlist[count-1]:
bb = list(one)
bb.append(item)
result.append( bb )
return result


def makeutf8(s):
#ESCAPE = re.compile(r'[\x00-\x17\\"\b\f\n\r\t]')
ESCAPE = re.compile(r'[\x00-\x1f]')
ESCAPE_DCT = {}
for i in range(32):
ESCAPE_DCT.setdefault(chr(i), ' ')
#ESCAPE_DCT.setdefault(chr(i), '\\%02x' % (i,))
#ESCAPE_DCT.setdefault(chr(i), '^%c' % (ord('@')+i) )

def replace(match):
return ESCAPE_DCT[match.group(0)]

s = ESCAPE.sub(replace, s)
if isinstance(s, unicode):
s = s.encode('utf-8')
return s

def generate_pinyin_canadiate(chinese_word,fd):
listlist = []
for uchar in chinese_word:
if pinyin_map.has_key(uchar) and len(pinyin_map[uchar]):
listlist.append( pinyin_map[uchar] )
else:
listlist.append( uchar )
listlist = listlist_enumerate(listlist, len(listlist))
for alist in listlist:
canadiate = ''.join(alist)
canadiate = canadiate.lower()
print canadiate
query = ""
try:
query = urllib.urlencode({'wd' : canadiate})
except:
pass
url = 'http://www.baidu.com/s?%s'%(query)
search_results = urllib.urlopen(url).read()
search_results = transcode(search_results)
print url, canadiate #, search_results
r_suggest = re.compile('<a href="s\?wd=(.*)&f=12&oq='%dict(pinyin=canadiate), re.I|re.S)
re_match = r_suggest.findall( search_results )
if re_match:
try:
s = urllib.unquote(re_match[0])
s= unicode(s,'gb18030')
fd.write(canadiate + " " + makeutf8(s) + "\n")
fd.flush()
if s == chinese_word:
print "one hit: ", canadiate, chinese_word
#fd.write(canadiate + " " + makeutf8(chinese_word) + "\n")
fd.flush()
return canadiate
except:
pass
return None

def run( input_file, out_file):
generate_pinyin_map( )
generate_synonym( input_file, out_file )

if 1 and __name__ == "__main__":
args = sys.argv[1:]
if len(args) < 2:
print "USage: ./synonym.py <input_wordlist> <output_file> "
else:
run(args[0], args[1]);
  • 0
    点赞
  • 0
    收藏
    觉得还不错? 一键收藏
  • 0
    评论

“相关推荐”对你有帮助么?

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值