1 # -*- coding: utf-8 -*-
2 """
3 config.py
4
5 Created by Manabu Terada, CMScom on 2009-08-08.
6 """
7 import re
8
9 STOP_WORD = []
10
11 ## Setting, adding langs.
12 rangetable = dict(
13 # ascii=u"a-zA-Z0-9_",
14 # digit=u"\d",
15
16 # U+AC00-D7AF Hangul Syllables ハングル音節文字
17 hangul=u"\uAC00-\uD7AF",
18
19 # U+30A0-30FF Katakana 片仮名
20 # U+3040-309F Hiragana 平仮名
21 # kana=u"\u3040-\u30FF",
22 # hiragana=u"\u3040-\u309F\u30FC",
23 # katakana=u"\u30A0-\u30FF",
24
25 # U+4E00-9FFF CJK Unified Ideographs CJK統合漢字
26 # U+3400-4DBF CJK Unified Ideographs Extension A CJK統合漢字拡張A
27 # U+F900-FAFF CJK Compatibility Ideographs CJK互換漢字
28 # ideo=u"\u4E00-\u9FFF\u3400-\u4DBF\uF900-\uFAFF",
29
30 cj=u"\u3040-\u30FF\u4E00-\u9FFF\u3400-\u4DBF\uF900-\uFAFF",
31 thai=u"\u0E00-\u0E7F", # U+0E00-0E7F Thai タイ文字
32 )
33 ## End of setting.
34
35
36 ## Splitting core.
37 ps = rangetable.values()
38 allp = u"".join(ps)
39 glob_true = u"[^%s]([^%s]|[\*\?])*|" % (allp, allp) + u"|".join([u"[%s]+" % (x, ) for x in ps])
40
41 glob_false = u"[^%s]+|" % allp + u"|".join(u"[%s]+" % x for x in ps)
42
43 rx_all = re.compile(ur"[%s]" % allp, re.UNICODE)
44 rx_U = re.compile(r"\w+", re.UNICODE)
45 rxGlob_U = re.compile(r"\w+[\w*?]*", re.UNICODE)
46
47 rx_L = re.compile(r"\w+", re.LOCALE)
48 rxGlob_L = re.compile(r"\w+[\w*?]*", re.LOCALE)
49
50 # pattern = re.compile(u"[a-zA-Z0-9_]+|[\uac00-\ud7af]+|[\u4E00-\u9FFF\u3400-\u4dbf\uf900-\ufaff\u3040-\u30ff]+", re.UNICODE)
51 # pattern_g = re.compile(u"[a-zA-Z0-9_]+[*?]*|[\u4E00-\u9FFF\u3400-\u4dbf\uf900-\ufaff\u3040-\u30ff\uac00-\ud7af]+[*?]*", re.UNICODE)
52
53 pattern = re.compile(glob_false, re.UNICODE)
54 pattern_g = re.compile(glob_true, re.UNICODE)
55
2 """
3 config.py
4
5 Created by Manabu Terada, CMScom on 2009-08-08.
6 """
7 import re
8
9 STOP_WORD = []
10
11 ## Setting, adding langs.
12 rangetable = dict(
13 # ascii=u"a-zA-Z0-9_",
14 # digit=u"\d",
15
16 # U+AC00-D7AF Hangul Syllables ハングル音節文字
17 hangul=u"\uAC00-\uD7AF",
18
19 # U+30A0-30FF Katakana 片仮名
20 # U+3040-309F Hiragana 平仮名
21 # kana=u"\u3040-\u30FF",
22 # hiragana=u"\u3040-\u309F\u30FC",
23 # katakana=u"\u30A0-\u30FF",
24
25 # U+4E00-9FFF CJK Unified Ideographs CJK統合漢字
26 # U+3400-4DBF CJK Unified Ideographs Extension A CJK統合漢字拡張A
27 # U+F900-FAFF CJK Compatibility Ideographs CJK互換漢字
28 # ideo=u"\u4E00-\u9FFF\u3400-\u4DBF\uF900-\uFAFF",
29
30 cj=u"\u3040-\u30FF\u4E00-\u9FFF\u3400-\u4DBF\uF900-\uFAFF",
31 thai=u"\u0E00-\u0E7F", # U+0E00-0E7F Thai タイ文字
32 )
33 ## End of setting.
34
35
36 ## Splitting core.
37 ps = rangetable.values()
38 allp = u"".join(ps)
39 glob_true = u"[^%s]([^%s]|[\*\?])*|" % (allp, allp) + u"|".join([u"[%s]+" % (x, ) for x in ps])
40
41 glob_false = u"[^%s]+|" % allp + u"|".join(u"[%s]+" % x for x in ps)
42
43 rx_all = re.compile(ur"[%s]" % allp, re.UNICODE)
44 rx_U = re.compile(r"\w+", re.UNICODE)
45 rxGlob_U = re.compile(r"\w+[\w*?]*", re.UNICODE)
46
47 rx_L = re.compile(r"\w+", re.LOCALE)
48 rxGlob_L = re.compile(r"\w+[\w*?]*", re.LOCALE)
49
50 # pattern = re.compile(u"[a-zA-Z0-9_]+|[\uac00-\ud7af]+|[\u4E00-\u9FFF\u3400-\u4dbf\uf900-\ufaff\u3040-\u30ff]+", re.UNICODE)
51 # pattern_g = re.compile(u"[a-zA-Z0-9_]+[*?]*|[\u4E00-\u9FFF\u3400-\u4dbf\uf900-\ufaff\u3040-\u30ff\uac00-\ud7af]+[*?]*", re.UNICODE)
52
53 pattern = re.compile(glob_false, re.UNICODE)
54 pattern_g = re.compile(glob_true, re.UNICODE)
55