文本清洗代码

#!/usr/bin/env python2.7
# -*-encoding=utf8-*-
# @Time    : 2018/4/9 16:52
# @Author  : syj
# @File    : text_clean_helper.py

import sys
import re

try:
    # Wide UCS-4 build
    emoji_re = re.compile(u'['
        u'\U0001F300-\U0001F64F'
        u'\U0001F680-\U0001F6FF'
        u'\u2600-\u2B55]+',
        re.UNICODE)
except re.error:
    # Narrow UCS-2 build
    emoji_re = re.compile(u'('
        u'\ud83c[\udf00-\udfff]|'
        u'\ud83d[\udc00-\ude4f\ude80-\udeff]|'
        u'[\u2600-\u2B55])+',
        re.UNICODE)

def clean_emoji(query):
    result = emoji_re.sub("", query)
    return result

def clean_200B(query):
    result = re.sub(u"\u200B", "", query)
    return result

def clean_punctuation(query):
    PAT1 = "[\s+\!_$%^*(+\"\')]+|[+——()?【】。“”！？、~@#￥%……&*（）；：{}~……《》<>「」]+"
    PAT2 = "[\s+,.，/]+"
    PAT = PAT1 + "|" + PAT2
    prog1 = re.compile(PAT)
    result = prog1.sub(" ", query)
    return result

def clean_punctuation_query_utf8(query):
    PAT1 = "[\s+\!_$%^*(+\"\')]+|[+——()?【】。“”！？、~@#￥%……&*（）；：{}~……《》<>「」]+"
    PAT2 = "[\s+,.，/]+"
    PAT = PAT1 + "|" + PAT2
    PAT = PAT.decode("utf8")
    prog1 = re.compile(PAT)
    result = prog1.sub(" ", query)
    result = result.strip()
    return result