#!/usr/bin/env python2.7
# -*-encoding=utf8-*-
# @Time : 2018/4/9 16:52
# @Author : syj
# @File : text_clean_helper.py
import sys
import re
try:
# Wide UCS-4 build
emoji_re = re.compile(u'['
u'\U0001F300-\U0001F64F'
u'\U0001F680-\U0001F6FF'
u'\u2600-\u2B55]+',
re.UNICODE)
except re.error:
# Narrow UCS-2 build
emoji_re = re.compile(u'('
u'\ud83c[\udf00-\udfff]|'
u'\ud83d[\udc00-\ude4f\ude80-\udeff]|'
u'[\u2600-\u2B55])+',
re.UNICODE)
def clean_emoji(query):
result = emoji_re.sub("", query)
return result
def clean_200B(query):
result = re.sub(u"\u200B", "", query)
return result
def clean_punctuation(query):
PAT1 = "[\s+\!_$%^*(+\"\')]+|[+——()?【】。“”!?、~@#¥%……&*();:{}~……《》<>「」]+"
PAT2 = "[\s+,.,/]+"
PAT = PAT1 + "|" + PAT2
prog1 = re.compile(PAT)
result = prog1.sub(" ", query)
return result
def clean_punctuation_query_utf8(query):
PAT1 = "[\s+\!_$%^*(+\"\')]+|[+——()?【】。“”!?、~@#¥%……&*();:{}~……《》<>「」]+"
PAT2 = "[\s+,.,/]+"
PAT = PAT1 + "|" + PAT2
PAT = PAT.decode("utf8")
prog1 = re.compile(PAT)
result = prog1.sub(" ", query)
result = result.strip()
return result
文本清洗代码
最新推荐文章于 2024-07-23 17:46:47 发布