import unicodedata
defremove_control_character(content):
result =""for ch in content:if ch notin("\t","\n","\r")and unicodedata.category(ch)in("Cc","Cf"):continue#0xfffd主要是windows的一些字符编码在unicode中没有对应的,相当于[UNK]iford(ch)in(0,0xfffd):continue
result += ch
return result
defformat_white_space(content):
result =""
last =1
content =list(content)for index inrange(len(content)):
ch = content[index]if ch in("\n","\t","\r")or unicodedata.category(ch)in("Zs","Zp","Zl"):if last ==0and index >0and is_punctuation(content[index-1])==False:#可以换成其他字符串
ch ="# "else:
ch =""
last =1else:
last =0
result += ch
return result
控制字符去除import unicodedatadef remove_control_character(content): result = "" for ch in content: if ch not in ("\t", "\n", "\r") and unicodedata.category(ch) in ("Cc", "Cf"): continue if ord(ch) in (0, 0xfffd): .