数据处理-清洗

 ---分词--

使用Python清洗文本数据_小北的北的博客-CSDN博客

readandfenci_test1

# # In case of import errors
# ! pip install nltk
# ! pip install textblob
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import re
import nltk
import string
from nltk.corpus import stopwords
# # In case of any corpus are missing 
# download all-nltk
#nltk.download()
#df = pd.read_csv('train.csv')
df =  "America like South Africa is a traumatised sick country - in different ways of course - but still messed up."
stop_words = stopwords.words("english")
wordnet = WordNetLemmatizer()
def text_preproc(x):
  x = x.lower()
  x = ' '.join([word for word in x.split(' ') if word not in stop_words])
  x = x.encode('ascii', 'ignore').decode()
  x = re.sub(r'https*\S+', ' ', x)
  x = re.sub(r'@\S+', ' ', x)
  x = re.sub(r'#\S+', ' ', x)
  x = re.sub(r'\'\w+', '', x)
  x = re.sub('[%s]' % re.escape(string.punctuation), ' ', x)
  x = re.sub(r'\w*\d+\w*', '', x)
  x = re.sub(r'\s{2,}', ' ', x)
  return x
res = text_preproc(text)
res
###df['clean_text'] = df.text.apply(text_preproc)##apply有多个返回值

输出里面有换行--替换操作---

text = text.replace('\n', ' ')

anaconda 下安装nltk - 知乎

如何安装NLTK（MacBook, Anaconda) - 知乎