使用NLTK进行英文分词

使用NLTK进行英文分词

import nltk
import re
english='C:\\Users\\pc\\CapStone\\english.txt'
with open(english,'r',encoding='utf-8') as file:
    u=file.read()
str=re.sub('[^\w ]','',u)
print(nltk.word_tokenize(str))
print(nltk.pos_tag(nltk.word_tokenize(str))) #对分完词的结果进行词性标注

#result

['Trump', 'was', 'born', 'and', 'raised', 'in', 'the', 'New', 'York', 'City', 'borough', 'of', 'Queens', 'and', 'received', 'an', 'economics', 'degree', 'from', 'the', 'Wharton', 'School', 'He', 'was', 'appointed', 'president', 'of', 'his', 'familys', 'real', 'estate', 'business', 'in', '1971', 'renamed', 'it', 'The', 'Trump', 'Organization', 'and', 'expanded', 'it', 'from', 'Queens', 'and', 'Brooklyn', 'into', 'Manhattan', 'The', 'company', 'built', 'or', 'renovated', 'skyscrapers', 'hotels', 'casinos', 'and', 'golf', 'courses', 'Trump', 'later', 'started', 'various', 'side', 'ventures', 'including', 'licensing', 'his', 'name', 'for', 'real', 'estate', 'and', 'consumer', 'products', 'He', 'managed', 'the', 'company', 'until', 'his', '2017', 'inauguration', 'He', 'coauthored', 'several', 'books', 'including', 'The', 'Art', 'of', 'the', 'Deal', 'He', 'owned', 'the', 'Miss', 'Universe', 'and', 'Miss', 'USA', 'beauty', 'pageants', 'from', '1996', 'to', '2015', 'and', 'he', 'produced', 'and', 'hosted', 'The', 'Apprentice', 'a', 'reality', 'television', 'show', 'from', '2003', 'to', '2015', 'Forbes', 'estimates', 'his', 'net', 'worth', 'to', 'be', '31', 'billion']
[('Trump', 'NNP'), ('was', 'VBD'), ('born', 'VBN'), ('and', 'CC'), ('raised', 'VBN'), ('in', 'IN'), ('the', 'DT'), ('New', 'NNP'), ('York', 'NNP'), ('City', 'NNP'), ('borough', 'NN'), ('of', 'IN'), ('Queens', 'NNP'), ('and', 'CC'), ('received', 'VBD'), ('an', 'DT'), ('economics', 'NNS'), ('degree', 'NN'), ('from', 'IN'), ('the', 'DT'), ('Wharton', 'NNP'), ('School', 'NNP'), ('He', 'PRP'), ('was', 'VBD'), ('appointed', 'VBN'), ('president', 'NN'), ('of', 'IN'), ('his', 'PRP$'), ('familys', 'JJ'), ('real', 'JJ'), ('estate', 'NN'), ('business', 'NN'), ('in', 'IN'), ('1971', 'CD'), ('renamed', 'VBD'), ('it', 'PRP'), ('The', 'DT'), ('Trump', 'NNP'), ('Organization', 'NNP'), ('and', 'CC'), ('expanded', 'VBD'), ('it', 'PRP'), ('from', 'IN'), ('Queens', 'NNP'), ('and', 'CC'), ('Brooklyn', 'NNP'), ('into', 'IN'), ('Manhattan', 'NNP'), ('The', 'DT'), ('company', 'NN'), ('built', 'VBD'), ('or', 'CC'), ('renovated', 'VBD'), ('skyscrapers', 'NNS'), ('hotels', 'NNS'), ('casinos', 'NNS'), ('and', 'CC'), ('golf', 'NN'), ('courses', 'NNS'), ('Trump', 'NNP'), ('later', 'RB'), ('started', 'VBD'), ('various', 'JJ'), ('side', 'NN'), ('ventures', 'NNS'), ('including', 'VBG'), ('licensing', 'VBG'), ('his', 'PRP$'), ('name', 'NN'), ('for', 'IN'), ('real', 'JJ'), ('estate', 'NN'), ('and', 'CC'), ('consumer', 'NN'), ('products', 'NNS'), ('He', 'PRP'), ('managed', 'VBD'), ('the', 'DT'), ('company', 'NN'), ('until', 'IN'), ('his', 'PRP$'), ('2017', 'CD'), ('inauguration', 'NN'), ('He', 'PRP'), ('coauthored', 'VBD'), ('several', 'JJ'), ('books', 'NNS'), ('including', 'VBG'), ('The', 'DT'), ('Art', 'NN'), ('of', 'IN'), ('the', 'DT'), ('Deal', 'NNP'), ('He', 'PRP'), ('owned', 'VBD'), ('the', 'DT'), ('Miss', 'NNP'), ('Universe', 'NNP'), ('and', 'CC'), ('Miss', 'NNP'), ('USA', 'NNP'), ('beauty', 'NN'), ('pageants', 'NNS'), ('from', 'IN'), ('1996', 'CD'), ('to', 'TO'), ('2015', 'CD'), ('and', 'CC'), ('he', 'PRP'), ('produced', 'VBD'), ('and', 'CC'), ('hosted', 'VBD'), ('The', 'DT'), ('Apprentice', 'NNP'), ('a', 'DT'), ('reality', 'NN'), ('television', 'NN'), ('show', 'NN'), ('from', 'IN'), ('2003', 'CD'), ('to', 'TO'), ('2015', 'CD'), ('Forbes', 'NNP'), ('estimates', 'VBZ'), ('his', 'PRP$'), ('net', 'JJ'), ('worth', 'NN'), ('to', 'TO'), ('be', 'VB'), ('31', 'CD'), ('billion', 'CD')]

 

 

english.txt内容：

Trump was born and raised in the New York City borough of Queens and received an economics degree from the Wharton School. He was appointed president of his family's real estate business in 1971, renamed it The Trump Organization, and expanded it from Queens and Brooklyn into Manhattan. The company built or renovated skyscrapers, hotels, casinos, and golf courses. Trump later started various side ventures, including licensing his name for real estate and consumer products. He managed the company until his 2017 inauguration. He co-authored several books, including The Art of the Deal. He owned the Miss Universe and Miss USA beauty pageants from 1996 to 2015, and he produced and hosted The Apprentice, a reality television show, from 2003 to 2015. Forbes estimates his net worth to be $3.1 billion.