Transformers学习4

from transformers import pipeline
from pprint import pprint

"""
掩码语言建模
"""
nlp = pipeline("fill-mask")
pprint(nlp(f"HuggingFace is creating a {nlp.tokenizer.mask_token} that the community uses to solve NLP tasks."))

from transformers import AutoModelWithLMHead,AutoTokenizer
import torch

tokenizer = AutoTokenizer.from_pretrained("distilbert-base-cased")
model = AutoModelWithLMHead.from_pretrained("distilbert-base-cased",return_dict=True)
sequence = f"Distilled models are smaller than the models they mimic. Using them instead of the large versions would help {tokenizer.mask_token} our carbon footprint."
input = tokenizer.encode(sequence,return_tensors="pt")
mask_token_index = torch.where(input==tokenizer.mask_token_id)[1]
token_logits = model(input).logits
mask_token_logits = token_logits[0,mask_token_index,:]
top_5_tokens = torch.topk(mask_token_logits,5,dim=1).indices[0].tolist()
for token in top_5_tokens:
    print(sequence.replace(tokenizer.mask_token,tokenizer.decode([token])))

"""
因果语言建模
"""
from transformers import AutoModelWithLMHead,AutoTokenizer,top_k_top_p_filtering
import torch
import torch.nn.functional as F

tokenizer = AutoTokenizer.from_pretrained("gpt2")
model = AutoModelWithLMHead.from_pretrained("gpt2",return_dict=True)
sequence = f"Hugging Face is based in DUMBO, New York City, and "
input_ids = tokenizer.encode(sequence,return_tensors="pt")
next_token_logits = model(input_ids).logits[:,-1,:]
filtered_next_token_logits = top_k_top_p_filtering(next_token_logits,top_k=50,top_p=1.0)
probs = F.softmax(filtered_next_token_logits,dim=-1)
next_token = torch.multinomial(probs,num_samples=1)
generated = torch.cat([input_ids,next_token],dim=-1)
resulting_string = tokenizer.decode(generated.tolist()[0])
print(resulting_string)

"""
文字生成
"""
from transformers import pipeline
text_generator = pipeline("text-generation")
print(text_generator("As far as I am concerned, I will",max_length=50,do_sample=False))

from transformers import AutoModelWithLMHead,AutoTokenizer
model = AutoModelWithLMHead.from_pretrained("xlnet-base-cased",return_dict=True)
tokenizer = AutoTokenizer.from_pretrained("xlnet-base-cased")
PADDING_TEXT = """In 1991, the remains of Russian Tsar Nicholas II and his family
 (except for Alexei and Maria) are discovered.
The voice of Nicholas's young son, Tsarevich Alexei Nikolaevich, narrates the
remainder of the story. 1883 Western Siberia,
a young Grigori Rasputin is asked by his father and a group of men to perform magic.
Rasputin has a vision and denounces one of the men as a horse thief. Although his
 father initially slaps him for making such an accusation, Rasputin watches as the
 man is chased outside and beaten. Twenty years later, Rasputin sees a vision of
 the Virgin Mary, prompting him to become a priest. Rasputin quickly becomes famous,
 with people, even a bishop, begging for his blessing. <eod> </s> <eos>"""
prompt = "Today the weather is really nice and I am planning on "
inputs = tokenizer.encode(PADDING_TEXT + prompt,add_special_tokens=False,return_tensors="pt")
prompt_length = len(tokenizer.decode(inputs[0],skip_special_tokens=True,clean_up_tokenization_spaces=True))
outputs = model.generate(inputs,max_length=250,do_sample=True,top_p=0.95,top_k=60)
generated = prompt + tokenizer.decode(outputs[0])[prompt_length:]
print(generated)

"""
命名实体识别
"""
from transformers import pipeline
nlp = pipeline("ner")
sequence = "Hugging Face Inc. is a company based in New York City. Its headquarters are in DUMBO, therefore very close to the Manhattan Bridge which is visible from the window."
print(nlp(sequence))

from transformers import AutoModelForTokenClassification,AutoTokenizer
import torch
model = AutoModelForTokenClassification.from_pretrained("dbmdz/bert-large-cased-finetuned-conll03-english", return_dict=True)
tokenizer = AutoTokenizer.from_pretrained("bert-base-chinese")
label_list = [
     "O",       # Outside of a named entity
     "B-MISC",  # Beginning of a miscellaneous entity right after another miscellaneous entity
     "I-MISC",  # Miscellaneous entity
     "B-PER",   # Beginning of a person's name right after another person's name
     "I-PER",   # Person's name
     "B-ORG",   # Beginning of an organisation right after another organisation
     "I-ORG",   # Organisation
     "B-LOC",   # Beginning of a location right after another location
     "I-LOC"    # Location
    ]
sequence = "Hugging Face Inc. is a company based in New York City. Its headquarters are in DUMBO, therefore very close to the Manhattan Bridge which is visible from the window."
tokens = tokenizer.tokenize(tokenizer.decode(tokenizer.encode(sequence)))
inputs = tokenizer.encode(sequence,return_tensors="pt")
outputs = model(inputs).logits
predictions = torch.argmax(outputs,dim=2)
print([(token, label_list[prediction]) for token, prediction in zip(tokens, predictions[0].numpy())])

"""
自动摘要
"""
from transformers import pipeline
summarizer = pipeline("summarization")
ARTICLE = """Foreign Network November 19(cnn)18 reported that several Trump administration officials have begun to quietly contact
the Biden transition team. In addition to current Trump administration officials,
there are some who have just left in recent months.
American media believe that this phenomenon shows that Trump's refusal to give in to the election
and the continued obstruction of the White House have begun to frustrate those under the Trump administration.
The U.S. General Administration has yet to acknowledge Biden's victory,
which is why Biden and his team have been barred from contacting federal agencies,
have no access to funds to support new government appointments and access confidential intelligence information.
A Trump administration official confirmed to cnn on the 18th that Trump administration insiders have had informal contact with the Biden team ,
" they know what we mean, what we can say, and what we can't say ."
In addition, former Trump administration officials have said they regard their ties to the Biden team as national responsibility,
 not partisan. The talks weren't as detailed as previous information reports during the formal transition period,
 but could at least help Biden's transition team understand what they might be dealing with later, sources said.
 Former White House officials who left the White House a few months ago also revealed that he privately emailed people and said he was willing to help Biden. """
print(summarizer(ARTICLE, max_length=20, min_length=5, do_sample=False))

from transformers import AutoModelWithLMHead,AutoTokenizer
model = AutoModelWithLMHead.from_pretrained("t5-base",return_dict=True)
tokenizer = AutoTokenizer.from_pretrained("t5-base")
inputs = tokenizer.encode("summarize:"+ARTICLE,return_tensors="pt",max_length=1000)
outputs = model.generate(inputs,max_length=20,min_length=10,length_penalty=2.0,num_beams=4,early_stopping=True)
print(outputs)