第四章 新闻爬取

import requests
from lxml import etree
import json
url="https://www.ptpress.com.cn/newsInfo/getCurrentAffairsForPortal"
headers = {
        "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36",

        }
response=requests.get(url,headers=headers)
content=response.content.decode("utf-8")
print(content)
data=json.loads(content)
data_1=data['data']
data_2=data_1['news6']



##存储
import pymongo
client=pymongo.MongoClient("mongodb://localhost:27017/")
db=client.pythondb

MONGO_URL='localhost'
client=pymongo.MongoClient(MONGO_URL)
db=client['pythondb']
collection=db.test
for i in data_2:
    data_time=i['created']
    mainTitle=i['mainTitle']
    a={'created':data_time,
       'mainTitle':mainTitle
            
            }
    collection.insert_one(a)
    print(a)

MongoDB需要自行下载