from urllib.parse import urlencode
from requests.exceptions import RequestException
import requests
import json
import re
from bs4 import BeautifulSoup
from config import *
import pymongo
client = pymongo.MongoClient(MONGO_URL)
db = client(MONGO_DB)
def save_to_mogo(result):
if db[MONGO_TABLE].insert(result):
print("储存成功",result)
return True
return False
def get_page_index(offset,keyword):
print(1)
data={
'offset': offset,
'format': 'json',
'keyword': keyword,
'autoload': 'true',
'count': '20',
'cur_tab': 1,
}
url ='https://www.toutiao.com/search_content/?'+urlencode(data)
r=requests.get(url)
try:
if r.status_code ==200:
r.encoding = r.apparent_encoding
return r.text
return None
except RequestException:
print("请求页面错误")
return None
def parse_page_index(html):
data =json.loads(html)
if data and "data" in data.keys():
for item in data.get("data"):
yield item.get("article_url")
def get_pic_url(url):
try:
r=requests.get(url)
if r.status_code==200:
return r.text
return None
except RequestException:
return None
def get_pic(html,url):
soup=BeautifulSoup(html,'html.parser')
title=soup.select('title')[0].get_text()
images_pattern = re.compile('gallery = (.*?);',re.S)
result = re.search(images_pattern,html)
if result:
data = json.loads(result.group(1))
if data and 'sub_images' in data.keys():
sub_images = data.get('sub_images')
images=[item.get('url') for item in sub_images]
return {
'title':title,
'images':images,
'url':url
}
def main():
html= get_page_index(0, "街拍")
parse_page = parse_page_index(html)
for url in parse_page:
html =get_pic_url(url)
if html:
if get_pic(html,url) is not None:
result = get_pic(html,url)
save_to_mogo(result)
if __name__ == '__main__':
main()
python_爬虫今日头条
最新推荐文章于 2024-07-21 18:03:04 发布