爬虫requests、BeautifulSoup 自我总结

import requests, random, bs4

for x in range(10):
    url = 'https://movie.douban.com/top250?start=' + str(x*25) + '&filter='
    res = requests.get(url)
    bs = bs4.BeautifulSoup(res.text, 'html.parser')
    bs = bs.find('ol', class_="grid_view")
    for titles in bs.find_all('li'):
        num = titles.find('em',class_="").text
        #查找序号
        title = titles.find('span', class_="title").text
        #查找电影名
        tes = titles.find('span',class_="inq").text
        #查找推荐语
        comment = titles.find('span',class_="rating_num").text
        #查找评分
        url_movie = titles.find('a')['href']

        print(num + '.' + title + '——' + comment + '\n' + '推荐语：' + tes +'\n' + url_movie)

下厨房爬虫

import requests
#引用requests库
from bs4 import BeautifulSoup
引用BeautifulSoup库

res_foods = requests.get('http://www.xiachufang.com/explore/')# 获取数据
bs_foods = BeautifulSoup(res_foods.text,'html.parser')# 解析数据

tag_name = bs_foods.find_all('p',class_='name')
# 查找包含菜名和URL的<p>标签
tag_ingredients = bs_foods.find_all('p',class='ing ellipsis')
# 查找包含食材的<p>标签

list_all = []
# 创建一个空列表，用于存储信息

for i in range(len(tag_name)):
# 启动一个循环，次数等于菜名的数量
    list_food = [tag_name[x].text[18:-14],tag_name[x].find('a')['href'],tag_ingredients[x].tsxt[1:-1]]
# 提取信息，封装为列表。注意此处[18:-14]切片和之前不同，是因为此处使用的是<p>标签，而之前是<a>  
list_all.append(list_food)
# 将信息添加进list_all
print(list_all)
# 打印

#以下是另外一种解法
list_foods = bs_foods.find_all('div',class_='info pure-u')
#查找最小父级标签
list_all = []
#创建
for food in list_foods:

    tag_a = food.find('a')
    # 提取第0个父级标签中的<a>标签
    name = tag_a.text[17:-13]
    # 菜名，使用[17:-13]切掉了多余的信息
    URL = 'http://www.xiachufang.com'+tag_a['href']
    # 获取URL
    tag_p = food.find('p',class_='ing ellipsis')
    # 提取第0个父级标签中的<p>标签
    ingredients = tag_p.text[1:-1]
    # 食材，使用[1:-1]切掉了多余的信息
    list_all.append([name,URL,ingredients])
    # 将菜名、URL、食材，封装为列表，添加进list_all

print(list_all)

豆瓣网爬虫

import requests
from bs4 import BeautifulSoup
for x in range(10):
    res = requests.get('https://movie.douban.com/top250?start=' + str(x*25) + '&filter=')
    bs = BeautifulSoup(res.text,;html.parser')
   tag_num = bs.find_all('div',class_='item')
   tag_comment = bs.find_all('div',class_='str')
   tag_word = bs.find_all('span',class_='inq')
   list_all = []
    for x in range(len(tag_num)):
        if tag_num[x].text[2:5] =='' or tag_num[x].text[2:5]=='':
            list_movie = [tag_num[x].text[2:5],tag_num[x].find('img')('alt'),tag_comment[x].text[2:5],tag_num[x].find('a')['href']]
        else:
            list_movie = [tag_num[x].text[2:5],tag_num[x].find('img')('alt'),tag_comment[x].text[2:5],tag_word[x].text,tag_num[x].find('a')['href']]
        list_all.append(list_movie)
        print(list_all)