网络数据采集 (爬虫)----＞ CSS,Excel(2)

网络数据采集 (爬虫)----> CSS,Excel(2)

练习：获取搜狐首页的新闻标题和链接

通过CSS选择器获取页面元素
首先安装pip install BeautifulSoup4
BeautifulSoup ----> select/ select_one

import requests
# 导入bs4
import bs4

resp = requests.get('https://www.sohu.com/')
if resp.status_code == 200:
    # 创建bs4对象，获得页面的html代码，用html.parser默认的html解析器解析
    soup = bs4.BeautifulSoup(resp.text, 'html.parser')
    # 在返回对象中用select选取a标签，select可以选取多个标签，select_one选一个标签
    a_list = soup.select('a')
    for a in a_list:
        # 判断属性href和title是否在选出的列表中
        if 'href' in a.attrs and 'title' in a.attrs:
            # 写出属性所对应的值
            print(a.attrs['href'], a.attrs['title'])

练习：获取豆瓣电影top250的电影标题
获取豆瓣电影Top250电影名称

BeautifulSoup
    ~ select ---> 根据选择器获取元素的列表
    ~ select_one ---> 根据选择器获取第一个匹配的元素

bs4.element.Tag ---> Tag对象就对应页面上的一个标签
    ~ text ---> 获取标签里面的文本内容
    ~ attrs ---> 获取标签的属性（字典）

官方文档：https://beautifulsoup.readthedocs.io/zh_CN/v4.4.0/

CSV文件 - Comma Seperate Value(逗号分隔值文件)
    ~ import csv
    ~ writer ---> writerow([...])  -------->写一行
    ~ reader --->用for .. in ..循环，一行一行读下来

# 获取User-Agent的方法：首先打开你的浏览器输入：about:version。
headers = {
    'User-Agent': '--------- '
}
url = f'https://movie.douban.com/top250?'

for page in range(10):
    resp = requests.get(url=url, headers=headers)
    if resp.status_code == 200:
        soup = bs4.BeautifulSoup(resp.text, 'html.parser')
# 在原网页通过检查，右键copy, copy.select
        spans_list = soup.select('div > div.info > div.hd > a > span:nth-child(1)')
        for span in spans_list:
            print(span.text)
    time.sleep(random.randint(2, 5))

练习：将数据写入Excel文件
Excel 2007 以前版本 —> xls ------->xlwt / xlrd
Excel 2007 以后版本 ----->xlsx ------> openpyxl(新式的Excel表）
首先要安装包，在Terminal中输入install xlwt

import random
import time
import xlwt
import bs4
import requests

wb = xlwt.Workbook()
# 通过工作簿的add_sheet方法添加工作表
sheet = wb.add_sheet('Top250')
# 向表中写入数据
sheet.write(0, 0, '标题')
sheet.write(0, 1, '评分')
sheet.write(0, 2, '名言')
row = 1
for page in range(10):
   resp = requests.get(
       url=f'https://movie.douban.com/top250?start={page * 25}',
       headers={
           'User-Agent': '------------- '
                     
       }
   )
   if resp.status_code == 200:
       soup = bs4.BeautifulSoup(resp.text, 'html.parser')
       title_spans_list = soup.select('div.info > div.hd > a > span:nth-child(1)')
       rating_spans_list = soup.select('div.info > div.bd > div > span.rating_num')
       saying_spans_list = soup.select('div > div.info > div.bd > p.quote > span')
       for title_span, rating_span, saying_span in zip(title_spans_list, rating_spans_list, saying_spans_list ):
           sheet.write(row, 0, title_span.text)
           sheet.write(row, 1, rating_span.text)
           sheet.write(row, 2, saying_span.text)
           row += 1
   else:
       print('无法获取数据！')
   time.sleep(random.randint(1, 3))
# 保存工作簿
wb.save('豆瓣电影.xls')

作业：获取豆瓣电影top250,电影的编号，标题，评分，名言，时长，类型

import random
import time
import xlwt
# import openpyxl
import bs4
import requests

url = 'https://movie.douban.com/top250'
headers = {
            'User-Agent': '----------------',
}


def fetch_movie_detail(url):
    """根据指定的url抓取电影的详细信息（类型和时长）"""
    resp = requests.get(url=url, headers=headers)
    """两个soup都是函数里的局部变量"""
    soup = bs4.BeautifulSoup(resp.text, 'html.parser')
    """要求span标签有property属性且值等于v:genre"""
    genre_spans = soup.select('span[property="v:genre"]')
    """这个标签用select_one取，因为电影时长只有一个"""
    run_time_span = soup.select_one('span[property="v:runtime"]')
    """用/把多个字符串合成一个字符串"""
    genre = '/'.join([genre_span.text for genre_span in genre_spans])
    runtime = run_time_span.attrs['content']
    return [genre, runtime]


def main():
    wb = xlwt.Workbook()
    # wb = openpyxl.Workbook() 
    sheet = wb.add_sheet('Top250')
    # sheet = wb.create_sheet('Top250')
    number = 0
    headers1 = ('编号', '片名', '评分', '名言', '类型', '时长')
    for col, header in enumerate(headers1):
        sheet.write(number, col, header)
        # sheet.cell(number + 1,col + 1, header)
        # openpyxl的行和列都是从一开始数的，因此 + 1
   
    try:
        for page in range(10):
            resp = requests.get(url=url, headers=headers)
		    if resp.status_code == 200:
		        soup = bs4.BeautifulSoup(resp.text, 'html.parser')
		        anchors = soup.select('div.info > div.hd> a')
		        title_spans_list = soup.select('div.info > div.hd > a > span:nth-child(1)')
		        rating_spans_list = soup.select('div.info > div.bd > div > span.rating_num')
		        quote_spans_list = soup.select('div > div.info > div.bd > p.quote > span')
		        for anchor, title_span, rating_span, quote_span in zip(anchors, title_spans_list, rating_spans_list, quote_spans_list ):
		            number += 1
		            detail_url = anchor.attrs['href']
		            movie_infos = [number, title_span.text, rating_span.text, quote_span.text]
		            movie_infos += fetch_movie_detail(detail_url)
		            print(movie_infos)
		            for col, info in enumerate(movie_infos):
		                sheet.write(number, col, info)
		                # sheet.cell(number + 1,col + 1, info)
		    else:
		    # 格式化，请求失败的话，发响应状态码
		        print(f'请求失败，响应状态码：{resp.status_code}')
		    time.sleep(random.randint(1, 3))
    # 保存工作簿
    wb.save('豆瓣电影.xls')
    # wb.save('豆瓣电影.xlsx')


if __name__ == '__main__':
    main()