爬虫案例4: parsel 模块的运用

最新推荐文章于 2024-05-09 09:40:15 发布

刹那间的回眸x.y

最新推荐文章于 2024-05-09 09:40:15 发布

阅读量960

点赞数 4

分类专栏： python爬虫案例文章标签：爬虫

本文链接：https://blog.csdn.net/weixin_47817248/article/details/136635490

版权

python爬虫案例专栏收录该内容

5 篇文章 0 订阅

订阅专栏

目标页面: https://www.shanghairanking.cn/rankings/bcur/2023
打印在终端

import requests
import json
from urllib.parse import urljoin
from parsel import Selector

url = 'https://www.shanghairanking.cn/rankings/bcur/2023'

headers = {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36'
}

data = []

def parse():
    resp = requests.get(url=url, headers=headers)
    resp.encoding = 'utf8' 
    if resp.status_code == 200:
        html_data = resp.text
        selector = Selector(text=html_data)
        
        scool_list = selector.xpath('//*[@id="content-box"]/div[2]/table/tbody/tr').getall()
        for itemn in scool_list:
            scool = Selector(text=itemn) # 这里item是字符串的html
            data.append({
                '排名': scool.xpath('.//td[1]//text()').get().strip(),
                '校名': scool.css('.name-cn::text').get().strip(),
                '详情': urljoin(url, scool.css('a::attr(href)').get().strip()),
                '城市': scool.xpath('.//td[3]/text()').get().strip(),
                '评分': scool.xpath('.//td[5]/text()').get().strip()
            }) 
        print(json.dumps(data, indent=4, ensure_ascii=False))
    else:
        print('状态码不为200: ', resp.text)


if __name__ == '__main__':
    parse()

在这里插入图片描述

目标页面: https://movie.douban.com/top250
打印终端

import requests
import json
from urllib.parse import urljoin
from parsel import Selector


url = 'https://movie.douban.com/top250?'

headers = {
    'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36'
}

data = []

def parse():
    resp = requests.get(url, headers=headers)
    if resp.status_code == 200:
        html_data = resp.text
        selector = Selector(text=html_data)
        move_html_list = selector.css('#content > div > div.article > ol > li').getall()
        for item in move_html_list:
            move = Selector(text=item)
            data.append({
                '排名': move.xpath('.//em/text()').get().strip(),
                '电影名称': move.css('div > div.info > div.hd > a > span:nth-child(1)::text').get().strip(),
                '评分': move.css('div > div.info > div.bd > div > span.rating_num::text').get().strip(),
                '详情链接': move.css('div > div.pic > a::attr(href)').get().strip(),
                '图片链接': move.css('div > div.pic > a > img::attr(src)').get().strip()
            })
        print(json.dumps(data, indent=4, ensure_ascii=False))
    else:
        print('状态码不为200', resp.text)


if __name__ == '__main__':
    parse()