Python 爬虫基础学习（three）

最新推荐文章于 2024-06-22 16:56:29 发布

大大枫free

最新推荐文章于 2024-06-22 16:56:29 发布

阅读量247

点赞数

分类专栏： python基础学习

本文链接：https://blog.csdn.net/qq_37397335/article/details/106008065

版权

python基础学习专栏收录该内容

10 篇文章 0 订阅

订阅专栏

Python 爬虫基础学习（three）

1.正则表达式(回顾）

案例：糗事百科图片下载

# 1 创建request对象
# 2 获取网页源码
# 3 解析出图片的路径
# 4 下载图片
import urllib.request
import re

# 1 创建request对象
def create_request(page):
    url = 'https://www.qiushibaike.com/hot/page/' + str(page) + '/'
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36',
    }
    request = urllib.request.Request(url=url,headers=headers)
    return request

# 2 获取网页源码
def get_content(request):
    response = urllib.request.urlopen(request)
    content = response.read().decode('utf-8')
    return content

# 3 解析出图片的路径
# <div class="thumb">
    # <a href="/article/122119754" target="_blank">
        # <img src="//pic.qiushibaike.com/system/pictures/12211/122119754/medium/S18OUY90ZVAU3WSK.jpg" alt="糗事#122119754" class="illustration" width="100%" height="auto">
    # </a>
# </div>
def parse_srclist(content):
    pattern = re.compile('<div class="thumb">.*?<img src="(.*?)" alt=".*?"',re.S)
    src_list = pattern.findall(content)
    return src_list

# 4 下载图片
def down_load(src_list):
    for src in src_list:
        name = src.split('/')[-1]
        filename = './qiubai/' + name
        url = 'https:' + src
        urllib.request.urlretrieve(url=url,filename=filename)



if __name__ == '__main__':
    start_page = int(input('请输入起始页码'))
    end_page = int(input('请输入结束页码'))
    for page in range(start_page,end_page+1):
        request = create_request(page)
        content = get_content(request)
        src_list = parse_srclist(content)
        down_load(src_list)

2.xpath

xpath基本语法：
	1.路径查询
		//：查找所有子孙节点，不考虑层级关系
		/ ：找直接子节点
	2.谓词查询
		//div[@id]  
		//div[@id="maincontent"]    
	3.属性查询
		//@class         
	4.模糊查询
		//div[contains(@id, "he")]   
		//div[starts-with(@id, "he")] 
	5.内容查询
		//div/h1/text()
	6.逻辑运算
		//div[@id="head" and @class="s_down"]
		//title | //price
	7.索引+1：
	//tbody[@id='datalist']/tr/td[3]/span/text()
	
	td[3] : td同级的第三个

xpath使用：
	注意：提前安装xpath插件
	1.安装lxml库      
			pip install lxml -i https://pypi.douban.com/simple
	2.导入lxml.etree  
			from lxml import etree
	3.etree.parse()   解析本地文件
		    html_tree = etree.parse('XX.html')	
	4.etree.HTML()    服务器响应文件
		    html_tree = etree.HTML(response.read().decode('utf-8')	
	4.html_tree.xpath(xpath路径)

应用案例：1.使用xpath解析糗事百科页面中所有的图片路径 --》xpath和正则的对比

# 1 创建request对象
# 2 获取页面源码
# 3 解析路径
# 4 下载

import urllib.request
from lxml import etree

url = 'https://www.qiushibaike.com/hot/'
headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36',
    }
# 1 创建request对象
request = urllib.request.Request(url=url,headers=headers)
# 2 获取页面源码
response = urllib.request.urlopen(request)
content = response.read().decode('utf-8')
# 3 解析路径
tree = etree.HTML(content)
src_list = tree.xpath('//div[@id="content-left"]/div/div[@class="author clearfix"]/a/img/@src')
name_list = tree.xpath('//div[@id="content-left"]/div/div[@class="author clearfix"]/a[2]/h2/text()')
for i in range(len(src_list)):
    url = 'https:' + src_list[i]
    name = name_list[i].strip('\n')
    filename = './qbtouxiang/'+name+'.jpg'
    # print(filename)
    urllib.request.urlretrieve(url=url,filename=filename)

2.使用xpath解析糗事百科的用户信息（用户头像，用户名字，用户等级）–》用户信息不对等

# 需求：下载前十页的用户信息
# 解决方案:1 创建request对象
#        2 获取页面源码
#        3 解析
#        4 保存数据
import urllib.request
from lxml import etree

def create_request(page):
    url = 'https://www.qiushibaike.com/hot/page/' + str(page) + '/'
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36',
    }
    request = urllib.request.Request(url=url,headers=headers)
    return request

def get_content(request):
    response = urllib.request.urlopen(request)
    content = response.read().decode('utf-8')
    return content


def save_content(content):
    tree = etree.HTML(content)
    # //div[@id="content-left"]/div/div[@class="author clearfix"]//img/@src
    src_list = tree.xpath('//div[@id="content-left"]/div/div[@class="author clearfix"]//img/@src')
    # //div[@id="content-left"]/div/div[@class="author clearfix"]//img/@alt
    name_list = tree.xpath('//div[@id="content-left"]/div/div[@class="author clearfix"]//img/@alt')
    # //div[@id="content-left"]/div/div[@class="author clearfix"]/div/text()
    level_list = tree.xpath('//div[@id="content-left"]/div/div[@class="author clearfix"]/div/text()')

    users = []

    for i in range(len(src_list)):
        src = src_list[i]
        name = name_list[i]

        # 如果src等于这个值  那么在列表中插入一个空串
        if src == '//static.qiushibaike.com/images/thumb/anony.png?v=b61e7f5162d14b7c0d5f419cd6649c87':
            level_list.insert(i,'')

        level = level_list[i]

        user = {}

        user['src']=src
        user['name']=name
        user['level']=level

        users.append(str(user))

    with open('user.txt','w',encoding='utf-8')as fp:
        fp.write(str(users))




if __name__ == '__main__':
    start_page = int(input('请输入起始页码'))
    end_page = int(input('请输入结束页码'))
    for page in range(start_page,end_page+1):
        request = create_request(page)
        content = get_content(request)
        save_content(content)

3.站长素材图片抓取并且下载（http://sc.chinaz.com/tupian/shuaigetupian.html）–》懒加载

原理：先将img标签中的src链接设为同一张图片（空白图片），将其真正的图片地址存储再img标签的自定义属性中（比如data-src）。当js监听到该图片元素进入可视窗口时，即将自定义属性中的地址存储到src属性中，达到懒加载的效果。优点：这样做能防止页面一次性向服务器响应大量请求导致服务器响应慢，页面卡顿或崩溃等问题。

既然懒加载的原理是基于判断元素是否出现在窗口可视范围内,图片在可视范围内就加载，不在可视范围内就先不加载




# //div[@id="container"]/div/div/a/img/@alt
import urllib.request

def create_request(page):
    # http://sc.chinaz.com/tag_tupian/HeiRen_3.html
    base_url = 'http://sc.chinaz.com/tag_tupian/HeiRen'
    if page == 1:
        url = base_url + '.html'
    else:
        url = base_url + '_' + str(page) + '.html'

    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36',
    }
    request = urllib.request.Request(url=url,headers=headers)
    return request


def get_content(request):
    response = urllib.request.urlopen(request)
    content = response.read().decode('utf-8')
    return content

from lxml import etree
def save_content(content):
   tree = etree.HTML(content)
   alt_list = tree.xpath('//div[@id="container"]/div/div/a/img/@alt')
   src_list = tree.xpath('//div[@id="container"]/div/div/a/img/@src2')
   for i in range(len(src_list)):
       alt = alt_list[i]
       src = src_list[i]
       suffix = src.split('.')[-1]
       filename = './blackwomen/' + alt + '.' +suffix
       urllib.request.urlretrieve(url=src,filename=filename)


if __name__ == '__main__':
    start_page = int(input('请输入起始页码'))
    end_page = int(input('请输入结束页码'))
    for page in range(start_page,end_page+1):
        request = create_request(page)
        content = get_content(request)
        save_content(content)

3.JsonPath

jsonpath的安装及使用方式：
			pip安装： 
				   pip install jsonpath
			jsonpath的使用：
                    obj = json.load(open('json文件', 'r', encoding='utf-8'))
                    ret = jsonpath.jsonpath(obj, 'jsonpath语法')

json对象的转换
	json.loads()
		是将字符串转化为python对象
	json.dumps()
		将python对象转化为json格式的字符串
		
#注意此处的url为network查找的接口，否则不能转化成json类型

#多页查询怎么办（page）？
	json.load()
		读取json格式的文本，转化为python对象
		json.load(open(a.json))
	json.dump()
		将python对象写入到文本中

教程连接（http://blog.csdn.net/luxideyao/article/details/77802389）

应用案例：智联招聘（薪水，公司名称，职位需求）

# 爬取智联招聘的职位，薪水，公司名称

import urllib.request

url = 'https://fe-api.zhaopin.com/c/i/sou?pageSize=90&cityId=538&salary=0,0&workExperience=-1&education=-1&companyType=-1&employmentType=-1&jobWelfareTag=-1&kw=%E7%88%AC%E8%99%AB%E5%B7%A5%E7%A8%8B%E5%B8%88&kt=3&=0&_v=0.29127774&x-zp-page-request-id=177fd3f08d9942488c2f482baaad9a3d-1565772375633-677458&x-zp-client-id=94df40c0-69a5-40f2-9d66-0bbe28f523a4'

headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36',
    }
request = urllib.request.Request(url=url,headers=headers)
response = urllib.request.urlopen(request)
content = response.read().decode('utf-8')

with open('zhilian.json','w',encoding='utf-8')as fp:
    fp.write(content)

import json
import jsonpath

obj = json.load(open('zhilian.json','r',encoding='utf-8'))

jobName_list = jsonpath.jsonpath(obj,'$..jobName')
salary_list = jsonpath.jsonpath(obj,'$..salary')
companyName_list = jsonpath.jsonpath(obj,'$..company.name')



for i in range(len(jobName_list)):
    jobname = jobName_list[i]
    salary = salary_list[i]
    companyname = companyName_list[i]

    job = {}

    job['jobname']=jobname
    job['salary']=salary
    job['companyname']=companyname

    with open('job.json','a',encoding='utf-8')as fp:
        fp.write(str(job))

大大枫free

关注

0
点赞
踩
1

收藏

觉得还不错? 一键收藏
打赏
0
评论
Python 爬虫基础学习（three）

Python 爬虫基础学习（three）1.正则表达式(回顾）案例：糗事百科图片下载# 1 创建request对象# 2 获取网页源码# 3 解析出图片的路径# 4 下载图片import urllib.requestimport re# 1 创建request对象def create_request(page): url = 'https://www.qiushibaike.com/hot/page/' + str(page) + '/' headers = {
复制链接

扫一扫