python爬虫

爬虫

1.mac壁纸网爬壁纸

利用xpath

import requests
from lxml import etree
for i in range(1,10):
    url='https://www.macapp.so/wallpaper/'+'index_'+str(i)+'.html'
    headers={

        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36 Edg/86.0.622.69'
    }

    response = requests.get(url=url,headers=headers)
    #print(response.text)

    data = etree.HTML(response.text)
    girls = data.xpath('/html/body/div[3]/ul//li/a/p/img')

    for girl in girls:
        img_url = girl.xpath('./@src')[0]
        img_url = img_url.replace('1280-800-','')
        name = girl.xpath('./@alt')[0]
        image = requests.get(url=img_url)
        with open('./图片/mac壁纸/%s.jpg' % name,'wb') as jpg:
            jpg.write(image.content)
        print('《%s》下载完毕'% name)

2.斗鱼爬图片

利用jsonpath

import requests
import jsonpath

url ='https://www.douyu.com/gapi/rknc/directory/yzRec/1'
headers = {
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36 Edg/86.0.622.69'
}
data = requests.get(url,headers=headers).json()

img_urls = jsonpath.jsonpath(data,'$..rs1')
names = jsonpath.jsonpath(data,'$..nn')

for name, img_url in zip(names,img_urls):
    response = requests.get(img_url,headers=headers).content
    filename ='图片/斗鱼/'+name+'.jpg'
    with open(filename,'wb')as f:
        f.write(response)
    print('%s下载完成'%name)

3.虎牙爬图片

利用jsonpath和xpath（注释掉的代码）

	import requests
from lxml import etree
import jsonpath

url='https://www.huya.com/cache.php?m=LiveList&do=getTmpLiveByPage&gameId=2168&tmpId=2609&page=1'
headers={

    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36 Edg/86.0.622.69'
}

response = requests.get(url=url,headers=headers).json()
urls = jsonpath.jsonpath(response,'$..screenshot')
names = jsonpath.jsonpath(response,'$..nick')
for name,url in zip(names,urls):
    image=requests.get(url)
    with open('./图片/虎牙/%s.jpg' % name, 'wb') as f:
        f.write(image.content)
    print('%s下载完成'% name)
# data = etree.HTML(response.text)
# girls = data.xpath('//*[@id="imgid"]/div/ul//li/div/a/img')

# for girl in girls:
#     img_url = girl.xpath('./@data-imgurl')[0]
#     img_url = img_url.split('?')[0]
#     name = girl.xpath('./@alt')[0]
#     image = requests.get(url=img_url)
#     with open('./图片/虎牙/%s.jpg' % name,'wb') as jpg:
#         jpg.write(image.content)
#     print('《%s》下载完毕'% name)