利用xpath爬取网址数据

1.爬取小说，修改url改为自己需要爬取的网址，然后修改头和divs

import  requests
from lxml import etree
import time
url=f‘爬取的网址'
#反爬机制
headers={'user-agent':'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Mobile Safari/537.36'}
resp=requests.get(url,headers=headers).text
html=etree.HTML(resp)
divs = html.xpath('/html/body/div[5]/dl/dd')
def fetch_contend(adress):
    resp = requests.get(adress, headers=headers).text
    html=etree.HTML(resp)
    t_t=html.xpath("/html/body/div[4]/div[3]/div[3]/text()")
    t_t=''.join(t_t).split()
    t_t=''.join(t_t)
    return t_t

for div in divs[2:]:
    if div.xpath('./a/text()')[0][0]=='第':
        title=div.xpath('./a/text()')[0]
        net_adress='https://www.bigee.cc'+div.xpath('./a/@href')[0]
        content=fetch_contend(net_adress)

    # time.sleep(0.5)
        with open(f'./爬虫练习_下载小说/txet_{title}.txt',"w",encoding='utf-8') as f:
            f.write(f'{content}')
            f.write('\n')
            print(title,'下载完成')

2.爬取图像或者表格

import  requests
from lxml import etree
url='爬取的网址'
resp=requests.get(url)
#解析
html=etree.HTML(resp.text)
def fetch_table_value(path_d='/html/body/div[2]/div[1]/div[1]/div[6]/table/tr'):
    divs = html.xpath(path_d)
    for div in divs:
        path = './td[1]/div/a/text()'
        lei_net = div.xpath(path)
        lei_nonet = div.xpath('./td[1]/div/text()')
        lei_net_w=div.xpath('./td[1]/div/a/present/span/text()')
        lei_nonet_w=div.xpath('./td[1]/div/present/span/text()')
        print(lei_nonet_w)
def fetch_health_effect(path_d='/html/body/div[2]/div[1]/div[1]/div[5]/p/text()'):
    divs = html.xpath(path_d)
    print(divs)
def fetch_title(path_d='/html/body/div[2]/div[1]/div[1]/h2/text()'):
    divs = html.xpath(path_d)
    print(divs)
def fetch_image(path_d='/html/body/div[2]/div[1]/div[1]/div[7]/ul/li',save_path="./save"):
    divs=html.xpath(path_d)
    for list in divs:
        div=list.xpath('./img/@data-src')[0]
        name=list.xpath("./img/@alt")
        img_data = requests.get(url=div).content
        img_path = save_path + f'/{name}' + '.png'
        with open(img_path, "wb") as f:
            f.write(img_data)
            print(divs, "下载完成")

这里可以将需求写成函数的形式爬取多种格式文件

例子：

import  requests
from lxml import etree
import time
for i in range(10):
    url=f'http://journal.whu.edu.cn/news/index/page/{i+1}'
    #反爬机制
    headers={'user-agent':'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Mobile Safari/537.36'}
    resp=requests.get(url,headers=headers).text
    html=etree.HTML(resp)
    divs = html.xpath('/html/body/div[3]/div/div/ul/li')
    for div in divs:
        title=div.xpath('./div[1]/a/img/@alt')[0]
        TM=div.xpath('./div[2]/div[2]/text()')[0]
        net_adress='http://journal.whu.edu.cn/news'+div.xpath('./div[1]/a/@href')[0]

        #预防被封ip
        # time.sleep(0.5)
        with open('./爬虫练习_wuda/新闻内容.txt',"a",encoding='utf-8') as f:
            f.write(f'{title},{TM},{net_adress}')
            f.write('\n')