Python爬虫实例——基于Xpath爬取西刺网站ip、端口信息

import requests
from parsel import Selector
# from bs4 import BeautifulSoup

def getOneHtmlPage(page):
    url=f'https://www.xicidaili.com/nn/{page}'
    url='https://www.xicidaili.com/nn/%s'%(page)
    header={
        'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36'
    }
    response=requests.get(url,headers=header)
    return response.text

def parseOneHtmlPage(text):
    selectors=Selector(text=text)
    #最好手工选取xpath路径，直接copy有可能出错，比如此案例中copy下来的/body实际网页源代码中没有
    selectors=selectors.xpath('//table[@id="ip_list"]/tr')
    for selector in selectors:
        ip=selector.xpath('./td[2]/text()').get()
        port=selector.xpath('./td[3]/text()').get()
        print(port,ip)
        item=f"{ip},{port}"
        if 'none' not in item:
            pipelinesCSV(item)

def pipelinesCSV(item):
    with open('filename.csv','a',encoding='utf-8') as fp:
        fp.write(item+'\n')
        return item

def main():
	#爬取太频繁可能会被封ip,可以设置时间间隔或切换ip
    for page in range(1,10):
        text=getOneHtmlPage(page)
        parseOneHtmlPage(text)

if __name__=='__main__':
    main()