xpath爬取项目

扇贝爬取

import requests
from lxml import etree
import json

class ShanBei(object):
    def __init__(self,url,result):
        self.url = url
        self.result = result
        self.headers = {
            'user-agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36'
        }

    def get_html(self):
        try:
            response = requests.get(url=self.url,headers=self.headers)
            response.raise_for_status()
            html = response.content.decode('utf-8')
            # print(html)
            self.parse_html(html)
        except:
            return False
    def parse_html(self,html):
        xpath_html = etree.HTML(html)
        trs = xpath_html.xpath('//table[contains(@class,"table-bordered")]/tbody/tr')
        for tr in trs:
            item = {}
            word = tr.xpath('./td[@class="span2"]/strong/text()')[0]
            means = tr.xpath('./td[@class="span10"]/text()')[0]
            item[word] = means
            self.result.append(item)
        self.save_to_json(self.result)

    def save_to_json(self,result):
        with open('shanbei.json','w',encoding='utf-8') as fp:
            json.dump(result,fp)

if __name__ == '__main__':
    result = []
    base_url = 'https://www.shanbay.com/wordlist/110521/232414/?page={}'
    for i in range(1,4):
        url = base_url.format(i)
        shanbei = ShanBei(url,result)
        shanbei.get_html()

网易云歌手信息爬取

import requests
from lxml import etree
import json

class WangYiYun(object):
    def __init__(self):
        self.headers = {
            'user-agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36',
            'cookie': '_ntes_nnid=bde7c65abd5e06bbf64efe60691ef247,1566111466592; _ntes_nuid=bde7c65abd5e06bbf64efe60691ef247; __oc_uuid=7c038f50-c185-11e9-86f9-4bd3dd23d629; _iuqxldmzr_=32; WM_TID=cN5fXBJlrPpFQBFRFUM5pc68R2vwXqaF; UM_distinctid=16d9b2096ba1f6-0c3fa044fca013-3f385804-15f900-16d9b2096bb321; P_INFO=lovekk0502@163.com|1570539822|0|163|00&99|bej&1570265765&unireg#bej&null#10#0#0|&0|unireg|lovekk0502@163.com; mail_psc_fingerprint=db28de61b0efc44e30fad7491c2519e4; __root_domain_v=.163.com; _qddaz=QD.887vbv.pz0uqc.k1ut3v9f; hb_MA-BFF5-63705950A31C_source=www.baidu.com; JSESSIONID-WYYY=nCM3UFRCHU%2BN29I40JCdWqeUoK9ODAMvhatJvu55zI3N5TkvtqadSbtPdyhfp9%5C3%2BGu3evqCf6R%5ClFUng2S3yPCh92RD6%2FWvhk2AE6GsIoF%2By8m%5CV%5C%5CPIzx31iRMqK9pp%2B2dvcaNfTK5PmfV9kxB94U%5Cmt6k7MR%5C8bc%2BAS50YMZ6ijBN%3A1572501969558; WM_NI=t0uRHExyLcdgXOVHlLxRzq%2F3shycmWJQ15HvrwOf%2FvY%2F78m60wcmaPJFIVETBrWPHkprlp0M%2B53OC9cPIhhiyAFqmCWHOwIxdr9v3%2FltLFblCA1sqZCopftQL4zBBC3TUWw%3D; WM_NIKE=9ca17ae2e6ffcda170e2e6eeaddc67f79e8a97e55daab08ea6c54e969f9baeb87f86b88fbbce7cad9f81a3d22af0fea7c3b92ab8f0bbb3cb4fbca78484e76d9c919aa9ef48bc8eb692ae39b1b8b784f05d8a8fffd6cb7eb2a98187ae7e9bf1f9d9f03ef4a9ad8ee45c8a9a87b7c86082ae99b1cb61b4aea491b347e9ebfdb0c233a1908789eb63bb9f8a92f04995988198cc74a8aef797b225b0ac84a7e168819fff94f45281b1c087f75b90ecb6a3c149f18c9bb8f637e2a3',
            'upgrade-insecure-requests': '1'

        }
    def get_html(self,url):
        response = requests.get(url=url,headers=self.headers)
        html = response.content.decode('utf-8')
        xpath_html = etree.HTML(html)
        return xpath_html

    ## 进入歌手主页，获取歌手详细信息
    def parse_detail(self,url,item):
        html = self.get_html(url)
        description = html.xpath('//div[@class="n-artdesc"]/p/text()')
        item['description'] = ''.join(description).replace('\n','')
        self.save_to_json(item)


    ## 解析每个歌手的姓名和url
    def parse_singer(self,url):
        item = {}
        html = self.get_html(url)
        singer_name = html.xpath('//ul[@id="m-artist-box"]/li/p/a[1]/text()')
        singer_url = html.xpath('//ul[@id="m-artist-box"]/li/p/a[1]/@href|//ul[@id="m-artist-box"]/li/a[1]/@href')
        singer_url = [x.strip() for x in singer_url]
        for index,name in enumerate(singer_name):
            item['singer_name'] = name
            new_url = 'https://music.163.com' + singer_url[index]
            item['singer_url'] = new_url
            ## 艺人介绍的url
            url = 'https://music.163.com' + singer_url[index].split('?')[0]+'/desc?'+singer_url[index].split('?')[1]
            self.parse_detail(url,item)
            # print(url)



    ## 抓取按字母分类的url
    def parse_zimu(self,url):
        html = self.get_html(url)
        zimu_url = html.xpath('//ul[@id="initial-selector"]/li/a/@href')
        for one_url in zimu_url:
            new_url = 'https://music.163.com' + one_url
            self.parse_singer(new_url)


    ### 获取首页的内容，抓取地区分类的url
    def parse_index(self,url):
        html = self.get_html(url)
        area_url = html.xpath('//div[@class="blk"]/ul/li/a/@href')
        for one_url in area_url:
            new_url = 'https://music.163.com' + one_url
            self.parse_zimu(new_url)


    def save_to_json(result):
        with open('singer.json','w',encoding='utf-8') as fp:
            json.dump(result,fp)
if __name__ == '__main__':
    base_url = 'https://music.163.com/discover/artist'
    wangyiyun = WangYiYun()
    wangyiyun.parse_index(base_url)