pyhton爬虫--爬取扇贝单词

“”"

思路

爬取所有的单词
分页爬取
写入一个文件里面
然后写入json文件中
最后写入文档
“”"

import os
import requests
from lxml import etree
def python(page):
    #起始的url
    for page in range(1,int(page)+1):
        path='./english/python'
        if not os.path.exists(path):
            os.makedirs(path)
        base_url="https://www.shanbay.com/wordlist/110521/232414/?page={}".format(page)
        response=requests.get(base_url)
        html=response.text
        # print(html)
        #将字符串格式转换成xml格式
        html_xml=etree.HTML(html)
        #获取所有的单词
        python_list=html_xml.xpath('//strong/text()')
        print(python_list)
        #获取所有的中文解释
        china_list=html_xml.xpath('//td[@class="span10"]/text()')
        print(china_list)
        big_dic={
            '{}页数'.format(page):{}
        }
        for i in range(len(china_list)):
            dic = {}
            dic['英文']=python_list[i]
            dic['翻译']=china_list[i]
            # print(dic)
            # print(type(dic))#查看数据类型
            big_dic.get('{}页数'.format(page))[python_list[i]]=china_list[i]
        import json
        path1=path+str(page)+'.txt'
        with open(path1,'w',encoding='utf-8') as f:
            f.write(json.dumps(str(big_dic)))
if __name__ == '__main__':
    page=input('请输入要爬取的页数')
    python(page)

第二种方式

‘’’
爬取扇贝网python必背词汇表
接口地址：https://www.shanbay.com/wordlist/110521/232414/
要求：获取所有的python词汇数据，形成字典，然后存储数据

思路：
    第一页：https://www.shanbay.com/wordlist/110521/232414/
    第二页：https://www.shanbay.com/wordlist/110521/232414/?page=2

‘’’

import requests,re,os,json
from lxml import etree
class Shanbei:
    def __call__(self, *args, **kwargs):
        self.get_xml(*args)

    def get_xml(self,pages):
        '''
    获取扇贝英语python页 内容 转换为xml
    :return:获取扇贝英语python页的xml
    '''
    base_url = 'https://www.shanbay.com/wordlist/110521/232414/?page='

    for page in range(1,pages+1):
        base_html = requests.get('https://www.shanbay.com/wordlist/110521/232414/?page=1').text
        base_xml = etree.HTML(requests.get(base_url+str(page)).text)
        pages = page

        # print(base_xml)

        with open('贝壳应英语.html','w',encoding='utf-8') as f :
            f.write(base_html)
        self.get_python(base_html,base_xml,pages)


def get_python(self,base_html,base_xml,pages):
    '''
    处理python页面数据
    :param base_xml:
    :return:
    '''
    #获取标题
    # main_title = base_xml.xpath("/html/body/div[@class='container main-body ']/div[@class='row']/div[@class='span8']/div[@class='row'][1]/div[@class='span8']/div[@class='well']/div[@class='row']/div[@class='span6']/h4/text()")
    #胆子获取
    worlds_rule = re.compile(r'<h4>(.*?)</h4>',re.S)
    main_title = worlds_rule.findall(base_html)

    get_words = re.compile(r'[a-z][A-Z]+',re.I)
    world_title = (get_words.findall(main_title[0]))[0]

    # print(base_html)
    print(world_title)

    # #获取本页词条数量
    trs_length = len(base_xml.xpath("/html/body/div[@class='container main-body ']/div[@class='row']/div[@class='span8']/div[@class='row'][2]/div[@class='span8']/table[@class='table table-bordered table-striped']/tbody/tr[@class='row']"))
    print(trs_length)

    #准备大字典
    shell_word_dict = {
        '必备单词{}'.format(world_title):{},
    }

    #根据词条数量获取循环次数
    for line in range(int(trs_length)):
        word = base_xml.xpath("/html/body/div[@class='container main-body ']/div[@class='row']/div[@class='span8']/div[@class='row'][2]/div[@class='span8']/table[@class='table table-bordered table-striped']/tbody/tr[@class='row'][{}]/td[@class='span2']/strong/text()".format(line+1))
        word_description = base_xml.xpath("/html/body/div[@class='container main-body ']/div[@class='row']/div[@class='span8']/div[@class='row'][2]/div[@class='span8']/table[@class='table table-bordered table-striped']/tbody/tr[@class='row'][{}]/td[@class='span10']/text()".format(line+1))
        # print(word,word_description)

        shell_word_dict.get('必备单词{}'.format(world_title))[word[0]] = word_description[0]
        print(shell_word_dict)

    #储存文件
    path = './扇贝单词/'+'第'+str(pages)+'页' +'/'
    print(path)
    if not os.path.exists(path):
        os.makedirs(path)
    file_path = path+'第'+str(pages)+'页'

    with open(file_path,'w',encoding='utf-8')as f:
        f.write(json.dumps(shell_word_dict))


if __name__ == '__main__':
    # pages = int(input('请输入需要爬取的页数'))
    pages = 3
    shanbei = Shanbei()
    shanbei(pages)