Python项目实例——多线程爬【药房网】的相关信息

多线程爬药房网

import requests
from lxml.html import etree
import urllib.parse
import time
import re
import csv
import datetime
import threading
import math

cookies = {
   
    'real_ip': '125.120.62.56',
    '__jsluid_s': '59165490edcd8dd747b56e5ee043f5ff',
    'UM_distinctid': '1740a6dc632646-031c5399478b33-3b634404-1fa400-1740a6dc63437d',
    'hotkeywords': '999%23%230%23%230%23%23https%3A%2F%2Fwww.yaofangwang.com%2Fsearch%2F13791.html%40%40%E7%89%87%E4%BB%94%E7%99%80%23%230%23%230%23%23https%3A%2F%2Fwww.yaofangwang.com%2Fsearch%2F39735.html%40%40%E9%98%BF%E8%83%B6%23%231%23%230%23%23https%3A%2F%2Fwww.yaofangwang.com%2Fsearch%2F11442.html%40%40%E9%87%91%E6%88%88%23%230%23%230%23%23https%3A%2F%2Fwww.yaofangwang.com%2Fsearch%2F30642.html%40%40%E6%B1%A4%E8%87%A3%E5%80%8D%E5%81%A5%23%230%23%230%23%23https%3A%2F%2Fwww.yaofangwang.com%2Fsearch%2F50493.html',
    'CNZZDATA1261831897': '1737390408-1597898402-https%253A%252F%252Fwww.baidu.com%252F%7C1597903807',
    'Hm_lvt_e5f454eb1aa8e839f8845470af4667eb': '1597902800,1597906702',
    'historysearch': '%E4%BB%A5%E5%B2%AD%7C%7C999%E6%84%9F%E5%86%92%E8%8D%AF%7C%7C%E6%84%9F%E5%86%92%E8%8D%AF',
    'curentSearch': 'search_Yao',
    'Hm_lpvt_e5f454eb1aa8e839f8845470af4667eb': '1597906860',
}

headers = {
   
    'Connection': 'keep-alive',
    'Cache-Control': 'max-age=0',
    'Upgrade-Insecure-Requests': '1',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
    'Sec-Fetch-Site': 'same-origin',
    'Sec-Fetch-Mode': 'navigate',
    'Sec-Fetch-User': '?1',
    'Sec-Fetch-Dest': 'document',
    'Referer': 'https://www.yaofangwang.com/search/78611.html?aliascn=%E4%BB%A5%E5%B2%AD',
    'Accept-Language': 'zh-CN,zh;q=0.9',
}


class YaofangwangSpider():
    def __init__(self, task_id=None, key=None, sort='销量', brand_id=None, *args, **kwargs):
        self.task_id = task_id
        self.platform = '药房网'
        self.search_key = urllib.parse.quote(key)    # 将中文转换为’utf-8‘编码格式
        self.url = f'https://www.yaofangwang.com/search.html?keyword={self.search_key}'
        #self.url = 'https://www.yaofangwang.com/search.html?keyword=999%E6%84%9F%E5%86%92%E7%81%B5%E9%A2%97%E7%B2%92&price=1'
        pass

    def num_page(self):
        response = requests.get(self.url, headers=headers, cookies=cookies )
        #print(response.text)
        res_html = etree.HTML(response.text)
        num = res_html.xpath('//span[@class="num"]/text()')[-1].replace('/','').strip()    #总页数
        print(f'药品种类一共{num}页')
        self.get_goods_id(num)

    #拿medicine列表的药品ID
    def get_goods_id(self,num):
        goods_id_list = []
        for page in range(int(num)+1):
            goods_list_url = f'https://www.yaofangwang.com/search.html?keyword={self.search_key}&page={page}'
            try:
                response = requests.get(goods_list_url, headers=headers, )  # cookies=cookies
                res_html = etree.HTML(response.text)
                goodslists = res_html.xpath('//ul[@class="goodlist_search clearfix"]/li')
                for goodslist in goodslists:
                    goods_id = re.findall('medicine-(.*?).html',goodslist.xpath('div/a[1]/@href')[0])[0]
                    goods_id_list.append(goods_id)
            except:
                print('药品列表ID网页访问失败')
        print(f'一共有{len(goods_id_list)}种药品')
        new_goods_id_list