大***Spider

仅供学习。

# coding=utf-8
import requests
from lxml import etree
import time
import csv

def getLink(url):
    headers = {
        "Referer": "http://www.dpfile.com/",
        "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36",
    }
    r = requests.get(url,headers=headers)
    element_obj = etree.HTML(r.text)
    link_list = element_obj.xpath('//*[@id="shop-all-list"]/ul/li/div[2]/div[1]/a[1]/@href')
    print(link_list)
    for link in link_list:
        getData(link)
        time.sleep(3)

def getData(link):
    data_dict = {}
    headers = {
        "Cookie": "fspop=test; cy=2; cye=beijing; _lx_utm=utm_source%3DBaidu%26utm_medium%3Dorganic; _lxsdk_cuid=1790c10cbebc8-005b694c4a01cc-67121d7a-100200-1790c10cbebc8; _lxsdk=1790c10cbebc8-005b694c4a01cc-67121d7a-100200-1790c10cbebc8; Hm_lvt_602b80cf8079ae6591966cc70a3940e7=1619405098; _hc.v=bfdb8286-3357-436c-4418-e2f66689711b.1619405098; s_ViewType=10; Hm_lpvt_602b80cf8079ae6591966cc70a3940e7=1619405802; _lxsdk_s=1790c10cbed-a2e-57d-0f2%7C%7C104",
        "Host": "www.dianping.com",
        "Referer": "http://www.dianping.com/beijing/ch85/g235",
        "Upgrade-Insecure-Requests": "1",
        "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36",
    }

    r = requests.get(link,headers=headers)
    element_obj = etree.HTML(r.text)
    name = element_obj.xpath('//*[@id="basic-info"]/h1/text()')[0]
    address = element_obj.xpath('//*[@id="basic-info"]/div[2]/span[2]/text()')[0]
    telphone = element_obj.xpath('//*[@id="basic-info"]/p/span/text()')[1:]
    price = element_obj.xpath('//*[@id="avgPriceTitle"]/text()')[0]
    data_dict.update({"药店":name,"药店地址":address,"联系电话":','.join(telphone),"人均价格":price.split(":")[1]})
    print(data_dict)
    data_dict_list.append(data_dict)


def saveData(l):
    with open(r'C:\Users\EDZ\Desktop\药店信息.csv', 'w',encoding='utf-8',newline='') as csvfile:
        fieldnames = ['药店', '药店地址',"联系电话","人均价格"]
        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
        writer.writeheader()
        for d in data_dict_list:
            writer.writerow(d)

if __name__ == "__main__":
    data_dict_list = []
    url = 'http://www.dianping.com/beijing/ch85/g235'
    getLink(url)
    saveData(data_dict_list)