Python爬虫企查查

#爬取企查查公司的网址和名字

Cookie为你登录企查查官网后的，
程序运行后输入需要搜索的字段即可湖区相关企业信息
eg:游戏，餐饮等字眼

import time
import urllib
from urllib import request
from lxml import etree

for PageNum in range(1,2):
    search = input("请输入需要搜索的字段:")
    search1 = urllib.parse.quote(search)
    url = r"https://www.qichacha.com/search?key="+ search1+"#p:"+str(PageNum)+"&"
    headers = {
        'Host': 'www.qichacha.com',
        'Connection': 'keep-alive',
        'Accept': r'text/html, */*; q=0.01',
        'X-Requested-With': 'XMLHttpRequest',
        'User-Agent': r'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1',
        'Referer': url,
        'Accept-Language': 'zh-CN,zh;q=0.9',
        'Cookie': "UM_distinctid=16c4ffec9184b4-06bf1ab467097d-c343162-100200-16c4ffec919402; zg_did=%7B%22did%22%3A%20%2216c4ffecad927b-02c0fc68d3cecc-c343162-100200-16c4ffecada522%22%7D; _uab_collina=156471019499611887695946; acw_tc=7cc1e21615680865761297886e663a64c8dd44f26a2658e07f2c53ffbb; QCCSESSID=rl94fffour31is6mqkpvsvrg96; hasShow=1; Hm_lvt_3456bee468c83cc63fb5147f119f1075=1568698791,1568699293,1568700240,1568700608; CNZZDATA1254842228=926297394-1564705055-https%253A%252F%252Fwww.baidu.com%252F%7C1568701992; Hm_lpvt_3456bee468c83cc63fb5147f119f1075=1568704483; zg_de1d1a35bfa24ce29bbf2c7eb17e6c4f=%7B%22sid%22%3A%201568703637962%2C%22updated%22%3A%201568704483941%2C%22info%22%3A%201568698790826%2C%22superProperty%22%3A%20%22%7B%7D%22%2C%22platform%22%3A%20%22%7B%7D%22%2C%22utm%22%3A%20%22%7B%7D%22%2C%22referrerDomain%22%3A%20%22www.qichacha.com%22%2C%22cuid%22%3A%20%22f289610a9540a6c5a306d7ed743b5dd3%22%2C%22zs%22%3A%200%2C%22sc%22%3A%200%7D",
    }

    r = request.Request(url,headers=headers)#请求网址
    r1 = request.urlopen(r).read()#获取请求网址的HTML
    r1 = r1.decode("utf8")#解码
    html = etree.HTML(r1)#生成DOM树
    result = html.xpath("//tbody//tr//td/a//@href")#解析
    for ie in result:
        time.sleep(3)
        reurl = r"https://www.qichacha.com" + str(ie)#企业详细页面
        print(reurl)
        headers = {
            'User-Agent': r'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36',
        }
        rs = request.Request(reurl, headers=headers)  # 请求网址
        r2 = request.urlopen(rs).read()  # 获取请求网址的HTML
        r2 = r2.decode("utf8")  # 解码
        html2 = etree.HTML(r2)  # 生成DOM树
        Web = html2.xpath("//div[@class='dcontent']/div[1]/span[3]/a/text()")  # 官网
        Name = html2.xpath("//div[@class='content']/div[1]/h1/text()") #公司名字
        # Adress = html2.xpath("//tbody/tr[11]/td[2]/text()")#公司地址

        with open("qcc.txt","a+")as f:
            if not Web:
                print(Web)
                print(Name[0].strip())
                f.write("暂无")
                f.write("="*3)
                f.write(Name[0].strip())
                f.write("\n")
            else:
                print(Web[0].strip())
                print(Name[0].strip())
                f.write(Web[0].strip())
                f.write("="*3)
                f.write(Name[0].strip())
                f.write("\n")
            f.close()