qichacha/知乎/国家统计局最新4级地区划/百度地图API获取经纬度/Google play app评论等分数据爬取

最新推荐文章于 2024-05-23 14:58:26 发布
最新推荐文章于 2024-05-23 14:58:26 发布
阅读量2.1k 收藏 8
点赞数
分类专栏： 爬虫 文章标签： 爬虫 python
版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
本文链接：https://blog.csdn.net/qq_41589941/article/details/120037524
版权

1、企查查数据抓取

1.1 关键公司LOGO 

# -*-coding:utf-8-*-

import pandas as pd
import requests
import json
import random
import time
from lxml import etree
import re

cookies = pd.read_csv('C:/Desktop/cookies.txt')
# cookies2 = pd.read_csv('C:/Desktop/cookies2.txt')


# cookie = 'acw_tc={}; QCCSESSID={}'.format(acw_tc, QCCSESSID)
# 'User-Agent': random.choice(uas)


def get_companies(key):
    idx = random.randint(0, cookies.shape[0] - 1)
    acw_tc = cookies.iloc[idx, 0]
    QCCSESSID = cookies.iloc[idx, 1]
    cookie = 'acw_tc={}; QCCSESSID={}'.format(acw_tc, QCCSESSID)
    headers = {
        'Cookie': cookie,
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1623.0 Safari/537.36'}

    # proxy = {
    #     'http': pd.read_csv('E:/ip.txt', header=None, sep='\n', encoding='utf8').iloc[:, 0].to_list()[-1]}
    fnum = 0
    url = 'https://www.qcc.com/web/search?key={}'.format(key)
    try:
        # with requests.get(url, headers=self.headers, proxies=self.proxies) as response:
        response = requests.get(url, headers=headers)
        # response = requests.get(url, headers=self.headers)
        html = response.text
        parseHtml = etree.HTML(html)
        resNum = parseHtml.xpath('//div[@class="npanel-heading"]/h4/span[@class="text-danger"]/text()')
        resNum = int(resNum[0])
        if resNum > 0:
            print(resNum)
            imgUrl = parseHtml.xpath(
                '//div[@class="search-cell"]//tr[1]/td[@class="imgtd"]/div[@class="img"]/img/@src')
            print(imgUrl)
            return imgUrl

    except Exception as e:
        fnum = fnum + 1
        names.append(key)
        print('Connect fail')
        time.sleep(random.randint(10, 30))
        if fnum % 2 == 0:
            time.sleep(random.randint(10, 30))


def save_img(imgUrl, key):
    response = requests.get(imgUrl)
    filename = key + '.jpg'
    with open(filename, 'wb') as f:
        f.write(response.content)
    return key + 'Done'


if __name__ == "__main__":
    names = pd.read_excel('E:/爬取集团logo.xlsx').loc[:,'集团名称'].unique().tolist()
    # names = names[:2]
    total = 0
    count = 0
    with open('E:/groupimgUrl.txt', 'a', encoding='utf8') as fp:
        for i in names:
            total += 1
            key = i
            data= dict()
            imgUrl = get_companies(key)
            if imgUrl:
                data['key'] = key
                data['imgUrl'] = imgUrl
                data = json.dumps(data)
                fp.write(data + '\n')
                count += 1
                print(count)
                if count % 100 == 0:
                    time.sleep(random.randint(30, 60))
            else:
                print('NotFund')
            print('total=%s' % total)

1.2 获取股权穿透图

import pandas as pd
import requests
import time
import random
from lxml import etree
import json

cookies = pd.read_csv('C:/Desktop/cookies.txt')
cookies2 = pd.read_csv('C:/Desktop/cookies2.txt')


# 获取公司Id
def getCompanyId(key):
    idx = random.randint(0, cookies2.shape[0] - 1)
    acw_tc = cookies2.iloc[idx, 0]
    QCCSESSID = cookies2.iloc[idx, 1]
    cookie = 'acw_tc={}; QCCSESSID={}'.format(acw_tc, QCCSESSID)
    headers = {
        'Cookie': cookie,
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1623.0 Safari/537.36'}

    ip = pd.read_csv('E:/ip.txt', header=None, sep='\n', encoding='utf8').iloc[:, 0].to_list()[-1]
    proxy = {
        'http': pd.read_csv('E:/ip.txt', header=None, sep='\n', encoding='utf8').iloc[:, 0].to_list()[-1]}
    fnum = 0
    url = 'https://www.qcc.com/web/search?key={}'.format(key)
    try:
        # with requests.get(url, headers=self.headers, proxies=self.proxies) as response:
        response = requests.get(url, headers=headers, proxies=proxy)
        # response = requests.get(url, headers=self.headers)
        html = response.text
        parseHtml = etree.HTML(html)
        resNum = parseHtml.xpath('//div[@class="npanel-heading"]/h4/span[@class="text-danger"]/text()')
        resNum = int(resNum[0])
        if resNum > 0:
            id = parseHtml.xpath(
                '//div[@class="maininfo"]/a[@class="title"]/@href')
            return id[0]
        else:
            #names.append(key)
            pass

    except Exception as e:
        fnum = fnum + 1
        #names.append(key)
        print('Connect fail')
        time.sleep(random.randint(10, 30))
        if fnum % 2 == 0:
            time.sleep(random.randint(10, 30))


def getCompanyGuQuan(key):
    idx = random.randint(0, cookies.shape[0] - 1)
    acw_tc = cookies.iloc[idx, 0]
    QCCSESSID = cookies.iloc[idx, 1]
    cookie = 'acw_tc={}; QCCSESSID={}'.format(acw_tc, QCCSESSID)
    headers = {
        'Cookie': cookie,
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1623.0 Safari/537.36'}

    proxy = {
        'http': pd.read_csv('E:/ip.txt', header=None, sep='\n', encoding='utf8').iloc[:, 0].to_list()[-1]}
    fnum = 0
    url = "https://www.qcc.com/api/charts/getBeneficiaryData"
    postData = {'keyNo': key,"type": 2}
    try:
        res = requests.post(url=url, data=postData, headers=headers, proxies=proxy)
        if res.text:
            return res.text
        else:
            ids.append(id)
    except Exception as e:
        fnum = fnum + 1
        ids.append(id)
        print('Connect fail')
        time.sleep(random.randint(10, 30))
        if fnum % 2 == 0:
            time.sleep(random.randint(10, 30))


if __name__ == "__main__":
    # names = pd.read_excel('C:/Users/lele.ding/Desktop/test.xlsx').loc[:, 'keys'].unique().tolist()
    # ids = pd.read_excel('E:/companyIds.xlsx').loc[:,'id'].tolist()
    ids =['65765594ae6ec7793a279424e52c7dc2']
    # names = names[5222:]
    total = 0
    count = 0
    # with open('E:/companyIds.txt', 'a', encoding='utf8') as fp:
    #     for i in ids:
    #         total += 1
    #         key = i
    #         data = dict()
    #         id = getCompanyId(key)
    #         if id:
    #             data['key'] = key
    #             data['id'] = id
    #             data = json.dumps(data)
    #             fp.write(data + '\n')
    #             count += 1
    #             print(count)
    #             if count % 100 == 0:
    #                 time.sleep(random.randint(30, 60))
    #         else:
    #             print('NotFund')
    #         print('total=%s' % total)
    with open('E:/companyGuQuan.txt', 'a', encoding='utf8') as fp:
        for i in ids:
            total += 1
            key = i
            data = dict()
            id = getCompanyGuQuan(key)
            if id:
                data['key'] = key
                data['id'] = id
                data = json.dumps(data)
                fp.write(data + '\n')
                count += 1
                print(count)
                if count % 100 == 0:
                    time.sleep(random.randint(30, 60))
            else:
                print('NotFund')
            print('total=%s' % total)

1.3 VIP 集团相关数据

# -*-coding:utf-8-*-

import pandas as pd
import requests
import json
import random
import time
from lxml import etree

cookies = pd.read_csv('../cookies.txt')
# 'User-Agent': random.choice(uas)


def get_companies(key):
    idx = random.randint(0, cookies.shape[0]-1)
    acw_tc = cookies.iloc[idx, 0]
    QCCSESSID = cookies.iloc[idx, 1]
    cookie = 'acw_tc={}; QCCSESSID={}'.format(acw_tc, QCCSESSID)
    headers = {
        'Cookie': cookie,
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1623.0 Safari/537.36'}

    # proxy = {
    #     'http':pd.read_csv('E:/ip.txt',header=None,sep='\n',encoding='utf8').iloc[:,0].to_list()[-1]}
    fnum = 0
    url = 'https://www.qcc.com/web/search?key={}'.format(key)
    try:
        # with requests.get(url, headers=self.headers, proxies=self.proxies) as response:
        response = requests.get(url, headers=headers)
        # response = requests.get(url, headers=self.headers)
        html = response.text
        parseHtml = etree.HTML(html)
        resNum = parseHtml.xpath('//div[@class="npanel-heading"]/h4/span[@class="text-danger"]/text()')
        resNum = int(resNum[0])
        if resNum > 1:
            comurl = parseHtml.xpath('//div[@class="maininfo"]/a[@class="title"]/@href')
            com = dict()
            com['comurl'] = comurl[0]
            return com
        else:
            com = dict()
            com['temp']= 'temp'
            return com
    except Exception as e:
        fnum = fnum + 1
        names.append(key)
        print('Connect fail')
        time.sleep(random.randint(10, 30))
        if fnum % 2 == 0:
            time.sleep(random.randint(10, 30))


def get_group(url):
    idx = random.randint(0, cookies.shape[0]-1)
    acw_tc = cookies.iloc[idx, 0]
    QCCSESSID = cookies.iloc[idx, 1]
    cookie = 'acw_tc={}; QCCSESSID={}'.format(acw_tc, QCCSESSID)
    headers = {
        'Cookie': cookie,
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36 Edg/88.0.705.502 Safari/537.36'}
    # proxy = {
    #     'http': pd.read_csv('E:/ip.txt', header=None, sep='\n', encoding='utf8').iloc[:, 0].to_list()[-1]}
    fnum = 0
    data = dict()
    try:
        # with requests.get(url, headers=self.headers, proxies=self.proxies) as response:
        response = requests.get(url, headers=headers)
        # response = requests.get(url, headers=self.headers)
        html = response.text
        parseHtml = etree.HTML(html)
        temp = parseHtml.xpath(
            '//a[@class="oxin2"]//text()')
        print(temp)
        if temp:
            if temp[0] == "所属集团：" or temp[0] == "企业族群：":
                groupUrl = parseHtml.xpath(
                    '//a[@class="oxin2"][2]/@href')
                data['groupUrl'] = groupUrl
                data['label'] = 'Success'
        else:
            data['label'] = 'Nothing'
        return data
    except Exception as e:
        fnum = fnum + 1
        names.append(key)
        print('Connect fail')
        time.sleep(random.randint(10, 30))
        if fnum % 2 == 0:
            time.sleep(random.randint(10, 30))


if __name__ == "__main__":
    names = pd.read_excel('E:/典范雇主工商信息.xls').loc[:, '企业名称'].unique().tolist()
    # names = names[12348:]
    total = 0
    fnum = 0
    count = 0
    with open('E:/dianfan_groupName.txt', 'a', encoding='utf8') as fp:
        for i in names:
            total += 1
            key = i
            com = get_companies(key)
            if com:
                if com.get('comurl'):
                    data = get_group(com.get('comurl'))
                    if data:
                        data['key'] = key
                        data['groupUrl'] = com.get('groupUrl')
                        data = json.dumps(data)
                        fp.write(data + '\n')
                        count += 1
                        print(count)
                        if count % 100 == 0:
                            time.sleep(random.randint(30, 60))
                else:
                    print('NotFund')
            print('total=%s' % total)

2、Boss数据抓取

2.1 app端求职偏好设置问题爬取

import pandas as pd
import requests
import datetime
import json
import numpy as np

code_df = pd.read_csv('./workFuncCodeMap.txt')   #boss每个职能的code
codes = code_df['code'].unique().tolist()

d = datetime.datetime.now()
d = "%s%s%s" % (d.year, d.month, d.day)


def get_perInfo():
    headers = {'cookie': '_bl_uid=a9k10mag3he18U7jhybai70rIC7v; lastCity=101010100; __g=-; '
                         'Hm_lvt_194df3105ad7148dcf2b98a91b5e727a=1623835187; '
                         'wt2=D0Tcy2A_vz6-GkdeFUckL28Oa8gmlvcLmLMLn1bPrDV55Ic1yQwFY-4uU6Z9sIRcNldDJXsCLg4OaNLT6ZjznRw'
                         '~~; __l=l=%2Fwww.zhipin.com%2Fweb%2Fgeek%2Frecommend%3Frandom%3D1623835215696&r=&g=&s=3'
                         '&friend_source=0&s=3&friend_source=0; Hm_lpvt_194df3105ad7148dcf2b98a91b5e727a=1623835218; '
                         '__c=1623835187; __a=40920060.1623835187..1623835187.4.1.4.4; '
                         '__zp_stoken__=3638cKTB2PxgQK3wmdSxLGTEKbDZaVmo0CiUQM0p9Nw0SICRyW3p3YhhoJSJPVC0hfiBNSHcgLmUdZHVUG0FONE0LIDYCN1V8Uk46Jk0LNmJHYgMDEjFcXkABY1lvCidcTgNddV8gBhAYcjR0; geek_zp_token=V1R9MlGeD12VlgXdNrzBUbLi246TLVzQ~~',
               'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
                             'Chrome/91.0.4472.77 Safari/537.36'}
    with open('./boss偏好_%s.txt' % d, 'a', encoding='utf8') as f:
        for k, i in enumerate(codes):
            url = "https://m.zhipin.com/wapi/zpgeek/trait/questions.json?position=%s&source=0" % i
           	"关键url"
            res = requests.get(url=url, headers=headers)
            s = i + '|' + res.text + '\n'
            f.write(s)
            print(k)


def get_detail(x):
    detail = pd.DataFrame()
    for i in x['questions']:
        te = pd.DataFrame(i.get('options'))
        te.rename(columns={'encryptId': 'options_encryptId'}, inplace=True)
        te['code'] = x['code']
        te['title'] = x['title']
        te['subTitle'] = x['subTitle']
        te['encryptId'] = i.get('encryptId')
        te['questionTitle'] = i.get('questionTitle')
        te['questionType'] = i.get('questionType')
        te['otherAnswer'] = i.get('otherAnswer')
        te['optionLimit'] = i.get('optionLimit')
        detail = pd.concat([detail, te])
    return detail


def clean_data():
    boss = pd.read_csv('./boss偏好_%s.txt' % d, header=None, sep='|')
    boss.columns = ['code', 'info']
    boss['info'] = boss['info'].apply(lambda x: json.loads(x))
    boss['zpData'] = boss['info'].apply(lambda x: x.get('zpData'))
    boss['questions'] = boss['zpData'].apply(lambda x: x.get('questions'))
    boss['title'] = boss['zpData'].apply(lambda x: x.get('title'))
    boss['subTitle'] = boss['zpData'].apply(lambda x: x.get('subTitle'))
    boss = boss.iloc[:, [0, 3, 4, 5]]
    boss = boss[boss['questions'].notna()]
    detail = boss.apply(get_detail, axis=1)
    bossPer = pd.concat(detail.values)
    bossPer = bossPer.iloc[:, :-2]
    bossPer = bossPer.loc[:, ['code', 'title', 'subTitle', 'encryptId', 'questionTitle', 'questionType',
                              'options_encryptId', 'jumpQuestionIds', 'content', 'chosen']]
    bossPer['code'] = bossPer['code'].map(str)
    bossPer = code_df.merge(bossPer, on='code')
    bossPer = bossPer.iloc[:, 1:]
    bossPer.drop(columns='code', inplace=True)
    bossPer.drop(columns='chosen', inplace=True)
    bossPer['jumpQuestionIds'] = bossPer[bossPer['jumpQuestionIds'].notna()]['jumpQuestionIds'].apply(
        lambda x: x.replace('[', '').replace(']', ''))
    bossPer['jumpQuestionIds'] = bossPer['jumpQuestionIds'].apply(lambda x: x if x else np.nan)
    bossPer.drop(columns='chosen', inplace=True)
    bossPer.drop_duplicates(inplace=True)
    bossPer.to_excel('./boss偏好_%s.xlsx' % d, index=None)


if __name__ == "__main__":
    get_perInfo()
    clean_data()

2.2 网页端左侧职能类目数据爬取

这个比较简单，不需要登录，直接requests模块请求即可。

3、知乎数据抓取

知乎数据抓取的难点在于js加密的解密获得关键中间id。

拼接后的字符串先经过md5加密后再经过js加密获得关键中间id。后再去请求url或则你想要的数据

下面以爬取院校专业及开设课程为例进行过程说明。

import json
import hashlib
from urllib import parse
import requests
import pandas as pd

college_ = pd.read_excel('E:/知乎/专业.xlsx')

def get_fmd5():
    fmds = pd.DataFrame()
    for kw in college_['专业名称'].to_list():
        data1 = {'': kw}
        data1 = parse.urlencode(data1)
        url = '/api/v4/search_v3?t=general&q' + data1 + '&correction=1&offset=0&limit=20&lc_idx=0&show_all_topics=0'
        use_url = 'https://www.zhihu.com' + url
        d_c = '"ADAbkyZHcBGPTtV_mkiBdB_YUceORLSGQFI=|1592372620"'   #cookies中的值
        to_encrypt = '101_3_2.0+{}+{}'.format(url, d_c)
        baseurl = 'https://www.zhihu.com'
        fmd5 = hashlib.md5(to_encrypt.encode()).hexdigest()
        use_url = baseurl + url
        res = pd.DataFrame([kw, fmd5, use_url]).T
        fmds = pd.concat([fmds, res])
    fmds.to_csv('../mastor',index=None)
    with open('../mastor.json', 'a') as f:
        f.write(json.dumps(fmds.iloc[:, 1].to_list()))

因为我的环境不知道为什么调用不起来js脚本就将md5加密后的str，让前端同事帮忙调用js加密脚本获得最终的加密字符串即id。js加密脚本如下：

// const jsdom = require("jsdom");
// const { JSDOM } = jsdom;
// const dom = new JSDOM(`<!DOCTYPE html><p>Hello world</p>`);
// window = dom.window;
// document = window.document;
// XMLHttpRequest = window.XMLHttpRequest;


var exports = {}

function t(e) {
    return (t = "function" == typeof Symbol && "symbol" == typeof Symbol.A ? function (e) {
        return typeof e
    } : function (e) {
        return e && "function" == typeof Symbol && e.constructor === Symbol && e !== Symbol.prototype ? "symbol" : typeof e
    })(e)
}

Object.defineProperty(exports, "__esModule", {value: !0});
var A = "2.0", __g = {};

function s() {
}

function i(e) {
    this.t = (2048 & e) >> 11, this.s = (1536 & e) >> 9, this.i = 511 & e, this.h = 511 & e
}

function h(e) {
    this.s = (3072 & e) >> 10, this.h = 1023 & e
}

function a(e) {
    this.a = (3072 & e) >> 10, this.c = (768 & e) >> 8, this.n = (192 & e) >> 6, this.t = 63 & e
}

function c(e) {
    this.s = e >> 10 & 3, this.i = 1023 & e
}

function n() {
}

function e(e) {
    this.a = (3072 & e) >> 10, this.c = (768 & e) >> 8, this.n = (192 & e) >> 6, this.t = 63 & e
}

function o(e) {
    this.h = (4095 & e) >> 2, this.t = 3 & e
}

function r(e) {
    this.s = e >> 10 & 3, this.i = e >> 2 & 255, this.t = 3 & e
}

s.prototype.e = function (e) {
    e.o = !1
}, i.prototype.e = function (e) {
    switch (this.t) {
        case 0:
            e.r[this.s] = this.i;
            break;
        case 1:
            e.r[this.s] = e.k[this.h]
    }
}, h.prototype.e = function (e) {
    e.k[this.h] = e.r[this.s]
}, a.prototype.e = function (e) {
    switch (this.t) {
        case 0:
            e.r[this.a] = e.r[this.c] + e.r[this.n];
            break;
        case 1:
            e.r[this.a] = e.r[this.c] - e.r[this.n];
            break;
        case 2:
            e.r[this.a] = e.r[this.c] * e.r[this.n];
            break;
        case 3:
            e.r[this.a] = e.r[this.c] / e.r[this.n];
            break;
        case 4:
            e.r[this.a] = e.r[this.c] % e.r[this.n];
            break;
        case 5:
            e.r[this.a] = e.r[this.c] == e.r[this.n];
            break;
        case 6:
            e.r[this.a] = e.r[this.c] >= e.r[this.n];
            break;
        case 7:
            e.r[this.a] = e.r[this.c] || e.r[this.n];
            break;
        case 8:
            e.r[this.a] = e.r[this.c] && e.r[this.n];
            break;
        case 9:
            e.r[this.a] = e.r[this.c] !== e.r[this.n];
            break;
        case 10:
            e.r[this.a] = t(e.r[this.c]);
            break;
        case 11:
            e.r[this.a] = e.r[this.c] in e.r[this.n];
            break;
        case 12:
            e.r[this.a] = e.r[this.c] > e.r[this.n];
            break;
        case 13:
            e.r[this.a] = -e.r[this.c];
            break;
        case 14:
            e.r[this.a] = e.r[this.c] < e.r[this.n];
            break;
        case 15:
            e.r[this.a] = e.r[this.c] & e.r[this.n];
            break;
        case 16:
            e.r[this.a] = e.r[this.c] ^ e.r[this.n];
            break;
        case 17:
            e.r[this.a] = e.r[this.c] << e.r[this.n];
            break;
        case 18:
            e.r[this.a] = e.r[this.c] >>> e.r[this.n];
            break;
        case 19:
            e.r[this.a] = e.r[this.c] | e.r[this.n];
            break;
        case 20:
            e.r[this.a] = !e.r[this.c]
    }
}, c.prototype.e = function (e) {
    e.Q.push(e.C), e.B.push(e.k), e.C = e.r[this.s], e.k = [];
    for (var t = 0; t < this.i; t++) e.k.unshift(e.f.pop());
    e.g.push(e.f), e.f = []
}, n.prototype.e = function (e) {
    e.C = e.Q.pop(), e.k = e.B.pop(), e.f = e.g.pop()
}, e.prototype.e = function (e) {
    switch (this.t) {
        case 0:
            e.u = e.r[this.a] >= e.r[this.c];
            break;
        case 1:
            e.u = e.r[this.a] <= e.r[this.c];
            break;
        case 2:
            e.u = e.r[this.a] > e.r[this.c];
            break;
        case 3:
            e.u = e.r[this.a] < e.r[this.c];
            break;
        case 4:
            e.u = e.r[this.a] == e.r[this.c];
            break;
        case 5:
            e.u = e.r[this.a] != e.r[this.c];
            break;
        case 6:
            e.u = e.r[this.a];
            break;
        case 7:
            e.u = !e.r[this.a]
    }
}, o.prototype.e = function (e) {
    switch (this.t) {
        case 0:
            e.C = this.h;
            break;
        case 1:
            e.u && (e.C = this.h);
            break;
        case 2:
            e.u || (e.C = this.h);
            break;
        case 3:
            e.C = this.h, e.w = null
    }
    e.u = !1
}, r.prototype.e = function (e) {
    switch (this.t) {
        case 0:
            for (var t = [], n = 0; n < this.i; n++) t.unshift(e.f.pop());
            e.r[3] = e.r[this.s](t[0], t[1]);
            break;
        case 1:
            for (var r = e.f.pop(), o = [], i = 0; i < this.i; i++) o.unshift(e.f.pop());
            e.r[3] = e.r[this.s][r](o[0], o[1]);
            break;
        case 2:
            for (var a = [], c = 0; c < this.i; c++) a.unshift(e.f.pop());
            e.r[3] = new e.r[this.s](a[0], a[1])
    }
};
var k = function (e) {
    for (var t = 66, n = [], r = 0; r < e.length; r++) {
        var o = 24 ^ e.charCodeAt(r) ^ t;
        n.push(String.fromCharCode(o)), t = o
    }
    return n.join("")
};

function Q(e) {
    this.t = (4095 & e) >> 10, this.s = (1023 & e) >> 8, this.i = 1023 & e, this.h = 63 & e
}

function C(e) {
    this.t = (4095 & e) >> 10, this.a = (1023 & e) >> 8, this.c = (255 & e) >> 6
}

function B(e) {
    this.s = (3072 & e) >> 10, this.h = 1023 & e
}

function f(e) {
    this.h = 4095 & e
}

function g(e) {
    this.s = (3072 & e) >> 10
}

function u(e) {
    this.h = 4095 & e
}

function w(e) {
    this.t = (3840 & e) >> 8, this.s = (192 & e) >> 6, this.i = 63 & e
}

function G() {
    this.r = [0, 0, 0, 0], this.C = 0, this.Q = [], this.k = [], this.B = [], this.f = [], this.g = [], this.u = !1, this.G = [], this.b = [], this.o = !1, this.w = null, this.U = null, this.F = [], this.R = 0, this.J = {
        0: s,
        1: i,
        2: h,
        3: a,
        4: c,
        5: n,
        6: e,
        7: o,
        8: r,
        9: Q,
        10: C,
        11: B,
        12: f,
        13: g,
        14: u,
        15: w
    }
}

Q.prototype.e = function (e) {
    switch (this.t) {
        case 0:
            e.f.push(e.r[this.s]);
            break;
        case 1:
            e.f.push(this.i);
            break;
        case 2:
            e.f.push(e.k[this.h]);
            break;
        case 3:
            e.f.push(k(e.b[this.h]))
    }
}, C.prototype.e = function (A) {
    switch (this.t) {
        case 0:
            var t = A.f.pop();
            A.r[this.a] = A.r[this.c][t];
            break;
        case 1:
            var s = A.f.pop(), i = A.f.pop();
            A.r[this.c][s] = i;
            break;
        case 2:
            var h = A.f.pop();
            A.r[this.a] = eval(h)
    }
}, B.prototype.e = function (e) {
    e.r[this.s] = k(e.b[this.h])
}, f.prototype.e = function (e) {
    e.w = this.h
}, g.prototype.e = function (e) {
    throw e.r[this.s]
}, u.prototype.e = function (e) {
    var t = this, n = [0];
    e.k.forEach(function (e) {
        n.push(e)
    });
    var r = function (r) {
        var o = new G;
        return o.k = n, o.k[0] = r, o.v(e.G, t.h, e.b, e.F), o.r[3]
    };
    r.toString = function () {
        return "() { [native code] }"
    }, e.r[3] = r
}, w.prototype.e = function (e) {
    switch (this.t) {
        case 0:
            for (var t = {}, n = 0; n < this.i; n++) {
                var r = e.f.pop();
                t[e.f.pop()] = r
            }
            e.r[this.s] = t;
            break;
        case 1:
            for (var o = [], i = 0; i < this.i; i++) o.unshift(e.f.pop());
            e.r[this.s] = o
    }
}, G.prototype.D = function (e) {
    console.log(window.atob(e));
    for (var t = window.atob(e), n = t.charCodeAt(0) << 8 | t.charCodeAt(1), r = [], o = 2; o < n + 2; o += 2) r.push(t.charCodeAt(o) << 8 | t.charCodeAt(o + 1));
    this.G = r;
    for (var i = [], a = n + 2; a < t.length;) {
        var c = t.charCodeAt(a) << 8 | t.charCodeAt(a + 1), s = t.slice(a + 2, a + 2 + c);
        i.push(s), a += c + 2
    }
    this.b = i
}, G.prototype.v = function (e, t, n) {
    for (t = t || 0, n = n || [], this.C = t, "string" == typeof e ? this.D(e) : (this.G = e, this.b = n), this.o = !0, this.R = Date.now(); this.o;) {
        var r = this.G[this.C++];
        if ("number" != typeof r) break;
        var o = Date.now();
        if (500 < o - this.R) return;
        this.R = o;
        try {
            this.e(r)
        } catch (e) {
            this.U = e, this.w && (this.C = this.w)
        }
    }
}, G.prototype.e = function (e) {
    var t = (61440 & e) >> 12;
    new this.J[t](e).e(this)
}, "undefined" != typeof window && (new G).v("AxjgB5MAnACoAJwBpAAAABAAIAKcAqgAMAq0AzRJZAZwUpwCqACQACACGAKcBKAAIAOcBagAIAQYAjAUGgKcBqFAuAc5hTSHZAZwqrAIGgA0QJEAJAAYAzAUGgOcCaFANRQ0R2QGcOKwChoANECRACQAsAuQABgDnAmgAJwMgAGcDYwFEAAzBmAGcSqwDhoANECRACQAGAKcD6AAGgKcEKFANEcYApwRoAAxB2AGcXKwEhoANECRACQAGAKcE6AAGgKcFKFANEdkBnGqsBUaADRAkQAkABgCnBagAGAGcdKwFxoANECRACQAGAKcGKAAYAZx+rAZGgA0QJEAJAAYA5waoABgBnIisBsaADRAkQAkABgCnBygABoCnB2hQDRHZAZyWrAeGgA0QJEAJAAYBJwfoAAwFGAGcoawIBoANECRACQAGAOQALAJkAAYBJwfgAlsBnK+sCEaADRAkQAkABgDkACwGpAAGAScH4AJbAZy9rAiGgA0QJEAJACwI5AAGAScH6AAkACcJKgAnCWgAJwmoACcJ4AFnA2MBRAAMw5gBnNasCgaADRAkQAkABgBEio0R5EAJAGwKSAFGACcKqAAEgM0RCQGGAYSATRFZAZzshgAtCs0QCQAGAYSAjRFZAZz1hgAtCw0QCQAEAAgB7AtIAgYAJwqoAASATRBJAkYCRIANEZkBnYqEAgaBxQBOYAoBxQEOYQ0giQKGAmQABgAnC6ABRgBGgo0UhD/MQ8zECALEAgaBxQBOYAoBxQEOYQ0gpEAJAoYARoKNFIQ/zEPkAAgChgLGgkUATmBkgAaAJwuhAUaCjdQFAg5kTSTJAsQCBoHFAE5gCgHFAQ5hDSCkQAkChgBGgo0UhD/MQ+QACAKGAsaCRQCOYGSABoAnC6EBRoKN1AUEDmRNJMkCxgFGgsUPzmPkgAaCJwvhAU0wCQFGAUaCxQGOZISPzZPkQAaCJwvhAU0wCQFGAUaCxQMOZISPzZPkQAaCJwvhAU0wCQFGAUaCxQSOZISPzZPkQAaCJwvhAU0wCQFGAkSAzRBJAlz/B4FUAAAAwUYIAAIBSITFQkTERwABi0GHxITAAAJLwMSGRsXHxMZAAk0Fw8HFh4NAwUABhU1EBceDwAENBcUEAAGNBkTGRcBAAFKAAkvHg4PKz4aEwIAAUsACDIVHB0QEQ4YAAsuAzs7AAoPKToKDgAHMx8SGQUvMQABSAALORoVGCQgERcCAxoACAU3ABEXAgMaAAsFGDcAERcCAxoUCgABSQAGOA8LGBsPAAYYLwsYGw8AAU4ABD8QHAUAAU8ABSkbCQ4BAAFMAAktCh8eDgMHCw8AAU0ADT4TGjQsGQMaFA0FHhkAFz4TGjQsGQMaFA0FHhk1NBkCHgUbGBEPAAFCABg9GgkjIAEmOgUHDQ8eFSU5DggJAwEcAwUAAUMAAUAAAUEADQEtFw0FBwtdWxQTGSAACBwrAxUPBR4ZAAkqGgUDAwMVEQ0ACC4DJD8eAx8RAAQ5GhUYAAFGAAAABjYRExELBAACWhgAAVoAQAg/PTw0NxcQPCQ5C3JZEBs9fkcnDRcUAXZia0Q4EhQgXHojMBY3MWVCNT0uDhMXcGQ7AUFPHigkQUwQFkhaAkEACjkTEQspNBMZPC0ABjkTEQsrLQ==");
var abc = function (e) {
    console.log(__g._encrypt(encodeURIComponent(e)))
    // return __g._encrypt(encodeURIComponent(e))
};

// function run(e) {
//     return __g._encrypt(encodeURIComponent(e))
// }

// module.exports = {run}

得到id后就可以愉快获取需要的信息拉。

# 计算机科学与技术为例，拼接必要信息获得md5加密调用js获得id,下面是test

test = 'a0F8o7L8r8Yf6RY88RY0eAHqFBFYeLtyKXO02Ae0F9SX'
use_url = 'https://www.zhihu.com/api/v4/search_v3?t=general&q=计算机科学与技术&correction=1&offset=0&limit=20&lc_idx=0&show_all_topics=0'
headers = {
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36',
        'cookie': 'd_c0=%s;' % (d_c),
        # 'referer': referer,
        'x-zse-93': '101_3_2.0',
        'x-zse-96': '2.0_%s' % test,  # js加密而来
    }

res = requests.get(use_url, headers=headers)

不过最近知乎已经没有原来的大学和专业开设院校、就业之类的信息了。。。。。

4、梧桐果数据抓取

scrapy框架多层调用url的方法，见下面7吧。

5、Google Play app评论爬取

软件：西柚（seeyou）

主要模块：python 3.6的requests

利用requests获得网页源码后，难点主要在于信息提取。

信息提取两种方式：

  • 文本转html，利用xpath提取（不适用于页面展示的源码和你请求的得到的源码不一致的情况，这次）
  • 利用正则提取（相对于xpath,难度会大一点）。

Google Play app评论中的

用户打分、评论数据在AF_initDataCallback中key:\'ds:25\'对应的data里(不同的app ds对应的值不一样，注意找下规律)，定位好所需信息位置，利用正则提取。

import re,demjson
import pandas as pd

test = open('../google_facebook.txt',encoding='utf8').read()  # facebook的requests结果。
p = re.compile(r"AF_initDataCallback\({key: \'ds:25.+?, data:.+?, sideChannel: {}}\)",re.DOTALL)
tt = p.findall(test)
tt_dict = demjson.decode(tt[0].replace('AF_initDataCallback(','').replace(')',''))
test_res=[]
for i,v in enumerate(tt_dict ["data"][0]):
    d = dict()
    d['name'] = v[1][0]
    d['star'] = v[2]
    d['contents'] = v[4]
    test_res.append(d)
    
    
pd.DataFrame(test_res)

6、py调用百度地图API获得经纬度

  1. 获取调去API的秘钥ak：
    1. 注册成为开发者。
    2. 创建应用（浏览器端，允许访问IP填*）
    1. 具体步骤如下图

  1. 利用requests模块调用获得输入地址的经纬度。
import requests
import json


def getGeo(p):
    url = 'https://api.map.baidu.com/geocoding/v3/'
    params = {'address': p,  
              'ak': '2gDvpo219bUG91R8VXGK5rhXYg7GKg8o',  # 你创建的应用对应的百度密钥
              'output': 'json'}  # 输出结果设置为json格式
    res = requests.get(url, params)
    jd = json.loads(res.text)
    return jd

7、利用scrapy 获取国家统计局5级行政区划分

http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/

2020年数据：http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2020/index.html

7.1 创建一个工程

scrapy startproject getGeoInfoFromStatsGov

cd getGeoInfoFromStatsGov

scrapy genspider geoInfo stats.gov.cn
  • -t:指定使用哪种模板，默认是“basic”

7.2 修改参数配置

7.2.1 pipelines

开始生成时，只有process_item这个函数，是用来接收item（主程序爬取return or yiled的数据）

为了写入文件，并减少文件的频繁打开与关闭，我们在process_item函数上下分别加入文件打开函数和文件关闭函数。

class GetgeoinfofromstatsgovPipeline:
    def open_spider(self, spider):
        # 打开文件
        self.fp = open('../geoInfo.txt', 'a', encoding='utf8')

    def process_item(self, item, spider):
        self.fp.write(str(item))      # 数据写入
        return item

    def close_spider(self, spider):
        # 关闭文件
        self.fp.close()

7.2.2 items

定义你的变量Field实例。

你yield or return到的变量都要定义。

7.2.3 settings

这里是进行一些常规设置：

是否遵循机器人协议（遵循了还爬个鬼）

放开pipelines的注释（也可以写多个pipelines,只要在settings中注册，并给定相应的权重，默认的是300，这个数值你可以随意给，只要能体现出彼此之间的大小就行。）

定义控制台输出的日志信息等级（也可以不定义，只是控制台输出的信息会很多，包括debug级别、warning级别等）

中间件注册

操作数据库相关

分布式爬虫相关

这里我只做了日志等级修改及输出

放开pipelines注释。

7.2.4 middlewares

定义自己的中间件是要先搞清楚网络请求的内部过程的。这属于太高阶，一般没写过。

要写中间件：

  • 首先在settings中放开中间件的注释，并把自己写的中间件注册（按已有的格式写好）
  • 中间件的一个添加随机UA的示例

settings中的修改

如果没记错的话，应该是数越小越先执行。

7.2.5 爬取数据存入数据库

我没存过，看网上写的比较简单，可自行百度。

7.3 写主程序爬虫代码

geoInfo.py就是你的主程序，scrapy genspider geoInfo就是生成的这个文件，注意该名称不可与你的爬虫程序同名。

7.3.1 一级url爬取

自然生成的geoInfo.py中就只有一个 parse函数。

这就会把国家统计局这张图中的各个省份的名字提取并传到pipelines里。

显而易见，每个省份下都链接他的下一级信息页面也即市级别的信息，在市级别的信息又链接下一级别的县层级信息。。。这就涉及多级url爬取

7.3.2 多级url爬取

yield就要callback下一个函数去解析下一级的信息，并将该层级的信息传递出去。

利用scrapy.Request()

参数：

  • url:就是下一层函数要请求的url
  • callback:就是你要调起的下一级解析函数的名字
  • meta:用来传递给下一级解析函数信息，字典的形式

具体代码如下

import scrapy
from ..items import GetgeoinfofromstatsgovItem


class GeoinfoSpider(scrapy.Spider):
    name = 'geoInfo'
    allowed_domains = ['stats.gov.cn']
    start_urls = [
        'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2020/']  # http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2020/index.html

    def parse(self, response):
        firstLevels = response.xpath('//tr[@class="provincetr"]//td//a')
        for i in firstLevels:
            province = i.xpath('./text()').extract_first()
            p_url = i.xpath('./@href').extract_first()
            p_url = 'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2020/' + p_url
            # if province in ['新疆维吾尔自治区', '云南省', '安徽省', '黑龙江省', '山西省', '河北省', '宁夏回族自治区', '四川省', '广东省', '内蒙古自治区', '上海市',
            #                 '湖北省', '青海省', '广西壮族自治区', '辽宁省', '江苏省', '天津市', '北京市','甘肃省', '江西省', '福建省', '陕西省', '湖南省', '吉林省', '山东省','西藏自治区', '贵州省', '重庆市']:
            #     continue
            yield scrapy.Request(url= p_url, callback=self.secondParse,
                                 meta={'info': (province, p_url)})

    def secondParse(self, response):
        province, p_url = response.meta.get('info')
        print(2, province, p_url)
        cityLevels = response.xpath('//table[@class="citytable"]//tr[@class="citytr"]')
        for c in cityLevels:
            cityCode = c.xpath('./td[1]//text()').extract_first()
            cityName = c.xpath('./td[2]//text()').extract_first()
            cityUrl = c.xpath('./td[2]//@href').extract_first()
            c_url = 'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2020/' + cityUrl
            yield scrapy.Request(url=c_url, callback=self.thirdParse,
                                 meta={'info': (province, cityCode, cityName, cityUrl)})

    def thirdParse(self, response):
        province, cityCode, cityName, cityUrl = response.meta.get('info')
        cityUrl = cityUrl.strip().split('/')[0]
        countyLevels = response.xpath('//table[@class="countytable"]//tr[@class="countytr"]')
        for co in countyLevels:
            countyCode = co.xpath('./td[1]//text()').extract_first()
            countyName = co.xpath('./td[2]//text()').extract_first()
            countyUrl = co.xpath('./td[2]//@href').extract_first()
            if countyUrl:
                co_url = 'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2020/' + cityUrl + '/' + countyUrl
                yield scrapy.Request(url=co_url, callback=self.fourParse,
                                     meta={'info': (
                                         province, cityCode, cityName, countyCode, countyName, cityUrl, countyUrl)})
            else:
                geo = GetgeoinfofromstatsgovItem(province=province, cityCode=cityCode, cityName=cityName,
                                                 countyCode=countyCode, countyName=countyName)
                yield geo

    def fourParse(self, response):
        province, cityCode, cityName, countyCode, countyName, cityUrl, countyUrl = response.meta.get('info')
        countyUrl = countyUrl.strip().split('/')[0]
        townLevels = response.xpath('//table[@class="towntable"]//tr[@class="towntr"]')
        for t in townLevels:
            townCode = t.xpath('./td[1]//text()').extract_first()
            townName = t.xpath('./td[2]//text()').extract_first()
            townUrl = t.xpath('./td[2]//@href').extract_first()
            if townUrl:
                t_url = 'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2020/' + cityUrl + '/' + countyUrl + '/' + townUrl
                yield scrapy.Request(url=t_url, callback=self.fiveParse,
                                     meta={'info': (
                                         province, cityCode, cityName, countyCode, countyName, townCode, townName)})
            else:
                geo = GetgeoinfofromstatsgovItem(province=province, cityCode=cityCode, cityName=cityName,
                                                 countyCode=countyCode, countyName=countyName, townCode=townCode,
                                                 townName=townName)
                yield geo

    def fiveParse(self, response):
        province, cityCode, cityName, countyCode, countyName, townCode, townName = response.meta.get('info')
        villageLevels = response.xpath('//table[@class="villagetable"]//tr[@class="villagetr"]')
        for v in villageLevels:
            vCode = v.xpath('./td[1]/text()').extract_first()
            vName = v.xpath('./td[3]/text()').extract_first()
            geo = GetgeoinfofromstatsgovItem(province=province, cityCode=cityCode, cityName=cityName,
                                             countyCode=countyCode, countyName=countyName, townCode=townCode,
                                             townName=townName, vCode=vCode, vName=vName)
            yield geo

7.4 数据清洗

import demjson
import pandas as pd

with open('E:/programDll/geoInfoNew.txt', 'a', encoding='utf8') as fp:
    with open('E:/programDll/geoInfo.txt', encoding='utf8') as f:
        for l in f.readlines():
            l = l.strip()
            if len(l.split('}')) == 2:
                l1 = l.split('}')[0] + "}" + "\n"
                fp.write(l1)
                if l.split('}')[1]:
                    l2 = l.split('}')[1]
                    fp.write(l2)
                else:
                    pass
            elif l.split('{')[0] == "":
                fp.write(l)
            else:
                fp.write(l)

geoInfo = pd.read_csv('E:/programDll/geoInfoNew.txt', header=None, sep='\n')
geoInfo.columns = ['info']
# geoInfo['info'] = geoInfo['info'].apply(lambda x: x.replace(',{', '{').replace(",,", ","))
geoInfo['info'] = geoInfo['info'].apply(lambda x: demjson.decode(x))
geoInfo['vCode'] = geoInfo['info'].apply(lambda x: x.get('vCode'))
geoInfo['vName'] = geoInfo['info'].apply(lambda x: x.get('vName'))
geoInfo['townCode'] = geoInfo['info'].apply(lambda x: x.get('townCode'))
geoInfo['townName'] = geoInfo['info'].apply(lambda x: x.get('townName'))
geoInfo['countyCode'] = geoInfo['info'].apply(lambda x: x.get('countyCode'))
geoInfo['countyName'] = geoInfo['info'].apply(lambda x: x.get('countyName'))
geoInfo['cityCode'] = geoInfo['info'].apply(lambda x: x.get('cityCode'))
geoInfo['cityName'] = geoInfo['info'].apply(lambda x: x.get('cityName'))
geoInfo['province'] = geoInfo['info'].apply(lambda x: x.get('province'))

geoInfo = geoInfo.iloc[:, 1:]
geoInfo.drop_duplicates(inplace=True)
geoInfo.to_csv('E:/programDll/getGeoInfoFromStatsGov/2020年五级地理位置划分.txt', index=None)

数据示例

Galaxy_saturn
关注
  • 0
    点赞
  • 踩
  • 8
    收藏
    觉得还不错? 一键收藏
  • 1
    评论
专栏目录
国家统计局 省市县镇村 5级地域名称及经纬度等 .sql 数据
04-26
国家统计局 省市县镇村 5级地域名称及经纬度等 .sql 数据国家统计局 省市县镇村 5级地域名称及经纬度等 .sql 数据国家统计局 省市县镇村 5级地域名称及经纬度等 .sql 数据
Mysql 2018国家统计局区划和城乡划分数据库(包含经纬度数据，以及数据来源，提供大家参考)
点滴记录
05-05 4630
-- ---------------------------- -- Table structure for districts -- ---------------------------- DROP TABLE IF EXISTS `districts`; CREATE TABLE `districts` ( `code` varchar(32) CHARACTER SET utf8m...
探数API统计分享-中国各省人均消费支出
最新发布
tanshuapi的博客
05-23 162
我国各省（市、区）的人均消费支出情况各不相同。其中，上海的人均消费支出最高，达到了2021年的48879元，位居全国之首。紧随其后的是北京，人均消费支出为43640元。相比之下，西藏的人均消费支出最低，2021年的数据为15343元，仅为上海的31%。这些数据均来源于国家统计局，不包括港澳台地区的统计。根据2017年至2021年的。
国家统计局 省市县镇乡村等信息。
郭勤靖
06-23 1422
国家统计局 省市县镇乡村等信息。http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2021/
vue添加百度统计
猫先生的专栏
04-12 4965
申请百度统计后，会得到一段JS代码，需要插入到每个网页中去，在Vue.js项目首先想到的可能就是，把统计代码插入到index.html入口文件中，这样就全局插入，每个页面就都有了;这样做就涉及到一个问题，Vue.js项目是单页应用，每次用户浏览网站时，访问内页时页面是不会刷新的，也就意味着不会触发百度统计代码；所以最终在百度统计后台看到的效果就是只统计到了网页入口的流量，却无法统计到内页的访问流量...
Java 爬取2023年国家统计局的数据，更新全国的省市区地址和编码
weixin_622464的博客
09-04 929
全国的省市区名称和编码
基于Python的企查查爬虫，爬取完整的公司数据+源代码+文档说明
12-01
基于Python的企查查爬虫，爬取完整的公司数据 -------- 该资源内项目源码是个人的毕设，代码都测试ok，都是运行成功后才上传资源，答辩评审平均分达到96分，放心下载使用！ <项目介绍> 1、该资源内项目代码都经过...
qichacha.zip
12-20
在本项目中，"qichacha.zip"是一个包含使用Spring Boot框架实现的企查查（Qichacha）数据抓取程序的压缩包。这个小规模的示例项目旨在展示如何利用编程技术从企查查网站获取数据，为技术交流提供参考。以下是该项目...
qichacha.csv
09-06
爬取所需要的数据
基于Python的qichacha重庆成都2000家企业历史信息查询设计源码
05-23
qichacha是一个基于Python开发的重庆成都2000家企业历史信息查询系统，包含410个文件，其中包括123个PNG图片文件、73个CSS文件、38个LESS文件、38个SCSS文件、37个JavaScript文件、13个Python文件、11个EOT字体文件...
company-crawler:天眼查爬虫&企查查爬虫，指定关键字爬取公司信息
05-12
天眼查、企查查 公司信息爬虫 使用说明 设置数据源 MysqlConfig = { 'develop': { 'host': '192.168.1.103', ...设置爬取关键字qichacha&tianyancha keys = ['Google'] # 设置爬取列表 crawler.load_keys(keys) c
网站搭建简要记述
平平酱的博客
03-13 3177
H96 Max小盒子一、建站1、域名（解析）2、内网穿透3、新建站点二、新建网站-博客、可道云1、搭建2、 虽然搭建了自己的博客，但我还是相信CSDN保存文章更安全，因此规划： CSDN发布技术型文章； 自建博客发布日常型文章。 本文记录一下从建站到安装博客再到优化界面的大体流程。 最终展示：桃子七七 一、建站 如今，拥有一个属于自己的网站是比较酷的事，网上的技术也日渐成熟，即使不清楚前端开发，有一定的技术思路，同样能拼拼凑凑搭建一个花里胡哨的网站。从TB上购买了一台200大洋的小型服务器—小盒子`，看处理
从国家统计局爬取行政区划数据
qqqqgmd的专栏
06-13 966
import requests from bs4 import BeautifulSoup _main_url = 'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2020/' def create_parser(url, encoding='gbk'): response = requests.get(url) response.encoding = encoding return BeautifulSoup(respon.
企查查导出excel数据添加WGS84坐标系
sumuses的博客
11-29 753
使用前准备 删除表格的第一行“企查查”图片，在最后两列添加新的字段（经度和纬度），然后另存为xlsx格式。 代码可直接赋值粘贴使用，只需修改 可能要替换的位置 直接对原excel进行操作——谨慎操作. from urllib.request import urlopen, quote import time from openpyxl.reader.excel import load_workbook import re from urllib import parse import hashlib f
从国家统计局获取行政区划代码并从高德地图/百度地图中获取对应的经纬度
qq_57703172的博客
03-23 1040
【代码】从国家统计局获取行政区划代码并从高德地图/百度地图中获取对应的经纬度。
全国经纬度大全
范天缘的博客
05-24 3万+
北京市 116.4 39.9 天津市 117.2 39.12 河北省 石家庄市 114.52 38.05 河北省 唐山市 118.2 39.63 河北省 秦皇岛市 119.6 39.93 河北省 邯郸市 114.48 36.62 河北省 邢台市 114.48 37.07 河北省 保定市 115.47 38.87 河北省 张家口市 114.88 40.82 河北省 承德市 117.93 40.97 河北省 沧州市 116.83 38.3 河北省 廊坊市 116.7 39.52 河北省 衡水市
百度统计的JS脚本原理分析
热门推荐
每天进步一点点 - MinCase
05-02 3万+
<!-- p {text-indent:2em} --> 首先，百度统计会要求我们在要统计的页面中嵌入一段js语句，类似如下： var _bdhmProtocol = ((“https:” == document.location.protocol) ? ” https://” : ” http://”); document.write(unescape(“%3C
全国行政区域数据及导入及接口实现及四级联动
qq_45614471的博客
07-22 1045
一、先将SQL导入数据库 https://download.csdn.net/download/qq_45614471/20419356 https://download.csdn.net/download/qq_45614471/20419356 二、接口全量查询所有四级父子行政区域数据 1、实体类 /** * 行政区域 VO * @author rambo */ @ApiModel(value = "行政区域 VO", description = "行政区域 VO") @Getter @Sette
python 爬取企查查数据
06-28
### 回答1： 要使用Python爬取企查查数据，可以按照以下步骤进行操作： 1. 导入所需的Python库，如requests、beautifulsoup和pandas。 ``` import requests from bs4 import BeautifulSoup import pandas as pd ``` 2. 构造请求URL，可以通过企查查网站检索得到相应公司的URL，或者直接拼接URL。 ``` url = "https://www.qichacha.com/company_search?key=关键词" ``` 3. 发送HTTP请求，获取网页内容。 ``` headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3"} response = requests.get(url, headers=headers) ``` 4. 使用BeautifulSoup解析网页内容，提取所需数据。 ``` soup = BeautifulSoup(response.text, 'html.parser') # 定位到要提取的数据的HTML标签 data = soup.find_all('div', class_='search_list')[0].find_all('tr') ``` 5. 将提取的数据存储到DataFrame中，方便后续处理和分析。 ``` records = [] for tr in data: record = [] for td in tr.find_all('td'): record.append(td.text.strip()) records.append(record) df = pd.DataFrame(records) ``` 6. 对DataFrame进行必要的数据清洗和处理。 ``` df.columns = df.iloc[0] # 将第一行作为列名 df = df[1:] # 去除第一行数据 ``` 7. 可以选择将处理后的数据保存到本地文件，或者进行进一步的分析和可视化展示。 ``` df.to_csv('企查查数据.csv', index=False) ``` 注意：爬取网站数据要遵守相关法律法规和网站的使用条款，尊重网站规则并使用爬虫技术进行合法合规的数据获取。 ### 回答2： Python可以使用多种库进行网页数据爬取，其中比较常用的是BeautifulSoup和Selenium。 首先，我们需要安装相应的库，可以使用pip install进行安装。 ```python pip install beautifulsoup4 pip install selenium ``` 然后，我们需要导入所需的库。 ```python from selenium import webdriver from bs4 import BeautifulSoup import time ``` 接着，我们需要设置webdriver的路径，并实例化一个浏览器对象。 ```python # 设置webdriver的路径 driver_path = "chromedriver_path/chromedriver" # 实例化一个浏览器对象，这里以Chrome为例 browser = webdriver.Chrome(executable_path=driver_path) ``` 接下来，我们需要打开企查查的网页并进行相关操作，这里以搜索公司信息为例。 ```python # 打开企查查网页 url = "https://www.qcc.com" browser.get(url) # 找到搜索框并输入关键词 search_input = browser.find_element_by_id("headerKey") search_input.send_keys("公司名") # 找到搜索按钮并点击 search_button = browser.find_element_by_id("index-getData") search_button.click() # 等待页面加载 time.sleep(5) ``` 页面加载完成后，我们可以通过BeautifulSoup来解析所需的数据。 ```python # 获取页面源代码 page_source = browser.page_source # 使用BeautifulSoup解析数据 soup = BeautifulSoup(page_source, "html.parser") # 找到所需的数据，并进行相应的处理 data = soup.find_all("div", class_="result-list") ``` 最后，我们需要关闭浏览器。 ```python browser.quit() ``` 这样，我们就完成了使用Python爬取企查查数据的过程。可以根据需求对代码进行相应的修改和优化。

“相关推荐”对你有帮助么？

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论 1
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值