python360指数_Python批量采集360关键词指数-CSDN博客

Python批量查360好搜关键词指数，requests模块让爬虫更简单！脚本基础的功能都有，轮换UA，IP，verfied_ip.txt里的IP先提前用脚本验证过能正常使用！接下来爆爆爆！自己调整代码或者可以加入多线程，都比较easy，另外整体代码也比较简单，不作多说明了！

#encoding=utf-8

#@by--bigwayseo

#https://bigwayseo.com/

import requests

# import json

# import urllib

import random

# import linecache

import re

import time

#提取360关键词数据的正则表达式

recon=re.compile('{"query":"(.*?)","data":{.*?"week_index":([0-9]+),"month_index":([0-9]+)}}')

old_ip=[]

class sodata():

def __init__(self):

self.old_ip=[]

self.recon=re.compile('{"query":"(.*?)","data":{.*?"week_index":([0-9]+),"month_index":([0-9]+)}}')

#随机轮换user agent

def getua(self):

head=[

"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.81 Safari/537.36",

"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.27 (KHTML, like Gecko) Chrome/12.0.712.0 Safari/534.27",

"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.24 Safari/535.1",

"Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.120 Safari/535.2",

"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.36 Safari/535.7",

"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.81 Safari/537.36",

"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.27 (KHTML, like Gecko) Chrome/12.0.712.0 Safari/534.27",

"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.24 Safari/535.1",

"Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.120 Safari/535.2",

"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.36 Safari/535.7",

"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:2.0b4pre) Gecko/20100815 Minefield/4.0b4pre",

"Mozilla/5.0 (Windows; U; Windows NT 6.1; zh-CN) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5",

"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-GB; rv:1.9.1.17) Gecko/20110123 (like Firefox/3.x) SeaMonkey/2.0.12",

"Mozilla/5.0 (Windows NT 5.2; rv:10.0.1) Gecko/20100101 Firefox/10.0.1 SeaMonkey/2.7.1",

"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; zh-CN) AppleWebKit/532.8 (KHTML, like Gecko) Chrome/4.0.302.2 Safari/532.8",

"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; zh-CN) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.464.0 Safari/534.3",

"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_5; zh-CN) AppleWebKit/534.13 (KHTML, like Gecko) Chrome/9.0.597.15 Safari/534.13",

"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.186 Safari/535.1",

"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/125.2 (KHTML, like Gecko) Safari/125.8",

"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; fr-fr) AppleWebKit/312.5 (KHTML, like Gecko) Safari/312.3",

"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/418.8 (KHTML, like Gecko) Safari/419.3",

"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1 Camino/2.2.1",

"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0b6pre) Gecko/20100907 Firefox/4.0b6pre Camino/2.2a1pre",

"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",

"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.4 (KHTML like Gecko) Chrome/22.0.1229.79 Safari/537.4",

"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2; rv:10.0.1) Gecko/20100101 Firefox/10.0.1",

"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:16.0) Gecko/20120813 Firefox/16.0",

"Mozilla/5.0 (Macintosh; U; Intel Mac OS X; zh-CN) AppleWebKit/528.16 (KHTML, like Gecko, Safari/528.16) OmniWeb/v622.8.0.112941",

"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_6; zh-CN) AppleWebKit/528.16 (KHTML, like Gecko, Safari/528.16) OmniWeb/v622.8.0"

]

random_headers=random.choice(head)

return random_headers

#随机轮换ip

def getip(self):

for ip in open('verfied_ip.txt'):

if ip.strip() in self.old_ip:

continue

else:

ip=ip.strip()

self.old_ip.append(ip)

newip=random.choice(self.old_ip)

return newip

def myproxies(self):

newip=self.getip()

proxies={

"http": "https://%s"%newip.strip()

# "http": "https://182.90.30.245:80",

# "https": "https://119.188.94.145:80",

# "https": "https://119.97.164.48:8085",

# "https": "https://180.153.46.112:80",

# "http": "https://219.139.159.252:80",

# "http": "https://42.51.155.42:80",

# "http": "https://182.90.68.106:80"

}

return proxies

def mainpass(self):

for word in open('360word.txt'):

keyword=str(word)

self.url='https://index.haosou.com/index.php?a=overviewJson&q='+keyword+'&area=全国'

random_headers=self.getua()

headers={'User-Agent':'%s'%random_headers}

# data={

# 'a':'soIndexJson',

# 'q':'word',

# 'area':'全国'

# }

# time.sleep(1)

#https://bigwayseo.com/

t=1

while t:

try:

pro=self.myproxies()

print pro

html=requests.post(url=self.url,headers=headers,proxies=pro,timeout=10).content

if '360指数_访问异常出错' or 'verify' in html:

print '360指数_访问异常出错'

# time.sleep(600)

continue

else:

t=0

r=re.findall(self.recon,html)

for i in r:

print '%s,周平均指数%s，月平均指数%s'%(keyword,i[1],i[2])

except Exception,e:

print e

continue

if __name__ == '__main__':

p=sodata()

p.mainpass()

通过抓包工具，或者浏览器自身的F12开发者工具可以捕获:

https://index.haosou.com/index.php?a=overviewJson&q='+keyword+'&area=全国

看看这个链接对应的reponse选项，可以发现对应的json数据

通过requests模块post数据到这个链接即可返回一个关于关键词的json格式数据

通过json的方法提取或者正则提取等，按自己的顺手的方法来即可

逻辑很简单，百度指数，因为已经加密，所以要复杂许多，但是python批查询360指数已全部实现