7、批量关键字百度搜索结果url解码

import requests
from bs4 import BeautifulSoup
import re
import time

#coding:utf-8

with open('key.txt','r') as f:
    result = f.read()
keys = result.split('\n')
key_words = list(enumerate(keys, start=1))


for key in key_words:
    url = 'https://www.baidu.com/s?wd='+ key[1]

    header = {
        'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36',
        'Cookie':'PSTM=1476231684; BIDUPSID=4F526560482E2A5E68D69CC8B0998806; plus_cv=1::m:92e3c68f; BAIDUID=C5A710455602AEA5BEC3D1B13B26321B:FG=1;'
                 ' BDUSS=W5zS3JSeVYwSHZjVm5SdTdjQjlKNC1FLWJqbklvaEptZjVZVkl2bXhMN1o1amhZSVFBQUFBJCQAAAAAAAAAAAEAAACj2nZjanVleWluZ3MAAAAAAAAAAAAAAAAAAAA'
                 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAANlZEVjZWRFYT; BD_HOME=1; BD_UPN=12314353; sug=3; sugstore=0; ORIGIN=2; bdime=0;'
                 ' H_PS_645EC=78d5XI4%2Bj6NkSjLKSmkiYdx%2F5jHNa0c4UemYz6WwEpyczIPebiQwaLtzwnXd2gUHv28P; BDRCVFR[feWj1Vr5u3D]=I67x6TjHwwYf0; BD_CK_SAM=1;'
                 ' PSINO=6; H_PS_PSSID=1448_18288_21112_17001_20241_21455_21406_21394_21377_21192_20929; BDSVRTM=0'
    }

    web_db = requests.get(url,headers=header)
    time.sleep(2)
    soup = BeautifulSoup(web_db.text,'lxml')

    titles = soup.select('#content_left > div > h3 > a')
    ranks = [ i for i in range(1,11)]

    for title,link,rank in zip(titles,titles,ranks):


        baidu_url = link.get('href')
        if str(baidu_url).find('link?url=') > 0 :
            web_db2 = requests.get(baidu_url, allow_redirects=False)
            if web_db2.status_code == 200:
                soup = BeautifulSoup(web_db2.text, 'lxml')
                urls = soup.select('head > noscript')
                url2 = urls[0]
                url_math = re.search(r'\'(.*?)\'', str(url2), re.S)
                web_url = url_math.group(1)
            elif web_db2.status_code == 302:
                web_url = web_db2.headers['location']
            else:
                web_url = 'error'
        else:
            web_url = baidu_url


        data = {
            'key':key,
            'title':title.get_text(),
            'url':web_url.encode('utf-8'),
            'rank':rank,
        }
        with open('info.txt','a') as f:
            f.write(str(data)+'\n')
    print('已完成采集任务' + str(key[0]) + '**********总采集任务' + str(len(key_words)))