python模拟百度搜索点击链接_用 Python 获取百度搜索结果链接

前言

近期有许多项目需要这个功能，由于Python实现起来比较简单就这么做了，代码贴下来觉得好点个赞吧~

代码

# coding: utf-8

import os

import time

import requests

import urllib.parse

from bs4 import BeautifulSoup

from urllib.parse import urlparse

from fake_useragent import UserAgent

from multiprocessing.pool import ThreadPool

LOCATIONS = {}

GLOBAL_THREAD = 500

GLOBAL_TIMEOUT = 50

def get_links(keyword, generator, pages):

links = []

for page in range(int(pages.split("-")[0]), int(pages.split("-")[1]) + 1):

for genera in range(int(generator.split("-")[0]), int(generator.split("-")[1]) + 1):

links.append(

"http://www.baidu.com.cn/s?wd=" + urllib.parse.quote(keyword + str(genera)) + "&pn=" + str(page * 10))

return links

def get_page(url):

headers = {"user-agent": UserAgent().chrome}

req = requests.get(url, headers=headers)

req.encoding = "utf-8"

soup = BeautifulSoup(req.text, "lxml")

for link in soup.select("div.result > h3.t > a"):

req = requests.get(link.get("href"), headers=headers, allow_redirects=False)

if "=" in req.headers["location"]:

root = urlparse(req.headers["location"]).netloc

LOCATIONS[root] = req.headers["location"]

def baidu_search():

try:

os.system("cls")

print("-" * 56 + "\n")

print("| BaiduSearch Engine By 美图博客[https://www.meitubk.com/] |\n")

print("-" * 56 + "\n")

keyword = input("Keyword: ")

generator = input("Generator(1-10): ")

pages = input("Pages(0-10): ")

start = time.time()

pool = ThreadPool(processes=GLOBAL_THREAD)

pool.map(get_page, get_links(keyword, generator, pages))

pool.close()

pool.join()

end = time.time()

path = r"D:\Desktop\result.txt"

save_result(path)

print("\nSava in %s" % path)

print("Result count: %d" % len(LOCATIONS.values()))

print("Running time: %ds" % (end - start))

except:

print("\nInput Error!")

exit(0)

def save_result(path):

with open(path, "w") as file:

for url in list(LOCATIONS.values()):

file.write(url + "\n")

baidu_search()

使用