def __init__(self):
self.count = 1 # 记录当前爬第几条数据
self.company = []
self.desc_url_queue = Queue() # 线程池队列
self.pool = Pool(POOL_MAXSIZE) # 线程池管理线程,最大协程数
def work_spider(self):
"""
爬虫入口
"""
urls = [START_URL.format(p) for p in range(1, 16)]
for url in urls:
logger.info("爬取第 {} 页".format(urls.index(url) + 1))
html = requests.get(url, headers=HEADERS).content.decode("gbk")
bs = BeautifulSoup(html, "lxml").find("div", class_="dw_table").find_all(
"div", class_="el"
)
for b in bs:
try:
href, post = b.find("a")["href"], b.find("a")["title"]
locate = b.find("span", class_="t3").text
salary = b.find("span", class_="t4").text