import threading
import requests
import re
from queue import Queue
import os
import logging
import time
from Dao.query import Util
from bean.book import Company
class Product(threading.Thread):
pass
class MyProxy(threading.Thread):
def __init__(self, proxy_list: Queue, url, queue: Queue, num=0):
"""
初始化这个多线程需要几个具体的参数
:param proxy_list: 放了当前处于闲置状态的IP, 里面是一个列表有IPDict 和 当前Ip失败次数
:param url: 需要请求的URL
:param queue: 放置所有的需要请求的网页
:param num:
"""
super().__init__()
self.proxy_list = proxy_list
self.url = url
self.queue = queue
self.num = num
def getProxy(self):
return self.proxy_list
def run(self):
# if self.proxy_list.full():
# print("当前代理队列已经满了, 暂时不必加入新的ip")
# return
if self.proxy_list.qsize() <= 8:
logging.info("开始请求IP, 将ip加入ip队列")
self.create_ip()
logging.info("成功将ip加入队列中,当前队列中的IP数量是: " + str(self.proxy_list.qsize()))
else:
logging .info("Ip数量已经足够了,不需要新的ip进入啦!!")
def create_ip(self):
try:
ip_addr, port = self.get_proxy()
proxy = {
"http": "http://{}:{}".format(ip_addr, port),
"https": "http://{}:{}".format(ip_addr, port)
}
self.proxy_list.put([proxy, 0])
except Exception as ec:
logging.error("获取代理ip失败")
print(ec)
def get_proxy(self):
try:
response = requests.get(self.url)
print(response.json())
ip = response.json()["RESULT"][0]['ip']
port = response.json()["RESULT"][0]['port']
return ip, port
except Exception as ec:
print(ec)
class Customer(threading.Thread):
def __init__(self, thread_id, queue, header, proxy: list, proxy_queue:Queue):
super().__init__()
self.thread_id = thread_id
self.queue = queue
self.header = header
self.proxy = proxy
self.proxy_list = proxy_queue
def run(self) -> None:
print('启动线程:', self.name)
self.crawl_spider()
print('退出了该线程:', self.name)
def crawl_spider(self):
while True:
if self.queue.empty():
break
else:
company_id = self.queue.get().company_id
self.create_dir("./data/jobs/{}/".format(company_id))
try:
cookie, header = self.__getCookiesByCompanyId(company_id)
response = self.__postDataAndCookies(cookie, header, self.proxy[0], company_id, 0)
except Exception as e:
print("thread " + self.name + " dead!! Error: " + str(e))
self.proxy[1] += 1
self.queue.put(Company(company_id=company_id))
self.test_proxy()
return
end = self.calculation_page(response)
for j in range(1, end+2):
print("current thread: " + self.name + ", try url: " + str(company_id)+", pageNo is: "+str(j))
# 更新cookies和header
if j % 10 == 0:
cookies, header = self.__getCookiesByCompanyId(company_id)
try:
response = self.__postDataAndCookies(cookie, header, self.proxy[0], company_id, j)
except Exception as e:
print("thread " + self.name + " dead!! Error: " + str(e))
self.proxy[1] += 1
self.queue.put(Company(company_id=company_id))
self.test_proxy()
return
# 得到的信息如果字符大于100,保存这个文件
if len(response.text) >= 100:
self.save(response, j, company_id)
time.sleep(1)
def test_proxy(self):
if self.proxy[1] <= 10:
self.proxy_list.put(proxy)
@staticmethod
def calculation_page(response):
end = 30
# 计算公司招聘职位页数
text = re.search("\"totalCount\":\"(.*)\",", response.text)
jobs_num = text.group(1) if text is not None else "10"
page_no = eval(jobs_num) / 10
if end != int(page_no):
end = 300 if int(page_no) > 300 else int(page_no)
return end
@staticmethod
def __postDataAndCookies(cookies: dict,
header,
proxy,
company_id,
page_no,
city="",
salary="",
work_year="",
school_job=""):
search_position = "https://www.lagou.com/gongsi/searchPosition.json"
data = {
"companyId": "{}".format(company_id),
"positionFirstType": "全部",
"city": city,
"salary": salary,
"workYear": work_year,
"schoolJob": school_job,
"pageNo": page_no,
"pageSize": "10"
}
position_resp = requests.post(search_position, data=data, headers=header, cookies=cookies, proxies=proxy)
return position_resp
def __getCookiesByCompanyId(self, company_id):
"""
通过公司Id返回请求公司的构造信息
:param company_id: 公司ID
:return:
"""
company_url = 'https://www.lagou.com/gongsi/j{id}.html'.format(id=company_id)
home_resp = requests.get(company_url, headers=self.header, proxies=self.proxy[0], allow_redirects=False)
cookies = home_resp.cookies.get_dict()
header = self.__setHeaders(company_url, self.header, home_resp)
return cookies, header
@staticmethod
def __setHeaders(company_url, header, home_resp):
text = re.search(r"window.X_Anti_Forge_Token = '(.*?)'", home_resp.text)
header['X_Anti_Forge_Token'] = "0" if text is None else text.group(1)
text = re.search(r"window.X_Anti_Forge_Code = '(.*?)'", home_resp.text)
header['X_Anti_Forge_Code'] = "None" if text is None else text.group(1)
header['Content-Type'] = 'application/x-www-form-urlencoded; charset=UTF-8'
header['Referer'] = company_url
return header
def save(self, response, n, company_id):
with open("data/jobs/{}/{}.txt".format(company_id, n), "w", encoding="utf-8") as f:
f.write(response.text)
print("成功存入一个文件在: " + "data/jobs/{}/{}.txt".format(company_id, n))
@staticmethod
def create_dir(path):
if not os.path.exists(path):
os.mkdir(path)
@staticmethod
def get_proxy():
try:
response = requests.get(url)
print(response.json())
ip = response.json()["RESULT"][0]['ip']
port = response.json()["RESULT"][0]['port']
return ip, port
except Exception as ec:
print(ec)
if __name__ == '__main__':
proxy_queue = Queue(20)
headers = {
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75Safari/537.36",
"host": "www.lagou.com",
"referer": "https://www.lagou.com/gongsi/62.html"
}
session = Util.getDBSession("root", "chuanzhi", "lagou")()
company_list = list(set(Util.selectAllCompany(session)))
session.close()
company_queue = Queue()
for company in company_list:
company_queue.put(company)
url = "你的代理获取方式"
i = 0
while not company_queue.empty():
MyProxy(proxy_queue, url, company_queue).start()
try:
i += 1
if not proxy_queue.empty():
# 拿到一个失败次数小于15次的可用代理IP.
while True:
proxy = proxy_queue.get()
if proxy[1] <= 15:
break
Customer(i, company_queue, headers, proxy, proxy_queue).start()
time.sleep(5)
except Exception as E:
print(E)
一个拉勾网的的爬虫_多线程版本
最新推荐文章于 2024-04-08 08:44:13 发布