爬虫笔记IP代理以及 boos直聘 实例

可以去某云某宝或其他渠道购买，具体使用看自己购买商家的API文档，查看使用方法。

ip_proxy.py

1. import requests

2.  
3.  

4. class ip_getter(object):

5. def __init__(self):

6. self.ip_proxy_str = get_ip_string()

7.  

8. def update_ip_proxy_str(self):

9. self.ip_proxy_str = get_ip_string()

10. print('get one ip : ' + self.ip_proxy_str)

11.  
12.  

13. def get_ip_string():

14. url = 'API接口'

15. response = requests.get(url)

16.  

17. return response.text

boos_bs4.py ( boos直聘 实例)

1. from bs4 import BeautifulSoup

2. import requests

3. import ip_proxy

4. from urllib import parse

5.  

6. headers = {

7. 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36',

8. }

9.  
10.  

11. def get_boss_info(my_ip, detailed_url):

12. # url = 'https://www.zhipin.com/job_detail/7e883f0c3a336cb51n142968FFM~.html?ka=search_list_1'

13.  

14. proxy = {

15. 'http': 'http://' + my_ip.ip_proxy_str,

16. 'https': 'http://' + my_ip.ip_proxy_str

17. }

18. response = requests.get(detailed_url, headers=headers, proxies=proxy, timeout=5)

19.  

20. soup = BeautifulSoup(response.text, 'lxml')

21. title = soup.find('h1').text

22. # div_ele = soup.find('div', class_="name")

23. # print(div_ele)

24. salary = soup.find('span', class_="badge").text.replace('\n', '').strip()

25. print(title)

26. print(salary)

27. gezhong_info = soup.select('div.info-primary > p')[0].text.replace('\n', '').strip()

28. print(gezhong_info)

29. gangwei_info = soup.select('div.text')[0].text

30. print(gangwei_info)

31.  
32.  

33. # 获取详情页的url

34. def get_detail_url(my_ip, url):

35. # url = 'https://www.zhipin.com/c101010100/h_101010100/?query=python&page=2&ka=page-2'

36. proxy = {

37. 'http': 'http://' + my_ip.ip_proxy_str,

38. 'https': 'http://' + my_ip.ip_proxy_str

39. }

40. response = requests.get(url, headers = headers, proxies=proxy, timeout=5)

41.  

42. soup = BeautifulSoup(response.text, 'lxml')

43. # a_ele_list = soup.select('h3.name > a')

44. a_ele_list = soup.select('div.job-list > ul > li div.info-primary > h3 > a')

45.  

46. for a_ele in a_ele_list:

47. # 属性值的获取可以通过类似字典的方式获取

48. a_href = a_ele['href']

49. # 拼接详情页的链接

50. href = parse.urljoin(url, a_href)

51. print('详情页的href: ' + href)

52. # 重试三次, 获取代理访问boss直聘, 三次没有成功访问就跳过

53. for i in range(0, 3):

54. try:

55. # 获取详情页的信息

56. get_boss_info(my_ip, href)

57. break

58. except Exception as e:

59. print(e)

60. my_ip.update_ip_proxy_str()

61.  
62.  

63. def get_all_info(my_ip):

64. base_url = 'https://www.zhipin.com/c101010100/h_101010100/?query=python&page=%s&ka=page-%s'

65. for i in range(1, 4):

66. # 每一个分页的url

67. url = base_url % (i, i)

68. # 循环处理, 如果proxy不好使, 就需要换代理, 如果重试4次依然不好使,就跳过

69. for i in range(0, 4):

70. try:

71. # 循环四次访问boss直聘的网站, 分页的内容

72. # get_detail_url(my_ip, url)

73. get_detail_url(my_ip, url)

74. break

75. except Exception as e:

76. print(e)

77. my_ip.update_ip_proxy_str()

78.  
79.  

80. if __name__ == '__main__':

81. my_ip = ip_proxy.ip_getter()

82. # 获取一个ip

83. # proxy_str = '36.27.143.72:21450'

84. # print(proxy_str)

85. # 获取所有的boss直聘信息

86. get_all_info(my_ip)

87.  

88. # with open('boss.html', 'wb') as f:

89. # f.write(response.content)