Python爬虫项目：抓取智联招聘信息

来自https://mp.weixin.qq.com/s/0SzLGqv2p0-IWSN3r8bOHA

'''
Python爬虫之五：抓取智联招聘基础版 
该文件运行后会产生一个代码，保存在这个Python文件所在的文件夹。

https://sou.zhaopin.com/jobs/searchresult.ashx?jl=%E5%8C%97%E4%BA%AC&kw=python%E5%B7%A5%E7%A8%8B%E5%B8%88&sm=0&isfilter=1&p=1&re=2005
searchresult.ashx?jl=北京&kw=python工程师&sm=0&isfilter=1&p=1&re=2005

url = 'https://sou.zhaopin.com/jobs/searchresult.ashx?' + urlencode(paras)

#构造请求地址
paras = {
	'j1' : '北京'				#搜索城市
	'kw' : 'Python工程师'			#搜索岗位
	'isadv' : '0'				#是否打开更详细搜索
	'isfilter' : '1'			#是否对结果过滤
	'p' : '1'					#显示页数
	're' : '2005'				#显示地区代码
}

url = 'https://sou.zhaopin.com/jobs/searchresult.ashx?' + urlencode(paras)


#请求头
headers = {
	'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'.
	'Host': 'sou.zhaopin.com',
	'Referer': 'https://www.zhaopin.com/',
	'Accept': 'text/html, application/xhtml + xml, application/xml; q = 0.9, image/webp, image/apng, */*; q = 0.8',
	'Accept-Encoding': 'gzip, deflate, br',
	'Accept-Language': 'zh-CN, zh; q= 0.9'
}



# 正则表达式进行解析
pattern = re.compile('<a style=.*? target = "_blank">(.*?)</a>.*?'
	'<td class = "gsmc"><a href= "(.*?)" target = "_blank">(.*?)</a>.*?'
	'<td class = "zwyx">(.*/)</td>', re.S)

# 匹配所有符合条件的内容
items = re.findall(pattern, html)

#解析后剔除job两侧的标签：<b>job</b>
for item in items:
	job_name = items[0].replace('<b>', '').replace('</b>', '')
	yield {
			'job' : job_name,
			'website' : item[1],
			'company' : item[2],
			'salary' : item[3]
	}


########写入CSV文件
import csv
def write_csv_headers(path, headers):
	#写入表头
	with open(path, 'a', encoding = 'gb18030', newline = '') as f:
		f_csv = csv.DictWriter(f, headers).writeheader()

def write_csv_rows(path, headers, rows):
	###写入行
	with open(path, 'a', encoding = 'gb18030', newline = '') as f:
		f_csv = csv.DictWriter(f, headers).writerows(rows)


#使用tqdm进行进度显示



from tqdm import tqdm
from time import sleep
for i iin tqdm(range(1000)):
	sleep(0.01)
'''

#################完整代码##################################
import re   			#正则表达式库
import csv 				#CSV格式库
import requests 		#网页解析
from tqdm import tqdm 	#进度条显示
from urllib.parse import urlencode 			#Python 3 的 urllib 模块是一堆可以处理 URL 的组件集合
from requests.exceptions import RequestException 		#方法 urlencode 可以向 URL 传输数据

def get_one_page(city, keyword, region, page):
    '''
    获取网页html内容并返回
    '''
    paras = {
        'jl': city,         # 搜索城市
        'kw': keyword,      # 搜索关键词 
        'isadv': 0,         # 是否打开更详细搜索选项
        'isfilter': 1,      # 是否对结果过滤
        'p': page,          # 页数
        're': region        # region的缩写，地区，2005代表海淀
    }

    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36',
        'Host': 'sou.zhaopin.com',
        'Referer': 'https://www.zhaopin.com/',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
        'Accept-Encoding': 'gzip, deflate, br',
        'Accept-Language': 'zh-CN,zh;q=0.9'
    }

    url = 'https://sou.zhaopin.com/jobs/searchresult.ashx?' + urlencode(paras)
    try:
        # 获取网页内容，返回html数据
        response = requests.get(url, headers=headers)
        # 通过状态码判断是否获取成功
        if response.status_code == 200:
            return response.text
        return None
    except RequestException as e:
        return None

def parse_one_page(html):
    '''
    解析HTML代码，提取有用信息并返回
    '''
    # 正则表达式进行解析
    pattern = re.compile('<a style=.*? target="_blank">(.*?)</a>.*?'        # 匹配职位信息
        '<td class="gsmc"><a href="(.*?)" target="_blank">(.*?)</a>.*?'     # 匹配公司网址和公司名称
        '<td class="zwyx">(.*?)</td>', re.S)                                # 匹配月薪      

    # 匹配所有符合条件的内容
    items = re.findall(pattern, html)   

    for item in items:
        job_name = item[0]
        job_name = job_name.replace('<b>', '')
        job_name = job_name.replace('</b>', '')
        yield {
            'job': job_name,
            'website': item[1],
            'company': item[2],
            'salary': item[3]
        }

def write_csv_file(path, headers, rows):
    '''
    将表头和行写入csv文件
    '''
    # 加入encoding防止中文写入报错
    # newline参数防止每写入一行都多一个空行
    with open(path, 'a', encoding='gb18030', newline='') as f:
        f_csv = csv.DictWriter(f, headers)
        f_csv.writeheader()
        f_csv.writerows(rows)

def write_csv_headers(path, headers):
    '''
    写入表头
    '''
    with open(path, 'a', encoding='gb18030', newline='') as f:
        f_csv = csv.DictWriter(f, headers)
        f_csv.writeheader()

def write_csv_rows(path, headers, rows):
    '''
    写入行
    '''
    with open(path, 'a', encoding='gb18030', newline='') as f:
        f_csv = csv.DictWriter(f, headers)
        f_csv.writerows(rows)

def main(city, keyword, region, pages):
    '''
    主函数
    '''
    filename = 'zl_' + city + '_' + keyword + '.csv'
    headers = ['job', 'website', 'company', 'salary']
    write_csv_headers(filename, headers)
    for i in tqdm(range(pages)):
        '''
        获取该页中所有职位信息，写入csv文件
        '''
        jobs = []
        html = get_one_page(city, keyword, region, i)
        items = parse_one_page(html)
        for item in items:
            jobs.append(item)
        write_csv_rows(filename, headers, jobs)

#只可以被独立执行，不可以被其他文件引用
if __name__ == '__main__':
    main('北京', 'python工程师', 2005, 10)