python爬虫----单线程爬取腾讯招聘内容【简单版】

慢速单线程爬取腾讯招聘内容

import time
from selenium import webdriver
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import os

def save_to_html(html_str,filename):
    dirname = os.path.dirname(filename)
    if not os.path.exists(dirname):
        os.mkdir(dirname)
    with open(filename,'w',encoding='utf-8') as fp:
        fp.write(html_str)
    print('下载完成！',filename)

def get_content_by_selenium(url):
    driver = webdriver.PhantomJS()
    wait = WebDriverWait(driver,20)
    driver.get(url)
    wait.until(EC.presence_of_all_elements_located((By.XPATH,'//div[@class="recruit-list"]')))
    html_str = driver.page_source
    return html_str

def main():
    base_url = 'https://careers.tencent.com/search.html?index=%s'
    for i in range(1,20):
        html_str = get_content_by_selenium(base_url %i)
        save_to_html(html_str,'./tencent/{}.html'.format(i))
if __name__ == '__main__':
    start = time.time()
    main()
    print(time.time()-start)