python 多进程抓取数据存入txt

复制粘贴直接用

res = 'https://m.qidian.com/category/detail?catId=21&subCatId=8&gender=male'


def pub(u, pa):
    cont = requests.get(u, header)
    home = etree.HTML(cont.text.encode('utf8'))
    ty = home.xpath(pa)
    return ty


def qd():
    ty = pub(res, '//*[@id="books-"]/li/a/@href')
    rel = []
    for i in ty:
        rel.append('https://m.qidian.com' + i)
    return rel


def qd3(url):
    ty = pub(url, '//*[@id="btnReadBook"]/@href')
    for i in ty:
        qd4('https://m.qidian.com' + i)


driver = webdriver.Chrome(
    r"D:\Anaconda3\pkgs\selenium-3.141.0-py37he774522_0\Lib\site-packages\selenium\webdriver\chrome\chromedriver.exe")


def qd4(url):
    if url:
        ty = pub(url, '//*[@id="btnLoadNextChapter"]')
        if ty:
            driver.get(url)
            cli(c=1)


def cli(c):
    path = driver.find_element_by_xpath("//*[@id='btnLoadNextChapter']")
    home = etree.HTML(driver.page_source)
    tit = home.xpath('//*[@id="chapterContent"]/section[{}]/h3/text()'.format(c))
    jz = home.xpath('//*[@id="chapterContent"]/section[{}]/p/text()'.format(c))
    name = home.xpath('//*[@id="readCover"]/div[1]/div[1]/h2/text()')
    title = tit + jz

    for i in title:
        with open('{}.txt'.format(name[0]), 'a') as f:
            f.write((i + '\n').encode("gbk", 'ignore').decode("gbk", "ignore"))
    if path:
        driver.execute_script("arguments[0].click();", path)
        time.sleep(2)
        cli(c + 1)


if __name__ == '__main__':
    p = qd()
    for i in p:
        time.sleep(5)
        car = Process(target=qd3, args=(i,))
        car.start()