异步爬虫

异步爬虫

• 多线程
• 多进程
• 协程

多线程与多进程

进程：运行中的程序，每次我们执行一个程序，操作系统对自动的为这个程序准备一些必要的资源(如：分配内存，创建一个能够执行的线程)

线程：程序内，可以直接被CPU调度的执行过程，是操作系统能够进行运算调度的最小单位，它被包含在进程之中，是进程中的实际运作单位

# 两种写法

# 第一种
from threading import Thread


# 创建任务
def func(name):
    for i in range(10):
        print(name, i)


def main():
    # 创建线程
    t1 = Thread(target=func, args=("周杰伦",))
    t2 = Thread(target=func, args=("马斯克",))
    t3 = Thread(target=func, args=("周星驰",))

    # 启动线程
    t1.start()
    t2.start()
    t3.start()

main()

# 第二种 用方法来写 不会

多线程写法

# 两种写法

# 第一种
from threading import Thread


# 创建任务
def func(name):
    for i in range(10):
        print(name, i)


def main():
    # 创建线程
    t1 = Thread(target=func, args=("周杰伦",))
    t2 = Thread(target=func, args=("马斯克",))
    t3 = Thread(target=func, args=("周星驰",))

    # 启动线程
    t1.start()
    t2.start()
    t3.start()

main()

多线程练习

from threading import Thread
import requests
from bs4 import BeautifulSoup

def func(index):
    url = f"http://2chsck.cc/vodtype/1-{index}.html"
    resp = requests.get(url)
    html = resp.text

    page = BeautifulSoup(html, "html.parser")

    # img
    jpg = page.findAll("a", attrs={"class": "stui-vodlist__thumb lazyload"})

    for item in jpg:
        jpgHref = item.get("data-original")
        photo = requests.get(jpgHref)
        jpgName = item.get("title")
        with open(f"images/{index}- {jpgName}.jpg", mode="wb") as f:
            f.write(photo.content)


def main():
    for i in range(101, 201):
        Thread(target=func, args=(f"{i}",)).start()
main()

线程池

# import requests
# from bs4 import BeautifulSoup
# from concurrent.futures import ThreadPoolExecutor
#
#
# def func(page):
#     url = f"https://pic.netbian.com/4kmeinv/index_{page}.html"
#     resp = requests.get(url)
#     resp.encoding = "gbk"
#     html = resp.text
#     page = BeautifulSoup(html, "html.parser")
#     ul = page.find("div", attrs={"class": "slist"})
#     img = ul.findAll("img")
#
#     for item in img:
#         src = "https://pic.netbian.com" + item.get("src")
#         photo = requests.get(src)
#         # print(src)
#         try:
#             with open(f"{src}.jpg", mode="wb") as f:
#                 f.write(photo.content)
#         except:
#             print("error")
#
#
# def main():
#     # 线程池
#     with ThreadPoolExecutor(3) as t:
#         for i in range(2, 10):
#             t.submit(func, i)
#
#
# if __name__ == "__main__":
#     main()


# ----------------------       ^待查询^        -----------------------------

from concurrent.futures import ThreadPoolExecutor

线程池案例

import requests
from lxml import etree
from concurrent.futures import ThreadPoolExecutor
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36 Edg/112.0.1722.48",
}

f = open("新发地.csv", mode="a", encoding="utf-8")


def download(url):
    resp = requests.get(url, headers=headers)
    tree = etree.HTML(resp.text)
    a_list = tree.xpath("//li[@class='market-list-item']/a")
    for a in a_list:
        span = a.xpath("./span/text()")[:4]
        s = ",".join(span)
        f.write(s)
        f.write("\n")
    print(f"{url} 已完成")

for i in range(1, 2):
    with ThreadPoolExecutor(10) as t:
        for i in range(1, 100):

            # 新发地 请求过多需要验证 待解决 2023年4月18日13:16:28
            url = f"https://www.cnhnb.com/hangqing/cdlist-2003192-0-0-0-0-{i}/"
            t.submit(download, url)

多进程

from multiprocessing import Process


def func(name):
    for i in range(1000):
        print(name, i)


if __name__ == '__main__':
    p1 = Process(target=func, args=("周杰伦",))
    p2 = Process(target=func, args=("林俊杰",))
    p1.start()
    p2.start()


"""
何时使用多线程 何时使用多进程
1. 多线程：任务相对统一，互相特别的相似
2. 多进程：多个任务相互独立，很少有交集
"""