多线程爬取百事糗科_queue down-CSDN博客

本文链接：https://blog.csdn.net/qq_50958709/article/details/113346158

这个小小的练习没有进入详细页面去爬取完整的内容（内容就是首页的，会有一些残缺）但我的目标是多线程就没有深入
如果代码一些不懂得可以看一哈我前面写的这一片是我自己写的时候的一点多线程的注释，我也是刚开始一脸懵逼，后面才理解一点点在这里啦啦

import requests
from lxml import html
import threading
import queue
import csv
headers={'user-agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'}

def url():
    urls=[]
    for i in range(1,20):
        base_url=f'https://www.bskl.net/xiaohua/p{str(i)}.html'
        urls.append(base_url)
    return urls


class producer(threading.Thread):
    def __init__(self,queue_url,queue_down,*args,**kwargs):
        super().__init__(*args,**kwargs)
        self.queue_url=queue_url
        self.queue_down=queue_down

    def run(self) -> None:
        while not self.queue_url.empty():
            urls=self.queue_url.get()
            for x in range(0,len(urls)):
                resp = requests.get(urls[x], headers=headers).text
                txt = html.etree.HTML(resp)
                contents = txt.xpath('//div[@class="contar-wrap"]/div/div[1]/p/text()')
                titles = txt.xpath('//h3/a/text()')
                for i in range(0,len(titles)):
                    jokes={'title':titles[i],'content':contents[i]}
                    self.queue_down.put(jokes)
                print(f'已成功{x+1}页')


class consumer(threading.Thread):
    def __init__(self,queue_down,writer,fp,*args,**kwargs):
        super().__init__(*args,**kwargs)
        self.queue_down=queue_down
        self.writer=writer
        self.fp=fp

    def run(self) -> None:

        while True:
            try:
                joke=self.queue_down.get(timeout=10)
                self.writer.writerow(joke)
            except:
                break
        self.fp.close()
def main():
    queue_url=queue.Queue(10)
    queue_down=queue.Queue(100)

    urls=url()
    queue_url.put(urls)

    header = ('title', 'content')
    fp = open('joke.csv', 'a', newline='', encoding='utf-8')
    writer = csv.DictWriter(fp, header)
    writer.writeheader()

    for x in range(3):
        th=producer(queue_url,queue_down)
        th.start()
    for x in range(5):
        th=consumer(queue_down,writer,fp)
        th.start()

if __name__ == '__main__':
    main()

我写的时候对于写入文件的卡住啦：
1.开始版本这个文件中打开会出现五个 title 和 content 我刚开还不懂然后就缓存第二个版本

def run(self) -> None:

        while True:
         header = ('title', 'content')
    fp = open('joke.csv', 'a', newline='', encoding='utf-8')
    writer = csv.DictWriter(fp, header)
    writer.writeheader()
            try:
                joke=self.queue_down.get(timeout=10)
                self.writer.writerow(joke)
            except:
                break
        self.fp.close()

2.但是这个好像也没有改变还多了一个

 def run(self) -> None:
 	header = ('title', 'content')
    fp = open('joke.csv', 'a', newline='', encoding='utf-8')
    writer = csv.DictWriter(fp, header)
    writer.writeheader()
    while True:
    	try:
            joke=self.queue_down.get(timeout=10)
            self.writer.writerow(joke)
        except:
             break
        self.fp.close()