python农产品价格预测_python爬虫系列(5.1-多线程下载农产品产品存储到本地)

一、具体步骤

1、导包

import os

import random

import csv

import time

import queue

import threading

import requests

from lxml import etree

2、定义几个全局的变量(方便下面的类使用)

headers = {

'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36',

}

path = os.path.join(os.path.dirname(__file__), 'products.csv')

3、定义一个生产者,用来抓取网页上的数据

class Procuder(threading.Thread):

"""

创建一个生产者(获取全部的url地址)

"""

def __init__(self, page_queue, data_queue, *args, **kwargs):

super(Procuder, self).__init__(*args, **kwargs)

self.page_queue = page_queue

self.data_queue = data_queue

def run(self):

while True:

if self.page_queue.empty():

break

url = self.page_queue.get()

self.parse_page(url)

def parse_page(self, url):

"""

定义一个根据url请求数据的方法

:param url:

:return:

"""

response = requests.get(url=url, headers=headers)

time.sleep(random.randrange(3))

if response.status_code == 200:

html = etree.HTML(response.text)