爬虫—美食天下各类菜谱的菜单信息

文章仅用于学习，请勿肆意的爬取网站信息，作为非法用途
文章仅用于学习，请勿肆意的爬取网站信息，作为非法用途
文章仅用于学习，请勿肆意的爬取网站信息，作为非法用途

效果图


源码

import csv
import os
import random

import requests
from lxml import etree

# 请求头
head = ['Mozilla/5.0', 'Chrome/78.0.3904.97', 'Safari/537.36']
headers = {
    'user-agent': head[random.randint(0, 2)]
}


def makedir(path):
    path = path.strip()
    # 去除尾部 \ 符号
    path = path.rstrip("\\")
    if not os.path.exists(path):
        os.mkdir(path)
        return True
    else:
        return False


def getHtml(url):
    try:
        response = requests.request("GET", url=url, headers=headers)
        if response.status_code == 200:
            response.encoding = 'utf-8'
            return response.text
        else:
            return response.status_code
    except Exception as e:
        return e


def htmlToTree(html):
    return etree.HTML(html)


def elementToString(element):
    return etree.tostring(element, pretty_print=True, encoding='utf-8').decode('utf-8')


def parseHtml(html):
    allcategory = []
    tree = htmlToTree(html)
    path = '//div[@class="wrap"]//div[@class="category_box mt20"]/div[@class="category_sub clear"]'
    data = tree.xpath(path)
    for item in data:
        category_list = []
        div = elementToString(item)
        div_tree = htmlToTree(div)
        category = div_tree.xpath('//h3/text()')[0]
        category_name = div_tree.xpath('//ul/li/a/text()')
        category_url = div_tree.xpath('//ul/li/a/@href')
        category_list.append(category)
        category_list.append(dict(zip(category_name, category_url)))
        allcategory.append(category_list)
    return allcategory


def writerCsv(food_list, category_name, name):
    try:
        title = [key for key in food_list[0].keys()]  # 表头的代码去掉
        if '/' in name or category_name:
            name = name.replace('/', '')
            category_name = category_name.replace('/', '')
        path = os.path.join(os.getcwd() + '\\source\\{0}'.format(category_name))
        flag = makedir(path)
		csvdata = open('source/{0}/{1}.csv'.format(category_name, name), 'a', encoding='utf-8')
        dictwriter = csv.DictWriter(csvdata, fieldnames=title)
        dictwriter.writeheader()
        # for data in food_dict:
        #     dictWriter.writerow(data)
        dictwriter.writerows(food_list)
        csvdata.close()
        return True
    except Exception as e:
        return False


def parseRecipe(html, category_name, name):
    tree = htmlToTree(html)
    path = '//div[@class="wrap"]//div[@id="J_list"]/ul/li'
    data = tree.xpath(path)
    food_list = []
    if data is not None and data != []:
        for item in data[0:1]:
            food_dict = {}
            li = elementToString(item)
            li_tree = htmlToTree(li)
            food_name = li_tree.xpath('//div[@class="pic"]/a/@title')[0]
            food_detailUrl = li_tree.xpath('//div[@class="pic"]/a/@href')[0]
            food_content = str(
                li_tree.xpath('//div[@class="detail"]/p[@class="subcontent"]/text()')[0])[3:-1]
            food_dict["菜名"] = food_name
            food_dict["详情链接"] = food_detailUrl
            food_dict["原料"] = food_content
            food_list.append(food_dict)
    res = writerCsv(food_list, category_name, name)
    return res


def getRecipePerCate(data):
    res = bool()
    for items in data[:4]:
        category_name = items[0]
        for name, url in items[1].items():
            html = getHtml(url)
            res = parseRecipe(html, category_name, name)
    if res:
        return "写入成功"
    else:
        return "写入失败"


if __name__ == '__main__':
    url = 'https://home.meishichina.com/recipe-type.html'
    data = parseHtml(getHtml(url))
    res = getRecipePerCate(data)
    print(res)