python爬取好豆网菜谱

爬取过程分析：
1、分析网页，发现好豆网的菜谱的url均为：https://www.haodou.com/recipe/id/
2、用beautifulsoup解析网页获得菜名，图片及菜谱，去除不必要的网页代码信息
3、将菜的图片存至本地，以菜名命名，将菜谱以文本方式存至本地

具体代码如下：

import requests
import urllib
import re
import lxml
from bs4 import BeautifulSoup

url = 'https://www.haodou.com/recipe'

id = 30
err_num = 0
while(id<200):
    id +=1
    try:
        headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36'}
        url = 'https://www.haodou.com/recipe/'
        resp = requests.get(url+str(id)+'/').text
        # resp.encoding = resp.apparent_encoding
        # print(resp)

        soup = BeautifulSoup(resp,'lxml')
        # print(soup)
        receipe_name = soup.find(id='showcover').get('alt')
        print(receipe_name)
        img = soup.find(id='showcover').get('src')
        print(img)
        with open('./'+receipe_name+'.jpg','wb') as file:
            req = urllib.request.Request(url=img,headers=headers)
            try:
                image = urllib.request.urlopen(req,timeout=10)
                pic = image.read()
            except Exception as e:
                print(e)
                print(receipe_name+'下载图片失败'+img)
            file.write(pic)
            print('图片下载成功')

            drop_html = re.compile(r'<[^>]+>',re.S)
            full_text = []

        receipe_text = soup.find_all('dd')
        # print(receipe_text)
        for text_str in receipe_text:
            text = drop_html.sub('',str(text_str.find_all('p')))

            text = text.replace('[','')
            text = text.replace("]","")
            if text != '':
                print(text)
                full_text.append(text)

        with open('./'+receipe_name+'.txt','w') as file:
            file.writelines(str(full_text))


    except Exception as e:
        print(e)
        err_num += 1
    else:
        continue