爬虫小练习---爬药网数据

1、药网数据爬取：
url:https://www.111.com.cn/categories/953710?tp=10-1
要求：抓取50页
字段：总价，描述，评论数量，详情页链接
用正则爬取。
代码如下

import re,requests,json

base_url='https://www.111.com.cn/categories/953710?-j%s.html'
# https://www.111.com.cn/categories/953710-j7.html
headers={
    'user-agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'
}
info=[]
for i in range(1,51):
    item={}
    # 总价，描述，评论数量，详情页链接
    response = requests.get(base_url %i, headers=headers)
    # print(response.text)
    ul_pattern=re.compile('<ul id="itemSearchList" class="itemSearchList">(.*?)</ul>',re.S)
    ul_content=ul_pattern.search(response.text)
    if ul_content:
        ul_content=ul_content.group(1)
        # print(ul_content)
    li_pattern =re.compile(r'<li id="producteg_(.*?)</li>',re.S)
    li_content =li_pattern.findall(ul_content)
    # print(li_content)
    # print(type(li_content))
    for li in li_content:
    #爬取总价
        price_pattern = re.compile(r'<span>(.*?)</span>',re.S)
        price_content= price_pattern.search(li)
        if price_content:
            price_content = price_content.group(1).strip()
        # print(price_content)
    #爬取描述
        detail_pattern = re.compile(r'<span class=" list_lable_self"></span>(.*?)</a>',re.S)
        detail_content = detail_pattern.search(li)
        if detail_content:
            detail_content=detail_content.group(1).strip()
        # print(detail_content)
    #爬取评论数量
        commit_pattern = re.compile(r'评论 <em>(.*?)</em>条</a>',re.S)
        commit_content = commit_pattern.search(li)
        if commit_content:
            commit_content = commit_content.group(1).strip()
        # print(commit_content)
    #爬取详情连接
        xq_pattern = re.compile('href="//(.*?)" target="_blank"',re.S)
        xq_content = xq_pattern.search(li)
        if xq_content:
            xq_content = xq_content.group(1).strip()
        # print(xq_content)
        item['price_content']=price_content
        item['detail_content']=detail_content
        item['commit_content']=commit_content
        item['xq_content']=xq_content
        info.append(item)
with open('haha.json','w',encoding='utf-8') as fp:
    json.dump(info,fp)

经验总结：在使用正则时尽量多匹配字符，这样能够精确匹配，否则数据容易混淆