Python学习笔记——20170831

最新推荐文章于 2024-07-26 11:43:37 发布

蒋含竹

最新推荐文章于 2024-07-26 11:43:37 发布

阅读量415

点赞数

分类专栏： Python 文章标签： python 爬虫同城旅游网练习

本文链接：https://blog.csdn.net/alionsss/article/details/77767508

版权

Python 专栏收录该内容

39 篇文章 2 订阅

订阅专栏

同城旅游网爬虫练习

类库安装
- pip install requests
- pip install beautifulsoup4
代码

import requests
from bs4 import BeautifulSoup
import os
#
#
def get_html(url):
    """
    获取html源码

    :param url: 链接地址
    :return: html源码
    """

    headers = {"User-Agent": "Mozilla/5.0 (compatible; MSIE 6.0; Windows NT 5.1;SV1)"}
    response = requests.get(url, headers=headers)
    response.encoding = response.apparent_encoding
    if response.status_code == 200:
        return response.text
    else:
        print("网络访问出错")
#
#
def parse_html(html):
    """
    解析html源码

    :param html: html代码
    :return: 返回数据List
    """

    soup = BeautifulSoup(html, 'lxml')
    items = soup.select("#tagList > ul > li > div.line-info > div")
    infoList = []
    for each in items:
        each_soup = BeautifulSoup(str(each), 'lxml')
        info = {
            'type': each_soup.select('div.line-imgbox > span')[0].string,
            'title': each_soup.select('p.line-title > b')[0].string,
            'price': each_soup.select('div.line-pricebox > div > p')[0].em.next_sibling,
        }
        if len(each_soup.select('p.sat-num')) == 0:
            info['satisfied'] = 'None'
        else:
            info['satisfied'] = each_soup.select('p.sat-num')[0].em.previous_sibling
        infoList.append(info)
    return infoList
#
#
def save_file(path, text):
    """
    文本存储

    :param path: 存储路径
    :param text: 文本内容
    :return: None
    """

    file_name = path.split("/")[-1]
    dir_path = path.strip(file_name)
    if not os.path.exists(dir_path):
        os.mkdir(dir_path)

    with open(path, 'w', encoding='UTF-8') as file:
        file.write(text)
#
#
if __name__ == '__main__':
    url = 'https://www.ly.com/dujia/taiguo-lvyou/f394/'
    html = get_html(url)

    infoList = parse_html(html)

    save_file('./lyinfo.json', str(infoList).replace('\'', '\"'))