python第二十五天

# 1.xpath解析原理：
# 通过需要的标签在html/xml结构的标签路径去获取指定标签

# 2.xml数据格式
# xml和json一样是一种通用的数据格式，用于多语言程序之间的数据传输
# xml和json的比较：xml（更安全，方便加密）、json更轻量级（传输的时候更快）
# xml和html一样是以节点（标签）为基本单位来提供数据的，例如：
"""
保存一个商店中的书的信息：
json:
{
 name: '新华书店',
 books: [
    {
        name: '流程的Python'
        author: '张三'
        price: 120
        num: 20
    },
    {
        name: '代码整洁之道'
        author: '小明'
        price: 89
        num: 12
    },
    {
        name: '程序员修养之道- 从小工到专家'
        author: 'Bob'
        price: 78
        num:23
    },
    {
        name: 'Unix编程艺术'
        author: '小花'
        price: 102
        num: 9
    }
 ]
}

<bookStore>
    <name>新华书店</name>
    <books>
        <book>
            <name flag="hot">流程的Python</name>
            <author>张三</author>
            <price>120</price>
            <num>20</num>
        </book>
        <book>
            <name>代码整洁之道</name>
            <author>小明</author>
            <price>89</price>
            <num>12</num>
        </book>
        <book>
            <name>程序员修养之道- 从小工到专家</name>
            <author>Bob</author>
            <price>78</price>
            <num>23</num>
        </book>
        <book>
            <name>Unix编程艺术</name>
            <author>小花</author>
            <price>102</price>
            <num>9</num>
        </book>
    </books>
</bookStore>
"""

from lxml import etree

with open('data.xml', encoding='utf-8') as f:
    data = f.read()

# 1.获取树对应的根节点
# 树  -  整个xml/html
# 根节点  -  xml或者html数据中最外层的标签/节点
bookStore = etree.XML(data)
print(bookStore)

# 2.通过xPath解析数据
# 语法： 节点对象.xpath(路径)
# =======================1)路径的写法=========================
# 1) / - 从根节点开始的绝对路径(查找的时候跟前面的节点对象没有任何关系)
# /A/B   -  找根节点A下面的B(A如果不是根节点，就找不到)

result = bookStore.xpath('/bookStore/books/book/author')
print(result)

result = bookStore.xpath('/bookStore/name')
print(result)   # [<Element name at 0x10cfc0be0>]

book_list = bookStore.xpath('/bookStore/books/book')
print(book_list)

book_list = result[0].xpath('/bookStore/books/book')
print(book_list)

# 2) // - 从任意位置开始按路径查找
# //A/B   -   在整个xml或者html中找A节点下面的B节点
# //B     -   在整个xml或者html中找B节点
name_list = bookStore.xpath('//name')
print(name_list)

name_list = bookStore.xpath('//book/name')
print(name_list)

# 3)
# ./   -   从当前位置开始找
# 节点名称  -  从当前位置开始找
name_list = bookStore.xpath('./name')
print(name_list)

name_list = bookStore.xpath('./books/book/name')
print(name_list)

result = book_list[0].xpath('./name/text()')
print(result)

result = book_list[0].xpath('name/text()')
print(result)


# 4) ../  -  从当前节点的父节点开始查找
name_list = bookStore.xpath('//name')
name_1 = name_list[0]
result = name_1.xpath('..')
print(result)    # [<Element bookStore at 0x106d2c050>]

result = name_1.xpath('../books')
print(result)    # [<Element books at 0x105461190>]


# ======================2)标签内容和属性=================
# 1）text()  -  获取标签内容
store_name = bookStore.xpath('/bookStore/name/text()')
print(store_name)   # ['新华书店']

store_name = bookStore.xpath('//name')[0].xpath('./text()')
print(store_name)    # ['新华书店']

result = bookStore.xpath('//text()')
print(result)

# 2）@属性名  - 获取指定属性值
flag = bookStore.xpath('//book/name')[0].xpath('./@flag')
print(flag)   # ['hot']

ids = bookStore.xpath('//book/@id')
print(ids)    # ['b1', 'b2', 'b3', 'b4']

# 3)======================3)谓词==================
# 路径[谓词]  -  按照谓词对应的条件通过指定路径获取标签
# 1）[N]  -  获取第N个标签，从1开始
# //book[1]/name   -  第一本书下面的name
# //book/name[1]   -  所有的书后面的第一个name
name_1 = bookStore.xpath('//book[1]/name/text()')
print(name_1)   # ['流程的Python']

# 2）
# [last()]   -  获取最后一个标签
# [last()-N]  -  获取最后一个的前N个标签（获取倒数第 N+1 个标签）
name_last = bookStore.xpath('//book[last()]/name/text()')
print(name_last)   # ['Unix编程艺术']

name_last_2 = bookStore.xpath('//book[last()-1]/name/text()')
print(name_last_2)


# 3)
# [position()<N]  - 获取前N-1个标签
# [position()>N]  - 获取第N个以后的标签
book_list = bookStore.xpath('//books/book[position()<4]/author/text()')
print(book_list)   # ['张三', '小明', 'Bob']

book_list = bookStore.xpath('//books/book[position()>2]/author/text()')
print(book_list)    # ['Bob', '小花']

# 4) [@属性名]  -  获取拥有指定属性的标签
flags = bookStore.xpath('//name[@flag]/text()')
print(flags)   # ['流程的Python', '代码整洁之道']

# 5) [@属性名=值]   -   获取指定属性是指定值的标签
flags = bookStore.xpath('//name[@flag="hot"]/text()')
print(flags)   # ['流程的Python']

# [@属性名>数值]、[@属性名<数值]、[@属性名>=数值]、[@属性名<=数值]
book_list = bookStore.xpath('//books/book[@id>200]')
print(book_list)

# 6) [标签名=值]、[标签名>数值]、[标签名>=数值]、[标签名<数值]、[标签名<=数值]
# //book[name="流程的Python"]  -  获取子标签name标签值是 '流程的Python'的 book标签
book_list = bookStore.xpath('//book[name="流程的Python"]')
print(book_list)    # [<Element book at 0x10495a280>]

book_list = bookStore.xpath('//book[price>100]/name/text()')
print(book_list)   # ['流程的Python', 'Unix编程艺术']

# ==================4)通配符==========================
# * - 表示任意节点或者任意属性(不能表示任意值)
# //book[1]/*  -  获取第一个book节点中所有的子节点
result = bookStore.xpath('//book[1]/*/text()')
print(result)

# //*[@id]  -  获取设置了id属性的任意节点
result = bookStore.xpath('//*[@id]')
print(result)

# //*[@*]   -  获取所有设置了属性的节点
result = bookStore.xpath('//*[@*]')
print(result)

result = bookStore.xpath('//book[@id=100]')
print(result)

# ==================5）选取若干路径(分支)=================
# 路径1 | 路径2 | 路径3 | ...  -  同时选取每个路径对应的标签
# 注意：谓词不能使用分支
result = bookStore.xpath('//book[1]/name/text()|//book[1]/price/text()')
print(result)

from lxml import etree
import requests


def get_data():
    url = 'https://movie.douban.com/top250'
    headers = {
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36'
    }
    response = requests.get(url, headers=headers)
    if response.status_code == 200:
        xpath_analysis(response.text)
    else:
        print('请求失败!', response)


def xpath_analysis(data):

    html = etree.HTML(data)

    # 获取电影盒子
    li_list = html.xpath('//ol[@class="grid_view"]/li')

    all_f_info = []
    # 遍历拿到每个电影对应的盒子(li标签)
    for li in li_list:
        f_info = {}
        # 1.电影名
        names = li.xpath('./div/div[@class="info"]/div[@class="hd"]/a/span/text()')
        # print(names)
        f_info['name'] = names[0]
        # 2.导演
        director = li.xpath('./div/div[@class="info"]/div[@class="bd"]/p[1]/text()')
        # 字符串.strip()  -  去掉字符串开头和结尾的空白字符
        director = [x.strip() for x in director]
        # print(director)
        f_info['director'] = director[0]
        f_info['time_type'] = director[1]
        # print(f_info)

        # 3.评分
        score = li.xpath('./div/div[@class="info"]/div[@class="bd"]/div/span[@class="rating_num"]/text()')
        f_info['score'] = score[0]
        # print(f_info)

        all_f_info.append(f_info)

    print(all_f_info)


if __name__ == '__main__':
    get_data()

import csv


def write_data1():
    # ===============写列表数据========================
    # 1.打开文件
    # 参数newline在csv文件中必须添加
    with open('files/data1.csv', 'w', newline='')as f:
        # 2.创建writer对象
        # csv.writer(文件对象)
        writer = csv.writer(f)

        # 3.写数据
        # 一行一行的写
        writer.writerow(['姓名', '性别', '年龄', '电话'])
        # 一次写多行
        writer.writerows([
            ['张三', '男', '30', '123'],
            ['李四', '女', '22', '321']
        ])


def write_data2():
    # ====================将数据以字典的形式写入文件===================
    # 1.打开文件
    with open('files/data2.csv', 'w', encoding='utf-8', newline='') as f:
        # 2.创建writer
        # csv.DictWriter(文件对象, 表头列表)
        # 表头列表必须是要写入的字典的所有的key
        writer = csv.DictWriter(f, ['name', 'sex', 'age', 'tel'])
        # 将表头列表中的元素作为文件的第一行内容
        writer.writeheader()
        # 一行一行的写
        writer.writerow({
            'name': '张三',
            'sex': '男',
            'age': 30,
            'tel': '110'
        })
        # 一次写多行
        writer.writerows([
            {'name': '小明', 'sex': '男', 'age': 19, 'tel': '119'},
            {'name': '小红', 'sex': '女', 'age': 21, 'tel': '114'}
        ])

write_data2()

def reader_list():
    # ===================以列表的形式读====================
    # 1.打开文件
    with open('files/data1.csv', encoding='gbk', newline='') as f:
        # 2. 创建reader
        reader = csv.reader(f)
        print(reader)
        # 3.一行一行的获取数据
        # 获取每一行的内容和迭代器获取元素的方法一样
        print(next(reader))
        print(next(reader))
        print('============')
        for x in reader:
            print(x)


def reader_dict():
    # ===================以字典的形式读====================
    # 1.打开文件
    with open('files/data1.csv', encoding='gbk', newline='') as f:
        # 2.创建reader
        # csv.DictReader(文件对象, 字典的key列表)
        # 字典的key列表  -  不赋值的时候会把第一行的内容作为字典的key
        reader = csv.DictReader(f, ['name', 'sex', 'age', 'tel'])
        # 3.读数据
        next(reader)
        print(dict(next(reader)))

        for x in reader:
            print(dict(x))


reader_dict()