网络爬虫笔记

爬取数据保存为json

pachong.py

import json
from bs4 import BeautifulSoup
import requests

url = "https://xl.16888.com/factory.html"
response = requests.get(url)
response.encoding = 'utf-8'
soup = BeautifulSoup(response.text, 'html.parser')

# 找到表格中的所有行
rows = soup.find_all('tr')

# 初始化一个空列表用于存储表格数据
table_data = []

# 提取表头字段
header_row = rows[0]
headers = [th.text.strip() for th in header_row.find_all('th')]

# 提取每一行的数据
for row in rows[1:]:
    row_data = [td.text.strip() for td in row.find_all('td')]
    if row_data:  # 如果行数据不为空，则添加到表格数据列表中
        entry = {}
        for i in range(len(headers)):
            entry[headers[i]] = row_data[i] if i < len(row_data) else ""  # 用空字符串填充缺失的数据
        table_data.append(entry)

# 将表格数据保存为 JSON 文件
json_data = json.dumps(table_data, ensure_ascii=False)

with open("./test/未转译.json", "w", encoding="utf-8") as json_file:
    json_file.write(json_data)

print("JSON 文件已成功保存为 未转译.json")

运行截图

json文件处理

json_1.py

import json
import csv

import pandas as pd

# 读取 JSON 文件
with open("./test/未转译.json", "r", encoding="utf-8") as json_file:
    json_data = json.load(json_file)

# 定义 CSV 文件名
csv_file = "./test/转译.csv"

# 获取字段名（列名）
fieldnames = list(json_data[0].keys())

# 将 JSON 数据写入 CSV 文件
with open(csv_file, "w", newline="", encoding="utf-8") as csvfile:
    writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
    
    # 写入表头
    writer.writeheader()
    
    # 写入数据
    writer.writerows(json_data)

print("CSV 文件已成功保存为", csv_file)

运行截图

数据处理,可视化

import pandas as pd
import matplotlib.pyplot as plt

#中文乱码处理
plt.rcParams['font.sans-serif'] = ['SimHei']

# 读取 CSV 文件
data = pd.read_csv("./test/转译.csv")

# 删除第一列和第二列
data_cleaned = data.drop(data.columns[[0, 1]], axis=1)

# 删除最后一行
data_cleaned = data_cleaned.drop(data_cleaned.index[-1])

# 将百分比字符串转换为浮点数
data_cleaned['占销量份额'] = data_cleaned['占销量份额'].str.rstrip('%').astype('float')

# 计算每个扇形的占比
sales_data = data_cleaned['占销量份额']
total_sales = sales_data.sum()
percentages = sales_data / total_sales * 100

# 将占比低于2%的扇形的标签设置为None
labels = data_cleaned['厂商']
labels[percentages < 2] = None

# 绘制饼状图
plt.figure(figsize=(8, 8))
plt.pie(sales_data, labels=labels, autopct='%1.1f%%')
plt.title('Distribution of Sales')

# 添加图例，并将图例放在右边，过滤掉标签为None的项
plt.legend(labels[labels.notnull()], loc='center left', bbox_to_anchor=(1, 0.5))

#保存图片
plt.savefig('./test/销量分布.png')

plt.show()

运行截图