csv、xlsx、txt转json

1、csv、xlsx转json

import pandas as pd
import json
import random

#excel/csv转json
# df = pd.read_excel('archive/news_collection.xlsx')
df = pd.read_csv('archive/news_collection.csv')
# 提取前两列
df = df.iloc[:, :2]
# Convert the data to JSON format
json_data = df.to_json(orient='records')

# Load the JSON data into a Python dictionary
json_dict = json.loads(json_data)
# Save the JSON data to a file
file_path = 'archive/csv2json.json'
with open(file_path, 'w', encoding='utf-8') as f:
    json.dump(json_dict, f, ensure_ascii=False, indent=4)

2、txt文件转json

# txt转json
import json

# 读取文本文件
with open("text_file.txt", "r") as file:
    lines = file.readlines()

# 解析每一行并生成JSON数据
json_data = []
for line in lines:
    # 假设每一行都是一个JSON对象
    json_data.append(json.loads(line.strip()))

# 将JSON数据保存到文件
with open("json_file.json", "w") as file:
    json.dump(json_data, file, indent=4)

3、json文件信息提取

js = json.load(open('archive\hotpot_dev_distractor_v1.json', 'r', encoding='utf-8'))
# 提取json前100条数据
# js = js[:100]
# json随机提取100条数据
random_data = random.sample(js, 100)
# 提取json中指定的键值对
selected_data = []
for item in js:
    question = item.get("question")
    context = item.get("context")
    selected_data.append({"instruction": question, "output": context})
file_path = 'archive/json_random100.json'
with open(file_path, 'w', encoding='utf-8') as f:
    json.dump(selected_data, f, ensure_ascii=False, indent=4)

4、json数据画图
a）饼图

import matplotlib.pyplot as plt

# 定义数据
data = {'new': 50, 'zhishi': 30, 'yiliao': 80}

# 绘制饼图
plt.pie(data.values(), labels=data.keys(), autopct='%1.1f%%')

# 添加标题
plt.title("Data Distribution")

# 显示图表
plt.show()

b）直方图

import matplotlib.pyplot as plt

# 定义数据
data = {'new': 50, 'zhishi': 30, 'yiliao': 80}

# 绘制直方图
plt.bar(data.keys(), data.values())

# 添加标题和标签
plt.title("Data Distribution")
plt.xlabel("Category")
plt.ylabel("Value")

# 显示图表
plt.show()

5、json数据划分训练集、验证集

import json
import random

# 读取JSON数据
with open("json_file.json", "r") as file:
    data = json.load(file)

# 随机打乱数据
random.shuffle(data)

# 划分数据
train_size = int(0.8 * len(data))
train_data = data[:train_size]
val_data = data[train_size:]

# 将划分后的数据保存到文件
with open("train_data.json", "w") as file:
    json.dump(train_data, file, indent=4)

with open("val_data.json", "w") as file:
    json.dump(val_data, file, indent=4)

6、json数据转换成Qwen标准模板格式

import json

# 解析JSON数据
data = json.load(open('ruozhiba10.json', 'r', encoding='utf-8'))

# 生成文本
texts = []
for item in data:
    text = ""
    if item.get("system"):
        text += "<|im_start|>system\n"
        text += item["system"] + "\n"
        text += "<|im_end|>\n"
    if item.get("prompt"):
        text += "<|im_start|>user\n"
        text += item["prompt"] + "\n"
        text += "<|im_end|>\n"
    text += "<|im_start|>assistant\n"
    text += item["assistant"] + "\n"
    text += "<|im_end|>\n"
    texts.append(text)

# 保存为TXT文件
with open("output.txt", "w") as file:
    for text in texts:
        file.write(text)