文章目录



1.pandas的文本与序列化 


result_data = pd.DataFrame(json_data_list)

with open(jsonl_file_path, 'w', encoding='utf-8') as jsonl_file:
    result_data.to_json(orient='records', lines=True, force_ascii=False, path_or_buf=jsonl_file)
  • 1.
  • 2.
  • 3.
  • 4.
数据不换行
df.at[i, column_name_transcript] = df.at[i, column_name_transcript].split('\n')
  • 1.
  • 2.

pandas转序列化数据

data_dicts = df.to_dict(orient='records')

with open(jsonl_file_path, 'w', encoding='utf-8') as jsonl_file:
    for data in data_dicts:
        # 将字典转换为JSON字符串，ensure_ascii=False参数确保中文字符不会被转义
        # 写入文件时，每个JSON对象后面跟着一个换行符
        jsonl_file.write(json.dumps(data, ensure_ascii=False) + '\n')
  • 1.
  • 2.
  • 3.
  • 4.
  • 5.
  • 6.
  • 7.

pandas元素序列化

df['column_01'] = df['column_01'].apply(
        lambda x: json.dumps(x, ensure_ascii=False) if isinstance(x, str) else ''
    )
  • 1.
  • 2.
  • 3.
# 对"answer"列中的每个字符串元素去除空白并分割成单词列表
    df['question'] = df['question'].apply(lambda x: x.strip().split())
    df['answer'] = df['answer'].apply(lambda x: x.strip().split())

    # 序列化"answer"列中的每个元素为JSON格式的字符串
    df['question'] = df['question'].apply(lambda x: json.dumps(x, ensure_ascii=False) if isinstance(x, list) else x)
    df['answer'] = df['answer'].apply(lambda x: json.dumps(x, ensure_ascii=False) if isinstance(x, list) else x)
  • 1.
  • 2.
  • 3.
  • 4.
  • 5.
  • 6.
  • 7.