import os
import matplotlib. pyplot as plt
import pandas as pd
root_folder_path = '/data3/zhouqiang/LLM_scratch/dataset/COIG-CQIA'
domains = [ d for d in os. listdir( root_folder_path) if os. path. isdir( os. path. join( root_folder_path, d) ) ]
domain_counts = [ ]
for domain in domains:
domain_path = os. path. join( root_folder_path, domain)
json_files = [ f for f in os. listdir( domain_path) if f. endswith( '.jsonl' ) ]
domain_counts. append( len ( json_files) )
df = pd. DataFrame( {
'Domain' : domains,
'Counts' : domain_counts
} )
output_folder = '/data3/zhouqiang/LLM_scratch/dataset/output_images'
if not os. path. exists( output_folder) :
os. makedirs( output_folder)
for i, row in df. iterrows( ) :
plt. figure( )
plt. bar( row[ 'Domain' ] , row[ 'Counts' ] , color= 'skyblue' )
plt. title( f'Number of JSON Files in { row[ "Domain" ] } ' )
plt. xlabel( 'Domain' )
plt. ylabel( 'Number of JSON Files' )
plt. xticks( rotation= 45 )
plt. tight_layout( )
file_path = os. path. join( output_folder, f' { row[ "Domain" ] } .jpg' )
plt. savefig( file_path)
plt. close( )
import os
import random
k = 500
root_folder_path = '/data3/zhouqiang/LLM_scratch/dataset/COIG-CQIA'
domain_data_counts = { }
total_data_count = 0
for domain in os. listdir( root_folder_path) :
domain_path = os. path. join( root_folder_path, domain)
if os. path. isdir( domain_path) :
jsonl_files = [ f for f in os. listdir( domain_path) if f. endswith( '.jsonl' ) ]
domain_total = 0
for jsonl_file in jsonl_files:
file_path = os. path. join( domain_path, jsonl_file)
with open ( file_path, 'r' , encoding= 'utf-8' ) as file :
lines_count = sum ( 1 for line in file )
domain_total += lines_count
domain_data_counts[ domain] = domain_total
total_data_count += domain_total
output_file_path = '/data3/zhouqiang/LLM_scratch/dataset/inverse_sampled_data.jsonl'
os. makedirs( os. path. dirname( output_file_path) , exist_ok= True )
with open ( output_file_path, 'w' , encoding= 'utf-8' ) as output_file:
for domain, count in domain_data_counts. items( ) :
if count > 0 :
sample_rate = k / count
sample_rate = min ( sample_rate, 1 )
domain_path = os. path. join( root_folder_path, domain)
jsonl_files = [ f for f in os. listdir( domain_path) if f. endswith( '.jsonl' ) ]
for jsonl_file in jsonl_files:
file_path = os. path. join( domain_path, jsonl_file)
with open ( file_path, 'r' , encoding= 'utf-8' ) as file :
lines = file . readlines( )
sampled_lines = random. sample( lines, max ( 1 , int ( len ( lines) * sample_rate) ) )
output_file. writelines( sampled_lines)
else :
print ( f"No data to sample in domain: { domain} " )
print ( f"Inverse sampling completed and combined data is stored in: { output_file_path} " )
固定比率的采样
import os
import random
sample_rate = 0.05
root_folder_path = '/data3/zhouqiang/LLM_scratch/dataset/COIG-CQIA'
total_data_count = 0
file_line_counts = [ ]
for domain in os. listdir( root_folder_path) :
domain_path = os. path. join( root_folder_path, domain)
if os. path. isdir( domain_path) :
jsonl_files = [ f for f in os. listdir( domain_path) if f. endswith( '.jsonl' ) ]
for jsonl_file in jsonl_files:
file_path = os. path. join( domain_path, jsonl_file)
with open ( file_path, 'r' , encoding= 'utf-8' ) as file :
lines = file . readlines( )
total_data_count += len ( lines)
file_line_counts. append( ( file_path, lines) )
print ( "Total lines before sampling:" , total_data_count)
output_file_path = '/data3/zhouqiang/LLM_scratch/dataset/combined_sampled_data.jsonl'
os. makedirs( os. path. dirname( output_file_path) , exist_ok= True )
sampled_total_count = 0
with open ( output_file_path, 'w' , encoding= 'utf-8' ) as output_file:
for file_path, lines in file_line_counts:
sampled_lines = random. sample( lines, max ( 1 , int ( len ( lines) * sample_rate) ) )
sampled_total_count += len ( sampled_lines)
output_file. writelines( sampled_lines)
print ( "Total lines after sampling:" , sampled_total_count)
print ( f"Sampling completed and combined data is stored in: { output_file_path} " )