Pandas, Dask DataFrame根据列分割成多个文件

import pandas as pd

# Assuming df is your DataFrame
df = pd.DataFrame({
    'A': ['foo', 'bar', 'foo', 'bar', 'foo', 'bar', 'foo', 'bar', 'foo', 'bar'],
    'B': ['one', 'one', 'two', 'three', 'two', 'two', 'one', 'three', 'four', 'three'],
    'C': ['small', 'large', 'large', 'small', 'small', 'large', 'small', 'large', 'large', 'small']
})

# Split the DataFrame by column B
df_split = df.groupby('B')

# Write each group to a separate CSV file
for name, group in df_split:
    group.to_csv(f'{name}.csv', index=False)

示例代码中根据B列的值分割成one.csv, two.csv, three.csv 和four.csv共4个文件。

import dask.dataframe as dd
import pandas as pd
# Create a sample Dask DataFrame
data = {'A': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
        'B': ['X', 'Y', 'X', 'Z', 'Y', 'Z', 'X', 'Y', 'Z', 'X'],
        'C': [10, 20, 30, 40, 50, 60, 70, 80, 90, 100]}
ddf = dd.from_pandas(pd.DataFrame(data), npartitions=2)

# Get the unique values of column B
unique_values_b = ddf['B'].unique().compute()
# Define a function to filter the DataFrame based on the value of column B
def filter_by_b_value(df, b_value):
    return df[df['B'] == b_value]

# Filter the DataFrame for each unique value of column B
filtered_dfs = [filter_by_b_value(ddf, b_value) for b_value in unique_values_b]
# Save each filtered DataFrame to CSV
for i, filtered_df in enumerate(filtered_dfs):
    filtered_df.compute().to_csv(f'file_{unique_values_b[i]}.csv', index=False)

示例代码中根据B列的值分割成X.csv, Y.csv, Z.csv 共3个文件。