如何将mysql大数据量数据导出到多个文件

数据：

 mysql select 'xx1','xx2' union select xx1,xx2 from xx into outfile '/atmd/aa/bb.csv' fields terminated by '\t'

总的数据量有2500w

1在导出时，指定limit或者where条件，将数据在导出时就分别放在不同的文件里

2直接导出一个大的文件里的情况：

 1使用spark分别读取再写文件，因为不需要做处理，这种方式有点大材小用

 2 在excel可以打开的情况下，可以使用excel自带的代码工具VB，按照某一列去分割为多个文件：

  https://www.microsoft.com/zh-cn/download/confirmation.aspx?id=13255

https://jingyan.baidu.com/article/64d05a0219ceacde55f73bf6.html

表sheet只有一个，名为“数据源”，选择查看代码---插入--模块---写入一下内容，再回到excel 表格窗口，执行 按钮...

 3 使用linux命令拆分 split命令按行或者按大小拆分为多个文件

下面为方法2excel的vb方法模块

Sub CFGZB()

    Dim myRange As Variant

    Dim myArray

    Dim titleRange As Range

    Dim title As String

    Dim columnNum As Integer

    myRange = Application.InputBox(prompt:="请选择标题行：", Type:=8)

    myArray = WorksheetFunction.Transpose(myRange)

    Set titleRange = Application.InputBox(prompt:="请选择拆分的表头，必须是第一行，且为一个单元格，如：“姓名”", Type:=8)

    title = titleRange.Value

    columnNum = titleRange.Column

    Application.ScreenUpdating = False

    Application.DisplayAlerts = False

    Dim i&, Myr&, Arr, num&

    Dim d, k

    For i = Sheets.Count To 1 Step -1

        If Sheets(i).Name <> "数据源" Then

            Sheets(i).Delete

        End If

    Next i

    Set d = CreateObject("Scripting.Dictionary")

    Myr = Worksheets("数据源").UsedRange.Rows.Count

    Arr = Worksheets("数据源").Range(Cells(2, columnNum), Cells(Myr, columnNum))

    For i = 1 To UBound(Arr)

        d(Arr(i, 1)) = ""

    Next

    k = d.keys

    For i = 0 To UBound(k) 

        Set conn = CreateObject("adodb.connection")

        conn.Open "provider=Microsoft.ACE.OLEDB.12.0;extended properties=Excel 8.0;data source=" & ThisWorkbook.FullName

        Sql = "select * from [数据源$] where " & title & " = '" & k(i) & "'"

        Dim Nowbook As Workbook

        Set Nowbook = Workbooks.Add

        With Nowbook

            With .Sheets(1)

                .Name = k(i)

                For num = 1 To UBound(myArray)

                    .Cells(1, num) = myArray(num, 1)

                Next num

                .Range("A2").CopyFromRecordset conn.Execute(Sql)

            End With

        End With

        ThisWorkbook.Activate

        Sheets(1).Cells.Select

        Selection.Copy

        Workbooks(Nowbook.Name).Activate

        ActiveSheet.Cells.Select

        Selection.PasteSpecial Paste:=xlPasteFormats, Operation:=xlNone,SkipBlanks:=False, Transpose:=False

        Application.CutCopyMode = False

        Nowbook.SaveAs ThisWorkbook.Path & "\" & k(i)

        Nowbook.Close True

        Set Nowbook = Nothing

    Next i

    conn.Close

    Set conn = Nothing

    Application.DisplayAlerts = True

    Application.ScreenUpdating = True

End Sub