Seq 处理文件的效率差别怎么会这么大?
(*
fsc --standalone --codepage:936 -O --noframework testpseq.fsx
*)
//#r @"C:\Program Files\FSharpPowerPack-2.0.0.0\bin\FSharp.PowerPack.Parallel.Seq.dll"
//open Microsoft.FSharp.Collections
open System
open System.IO
(*
let testPseq =
let a = [1..20000]
let t1 = System.DateTime.Now
a |> PSeq.iter ( fun i -> printf "%A" i)
printfn "%A" 0
let t2 = System.DateTime.Now
printfn "%A" (System.DateTime.op_Subtraction (t2, t1))
a |> Seq.iter ( fun i -> printf "%A" i)
printfn "%A" 0
let t3 = System.DateTime.Now
printfn "%A" (System.DateTime.op_Subtraction(t3, t2))
*)
let testSeqiter =
let a = [1..20000]
let t1 = DateTime.Now
let rnd = new Random()
let writeToFile filename obj =
using ( File.AppendText(filename)) (fun f ->
f.WriteLine("{0}", obj.ToString() )
)
// a |> Seq.iter(fun _ -> writeToFile "out.txt" (rnd.Next()))
File.ReadLines (@"out.txt")
|> Seq.map( fun l -> l.ToString().Trim())
|> Seq.filter (fun l -> not (String.IsNullOrWhiteSpace(l) ) )
|> Seq.filter (fun l -> not ( l.StartsWith(@"<!--") && l.EndsWith (@"-->") ))
|> Seq.iter (fun i -> writeToFile @"out2.txt" i)
let t2 = DateTime.Now
printfn "%A" (DateTime.op_Subtraction(t2, t1))
// File.ReadLines (@"out.txt")
// |> PSeq.filter (fun i -> not (String.IsNullOrWhiteSpace(i.Trim())))
// |> Seq.iter (fun i -> writeToFile @"out3.txt" i)
// // PSeq.iter (fun i -> writeToFile @"out3.txt" i) 不行,说前面的文件正在打开。
// // 且整体运行效率还不如顺序版本
let readFile filename =
seq { use reader = new StreamReader (File.OpenRead(filename))
while not reader.EndOfStream do
yield reader.ReadLine()}
let Lines = readFile @"out.txt"
Lines
|> Seq.map( fun l -> l.ToString().Trim())
|> Seq.filter (fun l -> not (String.IsNullOrWhiteSpace(l) ) )
|> Seq.filter (fun l -> not ( l.StartsWith(@"<!--") && l.EndsWith (@"-->") ))
|> Seq.iter (fun l -> writeToFile @"out3.txt" l )
let t3 = DateTime.Now
printfn "%A" (DateTime.op_Subtraction(t3, t2))
[<EntryPoint>]
let main args =
testSeqiter
0
处理一个 10 兆左右的文件,快的时候只要 20 秒,慢时却要 2 分多钟。
如何才能快呢?搞不太清楚。
不过,把不需要的进程结束,就能加快程序的执行速度。最有效的进程,就是 explorer.exe,结束后,可以达到最快。
在不结束无关进程的情况下,把进程的优先进高为高,也可以达到提升运行速度的效果。
start /high testPseq.exe
(*
fsc --standalone --codepage:936 -O --noframework testpseq.fsx
*)
//#r @"C:\Program Files\FSharpPowerPack-2.0.0.0\bin\FSharp.PowerPack.Parallel.Seq.dll"
//open Microsoft.FSharp.Collections
open System
open System.IO
(*
let testPseq =
let a = [1..20000]
let t1 = System.DateTime.Now
a |> PSeq.iter ( fun i -> printf "%A" i)
printfn "%A" 0
let t2 = System.DateTime.Now
printfn "%A" (System.DateTime.op_Subtraction (t2, t1))
a |> Seq.iter ( fun i -> printf "%A" i)
printfn "%A" 0
let t3 = System.DateTime.Now
printfn "%A" (System.DateTime.op_Subtraction(t3, t2))
*)
let testSeqiter =
let a = [1..20000]
let t1 = DateTime.Now
let rnd = new Random()
let writeToFile filename obj =
using ( File.AppendText(filename)) (fun f ->
f.WriteLine("{0}", obj.ToString() )
)
// a |> Seq.iter(fun _ -> writeToFile "out.txt" (rnd.Next()))
File.ReadLines (@"out.txt")
|> Seq.map( fun l -> l.ToString().Trim())
|> Seq.filter (fun l -> not (String.IsNullOrWhiteSpace(l) ) )
|> Seq.filter (fun l -> not ( l.StartsWith(@"<!--") && l.EndsWith (@"-->") ))
|> Seq.iter (fun i -> writeToFile @"out2.txt" i)
let t2 = DateTime.Now
printfn "%A" (DateTime.op_Subtraction(t2, t1))
// File.ReadLines (@"out.txt")
// |> PSeq.filter (fun i -> not (String.IsNullOrWhiteSpace(i.Trim())))
// |> Seq.iter (fun i -> writeToFile @"out3.txt" i)
// // PSeq.iter (fun i -> writeToFile @"out3.txt" i) 不行,说前面的文件正在打开。
// // 且整体运行效率还不如顺序版本
let readFile filename =
seq { use reader = new StreamReader (File.OpenRead(filename))
while not reader.EndOfStream do
yield reader.ReadLine()}
let Lines = readFile @"out.txt"
Lines
|> Seq.map( fun l -> l.ToString().Trim())
|> Seq.filter (fun l -> not (String.IsNullOrWhiteSpace(l) ) )
|> Seq.filter (fun l -> not ( l.StartsWith(@"<!--") && l.EndsWith (@"-->") ))
|> Seq.iter (fun l -> writeToFile @"out3.txt" l )
let t3 = DateTime.Now
printfn "%A" (DateTime.op_Subtraction(t3, t2))
[<EntryPoint>]
let main args =
testSeqiter
0
处理一个 10 兆左右的文件,快的时候只要 20 秒,慢时却要 2 分多钟。
如何才能快呢?搞不太清楚。
不过,把不需要的进程结束,就能加快程序的执行速度。最有效的进程,就是 explorer.exe,结束后,可以达到最快。
在不结束无关进程的情况下,把进程的优先进高为高,也可以达到提升运行速度的效果。
start /high testPseq.exe