【Go】并发目录遍历

package main

import(
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
)

func walkDir(dir string, fileSizes chan<- int64){
	for _, entry := range dirents(dir){
		if entry.IsDir(){
			subDir := filepath.Join(dir, entry.Name())
			walkDir(subDir, fileSizes)
		}else{
			fileSizes <- entry.Size()
		}
	}
}

func dirents(dir string)[]os.FileInfo{
	entries, err := ioutil.ReadDir(dir)
	if err != nil{
		fmt.Fprintf(os.Stderr, "du1: %v\n", err)
		return nil
	}
	return entries
}

func main(){
	flag.Parse()
	roots := flag.Args()
	if len(roots) == 0{
		roots = []string{"."}
	}
	fileSizes := make(chan int64)
	go func(){
		for _, root := range roots{
			walkDir(root, fileSizes)
		}
		close(fileSizes)
	}()
	var nfiles, nbytes int64
	for size := range fileSizes{
		nfiles++
		nbytes += size
	}
	printDiskUsage(nfiles, nbytes)
}

func printDiskUsage(nfiles, nbytes int64){
	fmt.Printf("%d files %.1fGB\n", nfiles, float64(nbytes)/1e9)
}

通过定义fileSizes通道，阻塞主线程，计算大小

通过一个线程不断遍历文件目录，传输大小给fileSizes

总共使用2个线程

package main

import(
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"sync"
	"time"
)

func walkDir(dir string, n *sync.WaitGroup, fileSizes chan<- int64){
	defer n.Done()
	for _, entry := range dirents(dir){
		if entry.IsDir(){
			n.Add(1)
			subDir := filepath.Join(dir, entry.Name())
			go walkDir(subDir, n, fileSizes)
		}else{
			fileSizes <- entry.Size()
		}
	}
}
var sema = make(chan struct{}, 20)
func dirents(dir string)[]os.FileInfo{
	sema <- struct{}{}
	defer func(){ <- sema }()
	entries, err := ioutil.ReadDir(dir)
	if err != nil{
		fmt.Fprintf(os.Stderr, "du1: %v\n", err)
		return nil
	}
	return entries
}

func main(){
	flag.Parse()
	roots := flag.Args()
	if len(roots) == 0{
		roots = []string{"."}
	}
	fileSizes := make(chan int64)
	var n sync.WaitGroup
	for _, root := range roots{
		n.Add(1)
		go walkDir(root, &n, fileSizes)
	}
	go func(){
		n.Wait()
		close(fileSizes)
	}()
	var tick <-chan time.Time
	tick = time.Tick(500*time.Millisecond)
	var nfiles, nbytes int64
loop:
		for{
			select{
			case size, ok := <-fileSizes:
				if !ok{
					break loop
				}
				nfiles++
				nbytes += size
			case <-tick:
				printDiskUsage(nfiles, nbytes)
			}
		}
	printDiskUsage(nfiles, nbytes)
}

func printDiskUsage(nfiles, nbytes int64){
	fmt.Printf("%d files %.1fGB\n", nfiles, float64(nbytes)/1e9)
}

版本2利用20个大小的缓冲通道作为信号量，最高打开20个并发遍历目录的线程。

通过一个线程，观察waitgroup的线程是否结束，结束则关闭fileSizes通道。

主线程通过time.Tick创建一个通道，周期性的触发tick事件，输出当前已经遍历的文件大小

主线程通过fileSizes通道计算已遍历文件大小

package main

import(
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"sync"
	"time"
)
var done = make(chan struct{})

func cancelled()bool{
	select{
	case <-done:
		return true
	default:
		return false
	}
}

func walkDir(dir string, n *sync.WaitGroup, fileSizes chan<- int64){
	defer n.Done()
	if cancelled(){
		return
	}
	for _, entry := range dirents(dir){
		if entry.IsDir(){
			n.Add(1)
			subDir := filepath.Join(dir, entry.Name())
			go walkDir(subDir, n, fileSizes)
		}else{
			fileSizes <- entry.Size()
		}
	}
}
var sema = make(chan struct{}, 20)
func dirents(dir string)[]os.FileInfo{
	select{
	case sema <- struct{}{}:
	case <-done:
		return nil
	}
	defer func(){ <- sema }()
	entries, err := ioutil.ReadDir(dir)
	if err != nil{
		fmt.Fprintf(os.Stderr, "du1: %v\n", err)
		return nil
	}
	return entries
}

func main(){
	flag.Parse()
	roots := flag.Args()
	if len(roots) == 0{
		roots = []string{"."}
	}

	go func(){
		os.Stdin.Read(make([]byte, 1))
		close(done)
	}()
	
	fileSizes := make(chan int64)
	var n sync.WaitGroup
	for _, root := range roots{
		n.Add(1)
		go walkDir(root, &n, fileSizes)
	}
	
	go func(){
		n.Wait()
		close(fileSizes)
	}()
	
	tick := time.Tick(500*time.Millisecond)
	var nfiles, nbytes int64
loop:
		for{
			select{
			case <-done:
				for range fileSizes{

				}
				return
			case size, ok := <- fileSizes:
				if !ok{
					break loop
				}
				nfiles++
				nbytes += size
			case <-tick:
				printDiskUsage(nfiles, nbytes)
			}
		}
	printDiskUsage(nfiles, nbytes)
}

func printDiskUsage(nfiles, nbytes int64){
	fmt.Printf("%d files %.1fGB\n", nfiles, float64(nbytes)/1e9)
}

版本3，通过增加一个done信号量和一个监控命令行输入的线程，终止程序运行。

canceled通过select实现非阻塞的通道。

各子线程通过done通道来决定是否结束线程。

其余一样