写了一个Go程序，爬取了所有的RFC文档，共计9109篇

下载链接如下：

CSDN资源 

阿里云盘

---

程序如下：

package rfcDownload

import (
	"fmt"
	"os"
	"spider/tools"
	"time"
)

var finish chan int

func isExist(file string) (bool, error) {
	_, err := os.Stat(file)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return false, err
}

func Download(rfcNo int, p chan int) {
	defer func() {
		p <- 1
	}()
	url := "https://www.ietf.org/rfc/"
	filename := fmt.Sprintf("rfc%4.4d.txt", rfcNo)
	dir := "F:\\golang\\myGo\\src\\spider\\rfcDownload\\rfc\\"

	fmt.Println(dir + filename)
	if exist, _ := isExist(dir + filename); exist {
		fmt.Println(filename + "文件存在")
		return
	}
	html, err := tools.GetHtml(url + filename)
	if err != nil {
		fmt.Println("下载错误:", err)
		return
	}
	fd, err := os.OpenFile(dir+filename, os.O_CREATE|os.O_RDWR, 0666)
	if err != nil {
		fmt.Println("文件创建失败")
		return
	}
	defer fd.Close()
	fd.WriteString(string(html))

}

func BatchDownload() {
	rfcNo := 9110 //9110
	finish := make(chan int)
	t1 := time.Now()
	for i := 1; i < rfcNo; i++ {

		go Download(i, finish)
	}

	for i := 1; i < rfcNo; i++ {
		<-finish
	}
	t2 := time.Now()
	fmt.Println("经历的时间为：", t2.Sub(t1))
}