mysql marisa_使用Go并发下载图片资源

package img

import (

"database/sql"

"fmt"

_ "github.com/go-sql-driver/mysql"

"io"

"net/http"

"os"

"runtime"

"strconv"

"sync"

"time"

)

const (

DataRoot = "./tmp/" // 存放封面图的根目录

TimeoutLimit = 10 // 设置超时时间

)

// 表示章节ID和封面URL的对应关系

type VolumeCover struct {

VolumeID int

Url string

Lock sync.Mutex

Msg chan string

Flag bool    "timeout_flag"

}

// 将图片下载并保存到本地

func SaveImage(vc *VolumeCover) {

res, err := http.Get(vc.Url)

defer res.Body.Close()

if err != nil {

vc.Msg

}

// 创建文件

dst, err := os.Create(DataRoot + strconv.Itoa(vc.VolumeID) + ".jpg")

if err != nil {

vc.Msg

}

// 生成文件

io.Copy(dst, res.Body)

// goroutine通信

vc.Lock.Lock()

vc.Msg

vc.Flag = true

vc.Lock.Unlock()

}

func Start(name string, password string, limit int) error {

runtime.GOMAXPROCS(4)

sl, err := sql.Open("mysql", name+":"+password+"@/xxx?charset=utf8")

defer sl.Close()

if err != nil {

return err

}

// 构造SELECT语句并检索

queryStr := "SELECT VolumeID, ImageUrl FROM volume "

if limit > 0 {

queryStr += "limit " + strconv.Itoa(limit)

}

rows, err := sl.Query(queryStr)

defer rows.Close()

if err != nil {

return err

}

// 构建列表

i := 0

vclist := make([]*VolumeCover, limit)

for rows.Next() {

vc := &VolumeCover{}

if err := rows.Scan(&vc.VolumeID, &vc.Url); err != nil {

return err

}

vc.Msg = make(chan string, 1)

//vc.To = make(chan bool, 1)

vc.Lock = *new(sync.Mutex)

vclist[i] = vc

i++

}

// start goroutines

for i := 0; i < len(vclist); i++ {

go SaveImage(vclist[i])

go func(i int) {

time.Sleep(TimeoutLimit * time.Second)

if vclist[i].Flag == false {

vclist[i].Msg

}

}(i)

}

// 阻塞地获取结果

for i := 0; i < len(vclist); i++ {

func(c *VolumeCover) {

select {

case m :=

fmt.Println(m)

}

}(vclist[i])

}

return nil

}

对爬虫已爬到的数据进行下载，请求量与响应时间成正比。

还未做分量下载与日志记录。