此项目爬取酷我音乐盒的mv,只用于学习使用
golang框架:gin
运行以下代码,需要你们自己搭建好kafka。我就直接把我的代码贴上来了
main.go文件代码
package main
import (
"fmt"
UserKafka "gin/app/api/userKafka"
"gin/router"
_ "net/http/pprof"
"strconv"
"syscall"
"unsafe"
)
func main() {
var start int
var end int
fmt.Println("爬取酷我音乐盒mv : https://www.kuwo.cn/mvplay/")
fmt.Println("请输入开始id:")
fmt.Scanln(&start)
fmt.Println("请输入结束id:")
fmt.Scanln(&end)
SetCmdTitle("id:" + strconv.Itoa(start) + "-" + strconv.Itoa(end))
go UserKafka.ProducerFunc(start, end) // kafka生产者
go UserKafka.ConsumerFunc() // kafka消费者
select {} // 停止在cmd,避免cmd自动关闭
// Init()
}
func Init() {
router.RouterInit() // 启动http服务且初始化路由
}
// 修改 cmd 的标题
func SetCmdTitle(title string) {
kernel32, _ := syscall.LoadLibrary(`kernel32.dll`)
sct, _ := syscall.GetProcAddress(kernel32, `SetConsoleTitleW`)
syscall.Syscall(sct, 1, uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(title))), 0, 0)
syscall.FreeLibrary(kernel32)
}
userKafka.go文件代码
package UserKafka
import (
"encoding/json"
"fmt"
"gin/Model"
"net/http"
"strconv"
"strings"
"time"
"github.com/Shopify/sarama"
cluster "github.com/bsm/sarama-cluster"
"github.com/golang/glog"
"context"
"log"
"github.com/PuerkitoBio/goquery"
"github.com/chromedp/chromedp"
)
var (
topics = "heima"
)
type Colly struct {
Id string `json:"id" gorm:"primaryKey"`
Uri string
Download_uri string
Web_id string
Time string
Name string
Artist string
}
func (Colly) TableName() string {
return "ipfs_colly"
}
// kafka生产者
func ProducerFunc(start int, end int) {
fmt.Println("ProducerFunc start")
config := sarama.NewConfig()
config.Producer.Return.Successes = true
config.Producer.RequiredAcks = sarama.WaitForAll
config.Producer.Partitioner = sarama.NewRandomPartitioner
producer, err := sarama.NewSyncProducer([]string{"192.168.71.130:9092"}, config)
if err != nil {
panic(err)
}
defer producer.Close()
msg := &sarama.ProducerMessage{
Topic: topics,
Partition: int32(-1),
Key: sarama.StringEncoder("key"),
}
var value string
for i := start; i < end; i++ {
uri := "https://www.kuwo.cn/mvplay/" + strconv.Itoa(i)
resp, err := http.Get(uri)
if err != nil {
fmt.Println("http.Get err=", err)
continue
}
// defer resp.Body.Close() // 函数结束时关闭Body
resp.Body.Close()
if resp.StatusCode != 200 {
fmt.Printf("id: %d http.Get StatusCode=%d \n", i, resp.StatusCode)
continue
}
example, err7 := GetHttpHtmlContent(uri, ".mv_out", "document.querySelector('body')")
if err7 != nil {
fmt.Println("GetHttpHtmlContent err:", err7)
continue
}
dom, err4 := goquery.NewDocumentFromReader(strings.NewReader(example))
if err4 != nil {
fmt.Println("goquery.NewDocumentFromReader err:", err4)
continue
}
src, bool := dom.Find("#vjs_video_403_html5_api").Eq(0).Attr("src")
if !bool || src == "res not found" {
fmt.Printf("dom.Find -> id:%d empty \n", i)
continue
}
name := dom.Find(".mv_name").Text()
artist := dom.Find(".artist").Text()
// 判断是否存在,存在就不插入数据
ret := &Colly{}
_, count := Model.GetList(ret, "web_id = "+strconv.Itoa(i), "", "1")
if count > 0 {
fmt.Printf("id:%s is exist \n", msg.Value)
continue
}
ret.Web_id = strconv.Itoa(i)
ret.Uri = uri
ret.Download_uri = src
ret.Name = name
ret.Artist = artist
ret.Time = time.Now().Format("2006/01/02 15:04:05")
json, _ := json.Marshal(ret)
value = strings.Replace(string(json), "\n", "", -1)
msg.Value = sarama.ByteEncoder(value)
paritition, offset, err := producer.SendMessage(msg)
if err != nil {
fmt.Println("Send Message Fail")
}
fmt.Printf("Partion = %d, offset = %d\n", paritition, offset)
}
}
// kafka消费者
func ConsumerFunc() {
fmt.Println("ConsumerFunc start")
groupID := "test-consumer-group"
config := cluster.NewConfig()
config.Group.Return.Notifications = true
// config.Consumer.Offsets.AutoCommit.Enable = true
config.Consumer.Offsets.CommitInterval = 3 * time.Second
config.Consumer.Offsets.Initial = sarama.OffsetNewest //初始从最新的offset开始
cc, err9 := cluster.NewConsumer(strings.Split("192.168.71.130:9092", ","), groupID, strings.Split(topics, ","), config)
if err9 != nil {
glog.Errorf("Failed open consumer: %v", err9)
return
}
defer cc.Close()
go func(cc *cluster.Consumer) {
errors := cc.Errors()
noti := cc.Notifications()
for {
select {
case err8 := <-errors:
glog.Errorln(err8)
case <-noti:
}
}
}(cc)
for msg := range cc.Messages() {
// fmt.Fprintf(os.Stdout, "%s/%d/%d\t%s", msg.Topic, msg.Partition, msg.Offset, msg.Value)
ret := &Colly{}
json.Unmarshal(msg.Value, ret)
fmt.Printf("ret: %+v", ret)
err2 := Model.Add(ret) // 此处只是用于写入数据库,可以做自己需要的操作
if err2 != nil {
fmt.Println(err2)
}
cc.MarkOffset(msg, "") // 消费者处理完毕一条消息后,提交告知kafka
}
}
//得到具体的数据
func GetSpecialData(htmlContent string, selector string) (string, error) {
dom, err := goquery.NewDocumentFromReader(strings.NewReader(htmlContent))
if err != nil {
return "", err
}
var str string
dom.Find(selector).Each(func(i int, selection *goquery.Selection) {
str = selection.Text()
})
return str, nil
}
//获取网站上爬取的数据
func GetHttpHtmlContent(url string, selector string, sel interface{}) (string, error) {
options := []chromedp.ExecAllocatorOption{
chromedp.Flag("headless", true), // debug使用
chromedp.Flag("blink-settings", "imagesEnabled=false"),
chromedp.UserAgent(`Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36`),
}
//初始化参数,先传一个空的数据
options = append(chromedp.DefaultExecAllocatorOptions[:], options...)
c, _ := chromedp.NewExecAllocator(context.Background(), options...)
// create context
chromeCtx, cancels := chromedp.NewContext(c, chromedp.WithLogf(log.Printf))
defer cancels() // 不可少,否则一直开后台chrome进程,狂吃内存
// 执行一个空task, 用提前创建Chrome实例
chromedp.Run(chromeCtx, make([]chromedp.Action, 0, 1)...)
//创建一个上下文,超时时间为40s
timeoutCtx, cancel := context.WithTimeout(chromeCtx, 40*time.Second)
defer cancel() // 不可少,否则一直开后台chrome进程,狂吃内存
var htmlContent string
err := chromedp.Run(timeoutCtx,
chromedp.Navigate(url),
chromedp.WaitVisible(selector),
chromedp.OuterHTML(sel, &htmlContent, chromedp.ByJSPath),
)
if err != nil {
return "", err
}
//log.Println(htmlContent)
return htmlContent, nil
}
最后附上图片一直,让你们看我的目录结构