go爬虫和python爬虫_比较爬虫用的语言Python与Go

1 package main2

3 import(4 "fmt"

5 "net/http"

6 "net/url"

7 "os"

8 "strings"

9 "strconv"

10 "gopkg.in/redis.v4"

11 "github.com/PuerkitoBio/goquery"

12 )13

14 var loginUrl string = "https://manage.xxx.com.cn/home/login"

15 var dataUrl string = "https://manage.xxx.com.cn/?pageNo=1&pageSize=100"

16 var userName string = "xxx"

17 var passWord string = "xxx"

18 var redisServer string = "192.168.1.2"

19 var redisPort string = "6379"

20 var redisPass string = ""

21 var redisDB int =022

23 func main() {24 productList :=make(map[string] string)25 productList["椰油"] = "CL_Spot"

26 productList["咖啡"] = "COFFEE"

27 productList["工业铜"] = "COPPER"

28 volumeList :=make(map[string] int)29 volumeList["u_CL_Spot"] =030 volumeList["d_CL_Spot"] =031 volumeList["u_COFFEE"] =032 volumeList["d_COFFEE"] =033 volumeList["u_COPPER"] =034 volumeList["d_COPPER"] =035 jsessionid :=getCookie()36 doLogin(jsessionid)37

38 request, err := http.NewRequest("GET", dataUrl, nil)39 request.AddCookie(&http.Cookie{Name: "JSESSIONID", Value: jsessionid})40 client := &http.Client{}41 response, err :=client.Do(request)42 if err !=nil {43 fmt.Println(err.Error())44 os.Exit(0)45 }46 defer response.Body.Close()47 doc, err :=goquery.NewDocumentFromReader(response.Body)48 doc.Find("table").Eq(1).Find("tr").Each(func(i int, tr *goquery.Selection) {49 td := tr.Find("td")50 name := td.Eq(3).Text()51 dir := td.Eq(6).Text()52 if val, ok :=productList[name]; ok {53 buyNum, _ := strconv.Atoi(td.Eq(7).Text())54 buyUnit, _ := strconv.Atoi(td.Eq(8).Text())55 num := buyNum *buyUnit56 cacheKey := ""

57 if dir == "买"{58 cacheKey = fmt.Sprintf("u_%s", val)59 } else if dir == "卖"{60 cacheKey = fmt.Sprintf("d_%s", val)61 }62 volumeList[cacheKey] +=num63 }64 })65 redisClient := redis.NewClient(&redis.Options{66 Addr: fmt.Sprintf("%s:%s", redisServer, redisPort),67 Password: redisPass,68 DB: redisDB,69 })70 for k, v :=range volumeList {71 strKey := fmt.Sprintf("redis_order_count_%s", k)72 redisClient.Set(strKey, int(v), 0)73 }74 fmt.Println("puti volume get success")75 }76

77 func getCookie() string {78 jsessionid := ""

79 response, err :=http.Get(loginUrl)80 if err !=nil {81 fmt.Println(err.Error())82 os.Exit(0)83 }84 defer response.Body.Close()85 for _, val :=range response.Cookies() {86 if val.Name == "JSESSIONID"{87 jsessionid =val.Value88 }89 }90 returnjsessionid91 }92

93 func doLogin(jsessionid string) bool {94 data :=url.Values{}95 data.Set("userName", userName)96 data.Add("password", passWord)97 request, _ := http.NewRequest("POST", loginUrl, strings.NewReader(data.Encode()))98 request.Header.Add("Content-Type", "application/x-www-form-urlencoded")99 request.Header.Add("Content-Length", strconv.Itoa(len(data.Encode())))100 request.AddCookie(&http.Cookie{Name: "JSESSIONID", Value: jsessionid})101 client := &http.Client{}102 response, err :=client.Do(request)103 if err !=nil {104 fmt.Println(err.Error())105 os.Exit(0)106 }107 defer response.Body.Close()108 returntrue109 }