http

简单爬取百度贴吧例子

package main

import (
    "fmt"
    "strconv"
    "net/http"
    "io"
    "os"
)

func httpGet(url string) (result string, err error) {
    resp, err1 := http.Get(url)
    if err != nil {
        err = err1
        return
    }
    defer resp.Body.Close()

    buf := make([]byte, 4096)
    for {
        n, err2 := resp.Body.Read(buf)
        if n == 0 {
            fmt.Println("读取网页完成")
            break
        }
        if err2 != nil && err2 != io.EOF {
            err = err2
            return
        }
        result += string(buf[:n])
    }
    return
}

func spiderFun(url string, i int, page chan <- int)  {
    result, err := httpGet(url)
    if err != nil {
        fmt.Println("httpGet err:", err)
        return
    }
    // fmt.Println("result = ", result)
    f, err := os.Create("第"+ strconv.Itoa(i) + "页" + ".html")
    if err != nil {
        fmt.Println("os.Create err:", err)
        return
    }
    f.WriteString(result)
    f.Close()
    page <- i
}

func working(start, end int)  {

    page := make(chan int)
    for i := start; i <= end; i++ {
        url := "https://tieba.baidu.com/f?kw=lol&ie=utf-8&pn=" + strconv.Itoa((i-1)*50)
        // fmt.Println(url)
        go spiderFun(url, i, page)
    }

    // 与主go程完成同步
    for i := start; i <= end; i++ {
        fmt.Printf("爬取第%d个页面完成", <- page)
    }
}

func main()  {
    
    var start, end int
    fmt.Print("请输入爬取的起始页(>=1)")
    fmt.Scan(&start)
    fmt.Print("请输入爬取的起始页(>=start)")
    fmt.Scan(&end)

    working(start, end)
}

正则

package main

import(
    "regexp"
    "fmt"
) 

func main()  {
    str := "ssabnabw 2.3 344 5.4"
    str2 := `
<div>
    hello world
</div>
    `
    reg, _ := regexp.Compile(`\d+\.\d+`)
    ret := reg.FindAllStringSubmatch(str, -1)   //2.3 5.4
    fmt.Println(ret)

    // 将(?s:(.*?))元组放置在某一特征字符串中就可以提取这一字符串的内容
    reg, _ = regexp.Compile(`<div>(?s:(.*?))</div>`)
    ret = reg.FindAllStringSubmatch(str2, -1)
    fmt.Println(ret[])
    /*
<div>
        hello world
</div>
    */
}

转载于:https://www.cnblogs.com/huyuan1004/p/11324127.html