go 其它编码转UTF8编码

需要安装两个包:

gopm get -g -v golang.org/x/text

gopm get -g -v golang.org/x/net/html

package fetcher

import (
   "bufio"
   "golang.org/x/net/html/charset"
   "golang.org/x/text/encoding"
   "golang.org/x/text/encoding/unicode"
   "golang.org/x/text/transform"
   "io/ioutil"
   "net/http"
   "src/github.com/gpmgo/gopm/modules/log"
)

func Fetch(url string) ([]byte, error) {
   resp, err := http.Get(url)
   defer resp.Body.Close()
   if err != nil {
      return nil, err
   }

   reader := bufio.NewReader(resp.Body)
   e := determineEncoding(reader)
   utf8Reader := transform.NewReader(reader, e.NewDecoder())

   body, err := ioutil.ReadAll(utf8Reader)
   if err != nil {
      panic(err)
   }

   return body, nil
}

func determineEncoding(r *bufio.Reader) encoding.Encoding {
   b, err := r.Peek(1024)
   if err != nil {
      log.Error("get code error")
      return unicode.UTF8
   }
   e, _, _ := charset.DetermineEncoding(b, "")
   return e
}

转载于:https://my.oschina.net/wgc2010/blog/2247064