【Go】Go读书社区web开发与高性能架构优化 (beego+redis+es)

记录一下较为 核心的知识点吧

第5章 V1.5 优化数据层并发【MySQL并发优化】

mysql高可用

分表

拆表

根据id的奇偶数 来实现评论表的分表查找

链接表名称

go语言实现读写分离 主从配置

package sysinit

func init() {
	sysinit()
	dbinit()             //初始化主库
	dbinit("r")          //初始化从库
	dbinit("uaw", "uar") //初始化社区库
}

初始化数据库

package sysinit

import (
	_ "ziyoubiancheng/mbook/models"

	"github.com/astaxie/beego"
	"github.com/astaxie/beego/orm"
	_ "github.com/go-sql-driver/mysql"
)

//调用方式
//dbinit() 或 dbinit("w") 或 dbinit("default") //初始化主库
//dbinit("w","r")	//同时初始化主库和从库
//dbinit("w")
func dbinit(aliases ...string) {
	//如果是开发模式，则显示命令信息
	isDev := (beego.AppConfig.String("runmode") == "dev")

	if len(aliases) > 0 {
		for _, alias := range aliases {
			registDatabase(alias)
			//主库 自动建表
			if "w" == alias {
				orm.RunSyncdb("default", false, isDev)
			}
		}
	} else {
		registDatabase("w")
		orm.RunSyncdb("default", false, isDev)
	}

	if isDev {
		orm.Debug = isDev
	}
}

func registDatabase(alias string) {
	if len(alias) == 0 {
		return
	}
	//连接名称
	dbAlias := alias
	if "w" == alias || "default" == alias {
		dbAlias = "default"
		alias = "w"
	}
	//数据库名称
	dbName := beego.AppConfig.String("db_" + alias + "_database")
	//数据库连接用户名
	dbUser := beego.AppConfig.String("db_" + alias + "_username")
	//数据库连接用户名
	dbPwd := beego.AppConfig.String("db_" + alias + "_password")
	//数据库IP（域名）
	dbHost := beego.AppConfig.String("db_" + alias + "_host")
	//数据库端口
	dbPort := beego.AppConfig.String("db_" + alias + "_port")

	orm.RegisterDataBase(dbAlias, "mysql", dbUser+":"+dbPwd+"@tcp("+dbHost+":"+dbPort+")/"+dbName+"?charset=utf8", 30)

}

model层

package models

import (
	"errors"
	"strings"

	"github.com/astaxie/beego/orm"
)

type Category struct {
	Id     int
	Pid    int    //分类id
	Title  string `orm:"size(30);unique"`
	Intro  string //介绍
	Icon   string
	Cnt    int  //统计分类下图书
	Sort   int  //排序
	Status bool //状态，true 显示
}

func (m *Category) TableName() string {
	return TNCategory()
}

func (m *Category) GetCates(pid int, status int) (cates []Category, err error) {
	qs := GetOrm("r").QueryTable(TNCategory())
	if pid > -1 {
		qs = qs.Filter("pid", pid)
	}

	if status == 0 || status == 1 {
		qs = qs.Filter("status", status)
	}
	_, err = qs.OrderBy("-status", "sort", "title").All(&cates)
	return
}

//查询分类
func (m *Category) Find(id int) (cate Category) {
	cate.Id = id
	GetOrm("r").Read(&cate)
	return cate
}

//批量新增分类
func (m *Category) InsertMulti(pid int, cates string) (err error) {
	slice := strings.Split(cates, "\n")
	if len(slice) == 0 {
		return
	}

	o := GetOrm("w")
	for _, item := range slice {
		if item = strings.TrimSpace(item); item != "" {
			var cate = Category{
				Pid:    pid,
				Title:  item,
				Status: true,
			}
			if o.Read(&cate, "title"); cate.Id == 0 {
				_, err = o.Insert(&cate)
			}
		}
	}
	return
}

第6章 V1.8 搜索模块优化【搜索模块接入ElasticSearch】

Router

	//搜索
	beego.Router("/search", &controllers.ElasticsearchController{}, "get:Search")
	beego.Router("/search/result", &controllers.ElasticsearchController{}, "get:Result")

ElasticsearchController

package controllers

import (
	"fmt"
	"time"
	"ziyoubiancheng/mbook/models"
	"ziyoubiancheng/mbook/utils"

	"github.com/astaxie/beego"
)

type ElasticsearchController struct {
	BaseController
}

func (c *ElasticsearchController) Search() {
	c.TplName = "search/search.html"
}

func (c *ElasticsearchController) Result() {
	//获取关键词
	wd := c.GetString("wd")
	if "" == wd {
		c.Redirect(beego.URLFor("ElasticsearchController.Search"), 302)
	}
	c.Data["Wd"] = wd

	//搜文档&图书
	tab := c.GetString("tab", "doc")
	c.Data["Tab"] = tab

	//page&size
	page, _ := c.GetInt("page", 1)
	if page < 1 {
		page = 1
	}
	size := 10

	//开始搜索
	now := time.Now()
	if "doc" == tab {
		ids, count, err := models.ElasticSearchDocument(wd, size, page)
		c.Data["totalRows"] = count
		if nil == err && len(ids) > 0 {
			c.Data["Docs"], _ = models.NewDocumentSearch().GetDocsById(ids)
		}
	} else {
		ids, count, err := models.ElasticSearchBook(wd, size, page)
		c.Data["totalRows"] = count
		if nil == err && len(ids) > 0 {
			c.Data["Books"], _ = models.NewBook().GetBooksByIds(ids)
		}
	}

	if c.Data["totalRows"].(int) > size { //有分页
		urlSuffix := fmt.Sprintf("&tab=%v&wd=%v", tab, wd)
		html := utils.NewPaginations(4, c.Data["totalRows"].(int), size, page, beego.URLFor("ElasticsearchController.Result"), urlSuffix)
		c.Data["PageHtml"] = html
	} else {
		c.Data["PageHtml"] = ""
	}

	c.Data["SpendTime"] = fmt.Sprintf("%.3f", time.Since(now).Seconds())
	c.TplName = "search/result.html"
}

BookController 发布图书

 //发布图书.
func (c *BookController) Release() {
	identify := c.GetString("identify")
	bookId := 0
	if c.Member.IsAdministrator() {
		book, err := models.NewBook().Select("identify", identify)
		if err != nil {
			beego.Error(err)
		}
		bookId = book.BookId
	} else {
		book, err := models.NewBookData().SelectByIdentify(identify, c.Member.MemberId)
		if err != nil {
			c.JsonResult(1, "未知错误")
		}
		if book.RoleId != common.BookAdmin && book.RoleId != common.BookFounder && book.RoleId != common.BookEditor {
			c.JsonResult(1, "权限不足")
		}
		bookId = book.BookId
	}

	if exist := utils.BooksRelease.Exist(bookId); exist {
		c.JsonResult(1, "正在发布中，请稍后操作")
	}

	go func() {
		models.NewDocument().ReleaseContent(bookId, c.BaseUrl())
		models.ElasticBuildIndex(bookId)
	}()

	c.JsonResult(0, "已发布")
}

model/elasticsearch.go

package models

import (
	"fmt"
	"strconv"
	"strings"
	"ziyoubiancheng/mbook/utils"

	"github.com/PuerkitoBio/goquery"
	"github.com/astaxie/beego"
)

func ElasticSearchBook(kw string, pageSize, page int) ([]int, int, error) {
	var ids []int
	count := 0

	if page > 0 {
		page = page - 1
	} else {
		page = 0
	}
	queryJson := `
		{
		    "query" : {
		        "multi_match" : {
		        "query":"%v",
		        "fields":["book_name","description"]
		        }
		    },
		    "_source":["book_id"],
			"size": %v,
			"from": %v
		}
	`

	//elasticsearch api
	host := beego.AppConfig.String("elastic_host")
	api := host + "mbooks/datas/_search"
	queryJson = fmt.Sprintf(queryJson, kw, pageSize, page)

	sj, err := utils.HttpPostJson(api, queryJson)
	if nil == err {
		count = sj.GetPath("hits", "total").MustInt()
		resultArray := sj.GetPath("hits", "hits").MustArray()
		for _, v := range resultArray {
			if each_map, ok := v.(map[string]interface{}); ok {
				id, _ := strconv.Atoi(each_map["_id"].(string))
				ids = append(ids, id)
			}
		}
	}
	return ids, count, err
}

func ElasticSearchDocument(kw string, pageSize, page int, bookId ...int) ([]int, int, error) {
	var ids []int
	count := 0

	if page > 0 {
		page = page - 1
	} else {
		page = 0
	}
	//搜索全部
	queryJson := `
		{
		    "query" : {
		        "match" : {
		        	"release":"%v"
		        }
		    },
		    "_source":["document_id"],
			"size": %v,
			"from": %v
		}
	`
	queryJson = fmt.Sprintf(queryJson, kw, pageSize, page)

	//按图书搜索
	if len(bookId) > 0 && bookId[0] > 0 {
		queryJson = `
			{
				"query": {
					"bool": {
						"filter": [{
							"term": {
								"book_id":%v
							}
						}],
						"must": {
							"multi_match": {
								"query": "%v",
								"fields": ["release"]
							}
						}
					}
				},
				"from": %v,
				"size": %v,
				"_source": ["document_id"]
			}
		`

		queryJson = fmt.Sprintf(queryJson, kw, pageSize, page)
	}

	//elasticsearch api
	host := beego.AppConfig.String("elastic_host")
	api := host + "mdocuments/datas/_search"

	fmt.Println(api)
	fmt.Println(queryJson)

	sj, err := utils.HttpPostJson(api, queryJson)
	if nil == err {
		count = sj.GetPath("hits", "total").MustInt()
		resultArray := sj.GetPath("hits", "hits").MustArray()
		for _, v := range resultArray {
			if each_map, ok := v.(map[string]interface{}); ok {
				id, _ := strconv.Atoi(each_map["_id"].(string))
				ids = append(ids, id)
			}
		}
	}
	return ids, count, err
}

func ElasticBuildIndex(bookId int) {
	book, _ := NewBook().Select("book_id", bookId, "book_id", "book_name", "description")
	addBookToIndex(book.BookId, book.BookName, book.Description)

	//index documents
	var documents []Document
	fields := []string{"document_id", "book_id", "document_name", "release"}
	GetOrm("r").QueryTable(TNDocuments()).Filter("book_id", bookId).All(&documents, fields...)
	if len(documents) > 0 {
		for _, document := range documents {
			addDocumentToIndex(document.DocumentId, document.BookId, flatHtml(document.Release))
		}
	}
}

func addBookToIndex(bookId int, bookName string, description string) {
	queryJson := `
		{
			"book_id":%v,
			"book_name":"%v",
			"description":"%v"
		}
	`

	//elasticsearch api
	host := beego.AppConfig.String("elastic_host")
	api := host + "mbooks/datas/" + strconv.Itoa(bookId)

	//发起请求
	queryJson = fmt.Sprintf(queryJson, bookId, bookName, description)
	err := utils.HttpPutJson(api, queryJson)
	if nil != err {
		beego.Debug(err)
	}
}

func addDocumentToIndex(documentId, bookId int, release string) {
	queryJson := `
		{
			"document_id":%v,
			"book_id":%v,
			"release":"%v"
		}
	`

	//elasticsearch api
	host := beego.AppConfig.String("elastic_host")
	api := host + "mdocuments/datas/" + strconv.Itoa(documentId)

	//发起请求
	queryJson = fmt.Sprintf(queryJson, documentId, bookId, release)
	err := utils.HttpPutJson(api, queryJson)
	if nil != err {
		beego.Debug(err)
	}

}

func flatHtml(htmlStr string) string {
	htmlStr = strings.Replace(htmlStr, "\n", " ", -1)
	htmlStr = strings.Replace(htmlStr, "\"", "", -1)

	gq, err := goquery.NewDocumentFromReader(strings.NewReader(htmlStr))
	if err != nil {
		return htmlStr
	}
	return gq.Text()
}

第8章 V2.1动态缓存优化【基于Redis的动态缓存实践】

缓存

页面缓存

动态缓存

缓存本质

redis 缓存操作

初始化redis

package sysinit

import (
	"encoding/gob"
	//"os"
	"path/filepath"
	"strings"

	conf "ziyoubiancheng/mbook/common"
	"ziyoubiancheng/mbook/models"
	"ziyoubiancheng/mbook/utils"
	"ziyoubiancheng/mbook/utils/dynamicache"
	"ziyoubiancheng/mbook/utils/pagecache"
	"ziyoubiancheng/mbook/utils/store"

	"github.com/astaxie/beego"
)

func sysinit() {
	gob.Register(models.Member{}) //序列化Member对象,必须在encoding/gob编码解码前进行注册

	//uploads静态路径
	uploads := filepath.Join(conf.WorkingDirectory, "uploads")
	//os.MkdirAll(uploads, 0666)
	beego.BConfig.WebConfig.StaticDir["/uploads"] = uploads

	//注册前端使用函数
	registerFunctions()

	//初始化pagecache
	initPageCache()

	//初始化动态缓存
	initDynamicache()

	//初始化OSS
	initOss()
}

func initOss() {
	store.InitOss()
}

func initDynamicache() {
	dynamicache.MaxOpen = 128
	dynamicache.MaxIdle = 128
	dynamicache.ExpireSec = 10
	dynamicache.InitCache()
}

go 操作redis 封装方法

package dynamicache

import (
	"encoding/json"
	"strconv"
	"time"

	"github.com/astaxie/beego"
	"github.com/gomodule/redigo/redis"
)

var (
	pool      *redis.Pool = nil
	MaxIdle   int         = 0
	MaxOpen   int         = 0
	ExpireSec int64       = 0
)

func InitCache() {
	addr := beego.AppConfig.String("dynamicache_addrstr")
	if len(addr) == 0 {
		addr = "127.0.0.1:6379"
	}
	if MaxIdle <= 0 {
		MaxIdle = 256
	}
	pass := beego.AppConfig.String("dynamicache_passwd")
	
	if len(pass) == 0 {
		pool = &redis.Pool{
			MaxIdle:     MaxIdle,
			MaxActive:   MaxOpen,
			IdleTimeout: time.Duration(120),
			Dial: func() (redis.Conn, error) {
				return redis.Dial(
					"tcp",
					addr,
					redis.DialReadTimeout(1*time.Second),
					redis.DialWriteTimeout(1*time.Second),
					redis.DialConnectTimeout(1*time.Second),
				)
			},
		}
	} else {
		pool = &redis.Pool{
			MaxIdle:     MaxIdle,
			MaxActive:   MaxOpen,
			IdleTimeout: time.Duration(120),
			Dial: func() (redis.Conn, error) {
				return redis.Dial(
					"tcp",
					addr,
					redis.DialReadTimeout(1*time.Second),
					redis.DialWriteTimeout(1*time.Second),
					redis.DialConnectTimeout(1*time.Second),
					redis.DialPassword(pass),
				)
			},
		}
	}
}

func rdsdo(cmd string, key interface{}, args ...interface{}) (interface{}, error) {
	con := pool.Get()
	if err := con.Err(); err != nil {
		return nil, err
	}
	parmas := make([]interface{}, 0)
	parmas = append(parmas, key)

	if len(args) > 0 {
		for _, v := range args {
			parmas = append(parmas, v)
		}
	}
	return con.Do(cmd, parmas...)
}

func WriteString(key string, value string) error {
	_, err := rdsdo("SET", key, value)
	beego.Debug("redis set:" + key + "-" + value)
	rdsdo("EXPIRE", key, ExpireSec)
	return err
}

func ReadString(key string) (string, error) {
	result, err := rdsdo("GET", key)
	beego.Debug("redis get:" + key)
	if nil == err {
		str, _ := redis.String(result, err)
		return str, nil
	} else {
		beego.Debug("redis get error:" + err.Error())
		return "", err
	}
}

func WriteStruct(key string, obj interface{}) error {
	data, err := json.Marshal(obj)
	if nil == err {
		return WriteString(key, string(data))
	} else {
		return nil
	}
}
func ReadStruct(key string, obj interface{}) error {
	if data, err := ReadString(key); nil == err {
		return json.Unmarshal([]byte(data), obj)
	} else {
		return err
	}
}

func WriteList(key string, list interface{}, total int) error {
	realKeyList := key + "_list"
	realKeyCount := key + "_count"
	data, err := json.Marshal(list)
	if nil == err {
		WriteString(realKeyCount, strconv.Itoa(total))
		return WriteString(realKeyList, string(data))
	} else {
		return nil
	}
}

func ReadList(key string, list interface{}) (int, error) {
	realKeyList := key + "_list"
	realKeyCount := key + "_count"
	if data, err := ReadString(realKeyList); nil == err {
		totalStr, _ := ReadString(realKeyCount)
		total := 0
		if len(totalStr) > 0 {
			total, _ = strconv.Atoi(totalStr)
		}
		return total, json.Unmarshal([]byte(data), list)
	} else {
		return 0, err
	}
}

Beego cache操作

func initPageCache() {
	pagecache.BasePath = "./cache/staticpage"
	pagecache.ExpireSec = 10
	pagecache.InitCache()
}

package pagecache

import (
	"errors"
	"strings"
	"time"

	"github.com/astaxie/beego"
	"github.com/astaxie/beego/cache"
)

var (
	BasePath  string              = ""
	ExpireSec int64               = 0
	store     *cache.FileCache    = nil
	cacheMap  map[string]bool     = nil
	paramMap  map[string][]string = nil
)

func InitCache() {
	store = &cache.FileCache{CachePath: BasePath}
	pagecacheList := beego.AppConfig.Strings("pagecache_list")

	//初始化静态化配置列表
	cacheMap = make(map[string]bool)
	for _, v := range pagecacheList {
		cacheMap[strings.ToLower(v)] = true
	}

	paramMap = make(map[string][]string)
	pagecacheMap, _ := beego.AppConfig.GetSection("pagecache_param")
	for k, v := range pagecacheMap {
		sv := strings.Split(v, ";")
		paramMap[k] = sv
	}
}

func InCacheList(controllerName, actionName string) bool {
	keyname := cacheKey(controllerName, actionName)
	if f := cacheMap[keyname]; f {
		return f
	}
	return false
}

func NeedWrite(controllerName, actionName string, params map[string]string) bool {
	if InCacheList(controllerName, actionName) {
		keyname := cacheKey(controllerName, actionName, params)
		if len(store.Get(keyname).(string)) > 0 {
			return false
		} else {
			beego.Debug("need write :" + keyname)
			return true
		}
	}
	return false
}

func Write(controllerName, actionName string, content *string, params map[string]string) error {
	keyname := cacheKey(controllerName, actionName, params)
	if len(keyname) == 0 {
		return errors.New("未找到缓存key")
	}

	err := store.Put(keyname, *content, time.Duration(ExpireSec)*time.Second)

	return err
}

func Read(controllerName, actionName string, params map[string]string) (*string, error) {
	keyname := cacheKey(controllerName, actionName, params)
	if len(keyname) == 0 {
		return nil, errors.New("未找到缓存key")
	}

	content := store.Get(keyname).(string)

	return &content, nil
}

func cacheKey(controllerName, actionName string, paramArray ...map[string]string) string {
	if len(controllerName) > 0 && len(actionName) > 0 {
		rtnstr := strings.ToLower(controllerName + "_" + actionName)
		if len(paramArray) > 0 {
			for _, v := range paramMap[rtnstr] {
				rtnstr = rtnstr + "_" + strings.ReplaceAll(v, ":", "") + "_" + paramArray[0][v]
			}
		}
		return rtnstr
	}

	return ""
}

func ClearExpiredFiles() {
	for k, _ := range cacheMap {
		if store.IsExist(k) {
			content := store.Get(k).(string)
			if len(content) == 0 {
				store.Delete(k)
			}
		}
	}
}

第9章 V2.2 文件下载优化【文件服务拆分与CDN接入】

初始化oss配置

package store

import (
	"errors"

	"github.com/aliyun/aliyun-oss-go-sdk/oss"
	"github.com/astaxie/beego"
)

var (
	endpoint        string = ""
	accessKeyId     string = ""
	accessKeySecret string = ""
	bucket          string = ""
)

func InitOss() {
	endpoint = beego.AppConfig.String("oss_endpoint")
	accessKeyId = beego.AppConfig.String("oss_access_key_id")
	accessKeySecret = beego.AppConfig.String("oss_access_key_secret")
	bucket = beego.AppConfig.String("oss_bucket")
}

func getOssBucket() (*oss.Bucket, error) {
	if len(endpoint) == 0 || len(accessKeyId) == 0 || len(accessKeySecret) == 0 {
		beego.Error("oss param error")
		return nil, errors.New("oss param error")
	}
	client, err := oss.New(endpoint, accessKeyId, accessKeySecret)
	if nil != err {
		beego.Error("oss init error")
		return nil, errors.New("oss init error")
	}

	if len(bucket) == 0 {
		return nil, errors.New("get bucket error")
	}
	return client.Bucket(bucket)
}

func OssPutObject(ossPath, localFilePath string) error {
	bucket, err := getOssBucket()
	if nil != err {
		return err
	}

	err = bucket.PutObjectFromFile(ossPath, localFilePath)
	if nil != err {
		beego.Error(err.Error())
	}
	return err
}