一,安装用到的库
1,安装go-redis
liuhongdi@ku:~$ go get -u github.com/go-redis/redis
2,安装bigcache
liuhongdi@ku:~$ go get -u github.com/allegro/bigcache
说明:刘宏缔的go森林是一个专注golang的博客,
网站:https://blog.imgtouch.com
原文: go语言web开发系列之十:gin框架中通过订阅redis消息更新进程内缓存bigcache – 架构森林
说明:作者:刘宏缔 邮箱: 371125307@qq.com
二,演示项目的相关信息
1,项目地址:
GitHub - liuhongdi/digv10: gin框架中通过订阅redis消息更新进程内缓存bigcache
2,项目功能:演示redis+bigcache两级缓存,
通过订阅redis消息更新进程内缓存bigcache
3,项目结构:如图:
三,go代码说明
1,config/config.yaml
Database:
DBType: mysql
UserName: root
Password: password
Host: 127.0.0.1:3306
DBName: dig
Charset: utf8
ParseTime: True
MaxIdleConns: 10
MaxOpenConns: 30
Server:
RunMode: debug
HttpPort: 8000
ReadTimeout: 60
WriteTimeout: 60
Redis:
Addr: 127.0.0.1:6379
Password:
2,controller/setController.go
package controller
import (
"github.com/gin-gonic/gin"
"github.com/liuhongdi/digv10/global"
"github.com/liuhongdi/digv10/pkg/result"
)
type SetController struct{}
//new
func NewSetController() SetController {
return SetController{}
}
//发布一条消息到redis
func (u *SetController) Pub(c *gin.Context) {
resultRes := result.NewResult(c)
data:=c.Query("id")
err := global.RedisDb.Publish("articleMsg", data).Err()
if err != nil {
resultRes.Error(400,err.Error())
return
} else {
resultRes.Success("发送成功")
}
}
3,bigcache/article.go
package bigcache
import (
"encoding/json"
"fmt"
"github.com/liuhongdi/digv10/global"
"github.com/liuhongdi/digv10/model"
"strconv"
)
//bigcache中索引的名字
func getArticleCacheName(articleId uint64) (string) {
return "article_"+strconv.FormatUint(articleId,10)
}
//从bigcache得到一篇文章
func GetOneArticleBigCache(articleId uint64) (*model.Article,error) {
fmt.Println("bigcache:GetOneArticleBigCache")
key := getArticleCacheName(articleId);
val,err := global.BigCache.Get(key)
if (err != nil) {
return nil,err
} else {
article := model.Article{}
if err := json.Unmarshal([]byte(val), &article); err != nil {
return nil,err
}
return &article,nil
}
}
//向bigcache保存一篇文章
func SetOneArticleBigCache(articleId uint64,article *model.Article) (error) {
key := getArticleCacheName(articleId);
content,err := json.Marshal(article)
if (err != nil){
fmt.Println(err)
return err;
}
errSet := global.BigCache.Set(key,[]byte(content))
if (errSet != nil) {
return errSet
}
return nil
}
4,rediscache/article.go
package rediscache
import (
"encoding/json"
"fmt"
"github.com/go-redis/redis"
"github.com/liuhongdi/digv10/global"
"github.com/liuhongdi/digv10/model"
"strconv"
"time"
)
//cache的过期时长
const ArticleDuration = time.Minute * 10
//cache的名字
func getArticleCacheName(articleId uint64) (string) {
return "article_"+strconv.FormatUint(articleId,10)
}
//从redis cache得到一篇文章
func GetOneArticleRedisCache(articleId uint64) (*model.Article,error) {
fmt.Println("redis:GetOneArticleRedisCache")
key := getArticleCacheName(articleId);
val, err := global.RedisDb.Get(key).Result()
if (err == redis.Nil || err != nil) {
return nil,err
} else {
article := model.Article{}
if err := json.Unmarshal([]byte(val), &article); err != nil {
//t.Error(target)
return nil,err
}
return &article,nil
}
}
//向redis cache保存一篇文章
func SetOneArticleRedisCache(articleId uint64,article *model.Article) (error) {
key := getArticleCacheName(articleId);
content,err := json.Marshal(article)
if (err != nil){
fmt.Println(err)
return err;
}
errSet := global.RedisDb.Set(key, content, ArticleDuration).Err()
if (errSet != nil) {
return errSet
}
return nil
}
5,service/article.go
package service
import (
"fmt"
"github.com/liuhongdi/digv10/bigcache"
"github.com/liuhongdi/digv10/dao"
"github.com/liuhongdi/digv10/global"
"github.com/liuhongdi/digv10/model"
"github.com/liuhongdi/digv10/rediscache"
"strconv"
)
//得到一篇文章的详情
func GetOneArticle(articleId uint64) (*model.Article, error) {
//get from bigcache
article,err := bigcache.GetOneArticleBigCache(articleId);
if ( err != nil) {
//get from redis
article,errSel := rediscache.GetOneArticleRedisCache(articleId)
if (errSel != nil) {
//get from mysql
article,errSel := dao.SelectOneArticle(articleId);
if (errSel != nil) {
return nil,errSel
} else {
//set redis cache
errSetR := rediscache.SetOneArticleRedisCache(articleId,article)
if (errSetR != nil){
fmt.Println(errSetR)
}
//set bigcache
errSetB := bigcache.SetOneArticleBigCache(articleId,article)
if (errSetB != nil){
fmt.Println(errSetB)
}
return article,nil
}
//return nil,errSel
} else {
//set bigcache
errSet := bigcache.SetOneArticleBigCache(articleId,article)
if (errSet != nil){
return nil,errSet
} else {
return article,errSel
}
}
} else {
return article,err
}
}
func GetArticleSum() (int, error) {
return dao.SelectcountAll()
}
//得到多篇文章,按分页返回
func GetArticleList(page int ,pageSize int) ([]*model.Article,error) {
articles, err := dao.SelectAllArticle(page,pageSize)
if err != nil {
return nil,err
} else {
return articles,nil
}
}
//从redis更新bigcache
func UpdateArticleBigcache(articleId uint64) (error) {
//get from redis
article,errSel := rediscache.GetOneArticleRedisCache(articleId)
if (errSel != nil) {
return errSel
} else {
errSetB := bigcache.SetOneArticleBigCache(articleId, article)
if (errSetB != nil) {
fmt.Println(errSetB)
return errSetB
}
return nil
}
}
//订阅redis消息
func SubMessage(channel string) {
pubsub := global.RedisDb.Subscribe(channel)
fmt.Println("subscribe begin Receive")
_, err := pubsub.Receive()
if err != nil {
return
}
fmt.Println("subscribe begin channel")
ch := pubsub.Channel()
for msg := range ch {
fmt.Println("message:")
fmt.Println( msg.Channel, msg.Payload, "\r\n")
//把字符串转articleid
articleId,errUint := strconv.ParseUint(msg.Payload, 0, 64)
if (errUint != nil) {
fmt.Println(errUint)
} else {
//更新bigcache
errB := UpdateArticleBigcache(articleId)
if (errB != nil){
fmt.Println(errB)
}
}
}
}
6,main.go
package main
import (
"github.com/gin-gonic/gin"
_ "github.com/jinzhu/gorm/dialects/mysql"
"github.com/liuhongdi/digv10/global"
"github.com/liuhongdi/digv10/router"
"github.com/liuhongdi/digv10/service"
"log"
)
//init
func init() {
//setting
err := global.SetupSetting()
if err != nil {
log.Fatalf("init.setupSetting err: %v", err)
}
//mysql
err = global.SetupDBLink()
if err != nil {
log.Fatalf("init.setupDBEngine err: %v", err)
}
//redis
err = global.SetupRedisDb()
if err != nil {
log.Fatalf("init.SetupRedisDb err: %v", err)
}
//bigcache
err = global.SetupBigCache()
if err != nil {
log.Fatalf("init.SetupGlobalCache err: %v", err)
}
//redis sub
go service.SubMessage("articleMsg")
}
func main() {
//设置运行模式
gin.SetMode(global.ServerSetting.RunMode)
//引入路由
r := router.Router()
//run
r.Run(":"+global.ServerSetting.HttpPort)
}
7,global/bigcache.go
package global
import (
"github.com/allegro/bigcache"
"log"
"time"
)
//定义一个全局的bigcache
var (
BigCache *bigcache.BigCache
)
//创建一个全局的bigcache
func SetupBigCache() (error) {
config := bigcache.Config {
Shards: 1024, // 存储的条目数量,值必须是2的幂
LifeWindow: 3*time.Minute, // 超时后条目被处理
CleanWindow: 2*time.Minute, //处理超时条目的时间范围
MaxEntriesInWindow: 0, // 在 Life Window 中的最大数量,
MaxEntrySize: 0, // 条目最大尺寸,以字节为单位
HardMaxCacheSize: 0, // 设置缓存最大值,以MB为单位,超过了不在分配内存。0表示无限制分配
}
var initErr error
BigCache, initErr = bigcache.NewBigCache(config)
if initErr != nil {
log.Fatal(initErr)
return initErr
}
//BigCache.Stats().
return nil
}
8,global/redisDb.go
package global
import (
"github.com/go-redis/redis"
)
var (
RedisDb *redis.Client
)
//创建redis链接
func SetupRedisDb() (error) {
RedisDb = redis.NewClient(&redis.Options{
Addr: RedisSetting.Addr,
Password: RedisSetting.Password, // no password set
DB: 0, // use default DB
})
_, err := RedisDb.Ping().Result()
if err != nil {
return err
}
return nil
}
9,global/setting.go
package global
import (
"github.com/liuhongdi/digv10/pkg/setting"
"time"
)
//服务器配置
type ServerSettingS struct {
RunMode string
HttpPort string
ReadTimeout time.Duration
WriteTimeout time.Duration
}
//数据库配置
type DatabaseSettingS struct {
DBType string
UserName string
Password string
Host string
DBName string
Charset string
ParseTime bool
MaxIdleConns int
MaxOpenConns int
}
//redis配置
type RedisSettingS struct {
Addr string
Password string
}
//定义全局变量
var (
ServerSetting *ServerSettingS
DatabaseSetting *DatabaseSettingS
RedisSetting *RedisSettingS
)
//读取配置到全局变量
func SetupSetting() error {
s, err := setting.NewSetting()
if err != nil {
return err
}
err = s.ReadSection("Database", &DatabaseSetting)
if err != nil {
return err
}
err = s.ReadSection("Server", &ServerSetting)
if err != nil {
return err
}
err = s.ReadSection("Redis", &RedisSetting)
if err != nil {
return err
}
return nil
}
10,其他相关代码可访问github
四,测试效果
1,查看两级缓存的效果
访问url:
http://127.0.0.1:8000/article/getone/2
返回:
查看控制台:
bigcache:GetOneArticleBigCache
redis:GetOneArticleRedisCache
mysql:SelectOneArticle
(/data/liuhongdi/digv10/dao/article.go:13)
[2020-12-22 17:06:12] [99.30ms] SELECT articleId, subject, url FROM `article` WHERE (articleId=2) LIMIT 1
[1 rows affected or returned ]
len: 1
Capacity: 350
[GIN] 2020/12/22 - 17:06:12 | 200 | 156.31128ms | 127.0.0.1 | GET "/article/getone/2"
bigcache:GetOneArticleBigCache
[GIN] 2020/12/22 - 17:06:16 | 200 | 116.705µs | 127.0.0.1 | GET "/article/getone/2"
bigcache:GetOneArticleBigCache
[GIN] 2020/12/22 - 17:06:18 | 200 | 82.248µs | 127.0.0.1 | GET "/article/getone/2"
bigcache:GetOneArticleBigCache
[GIN] 2020/12/22 - 17:06:20 | 200 | 90.045µs | 127.0.0.1 | GET "/article/getone/2"
可以看到,第一次从数据库查询时用了100ms以上,
而从bigcache查询时,时间在100µs左右
重启一次应用,因为重启后bigcache已丢失,应用会从redis中读取数据
查看控制台:
bigcache:GetOneArticleBigCache
redis:GetOneArticleRedisCache
len: 1
Capacity: 350
[GIN] 2020/12/22 - 17:08:28 | 200 | 535.077µs | 127.0.0.1 | GET "/article/getone/2"
bigcache:GetOneArticleBigCache
[GIN] 2020/12/22 - 17:08:30 | 200 | 121.435µs | 127.0.0.1 | GET "/article/getone/2"
bigcache:GetOneArticleBigCache
[GIN] 2020/12/22 - 17:08:37 | 200 | 96.567µs | 127.0.0.1 | GET "/article/getone/2"
bigcache:GetOneArticleBigCache
[GIN] 2020/12/22 - 17:08:39 | 200 | 84.293µs | 127.0.0.1 | GET "/article/getone/2"
可以看到从redis中查询时,用时在500µs以上
2,通过订阅redis消息更新进程内缓存:
访问url:
http://127.0.0.1:8000/article/getone/2
返回:
我们手动更新redis,
然后通过push消息让本地的应用内缓存得到及时更新:
访问:
http://127.0.0.1:8000/set/pub?id=2
再次访问文章地址:
五,查看库的版本
module github.com/liuhongdi/digv10
go 1.15
require (
github.com/gin-gonic/gin v1.6.3
github.com/go-playground/universal-translator v0.17.0
github.com/go-playground/validator/v10 v10.2.0
github.com/jinzhu/gorm v1.9.16
github.com/magiconair/properties v1.8.4 // indirect
github.com/mitchellh/mapstructure v1.3.3 // indirect
github.com/pelletier/go-toml v1.8.1 // indirect
github.com/spf13/afero v1.4.1 // indirect
github.com/spf13/cast v1.3.1 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/spf13/viper v1.7.1
github.com/allegro/bigcache v1.2.1
github.com/go-redis/redis v6.15.9+incompatible
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 // indirect
golang.org/x/text v0.3.4 // indirect
gopkg.in/ini.v1 v1.62.0 // indirect
gopkg.in/yaml.v2 v2.3.0 // indirect
)