Skip to content

Commit

Permalink
feat: bcs-user-manager 支持 redis 哨兵和集群模式
Browse files Browse the repository at this point in the history
  • Loading branch information
yuyudeqiu committed Sep 30, 2024
1 parent efdc804 commit 4baa578
Show file tree
Hide file tree
Showing 7 changed files with 213 additions and 14 deletions.
13 changes: 13 additions & 0 deletions bcs-services/bcs-user-manager/app/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,19 @@ func parseConfig(op *options.UserManagerOptions) (*config.UserMgrConfig, error)
}
userMgrConfig.RedisDSN = string(redisDSN)

// RedisDSN 没有配置,检查 RedisConfig
if userMgrConfig.RedisDSN == "" {
userMgrConfig.RedisConfig.RedisMode = op.RedisConfig.RedisMode
userMgrConfig.RedisConfig.Addr = op.RedisConfig.Addr
userMgrConfig.RedisConfig.Password = op.RedisConfig.Password
userMgrConfig.RedisConfig.DialTimeout = op.RedisConfig.DialTimeout
userMgrConfig.RedisConfig.ReadTimeout = op.RedisConfig.ReadTimeout
userMgrConfig.RedisConfig.WriteTimeout = op.RedisConfig.WriteTimeout
userMgrConfig.RedisConfig.PoolSize = op.RedisConfig.PoolSize
userMgrConfig.RedisConfig.MinIdleConns = op.RedisConfig.MinIdleConns
userMgrConfig.RedisConfig.IdleTimeout = op.RedisConfig.IdleTimeout
}

userMgrConfig.VerifyClientTLS = op.VerifyClientTLS

// server cert directory
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,29 +15,64 @@ package cache

import (
"context"
"strings"
"time"

"github.com/go-redis/redis/extra/redisotel/v8"
"github.com/go-redis/redis/v8"

"github.com/Tencent/bk-bcs/bcs-common/common/redisclient"
"github.com/Tencent/bk-bcs/bcs-services/bcs-user-manager/config"
"github.com/go-redis/redis/extra/redisotel/v8"
)

// RDB is the global redis cache
var RDB Cache

// InitRedis init redis cache with url
func InitRedis(conf *config.UserMgrConfig) error {
options, err := redis.ParseURL(conf.RedisDSN)
var client redisclient.Client
var err error
if conf.RedisDSN != "" {
client, err = redisclient.NewSingleClientFromDSN(conf.RedisDSN)
} else {
redisConf := parseRedisConfig(conf)
client, err = redisclient.NewClient(redisConf)
}
if err != nil {
return err
}
client := redis.NewClient(options)
client.AddHook(redisotel.NewTracingHook())

client.GetCli().AddHook(redisotel.NewTracingHook())
RDB = &redisCache{client: client}
return nil
}

// parseRedisConfig parse Redis config
func parseRedisConfig(conf *config.UserMgrConfig) redisclient.Config {
redisConf := redisclient.Config{
Addrs: strings.Split(conf.RedisConfig.Addr, ","),
Password: conf.RedisConfig.Password,
DB: conf.RedisConfig.DB,
DialTimeout: time.Duration(conf.RedisConfig.DialTimeout) * time.Second,
ReadTimeout: time.Duration(conf.RedisConfig.ReadTimeout) * time.Second,
WriteTimeout: time.Duration(conf.RedisConfig.WriteTimeout) * time.Second,
PoolSize: conf.RedisConfig.PoolSize,
MinIdleConns: conf.RedisConfig.MinIdleConns,
IdleTimeout: time.Duration(conf.RedisConfig.IdleTimeout) * time.Second,
}

// redis mode
switch conf.RedisConfig.RedisMode {
case "sentinel":
redisConf.Mode = redisclient.SentinelMode
redisConf.MasterName = conf.RedisConfig.MasterName
case "cluster":
redisConf.Mode = redisclient.ClusterMode
default:
redisConf.Mode = redisclient.SingleMode
}

return redisConf
}

// Cache is the interface of redis cache
type Cache interface {
Set(ctx context.Context, key string, value interface{}, expiration time.Duration) (string, error)
Expand All @@ -51,36 +86,37 @@ type Cache interface {
var _ Cache = &redisCache{}

type redisCache struct {
client *redis.Client
client redisclient.Client
}

// Set implements Cache.Set
func (r *redisCache) Set(ctx context.Context, key string, value interface{}, expiration time.Duration) (string, error) {
return r.client.Set(ctx, key, value, expiration).Result()
return r.client.Set(ctx, key, value, expiration)
}

// SetNX implements Cache.SetNX
func (r *redisCache) SetNX(ctx context.Context, key string, value interface{}, expiration time.Duration) (bool, error) {
return r.client.SetNX(ctx, key, value, expiration).Result()
return r.client.SetNX(ctx, key, value, expiration)
}

// SetEX implements Cache.SetEX
func (r *redisCache) SetEX(ctx context.Context, key string, value interface{}, expiration time.Duration) (
string, error) {
return r.client.SetEX(ctx, key, value, expiration).Result()
return r.client.SetEX(ctx, key, value, expiration)
}

// Get implements Cache.Get
func (r *redisCache) Get(ctx context.Context, key string) (string, error) {
return r.client.Get(ctx, key).Result()
return r.client.Get(ctx, key)
}

// Del implements Cache.Del
func (r *redisCache) Del(ctx context.Context, key string) (uint64, error) {
return r.client.Del(ctx, key).Uint64()
count, err := r.client.Del(ctx, key)
return uint64(count), err
}

// Expire implements Cache.Expire
func (r *redisCache) Expire(ctx context.Context, key string, expiration time.Duration) (bool, error) {
return r.client.Expire(ctx, key, expiration).Result()
return r.client.Expire(ctx, key, expiration)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
package cache

import (
"context"
"testing"
"time"

"github.com/Tencent/bk-bcs/bcs-services/bcs-user-manager/config"
"github.com/stretchr/testify/assert"
)

var testRDB Cache

func setup() error {
conf := &config.UserMgrConfig{
RedisConfig: config.RedisConfig{
Addr: "127.0.0.1:7021,127.0.0.1:7022,127.0.0.1:7023,127.0.0.1:7024,127.0.0.1:7025,127.0.0.1:7026",
RedisMode: "cluster",
},
}
err := InitRedis(conf)
testRDB = RDB
return err
}

func TestInitRedis(t *testing.T) {
err := setup()
assert.Nil(t, err)
assert.NotNil(t, testRDB)
}

func TestRedisCache_Set(t *testing.T) {
err := setup()
assert.Nil(t, err)

key := "test:key"
value := "testValue"
expiration := time.Second * 10

result, err := testRDB.Set(context.Background(), key, value, expiration)
assert.NoError(t, err)
assert.Equal(t, "OK", result)

// 验证值是否正确设置
val, err := testRDB.Get(context.Background(), key)
assert.NoError(t, err)
assert.Equal(t, value, val)
}

func TestRedisCache_Get(t *testing.T) {
err := setup()
assert.Nil(t, err)
key := "test:key"
value := "testValue"
_, err = testRDB.Set(context.Background(), key, value, 0)
assert.NoError(t, err)

result, err := testRDB.Get(context.Background(), key)
assert.NoError(t, err)
assert.Equal(t, value, result)
}

func TestRedisCache_Del(t *testing.T) {
err := setup()
assert.Nil(t, err)
key := "test:key"
_, err = testRDB.Set(context.Background(), key, "value", 0)
assert.NoError(t, err)

count, err := testRDB.Del(context.Background(), key)
assert.NoError(t, err)
assert.Equal(t, uint64(1), count)

// 验证值是否已删除
val, err := testRDB.Get(context.Background(), key)
assert.Error(t, err)
assert.Equal(t, "", val)
}

func TestRedisCache_Expire(t *testing.T) {
err := setup()
assert.Nil(t, err)

key := "test:key"
value := "testValue"
_, err = testRDB.Set(context.Background(), key, value, 0)
assert.NoError(t, err)

// 设置过期时间
expired, err := testRDB.Expire(context.Background(), key, time.Second*1)
assert.NoError(t, err)
assert.True(t, expired)

// 等待过期
time.Sleep(2 * time.Second)

val, err := testRDB.Get(context.Background(), key)
assert.Error(t, err)
assert.Equal(t, "", val)
}
16 changes: 16 additions & 0 deletions bcs-services/bcs-user-manager/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,21 @@ type CertConfig struct {
IsSSL bool
}

// RedisConfig is configuration of Redis
type RedisConfig struct {
Addr string
Password string
DB int
MasterName string
RedisMode string
DialTimeout int
ReadTimeout int
WriteTimeout int
PoolSize int
MinIdleConns int
IdleTimeout int
}

// Encrypt define encrypt config
type Encrypt struct {
Enable bool `json:"enable" yaml:"enable"`
Expand Down Expand Up @@ -85,6 +100,7 @@ type UserMgrConfig struct {

DSN string
RedisDSN string
RedisConfig RedisConfig
EnableTokenSync bool
BootStrapUsers []options.BootStrapUser
TKE options.TKEOptions
Expand Down
7 changes: 6 additions & 1 deletion bcs-services/bcs-user-manager/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ module github.com/Tencent/bk-bcs/bcs-services/bcs-user-manager
go 1.20

replace (
github.com/Tencent/bk-bcs/bcs-common => ../../bcs-common
github.com/TencentBlueKing/iam-go-sdk => github.com/TencentBlueKing/iam-go-sdk v0.1.5
go.opentelemetry.io/otel/exporters/otlp/otlptrace => go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.17.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc => go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.17.0
Expand All @@ -21,7 +22,6 @@ require (
github.com/go-playground/locales v0.14.1
github.com/go-playground/universal-translator v0.18.1
github.com/go-redis/redis/extra/redisotel/v8 v8.11.5
github.com/go-redis/redis/v8 v8.11.5
github.com/go-resty/resty/v2 v2.12.0
github.com/golang-migrate/migrate/v4 v4.17.0
github.com/gorilla/schema v1.2.0
Expand All @@ -48,6 +48,8 @@ require (
github.com/TencentBlueKing/bk-audit-go-sdk v0.0.6 // indirect
github.com/TencentBlueKing/crypto-golang-sdk v1.0.0 // indirect
github.com/TencentBlueKing/gopkg v1.1.0 // indirect
github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302 // indirect
github.com/alicebob/miniredis v2.5.0+incompatible // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bitly/go-simplejson v0.5.0 // indirect
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
Expand Down Expand Up @@ -76,11 +78,13 @@ require (
github.com/go-openapi/swag v0.19.14 // indirect
github.com/go-playground/validator/v10 v10.19.0 // indirect
github.com/go-redis/redis/extra/rediscmd/v8 v8.11.5 // indirect
github.com/go-redis/redis/v8 v8.11.5 // indirect
github.com/go-sql-driver/mysql v1.7.1 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/gomodule/redigo v1.9.2 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/gopherjs/gopherjs v1.17.2 // indirect
Expand Down Expand Up @@ -127,6 +131,7 @@ require (
github.com/ugorji/go/codec v1.2.11 // indirect
github.com/urfave/cli/v2 v2.3.0 // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
github.com/yuin/gopher-lua v1.1.1 // indirect
go.etcd.io/etcd/api/v3 v3.5.2 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.2 // indirect
go.opentelemetry.io/otel/exporters/jaeger v1.3.0 // indirect
Expand Down
16 changes: 16 additions & 0 deletions bcs-services/bcs-user-manager/options/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ type UserManagerOptions struct {
VerifyClientTLS bool `json:"verify_client_tls" value:"false" usage:"verify client when brings up a tls server" mapstructure:"verify_client_tls"`
EnableTokenSync bool `json:"enable_token_sync" value:"" usage:"enable sync token to redis periodically"`
RedisDSN string `json:"redis_dsn" value:"" usage:"dsn for connect to redis"`
RedisConfig RedisConfig `json:"redis_config" value:"" usage:"redis config for connect to redis"`
DSN string `json:"mysql_dsn" value:"" usage:"dsn for connect to mysql"`
BootStrapUsers []BootStrapUser `json:"bootstrap_users"`
TKE TKEOptions `json:"tke"`
Expand Down Expand Up @@ -157,3 +158,18 @@ type Activity struct {
Interval string `json:"interval" yaml:"interval" usage:"timed tasks"`
ResourceType []string `json:"resource_type" yaml:"resource_type"`
}

// RedisConfig redis 配置
type RedisConfig struct {
RedisMode string `json:"redis_mode" usage:"Redis mode" mapstructure:"redis_mode" yaml:"redis_mode"`
Addr string `json:"addr" usage:"Redis server address" mapstructure:"addr" yaml:"addr"`
Password string `json:"password" usage:"Redis password" mapstructure:"password" yaml:"password"`
DB int `json:"db" usage:"Redis db" mapstructure:"db" yaml:"db"`
MasterName string `json:"master_name" usage:"Redis master name" mapstructure:"master_name" yaml:"master_name"`
DialTimeout int `json:"dial_timeout" usage:"Redis dial timeout" mapstructure:"dial_timeout" yaml:"dial_timeout"`
ReadTimeout int `json:"read_timeout" usage:"Redis read timeout" mapstructure:"read_timeout" yaml:"read_timeout"`
WriteTimeout int `json:"write_timeout" usage:"Redis write timeout" mapstructure:"write_timeout" yaml:"write_timeout"`
PoolSize int `json:"pool_size" usage:"Redis pool size" mapstructure:"pool_size" yaml:"pool_size"`
MinIdleConns int `json:"min_idle_conns" usage:"Redis min connect" mapstructure:"min_idle_conns" yaml:"min_idle_conns"`
IdleTimeout int `json:"idle_timeout" usage:"Redis idle timeout" mapstructure:"idle_timeout" yaml:"idle_timeout"`
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,19 @@
"verify_client_tls": false,
"mysql_dsn": "${coreDatabaseDsn}",
"redis_dsn": "${redisDsn}",
"redis_config": {
"addr": "${redisAddr}",
"redis_mode": "${redisMode}",
"password": "${redisPassword}",
"db": "${redisDB}",
"master_name": "${redisMasterName}",
"dial_timeout": 0,
"read_timeout": 0,
"write_timeout": 0,
"pool_size": 0,
"min_idle_conns": 0,
"idle_timeout": 0
},
"enable_token_sync": ${enableTokenSync},
"bootstrap_users": [
{
Expand Down

0 comments on commit 4baa578

Please sign in to comment.