// redis-cli对monitor指令进行特殊处理，只要monitor不断输出StatusReply，可以实现不间断的流输出
// 适用于海量数据的扫描输出，比如iterator扫描整个数据库
func (server *GoRedisServer) OnMONITOR(session *Session, cmd *Command) (reply *Reply) {
// 特殊使用，monitor输出全部key
if cmd.Len() > 1 {
switch strings.ToUpper(cmd.StringAtIndex(1)) {
case "KEYS":
server.monitorKeys(session, cmd)
default:
reply = ErrorReply("bad monitor command")
go func() {
time.Sleep(time.Millisecond * 100)
session.Close()
}()
}
return
}
session.WriteReply(StatusReply("OK"))
client := NewMonClient(session)
remoteHost := session.RemoteAddr().String()
go func() {
stdlog.Printf("[%s] monitor start\n", remoteHost)
// sleep一下，避免启动瞬间输出 +1394530022.495448 [0 127.0.0.1:51980] "monitor"
time.Sleep(time.Millisecond * 10)
server.monmgr.Put(remoteHost, client)
client.Start()
server.monmgr.Remove(remoteHost)
stdlog.Printf("[%s] monitor exit\n", remoteHost)
}()
return
}
// ./slaveof-proxy -src localhost:6379 -dest remote:6379 -pullrate 400 -pushrate 400
func main() {
runtime.GOMAXPROCS(4)
src := flag.String("src", "", "master")
dest := flag.String("dest", "", "slave")
pullrate := flag.Int("pullrate", 400, "pull rate in Mbits/s")
pushrate := flag.Int("pushrate", 400, "push rate in Mbits/s")
buffer := flag.Int("buffer", 100, "buffer x10000 records")
flag.Parse()
if *pullrate < 100 {
*pullrate = 100
}
if *pushrate < 100 {
*pushrate = 100
}
if *buffer < 100 {
*buffer = 100
} else if *buffer > 1000 {
*buffer = 1000
}
stdlog.Println("slaveof-proxy 1.0.3")
stdlog.Printf("from [%s] to [%s]\n", *src, *dest)
stdlog.Printf("pull [%d] buffer [%d]\n", *pullrate, *buffer)
stdlog.Println("SYNC ...")
client, err := NewClient(*src, *dest, *buffer)
if err != nil {
stdlog.Println("ERR", err)
return
}
client.SetPullRate(*pullrate / 8 * 1024 * 1024)
client.SetPushRate(*pushrate / 8 * 1024 * 1024)
client.Sync()
}
// nextseq，返回快照的下一条seq位置
func (server *GoRedisServer) sendSnapshot(session *Session) (nextseq int64, err error) {
server.Suspend() //挂起全部操作
snap := server.levelRedis.Snapshot() // 挂起后建立只读快照
defer snap.Close() // 必须释放
lastseq := server.synclog.MaxSeq() // 获取当前日志序号
server.Resume() // 唤醒，如果不调用Resume，整个服务器无法继续工作
if err = session.WriteCommand(NewCommand([]byte("SYNC_RAW_START"))); err != nil {
return
}
// scan snapshot
broken := false
sendcount := 0
sendfinish := false
go func() {
for {
time.Sleep(time.Second * 1)
if sendcount == -1 {
break // finish
} else if broken {
break // cancel
}
if sendfinish {
stdlog.Printf("[S %s] snap send finish, %d raw items\n", session.GetAttribute(S_HOST), sendcount)
break
} else {
stdlog.Printf("[S %s] snap send %d raw items\n", session.GetAttribute(S_HOST), sendcount)
}
}
}()
// gogogo
snap.RangeEnumerate([]byte{}, []byte{levelredis.MAXBYTE}, levelredis.IterForward, func(i int, key, value []byte, quit *bool) {
if bytes.HasPrefix(key, []byte(PREFIX)) {
return
}
cmd := NewCommand([]byte("SYNC_RAW"), key, value)
err = session.WriteCommand(cmd)
if err != nil {
broken = true
*quit = true
}
sendcount++
})
if broken {
return -1, err
}
sendfinish = true
curseq := server.synclog.MaxSeq()
if err = session.WriteCommand(NewCommand(formatByteSlice("SYNC_RAW_END", sendcount, lastseq, curseq)...)); err != nil {
return
}
nextseq = lastseq + 1
return nextseq, nil
}
// 发起主从同步请求
func (server *GoRedisServer) initSlaveOf() {
host, port := server.opt.SlaveOf()
if len(host) > 0 && port != 0 {
stdlog.Printf("init slaveof %s:%d\n", host, port)
// 模拟外部, session=nil
simulatedCmd := NewCommand(formatByteSlice("SLAVEOF", host, port)...)
reply := server.OnSLAVEOF(nil, simulatedCmd)
stdlog.Printf("slaveof: %s:%d, %s\n", host, port, reply)
}
}
func (s *SlaveClient) RdbRecvFinishCallback(r *bufio.Reader) {
stdlog.Printf("[M %s] rdb recv finish, start decoding... \n", s.src.RemoteAddr())
s.initlog()
go s.readAllReply()
// decode
dec := newRdbDecoder(s)
err := rdb.Decode(r, dec)
if err != nil {
// must cancel
stdlog.Printf("[M %s] decode error %s\n", s.src.RemoteAddr(), err)
s.Close()
}
return
}
// S: SYNC UID [UID] PORT [PORT] SNAP [1/0] SEQ [-1/...]
func (server *GoRedisServer) OnSYNC(session *Session, cmd *Command) (reply *Reply) {
stdlog.Printf("[S %s] %s\n", session.RemoteAddr(), cmd)
args := cmd.Args()[1:]
if len(args) < 2 || len(args)%2 != 0 {
session.Close()
return
}
for i := 0; i < len(args); i += 2 {
session.SetAttribute(string(args[i]), string(args[i+1]))
}
if !server.synclog.IsEnabled() {
stdlog.Println("synclog enable")
server.synclog.Enable()
}
// 使用从库端口代替socket端口，标识来源
h, _ := splitHostPort(session.RemoteAddr().String())
remoteHost := fmt.Sprintf("%s:%s", h, session.GetAttribute("PORT"))
session.SetAttribute(S_HOST, remoteHost)
session.SetAttribute(S_STATUS, REPL_WAIT)
go func() {
server.syncmgr.Put(remoteHost, session)
err := server.doSync(session, cmd)
if err != nil {
stdlog.Println("sync ", err)
}
session.Close()
server.syncmgr.Remove(remoteHost)
}()
return NOREPLY
}
// ServerHandler.On()
// 由GoRedis协议层触发，通过反射调用OnGET/OnSET等方法
func (server *GoRedisServer) On(session *Session, cmd *Command) (reply *Reply) {
// invoke & time
begin := time.Now()
// suspend & resume
server.rwlock.Lock()
server.rwlock.Unlock()
cmd.SetAttribute(C_SESSION, session)
// varify command
if err := verifyCommand(cmd); err != nil {
stdlog.Printf("[%s] bad command %s\n", session.RemoteAddr(), cmd)
return ErrorReply(err)
}
// invoke
reply = server.invokeCommandHandler(session, cmd)
elapsed := time.Now().Sub(begin)
cmd.SetAttribute(C_ELAPSED, elapsed)
// async: counter/sync/monitor
server.rwwait.Add(1)
server.cmdChan <- cmd
return
}
func (server *GoRedisServer) OnAOF(session *Session, cmd *Command) (reply *Reply) {
defer func() {
if v := recover(); v != nil {
stdlog.Printf("aof panic %s\n", cmd)
stdlog.Println(string(debug.Stack()))
}
}()
onoff := strings.ToUpper(cmd.StringAtIndex(1))
if onoff == "YES" {
if server.aofwriter != nil {
return ErrorReply("aof already inited")
}
if !server.synclog.IsEnabled() {
stdlog.Println("synclog enable")
server.synclog.Enable()
}
go func() {
err := server.aofStart()
if err != nil {
stdlog.Println("aof", err)
}
}()
} else if onoff == "NO" {
return server.onAOF_NO()
} else {
return ErrorReply("must be YES/NO")
}
return StatusReply("OK")
}
func (server *GoRedisServer) Init() (err error) {
server.initSignalNotify()
stdlog.Println("server init, version", VERSION, "...")
err = server.initLevelDB()
if err != nil {
return
}
err = server.initSyncLog()
if err != nil {
return
}
server.config = NewConfig(server.levelRedis, PREFIX+"config:")
// monitor
server.initCommandMonitor(server.directory + "/cmd.log")
server.initCommandCounterLog("string", []string{"GET", "SET", "MGET", "MSET", "INCR", "DECR", "INCRBY", "DECRBY"})
server.initCommandCounterLog("hash", []string{"HGETALL", "HGET", "HSET", "HDEL", "HMGET", "HMSET", "HINCRBY", "HLEN"})
server.initCommandCounterLog("set", []string{"SADD", "SCARD", "SISMEMBER", "SMEMBERS", "SREM"})
server.initCommandCounterLog("list", []string{"LPUSH", "RPUSH", "LPOP", "RPOP", "LINDEX", "LLEN", "LRANGE", "LTRIM"})
server.initCommandCounterLog("zset", []string{"ZADD", "ZCARD", "ZSCORE", "ZINCRBY", "ZRANGE", "ZRANGEBYSCORE", "ZRANK", "ZREM", "ZREMRANGEBYRANK", "ZREMRANGEBYSCORE", "ZREVRANGE", "ZREVRANGEBYSCORE", "ZREVRANK"})
server.initSeqLog(server.directory + "/seq.log")
server.initLeveldbIOLog(server.directory + "/leveldb.io.log")
server.initLeveldbStatsLog(server.directory + "/leveldb.stats.log")
server.initExecLog(server.directory + "/exec.time.log")
server.initSlowlog(server.directory + "/slow.log")
stdlog.Printf("init uid %s\n", server.UID())
server.initSlaveOf()
return
}
// 从主库获取数据
func (server *GoRedisServer) OnSLAVEOF(session *Session, cmd *Command) (reply *Reply) {
// 保障不会奔溃
defer func() {
if v := recover(); v != nil {
stdlog.Printf("[%s] slaveof panic %s\n", session.RemoteAddr(), cmd)
stdlog.Println(string(debug.Stack()))
}
}()
arg1, arg2 := cmd.StringAtIndex(1), cmd.StringAtIndex(2)
// SLAVEOF NO ONE
if strings.ToUpper(arg1) == "NO" && strings.ToUpper(arg2) == "ONE" {
return server.onSlaveOfNoOne(session, cmd)
}
// connect to master
hostPort := arg1 + ":" + arg2
conn, err := net.Dial("tcp", hostPort)
if err != nil {
return ErrorReply(err)
}
// check exists
remoteHost := conn.RemoteAddr().String()
if server.slavemgr.Contains(remoteHost) {
return ErrorReply("connection exists")
}
masterSession := NewSession(conn)
isgoredis, version, err := redisInfo(masterSession)
if err != nil {
return ErrorReply(err)
}
var client ISlaveClient
if isgoredis {
slavelog.Printf("[M %s] SLAVEOF %s GoRedis:%s\n", remoteHost, remoteHost, version)
if client, err = NewSlaveClientV2(server, masterSession); err != nil {
return ErrorReply(err)
}
} else {
slavelog.Printf("[M %s] SLAVEOF %s Redis:%s\n", remoteHost, remoteHost, version)
if client, err = NewSlaveClient(server, masterSession); err != nil {
return ErrorReply(err)
}
}
// async
go func() {
client.Session().SetAttribute(S_STATUS, REPL_WAIT)
server.slavemgr.Put(remoteHost, client)
err := client.Sync()
if err != nil {
slavelog.Printf("[M %s] sync broken %s\n", remoteHost, err)
}
client.Close()
server.slavemgr.Remove(remoteHost)
}()
return StatusReply("OK")
}
// ./slaveof-proxy -src localhost:6379 -dest remote:6379 -pullrate 400 -pushrate 400
func main() {
runtime.GOMAXPROCS(4)
src := flag.String("src", "", "master")
dest := flag.String("dest", "", "slave")
pullrate := flag.Int("pullrate", 400, "pull rate in Mbits/s")
pushrate := flag.Int("pushrate", 400, "push rate in Mbits/s")
buffer := flag.Int("buffer", 100, "buffer x10000 records")
dbpath := flag.String("dbpath", "/tmp", "rdb path")
flag.Parse()
if *pullrate < 100 {
*pullrate = 100
}
if *pushrate < 100 {
*pushrate = 100
}
if *buffer < 100 {
*buffer = 100
} else if *buffer > 1000 {
*buffer = 1000
}
stdlog.Println("slaveof-proxy 1.0.4")
if len(*src) == 0 || len(*dest) == 0 {
stdlog.Println("Usage: ./slaveof-proxy -src master:port -dest slave:6379 -pullrate 400 -pushrate 400 -buffer 100 -dbpath /tmp")
return
}
stdlog.Printf("from [%s] to [%s]\n", *src, *dest)
stdlog.Printf("pull [%d] buffer [%d]\n", *pullrate, *buffer)
stdlog.Println("SYNC ...")
client, err := NewClient(*src, *dest, *buffer)
if err != nil {
stdlog.Println("ERR", err)
return
}
client.SetPullRate(*pullrate / 8 * 1024 * 1024)
client.SetPushRate(*pushrate / 8 * 1024 * 1024)
client.SetDbPath(*dbpath)
err = client.Sync()
if err != nil {
panic(err)
}
}
func (server *GoRedisServer) doSync(session *Session, cmd *Command) (err error) {
// snapshot
var nextseq int64
if session.GetAttribute("SEQ") != nil {
nextseq, err = strconv.ParseInt(session.GetAttribute("SEQ").(string), 10, 64)
if err != nil {
return
}
}
remoteHost := session.GetAttribute(S_HOST).(string)
if session.GetAttribute("SNAP") != nil && session.GetAttribute("SNAP").(string) == "1" {
session.SetAttribute(S_STATUS, REPL_SEND_BULK)
if nextseq, err = server.sendSnapshot(session); err != nil {
stdlog.Printf("[S %s] snap send broken (%s)\n", remoteHost, err)
return
}
}
if nextseq < 0 {
nextseq = 0
}
if nextseq < server.synclog.MinSeq() || nextseq > server.synclog.MaxSeq()+1 {
stdlog.Printf("[S %s] seq %d not in (%d, %d), closed\n", remoteHost, nextseq, server.synclog.MinSeq(), server.synclog.MaxSeq())
return errors.New("bad seq range")
}
// 如果整个同步过程s
stdlog.Println("sync online ...")
session.SetAttribute(S_STATUS, REPL_ONLINE)
// 发送日志数据
err = server.syncRunloop(session, nextseq)
if err != nil {
stdlog.Printf("[S %s] sync broken (%s)\n", remoteHost, err)
}
return
}
func (s *SyncLog) initSeq() {
prefix := bytes.Join([][]byte{s.prefix, []byte(":id:")}, []byte(""))
s.db.PrefixEnumerate(prefix, levelredis.IterForward, func(i int, key, value []byte, quit *bool) {
s.minseq = s.splitSeqkey(key)
*quit = true
})
s.db.PrefixEnumerate(prefix, levelredis.IterBackward, func(i int, key, value []byte, quit *bool) {
s.seq = s.splitSeqkey(key)
*quit = true
})
s.enabled = s.seq != -1
if s.enabled {
stdlog.Printf("synclog enabled, seq (%d, %d)\n", s.minseq, s.seq)
}
}
// go run redis-proxy.go -p 1603 -master localhost:6379 -slave localhost:6389
func main() {
runtime.GOMAXPROCS(8)
// options
opt := goredis_proxy.NewOptions()
// flags
version := flag.Bool("v", false, "print goredis-proxy version")
flag.StringVar(&opt.Host, "h", "", "server host")
flag.IntVar(&opt.Port, "p", 1602, "server port")
flag.StringVar(&opt.MasterHost, "master", "", "master")
flag.StringVar(&opt.SlaveHost, "slave", "", "slave")
flag.StringVar(&opt.Mode, "mode", "rrw", "r/rr/rw/rrw, default rrw")
flag.IntVar(&opt.PoolSize, "poolsize", 100, "pool for remote server")
flag.Parse()
if *version {
fmt.Println("redis-proxy ", goredis_proxy.VERSION)
return
}
if len(opt.MasterHost) == 0 || len(opt.SlaveHost) == 0 {
stdlog.Println("bad master/slave")
return
}
stdlog.Println("redis-proxy ", goredis_proxy.VERSION)
stdlog.Printf("master:[%s], slave:[%s], mode:[%s], poolsize:[%d]\n", opt.MasterHost, opt.SlaveHost, opt.Mode, opt.PoolSize)
stdlog.Println("listen", opt.Addr())
// start
server := goredis_proxy.NewProxy(opt)
err := server.Init()
if err != nil {
panic(err)
}
err = server.Listen(opt.Addr())
if err != nil {
panic(err)
}
}
func (s *SlaveClient) recvRdb() (err error) {
session := s.src
var f *os.File
f, err = os.OpenFile(s.rdbfilename(), os.O_CREATE|os.O_RDWR|os.O_TRUNC, os.ModePerm)
if err != nil {
return
}
stdlog.Printf("[M %s] create rdb:%s\n", session.RemoteAddr(), s.rdbfilename())
session.ReadByte()
var size int64
size, err = session.ReadInt64()
if err != nil {
return
}
s.RdbSizeCallback(size)
// read
w := bufio.NewWriter(f)
// var written int64
_, err = iotool.RateLimitCopy(w, io.LimitReader(session, size), s.pullrate, func(written int64, rate int) {
s.RdbRecvProcessCallback(written, rate)
})
if err != nil {
return
}
w.Flush()
f.Seek(0, 0)
// 不阻塞进行接收command
go func() {
s.RdbRecvFinishCallback(bufio.NewReader(f))
filename := f.Name()
f.Close()
os.Remove(filename)
}()
return
}
func (server *GoRedisServer) Listen() error {
stdlog.Printf("listen %s\n", server.opt.Bind())
return server.RedisServer.Listen(server.opt.Bind())
}
func (server *GoRedisServer) aofStart() (err error) {
if server.aofwriter == nil {
filename := server.opt.LogPath() + "/appendonly.aof"
f, e := os.OpenFile(filename, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, os.ModePerm)
if e != nil {
return e
}
server.aofwriter = NewAOFWriter(bufio.NewWriter(f))
defer func() {
f.Close()
server.aofwriter.Close()
server.aofwriter = nil
}()
stdlog.Println("aof inited")
}
server.Suspend()
snap := server.levelRedis.Snapshot()
snapclosed := false // 避免关闭两次
defer func() {
if !snapclosed {
snap.Close()
}
}()
lastseq := server.synclog.MaxSeq()
server.Resume()
snap.KeyEnumerate([]byte(""), levelredis.IterForward, func(i int, key, keytype, value []byte, quit *bool) {
// stdlog.Println(i, string(key), string(keytype))
if server.aofwriter.IsClosed() {
*quit = true
return
}
switch string(keytype) {
case "zset":
server.aofwriter.AppendZSet(snap.GetSortedSet(string(key)))
case "hash":
server.aofwriter.AppendHash(snap.GetHash(string(key)))
case "set":
server.aofwriter.AppendSet(snap.GetSet(string(key)))
case "list":
server.aofwriter.AppendList(snap.GetList(string(key)))
case "string":
server.aofwriter.AppendString(key, value)
case "doc":
server.aofwriter.AppendDoc(snap.GetDoc(string(key)))
case "none":
stdlog.Println("bad key type", string(key), string(value))
default:
stdlog.Println("bad key type", string(key), string(keytype), string(value))
}
})
snap.Close()
snapclosed = true
if server.aofwriter.IsClosed() {
return errors.New("aof closed")
}
seq := lastseq + 1
deplymsec := 10
for {
if server.aofwriter.IsClosed() {
return errors.New("aof closed")
}
var val []byte
val, err = server.synclog.Read(seq)
if err != nil {
stdlog.Printf("aof synclog read error %s\n", err)
break
}
if val == nil {
time.Sleep(time.Millisecond * time.Duration(deplymsec))
deplymsec += 10
if deplymsec >= 10000 {
deplymsec = 10
}
continue
} else {
deplymsec = 10
}
server.aofwriter.Write(val)
server.aofwriter.Flush()
seq++
}
return nil
}
func (s *SlaveClient) RdbSizeCallback(totalsize int64) {
s.totalsize = totalsize
stdlog.Printf("[M %s] rdb size: %s\n", s.src.RemoteAddr(), bytesInHuman(totalsize))
}
func (server *GoRedisServer) Listen() error {
addr := fmt.Sprintf("%s:%d", server.opt.Host(), server.opt.Port())
stdlog.Printf("listen %s\n", addr)
return server.RedisServer.Listen(addr)
}
func (s *SlaveClient) RdbRecvProcessCallback(size int64, rate int) {
stdlog.Printf("[M %s] rdb recv: %s/%s, rate:%s\n", s.src.RemoteAddr(), bytesInHuman(size), bytesInHuman(s.totalsize), bytesInHuman(int64(rate)))
}
func (server *GoRedisServer) ExceptionCaught(err error) {
stdlog.Printf("exception %s\n", err)
stdlog.Println(string(debug.Stack()))
}
// go run goredis-server.go -h localhost -p 1602
// go run goredis-server.go -procs 8 -p 17600
// go run goredis-server.go -slaveof localhost:1603
// go run goredis-server.go -dbpath /data/ -logpath /home/logs/
func main() {
version := flag.Bool("v", false, "print version")
host := flag.String("h", "0.0.0.0", "server host")
port := flag.Int("p", 1602, "server port")
slaveof := flag.String("slaveof", "", "replication")
procs := flag.Int("procs", 8, "GOMAXPROCS, CPU")
repair := flag.Bool("repair", false, "repair rocksdb")
dbpath := flag.String("dbpath", "/data/", "rocksdb path, recommend use SSD")
logpath := flag.String("logpath", "/data/", "all logs, include synclog,aof")
flag.Parse()
if *version {
fmt.Println("goredis-server", goredis_server.VERSION)
return
}
if !dirExist(*dbpath) {
stdlog.Println("-dbpath", *dbpath, "not exist")
return
}
if !dirExist(*logpath) {
stdlog.Println("-logpath", *logpath, "not exist")
return
}
runtime.GOMAXPROCS(*procs)
// Options
opt := goredis_server.NewOptions()
opt.SetHost(*host)
opt.SetPort(*port)
opt.SetDBPath(joinGoRedisPath(*dbpath, *port))
opt.SetLogPath(joinGoRedisPath(*logpath, *port))
// ensure
os.Mkdir(opt.DBPath(), os.ModePerm)
os.Mkdir(opt.LogPath(), os.ModePerm)
// split -slaveof host:port
if len(*slaveof) > 0 {
hostPort := strings.Split(*slaveof, ":")
if len(hostPort) != 2 {
panic("bad slaveof")
}
p, e := strconv.Atoi(hostPort[1])
if e != nil {
panic(e)
}
opt.SetSlaveOf(hostPort[0], p)
}
// 重定向日志输出位置
if err := redirectStdout(opt.LogPath()); err != nil {
panic(err)
}
// repair
if *repair {
dbhome := filepath.Join(opt.DBPath(), "db0")
if !dirExist(dbhome) {
stdlog.Println("db not exist")
} else {
stdlog.Println("start repair", dbhome)
levelredis.Repair(dbhome)
stdlog.Println("repair finish")
}
return
}
stdlog.Println("========================================")
stdlog.Println("server init, version", goredis_server.VERSION, "...")
stdlog.Printf("dbpath:%s, logpath:%s\n", opt.DBPath(), opt.LogPath())
// GoRedis Server
server := goredis_server.NewGoRedisServer(opt)
if err := server.Init(); err != nil {
panic(err)
}
if err := server.Listen(); err != nil {
panic(err)
}
}