func (cmd *cmdRestore) RestoreRDBFile(reader *bufio.Reader, target, passwd string, nsize int64) {
pipe := newRDBLoader(reader, &cmd.rbytes, args.parallel*32)
wait := make(chan struct{})
go func() {
defer close(wait)
group := make(chan int, args.parallel)
for i := 0; i < cap(group); i++ {
go func() {
defer func() {
group <- 0
}()
c := openRedisConn(target, passwd)
defer c.Close()
var lastdb uint32 = 0
for e := range pipe {
if !acceptDB(e.DB) || !acceptKey(e.Key) {
cmd.ignore.Incr()
} else if skipKey(e.Key) {
log.Warnf("restore skip key: %s", e.Key)
cmd.ignore.Incr()
} else {
cmd.nentry.Incr()
if e.DB != lastdb {
lastdb = e.DB
selectDB(c, lastdb)
}
restoreRdbEntry(c, e)
}
}
}()
}
for i := 0; i < cap(group); i++ {
<-group
}
}()
for done := false; !done; {
select {
case <-wait:
done = true
case <-time.After(time.Second):
}
stat := cmd.Stat()
var b bytes.Buffer
if nsize != 0 {
fmt.Fprintf(&b, "total = %d - %12d [%3d%%]", nsize, stat.rbytes, 100*stat.rbytes/nsize)
} else {
fmt.Fprintf(&b, "total = %12d", stat.rbytes)
}
fmt.Fprintf(&b, " entry=%-12d", stat.nentry)
if stat.ignore != 0 {
fmt.Fprintf(&b, " ignore=%-12d", stat.ignore)
}
log.Info(b.String())
}
log.Info("restore: rdb done")
}
func (cmd *cmdSync) SyncCommand(reader *bufio.Reader, target, passwd string) {
c := openNetConn(target, passwd)
defer c.Close()
cr := openRedisConn(target, passwd)
defer cr.Close()
writer := bufio.NewWriterSize(stats.NewCountWriter(c, &cmd.wbytes), WriterBufferSize)
defer flushWriter(writer)
go func() {
p := make([]byte, ReaderBufferSize)
for {
iocopy(c, ioutil.Discard, p, len(p))
}
}()
go func() {
var bypass bool = false
for {
resp := redis.MustDecode(reader)
if scmd, args, err := redis.ParseArgs(resp); err != nil {
log.PanicError(err, "parse command arguments failed")
} else if scmd != "ping" {
if scmd == "select" {
if len(args) != 1 {
log.Panicf("select command len(args) = %d", len(args))
}
s := string(args[0])
n, err := parseInt(s, MinDB, MaxDB)
if err != nil {
log.PanicErrorf(err, "parse db = %s failed", s)
}
bypass = !acceptDB(uint32(n))
}
if bypass || (len(args) > 0 && !acceptKey(args[0])) {
cmd.nbypass.Incr()
continue
}
if skipKey(args[0]) {
log.Warnf("skip key: %s", args[0])
cmd.ignore.Incr()
continue
}
if aggregateKey(args[0]) && ((scmd == "lpush") || (scmd == "LPUSH")) {
log.Infof("Aggregate Key %s", args[0])
for i := 1; i < len(args); i++ {
_, err := cr.Do(aggregateCmd, aggregateTarget, args[i])
if err != nil {
log.Warnf("SyncAggregate err at: %s %s", args[0], args[i])
//log.PanicError(err, "sync aggregate error")
}
}
}
// set 2 sorted set in sync command
if set2sortedKey(args[0]) {
switch scmd {
default:
log.Panicf("set2sorted operate %s on key %s err", scmd, args[0])
case "sadd":
for i := 1; i < len(args); i++ {
_, err := cr.Do("zadd", args[0], 1, args[i])
if err != nil {
log.PanicErrorf(err, "set2sorted zadd %s 1 %s", args[0], args[i])
}
}
case "srem":
for i := 1; i < len(args); i++ {
_, err := cr.Do("zrem", args[0], args[i])
if err != nil {
log.PanicErrorf(err, "set2sorted zrem %s %s", args[0], args[i])
}
}
case "del":
_, err := cr.Do("del", args[0])
if err != nil {
log.PanicErrorf(err, "set2sorted del %s", args[0])
}
}
continue
}
if sorted2setKey(args[0]) {
switch scmd {
default:
log.Panicf("sorted2set operate %s on key %s err", scmd, args[0])
case "zadd":
for i := 1; i < len(args); i++ {
if string(args[i]) != "1" {
_, err := cr.Do("sadd", args[0], args[i])
if err != nil {
log.PanicErrorf(err, "sorted2set sadd %s %s", args[0], args[i])
}
} else {
continue
}
}
case "zrem":
for i := 1; i < len(args); i++ {
_, err := cr.Do("srem", args[0], args[i])
if err != nil {
log.PanicErrorf(err, "sorted2set srem %s %s", args[0], args[i])
}
}
case "del":
_, err := cr.Do("del", args[0])
if err != nil {
log.PanicErrorf(err, "sorted2set del %s", args[0])
}
}
continue
}
// Some commands like MSET may have multi keys, but we only use
// first for filter
}
cmd.forward.Incr()
redis.MustEncode(writer, resp)
flushWriter(writer)
}
}()
for lstat := cmd.Stat(); ; {
time.Sleep(time.Second)
nstat := cmd.Stat()
var b bytes.Buffer
fmt.Fprintf(&b, "sync: ")
fmt.Fprintf(&b, " +forward=%-6d", nstat.forward-lstat.forward)
fmt.Fprintf(&b, " +nbypass=%-6d", nstat.nbypass-lstat.nbypass)
fmt.Fprintf(&b, " +nbytes=%d", nstat.wbytes-lstat.wbytes)
log.Info(b.String())
lstat = nstat
}
}
func restoreRdbEntry(c redigo.Conn, e *rdb.BinEntry) {
var ttlms uint64
if e.ExpireAt != 0 {
now := uint64(time.Now().Add(args.shift).UnixNano())
now /= uint64(time.Millisecond)
if now >= e.ExpireAt {
ttlms = 1
} else {
ttlms = e.ExpireAt - now
}
}
toText := func(p []byte) string {
var b bytes.Buffer
for _, c := range p {
switch {
case c >= '#' && c <= '~':
b.WriteByte(c)
default:
b.WriteByte('.')
}
}
return b.String()
}
if aggregateKey(e.Key) {
log.Infof("Aggregate key %s", e.Key)
o, err := rdb.DecodeDump(e.Value)
if err != nil {
log.PanicError(err, "decode failed")
}
switch obj := o.(type) {
default:
log.Panicf("unknown object %v", o)
case rdb.List:
for _, ele := range obj {
_, err := c.Do(aggregateCmd, aggregateTarget, toText(ele))
if err != nil {
log.PanicError(err, "aggregate error")
}
}
case rdb.Set:
for _, ele := range obj {
_, err := c.Do(aggregateCmd, aggregateTarget, toText(ele))
if err != nil {
log.PanicError(err, "aggregate error")
}
}
}
}
if set2sortedKey(e.Key) {
o, err := rdb.DecodeDump(e.Value)
if err != nil {
log.PanicError(err, "decode failed")
}
switch obj := o.(type) {
default:
log.Panicf("set2sorted key %s type is not set err", e.Key)
log.Panicf("unknown object %v", o)
case rdb.Set:
for _, ele := range obj {
_, err := c.Do("zadd", e.Key, 1, toText(ele))
if err != nil {
log.PanicErrorf(err, "set2sorted zadd %s 1 %s", e.Key, toText(ele))
}
}
}
return
}
if sorted2setKey(e.Key) {
o, err := rdb.DecodeDump(e.Value)
if err != nil {
log.PanicError(err, "decode failed")
}
switch obj := o.(type) {
default:
log.Panicf("sorted2set key %s type is not sorted set err", e.Key)
case rdb.ZSet:
for _, ele := range obj {
_, err := c.Do("sadd", e.Key, toText(ele.Member))
if err != nil {
log.PanicErrorf(err, "sorted2set sadd %s %s", e.Key, toText(ele.Member))
}
}
}
return
}
if (restoreCmd == "del") || (restoreCmd == "DEL") {
_, err := redigo.String(c.Do(restoreCmd, e.Key))
if err != nil {
log.Warnf("delete key: '%s'", e.Key)
}
} else {
s, err := redigo.String(c.Do(restoreCmd, e.Key, ttlms, e.Value))
if err != nil {
log.Warnf("restore error, when '%s' '%s'", restoreCmd, e.Key)
//log.PanicError(err, "restore command error")
}
if s != "OK" {
//log.Panicf("restore command response = '%s', should be 'OK'", s)
log.Warnf("restore command response = '%s', should be 'OK'", s)
}
}
}
func (cmd *cmdRestore) RestoreCommand(reader *bufio.Reader, target, passwd string) {
c := openNetConn(target, passwd)
defer c.Close()
writer := bufio.NewWriterSize(c, WriterBufferSize)
defer flushWriter(writer)
go func() {
p := make([]byte, ReaderBufferSize)
for {
iocopy(c, ioutil.Discard, p, len(p))
}
}()
go func() {
var bypass bool = false
for {
resp := redis.MustDecode(reader)
if scmd, args, err := redis.ParseArgs(resp); err != nil {
log.PanicError(err, "parse command arguments failed")
} else if scmd != "ping" {
if scmd == "select" {
if len(args) != 1 {
log.Panicf("select command len(args) = %d", len(args))
}
s := string(args[0])
n, err := parseInt(s, MinDB, MaxDB)
if err != nil {
log.PanicErrorf(err, "parse db = %s failed", s)
}
bypass = !acceptDB(uint32(n))
}
if skipKey(args[0]) {
log.Warnf("skip key: %s", args[0])
cmd.ignore.Incr()
continue
}
if bypass || (len(args) > 0 && !acceptKey(args[0])) {
cmd.nbypass.Incr()
continue
}
// added for aggregating list or set
if aggregateKey(args[0]) {
cr := openRedisConn(target, passwd)
defer cr.Close()
for i := 1; i < len(args); i++ {
_, err := cr.Do(aggregateCmd, aggregateTarget, args[i])
if err != nil {
log.PanicError(err, "restore aggregate error")
}
}
}
}
cmd.forward.Incr()
redis.MustEncode(writer, resp)
flushWriter(writer)
}
}()
for lstat := cmd.Stat(); ; {
time.Sleep(time.Second)
nstat := cmd.Stat()
var b bytes.Buffer
fmt.Fprintf(&b, "restore: ")
fmt.Fprintf(&b, " +forward=%-6d", nstat.forward-lstat.forward)
fmt.Fprintf(&b, " +nbypass=%-6d", nstat.nbypass-lstat.nbypass)
log.Info(b.String())
lstat = nstat
}
}