func (cmd *cmdDump) Main() {
from, output := args.from, args.output
if len(from) == 0 {
log.Panic("invalid argument: from")
}
if len(output) == 0 {
output = "/dev/stdout"
}
log.Infof("dump from '%s' to '%s'\n", from, output)
var dumpto io.WriteCloser
if output != "/dev/stdout" {
dumpto = openWriteFile(output)
defer dumpto.Close()
} else {
dumpto = os.Stdout
}
master, nsize := cmd.SendCmd(from, args.passwd)
defer master.Close()
log.Infof("rdb file = %d\n", nsize)
reader := bufio.NewReaderSize(master, ReaderBufferSize)
writer := bufio.NewWriterSize(dumpto, WriterBufferSize)
cmd.DumpRDBFile(reader, writer, nsize)
if !args.extra {
return
}
cmd.DumpCommand(reader, writer, nsize)
}
func (cmd *cmdSync) SendPSyncCmd(master, passwd string) (pipe.Reader, int64) {
c := openNetConn(master, passwd)
br := bufio.NewReaderSize(c, ReaderBufferSize)
bw := bufio.NewWriterSize(c, WriterBufferSize)
runid, offset, wait := sendPSyncFullsync(br, bw)
log.Infof("psync runid = %s offset = %d, fullsync", runid, offset)
var nsize int64
for nsize == 0 {
select {
case nsize = <-wait:
if nsize == 0 {
log.Info("+")
}
case <-time.After(time.Second):
log.Info("-")
}
}
piper, pipew := pipe.NewSize(ReaderBufferSize)
go func() {
defer pipew.Close()
p := make([]byte, 8192)
for rdbsize := int(nsize); rdbsize != 0; {
rdbsize -= iocopy(br, pipew, p, rdbsize)
}
for {
n, err := cmd.PSyncPipeCopy(c, br, bw, offset, pipew)
if err != nil {
log.PanicErrorf(err, "psync runid = %s, offset = %d, pipe is broken", runid, offset)
}
offset += n
for {
time.Sleep(time.Second)
c = openNetConnSoft(master, passwd)
if c != nil {
log.Infof("psync reopen connection, offset = %d", offset)
break
} else {
log.Infof("psync reopen connection, failed")
}
}
authPassword(c, passwd)
br = bufio.NewReaderSize(c, ReaderBufferSize)
bw = bufio.NewWriterSize(c, WriterBufferSize)
sendPSyncContinue(br, bw, runid, offset)
}
}()
return piper, nsize
}
func (cmd *cmdDump) DumpRDBFile(reader *bufio.Reader, writer *bufio.Writer, nsize int64) {
var nread atomic2.Int64
wait := make(chan struct{})
go func() {
defer close(wait)
p := make([]byte, WriterBufferSize)
for nsize != nread.Get() {
nstep := int(nsize - nread.Get())
ncopy := int64(iocopy(reader, writer, p, nstep))
nread.Add(ncopy)
flushWriter(writer)
}
}()
for done := false; !done; {
select {
case <-wait:
done = true
case <-time.After(time.Second):
}
n := nread.Get()
p := 100 * n / nsize
log.Infof("total = %d - %12d [%3d%%]\n", nsize, n, p)
}
log.Info("dump: rdb done")
}
func (cmd *cmdRestore) Main() {
input, target := args.input, args.target
if len(target) == 0 {
log.Panic("invalid argument: target")
}
if len(input) == 0 {
input = "/dev/stdin"
}
log.Infof("restore from '%s' to '%s'\n", input, target)
var readin io.ReadCloser
var nsize int64
if input != "/dev/stdin" {
readin, nsize = openReadFile(input)
defer readin.Close()
} else {
readin, nsize = os.Stdin, 0
}
reader := bufio.NewReaderSize(readin, ReaderBufferSize)
cmd.RestoreRDBFile(reader, target, args.auth, nsize)
if !args.extra {
return
}
if nsize != 0 && nsize == cmd.rbytes.Get() {
return
}
cmd.RestoreCommand(reader, target, args.auth)
}
func (cmd *cmdSync) Main() {
from, target := args.from, args.target
if len(from) == 0 {
log.Panic("invalid argument: from")
}
if len(target) == 0 {
log.Panic("invalid argument: target")
}
log.Infof("sync from '%s' to '%s'\n", from, target)
var sockfile *os.File
if len(args.sockfile) != 0 {
sockfile = openReadWriteFile(args.sockfile)
defer sockfile.Close()
}
var input io.ReadCloser
var nsize int64
if args.psync {
input, nsize = cmd.SendPSyncCmd(from, args.passwd)
} else {
input, nsize = cmd.SendSyncCmd(from, args.passwd)
}
defer input.Close()
log.Infof("rdb file = %d\n", nsize)
if sockfile != nil {
r, w := pipe.NewFilePipe(int(args.filesize), sockfile)
defer r.Close()
go func(r io.Reader) {
defer w.Close()
p := make([]byte, ReaderBufferSize)
for {
iocopy(r, w, p, len(p))
}
}(input)
input = r
}
reader := bufio.NewReaderSize(input, ReaderBufferSize)
cmd.SyncRDBFile(reader, target, args.auth, nsize)
cmd.SyncCommand(reader, target, args.auth)
}
func (cmd *cmdDump) DumpCommand(reader *bufio.Reader, writer *bufio.Writer, nsize int64) {
var nread atomic2.Int64
go func() {
p := make([]byte, ReaderBufferSize)
for {
ncopy := int64(iocopy(reader, writer, p, len(p)))
nread.Add(ncopy)
flushWriter(writer)
}
}()
for {
time.Sleep(time.Second)
log.Infof("dump: total = %d\n", nsize+nread.Get())
}
}
func (cmd *cmdDecode) Main() {
input, output := args.input, args.output
if len(input) == 0 {
input = "/dev/stdin"
}
if len(output) == 0 {
output = "/dev/stdout"
}
log.Infof("decode from '%s' to '%s'\n", input, output)
var readin io.ReadCloser
var nsize int64
if input != "/dev/stdin" {
readin, nsize = openReadFile(input)
defer readin.Close()
} else {
readin, nsize = os.Stdin, 0
}
var saveto io.WriteCloser
if output != "/dev/stdout" {
saveto = openWriteFile(output)
defer saveto.Close()
} else {
saveto = os.Stdout
}
reader := bufio.NewReaderSize(readin, ReaderBufferSize)
writer := bufio.NewWriterSize(saveto, WriterBufferSize)
ipipe := newRDBLoader(reader, &cmd.rbytes, args.parallel*32)
opipe := make(chan string, cap(ipipe))
go func() {
defer close(opipe)
group := make(chan int, args.parallel)
for i := 0; i < cap(group); i++ {
go func() {
defer func() {
group <- 0
}()
cmd.decoderMain(ipipe, opipe)
}()
}
for i := 0; i < cap(group); i++ {
<-group
}
}()
wait := make(chan struct{})
go func() {
defer close(wait)
for s := range opipe {
cmd.wbytes.Add(int64(len(s)))
if _, err := writer.WriteString(s); err != nil {
log.PanicError(err, "write string failed")
}
flushWriter(writer)
}
}()
for done := false; !done; {
select {
case <-wait:
done = true
case <-time.After(time.Second):
}
stat := cmd.Stat()
var b bytes.Buffer
fmt.Fprintf(&b, "decode: ")
if nsize != 0 {
fmt.Fprintf(&b, "total = %d - %12d [%3d%%]", nsize, stat.rbytes, 100*stat.rbytes/nsize)
} else {
fmt.Fprintf(&b, "total = %12d", stat.rbytes)
}
fmt.Fprintf(&b, " write=%-12d", stat.wbytes)
fmt.Fprintf(&b, " entry=%-12d", stat.nentry)
log.Info(b.String())
}
log.Info("decode: done")
}
func (cmd *cmdSync) SyncCommand(reader *bufio.Reader, target, passwd string) {
c := openNetConn(target, passwd)
defer c.Close()
cr := openRedisConn(target, passwd)
defer cr.Close()
writer := bufio.NewWriterSize(stats.NewCountWriter(c, &cmd.wbytes), WriterBufferSize)
defer flushWriter(writer)
go func() {
p := make([]byte, ReaderBufferSize)
for {
iocopy(c, ioutil.Discard, p, len(p))
}
}()
go func() {
var bypass bool = false
for {
resp := redis.MustDecode(reader)
if scmd, args, err := redis.ParseArgs(resp); err != nil {
log.PanicError(err, "parse command arguments failed")
} else if scmd != "ping" {
if scmd == "select" {
if len(args) != 1 {
log.Panicf("select command len(args) = %d", len(args))
}
s := string(args[0])
n, err := parseInt(s, MinDB, MaxDB)
if err != nil {
log.PanicErrorf(err, "parse db = %s failed", s)
}
bypass = !acceptDB(uint32(n))
}
if bypass || (len(args) > 0 && !acceptKey(args[0])) {
cmd.nbypass.Incr()
continue
}
if skipKey(args[0]) {
log.Warnf("skip key: %s", args[0])
cmd.ignore.Incr()
continue
}
if aggregateKey(args[0]) && ((scmd == "lpush") || (scmd == "LPUSH")) {
log.Infof("Aggregate Key %s", args[0])
for i := 1; i < len(args); i++ {
_, err := cr.Do(aggregateCmd, aggregateTarget, args[i])
if err != nil {
log.Warnf("SyncAggregate err at: %s %s", args[0], args[i])
//log.PanicError(err, "sync aggregate error")
}
}
}
// set 2 sorted set in sync command
if set2sortedKey(args[0]) {
switch scmd {
default:
log.Panicf("set2sorted operate %s on key %s err", scmd, args[0])
case "sadd":
for i := 1; i < len(args); i++ {
_, err := cr.Do("zadd", args[0], 1, args[i])
if err != nil {
log.PanicErrorf(err, "set2sorted zadd %s 1 %s", args[0], args[i])
}
}
case "srem":
for i := 1; i < len(args); i++ {
_, err := cr.Do("zrem", args[0], args[i])
if err != nil {
log.PanicErrorf(err, "set2sorted zrem %s %s", args[0], args[i])
}
}
case "del":
_, err := cr.Do("del", args[0])
if err != nil {
log.PanicErrorf(err, "set2sorted del %s", args[0])
}
}
continue
}
if sorted2setKey(args[0]) {
switch scmd {
default:
log.Panicf("sorted2set operate %s on key %s err", scmd, args[0])
case "zadd":
for i := 1; i < len(args); i++ {
if string(args[i]) != "1" {
_, err := cr.Do("sadd", args[0], args[i])
if err != nil {
log.PanicErrorf(err, "sorted2set sadd %s %s", args[0], args[i])
}
} else {
continue
}
}
case "zrem":
for i := 1; i < len(args); i++ {
_, err := cr.Do("srem", args[0], args[i])
if err != nil {
log.PanicErrorf(err, "sorted2set srem %s %s", args[0], args[i])
}
}
case "del":
_, err := cr.Do("del", args[0])
if err != nil {
log.PanicErrorf(err, "sorted2set del %s", args[0])
}
}
continue
}
// Some commands like MSET may have multi keys, but we only use
// first for filter
}
cmd.forward.Incr()
redis.MustEncode(writer, resp)
flushWriter(writer)
}
}()
for lstat := cmd.Stat(); ; {
time.Sleep(time.Second)
nstat := cmd.Stat()
var b bytes.Buffer
fmt.Fprintf(&b, "sync: ")
fmt.Fprintf(&b, " +forward=%-6d", nstat.forward-lstat.forward)
fmt.Fprintf(&b, " +nbypass=%-6d", nstat.nbypass-lstat.nbypass)
fmt.Fprintf(&b, " +nbytes=%d", nstat.wbytes-lstat.wbytes)
log.Info(b.String())
lstat = nstat
}
}
func main() {
usage := `
Usage:
redis-port decode [--ncpu=N] [--parallel=M] [--input=INPUT] [--output=OUTPUT]
redis-port restore [--ncpu=N] [--parallel=M] [--input=INPUT] --target=TARGET [--auth=AUTH] [--extra] [--faketime=FAKETIME] [--filterdb=DB]
[--filterkeys=keys] [--skipkeys=keys] [--restorecmd=slotsrestore] [--aggregatetype=type] [--aggregatekeys=keys] [--aggregateTargetKey=key]
redis-port dump [--ncpu=N] [--parallel=M] --from=MASTER [--password=PASSWORD] [--output=OUTPUT] [--extra]
redis-port sync [--ncpu=N] [--parallel=M] --from=MASTER [--password=PASSWORD] --target=TARGET [--auth=AUTH] [--sockfile=FILE [--filesize=SIZE]] [--filterdb=DB] [--psync] [--force]
[--filterkeys=keys] [--skipkeys=keys] [--restorecmd=slotsrestore] [--aggregatetype=type] [--aggregatekeys=keys] [--aggregateTargetKey=key] [--set2sortedkeys=keys] [--sorted2setkeys=keys]
Options:
-n N, --ncpu=N Set runtime.GOMAXPROCS to N.
-p M, --parallel=M Set the number of parallel routines to M.
-i INPUT, --input=INPUT Set input file, default is stdin ('/dev/stdin').
-o OUTPUT, --output=OUTPUT Set output file, default is stdout ('/dev/stdout').
-f MASTER, --from=MASTER Set host:port of master redis.
-t TARGET, --target=TARGET Set host:port of slave redis.
-P PASSWORD, --password=PASSWORD Set redis auth password.
-A AUTH, --auth=AUTH Set auth password for target.
--faketime=FAKETIME Set current system time to adjust key's expire time.
--sockfile=FILE Use FILE to as socket buffer, default is disabled.
--filesize=SIZE Set FILE size, default value is 1gb.
-e, --extra Set ture to send/receive following redis commands, default is false.
--filterdb=DB Filter db = DB, default is *.
--filterkeys=keys Filter key in keys, keys is seperated by comma and supports regular expression.
--skipkeys=keys Skip key in keys, keys is seperated by comma and supports regular expression.
--restorecmd=slotsrestore Restore command, slotsrestore for codis, restore for redis, if the from and target server are the same, use '--restorecmd=del' will delete the keys, togegher with
filterkeys, it will delete the keys filtered in the server.
--aggregatetype=type Aggregate type: list or set.
--aggregatekeys=keys Aggregate key in keys, keys is seperated by comma and supports regular expression.
--aggregateTargetKey=key Target key for aggregating.
--set2sortedkeys=keys Convert set key in keys to sorted set, keys is seperated by comma and supports regular expression.
--sorted2setkeys=keys Convert sorted set key in keys to set, keys is seperated by comma and supports regular expression.
--psync Use PSYNC command.
--force Use force, do not need to enter "yes", you mustn't use it ,unless you konw what you are doing.
`
d, err := docopt.Parse(usage, nil, true, "", false)
if err != nil {
log.PanicError(err, "parse arguments failed")
}
if s, ok := d["--ncpu"].(string); ok && s != "" {
n, err := parseInt(s, 1, 1024)
if err != nil {
log.PanicErrorf(err, "parse --ncpu failed")
}
runtime.GOMAXPROCS(n)
}
ncpu := runtime.GOMAXPROCS(0)
if s, ok := d["--parallel"].(string); ok && s != "" {
n, err := parseInt(s, 1, 1024)
if err != nil {
log.PanicErrorf(err, "parse --parallel failed")
}
args.parallel = n
}
if ncpu > args.parallel {
args.parallel = ncpu
}
if args.parallel == 0 {
args.parallel = 4
}
args.input, _ = d["--input"].(string)
args.output, _ = d["--output"].(string)
args.from, _ = d["--from"].(string)
args.passwd, _ = d["--password"].(string)
args.auth, _ = d["--auth"].(string)
args.target, _ = d["--target"].(string)
args.extra, _ = d["--extra"].(bool)
args.psync, _ = d["--psync"].(bool)
args.force, _ = d["--force"].(bool)
args.sockfile, _ = d["--sockfile"].(string)
var input string
for {
if args.force {
goto CONTINUE
}
fmt.Printf("Are you sure to continue (yes/no)?\n")
_, err := fmt.Scanf("%s\r\n", &input)
if err != nil {
log.PanicError(err, "input yes/no err")
}
switch input {
default:
fmt.Printf("Input the wrong value, you should input yes or no")
continue
case "yes":
goto CONTINUE
case "no":
os.Exit(1)
}
}
CONTINUE:
if s, ok := d["--faketime"].(string); ok && s != "" {
switch s[0] {
case '-', '+':
d, err := time.ParseDuration(strings.ToLower(s))
if err != nil {
log.PanicError(err, "parse --faketime failed")
}
args.shift = d
case '@':
n, err := strconv.ParseInt(s[1:], 10, 64)
if err != nil {
log.PanicError(err, "parse --faketime failed")
}
args.shift = time.Duration(n*int64(time.Millisecond) - time.Now().UnixNano())
default:
t, err := time.Parse("2006-01-02 15:04:05", s)
if err != nil {
log.PanicError(err, "parse --faketime failed")
}
args.shift = time.Duration(t.UnixNano() - time.Now().UnixNano())
}
}
if s, ok := d["--filterdb"].(string); ok && s != "" && s != "*" {
n, err := parseInt(s, MinDB, MaxDB)
if err != nil {
log.PanicError(err, "parse --filterdb failed")
}
u := uint32(n)
acceptDB = func(db uint32) bool {
return db == u
}
}
if s, ok := d["--filterkeys"].(string); ok && s != "" && s != "*" {
keys := strings.Split(s, ",")
keyRegexps := make([]*regexp.Regexp, len(keys))
for i, key := range keys {
keyRegexps[i], err = regexp.Compile(key)
if err != nil {
log.PanicError(err, "parse --filterkeys failed")
}
}
acceptKey = func(key []byte) bool {
for _, reg := range keyRegexps {
if reg.Match(key) {
return true
}
}
return false
}
}
if s, ok := d["--skipkeys"].(string); ok && s != "" && s != "*" {
keys := strings.Split(s, ",")
keyRegexps := make([]*regexp.Regexp, len(keys))
for i, key := range keys {
keyRegexps[i], err = regexp.Compile(key)
if err != nil {
log.PanicError(err, "parse --skipkeys failed")
}
}
skipKey = func(key []byte) bool {
for _, reg := range keyRegexps {
if reg.Match(key) {
return true
}
}
return false
}
}
if s, ok := d["--restorecmd"].(string); ok && s != "" {
restoreCmd = strings.TrimSpace(s)
}
if s, ok := d["--aggregatekeys"].(string); ok && s != "" && s != "*" {
keys := strings.Split(s, ",")
keyRegexps := make([]*regexp.Regexp, len(keys))
for i, key := range keys {
keyRegexps[i], err = regexp.Compile(key)
if err != nil {
log.PanicError(err, "parse --aggregatekeys failed")
}
}
aggregateKey = func(key []byte) bool {
for _, reg := range keyRegexps {
if reg.Match(key) {
return true
}
}
return false
}
}
if s, ok := d["--aggregateTargetKey"].(string); ok && s != "" {
aggregateTarget = s
}
if s, ok := d["--aggregatetype"].(string); ok && s != "" {
aggregateType = s
switch s {
default:
aggregateCmd = "lpush"
case "list":
aggregateCmd = "lpush"
case "set":
aggregateCmd = "sadd"
}
}
if s, ok := d["--set2sortedkeys"].(string); ok && s != "" && s != "*" {
keys := strings.Split(s, ",")
keyRegexps := make([]*regexp.Regexp, len(keys))
for i, key := range keys {
keyRegexps[i], err = regexp.Compile(key)
if err != nil {
log.PanicError(err, "parse --set2sortedkeys failed")
}
}
set2sortedKey = func(key []byte) bool {
for _, reg := range keyRegexps {
if reg.Match(key) {
return true
}
}
return false
}
}
if s, ok := d["--sorted2setkeys"].(string); ok && s != "" && s != "*" {
keys := strings.Split(s, ",")
keyRegexps := make([]*regexp.Regexp, len(keys))
for i, key := range keys {
keyRegexps[i], err = regexp.Compile(key)
if err != nil {
log.PanicError(err, "parse --sorted2setkeys failed")
}
}
sorted2setKey = func(key []byte) bool {
for _, reg := range keyRegexps {
if reg.Match(key) {
return true
}
}
return false
}
}
if s, ok := d["--filesize"].(string); ok && s != "" {
if len(args.sockfile) == 0 {
log.Panic("please specify --sockfile first")
}
n, err := bytesize.Parse(s)
if err != nil {
log.PanicError(err, "parse --filesize failed")
}
if n <= 0 {
log.Panicf("parse --filesize = %d, invalid number", n)
}
args.filesize = n
} else {
args.filesize = bytesize.GB
}
log.Infof("set ncpu = %d, parallel = %d\n", ncpu, args.parallel)
switch {
case d["decode"].(bool):
new(cmdDecode).Main()
case d["restore"].(bool):
new(cmdRestore).Main()
case d["dump"].(bool):
new(cmdDump).Main()
case d["sync"].(bool):
new(cmdSync).Main()
}
}
func restoreRdbEntry(c redigo.Conn, e *rdb.BinEntry) {
var ttlms uint64
if e.ExpireAt != 0 {
now := uint64(time.Now().Add(args.shift).UnixNano())
now /= uint64(time.Millisecond)
if now >= e.ExpireAt {
ttlms = 1
} else {
ttlms = e.ExpireAt - now
}
}
toText := func(p []byte) string {
var b bytes.Buffer
for _, c := range p {
switch {
case c >= '#' && c <= '~':
b.WriteByte(c)
default:
b.WriteByte('.')
}
}
return b.String()
}
if aggregateKey(e.Key) {
log.Infof("Aggregate key %s", e.Key)
o, err := rdb.DecodeDump(e.Value)
if err != nil {
log.PanicError(err, "decode failed")
}
switch obj := o.(type) {
default:
log.Panicf("unknown object %v", o)
case rdb.List:
for _, ele := range obj {
_, err := c.Do(aggregateCmd, aggregateTarget, toText(ele))
if err != nil {
log.PanicError(err, "aggregate error")
}
}
case rdb.Set:
for _, ele := range obj {
_, err := c.Do(aggregateCmd, aggregateTarget, toText(ele))
if err != nil {
log.PanicError(err, "aggregate error")
}
}
}
}
if set2sortedKey(e.Key) {
o, err := rdb.DecodeDump(e.Value)
if err != nil {
log.PanicError(err, "decode failed")
}
switch obj := o.(type) {
default:
log.Panicf("set2sorted key %s type is not set err", e.Key)
log.Panicf("unknown object %v", o)
case rdb.Set:
for _, ele := range obj {
_, err := c.Do("zadd", e.Key, 1, toText(ele))
if err != nil {
log.PanicErrorf(err, "set2sorted zadd %s 1 %s", e.Key, toText(ele))
}
}
}
return
}
if sorted2setKey(e.Key) {
o, err := rdb.DecodeDump(e.Value)
if err != nil {
log.PanicError(err, "decode failed")
}
switch obj := o.(type) {
default:
log.Panicf("sorted2set key %s type is not sorted set err", e.Key)
case rdb.ZSet:
for _, ele := range obj {
_, err := c.Do("sadd", e.Key, toText(ele.Member))
if err != nil {
log.PanicErrorf(err, "sorted2set sadd %s %s", e.Key, toText(ele.Member))
}
}
}
return
}
if (restoreCmd == "del") || (restoreCmd == "DEL") {
_, err := redigo.String(c.Do(restoreCmd, e.Key))
if err != nil {
log.Warnf("delete key: '%s'", e.Key)
}
} else {
s, err := redigo.String(c.Do(restoreCmd, e.Key, ttlms, e.Value))
if err != nil {
log.Warnf("restore error, when '%s' '%s'", restoreCmd, e.Key)
//log.PanicError(err, "restore command error")
}
if s != "OK" {
//log.Panicf("restore command response = '%s', should be 'OK'", s)
log.Warnf("restore command response = '%s', should be 'OK'", s)
}
}
}